a
    
da,                     @   s   d dl mZ d dlmZmZmZmZmZ d dlZ	d dl
Z
d dlmZmZ d dlmZ d dlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZ G dd de	jZ dS )    )contextmanager)AnyDictListTupleUnionN)
ListConfig	OmegaConf)	load_file)LambdaLR   )UNCONDITIONAL_CONFIG)OPENAIUNETWRAPPER)LitEma)defaultdisabled_trainget_obj_from_strinstantiate_from_configlog_txt_as_imgc                       s  e Zd Zd1edeeef edeeef edeeef edeeef edeeef edef edef ee	e	eee
df eed fddZedd	d
dZdd Zdd Ze dd Ze dd Zdd ZeedddZdd Zdd Zdd Zed2dd Zd!d" Zd#d$ Ze d3eeedf eedee
f d&d'd(Ze eeed)d*d+Z e d4eeee
e ed.d/d0Z!  Z"S )5DiffusionEngineNFH.?      ?jpg)conditioner_configsampler_configoptimizer_configscheduler_configloss_fn_confignetwork_wrapper	ckpt_pathuse_emaema_decay_ratescale_factor	input_keylog_keysno_cond_logcompile_modelc                    s   t    || _|| _t|ddi| _t|}tt|	t||d| _	t|| _
|d ur`t|nd | _tt|t| _|| _| | |d urt|nd | _|| _| jrt| j	|d| _tdtt| j  d || _|| _|| _|
d ur| |
 d S )Ntargetztorch.optim.AdamW)r&   )decayzKeeping EMAs of .)super__init__r$   r#   r   r   r   r   r   modeldenoisersamplerr   conditionerr   _init_first_stageloss_fnr    r   	model_emaprintlenlistbuffersr"   disable_first_stage_autocastr%   init_from_ckpt)selfZnetwork_configZdenoiser_configZfirst_stage_configr   r   r   r   r   r   r   r    r!   r"   r7   r#   r$   r%   r&   r,   	__class__ [/var/www/html/stable-diffusion-webui/repositories/generative-models/sgm/models/diffusion.pyr+      sB    




zDiffusionEngine.__init__)pathreturnc                 C   s   | drtj|ddd }n| dr2t|}nt| j|dd\}}td| d	t| d
t| d t|dkrtd|  t|dkrtd|  d S )Nckptcpu)map_location
state_dictsafetensorsF)strictzRestored from z with z missing and z unexpected keysr   zMissing Keys: zUnexpected Keys: )endswithtorchloadload_safetensorsNotImplementedErrorload_state_dictr3   r4   )r9   r>   sdmissing
unexpectedr<   r<   r=   r8   U   s    


zDiffusionEngine.init_from_ckptc                 C   s0   t | }t|_| D ]
}d|_q|| _d S )NF)r   evalr   train
parametersrequires_gradfirst_stage_model)r9   configr,   paramr<   r<   r=   r0   i   s
    z!DiffusionEngine._init_first_stagec                 C   s
   || j  S Nr#   )r9   batchr<   r<   r=   	get_inputp   s    zDiffusionEngine.get_inputc                 C   sP   d| j  | }tjd| j d | j|}W d    n1 sB0    Y  |S )Nr   cudaenabled)r"   rG   autocastr7   rS   decode)r9   zoutr<   r<   r=   decode_first_stageu   s    *z"DiffusionEngine.decode_first_stagec                 C   sL   t jd| j d | j|}W d    n1 s40    Y  | j| }|S )NrZ   r[   )rG   r]   r7   rS   encoder"   )r9   xr_   r<   r<   r=   encode_first_stage|   s    *
z"DiffusionEngine.encode_first_stagec                 C   s0   |  | j| j| j||}| }d|i}||fS )Nloss)r1   r,   r-   r/   mean)r9   rc   rX   re   Z	loss_mean	loss_dictr<   r<   r=   forward   s    zDiffusionEngine.forward)rX   r?   c                 C   s4   |  |}| |}| j|d< | ||\}}||fS )Nglobal_step)rY   rd   ri   )r9   rX   rc   re   rg   r<   r<   r=   shared_step   s
    


zDiffusionEngine.shared_stepc                 C   sp   |  |\}}| j|ddddd | jd| jddddd | jd url|  jd d }| jd|ddddd |S )NTF)prog_barloggeron_stepon_epochri   r   lrZlr_abs)rj   log_dictlogri   r   
optimizersparam_groups)r9   rX   	batch_idxre   rg   ro   r<   r<   r=   training_step   s$    
	
zDiffusionEngine.training_stepc                 O   s    | j d u s| jd u rtdd S )Nz6Sampler and loss function need to be set for training.)r.   r1   
ValueErrorr9   argskwargsr<   r<   r=   on_train_start   s    zDiffusionEngine.on_train_startc                 O   s   | j r| | j d S rV   )r    r2   r,   rw   r<   r<   r=   on_train_batch_end   s    z"DiffusionEngine.on_train_batch_endc              
   c   s   | j r<| j| j  | j| j |d ur<t| d z8d V  W | j r| j| j  |d urt| d n0| j r| j| j  |d urt| d 0 d S )Nz: Switched to EMA weightsz: Restored training weights)r    r2   storer,   rQ   copy_tor3   restore)r9   contextr<   r<   r=   	ema_scope   s    zDiffusionEngine.ema_scopec                 C   s&   t |d |fd|i|dt S )Nr'   ro   params)r   getdict)r9   r   ro   cfgr<   r<   r=   !instantiate_optimizer_from_config   s    
z1DiffusionEngine.instantiate_optimizer_from_configc                 C   s   | j }t| j }| jjD ]}|jr|t|  }q| ||| j}| j	d urt
| j	}td t||jddddg}|g|fS |S )Nz Setting up LambdaLR scheduler...)	lr_lambdastep   )	schedulerinterval	frequency)learning_rater5   r,   rQ   r/   	embeddersis_trainabler   r   r   r   r3   r   schedule)r9   ro   r   embedderoptr   r<   r<   r=   configure_optimizers   s     


z$DiffusionEngine.configure_optimizers   )conduc
batch_sizeshapec           	         s>   t j|g|R  j} fdd}j||||d}|S )Nc                    s   j j| ||fi  S rV   )r-   r,   )inputsigmacry   r9   r<   r=   <lambda>   s   
z(DiffusionEngine.sample.<locals>.<lambda>)r   )rG   randntodevicer.   )	r9   r   r   r   r   ry   r   r-   samplesr<   r   r=   sample   s    	zDiffusionEngine.sample)rX   nr?   c                    sD  || j  jdd \}}t }| jjD ]}| jdu sB|j | jv r&| js&||j  d|  t tj	r 
 dkr fddt jd D  t||f |d d}nF 
 dkr fd	dt jd D  t||f |d
 d}nt nFt ttfr.t d tr&t||f |d
 d}nt nt |||j < q&|S )z
        Defines heuristics to log different conditionings.
        These can be lists of strings (text-to-image), tensors, ints, ...
        r   Nr   c                    s   g | ]}t  |  qS r<   )stritem.0irc   r<   r=   
<listcomp>       z5DiffusionEngine.log_conditionings.<locals>.<listcomp>r      )sizec                    s(   g | ] }d  dd  |  D qS )rc   c                 S   s   g | ]}t |qS r<   )r   )r   xxr<   r<   r=   r      r   z@DiffusionEngine.log_conditionings.<locals>.<listcomp>.<listcomp>)jointolistr   r   r<   r=   r      s      )r#   r   r   r/   r   r$   r%   
isinstancerG   Tensordimranger   rJ   r   r   r   )r9   rX   r   Zimage_hZimage_wrq   r   xcr<   r   r=   log_conditionings   s6    

z!DiffusionEngine.log_conditionings   T)rX   Nr   ucg_keysr?   c                    s  dd j jD |rBttfdd|sFJ d| d n}t }|}j j|tj jdkrr|ng d\}}	i }
t|j	d   |
jd   }||d	< |}||d
< ||  |D ]:t| tjrt fdd||	f\|< |	< q|rd4 j|f|j	dd  |	 d|
}W d    n1 sd0    Y  |}||d< |S )Nc                 S   s   g | ]
}|j qS r<   rW   )r   er<   r<   r=   r     r   z.DiffusionEngine.log_images.<locals>.<listcomp>c                    s   |  v S rV   r<   r   )conditioner_input_keysr<   r=   r     r   z,DiffusionEngine.log_images.<locals>.<lambda>z]Each defined ucg key for sampling must be in the provided conditioner input keys,but we have z vs. r   )force_uc_zero_embeddingsinputsreconstructionsc                    s   |  d    jS rV   )r   r   )y)r   kr9   r<   r=   r   7  r   ZPlottingr   )r   r   r   r   )r/   r   allmapr   rY   get_unconditional_conditioningr4   minr   r   r   rd   ra   updater   r   rG   r   r   r   )r9   rX   r   r   r   ry   rq   rc   r   r   Zsampling_kwargsr_   r   r<   )r   r   r   r9   r=   
log_images  sP    	


(&
zDiffusionEngine.log_images)NNNNNNNFr   r   Fr   NFF)N)Nr   N)r   TN)#__name__
__module____qualname__r   r   r   r	   r   boolfloatr   r+   r8   r0   rY   rG   no_gradra   rd   rh   r   rj   ru   rz   r{   r   r   r   r   intr   r   r   r   __classcell__r<   r<   r:   r=   r      s                  


@

   
&   r   )!
contextlibr   typingr   r   r   r   r   pytorch_lightningplrG   	omegaconfr   r	   safetensors.torchr
   rI   Ztorch.optim.lr_schedulerr   modulesr   Z!modules.diffusionmodules.wrappersr   Zmodules.emar   utilr   r   r   r   r   LightningModuler   r<   r<   r<   r=   <module>   s   	