a
    
dzZ                    @   s  d Z ddlZddlmZ ddlZddlZddlm	Z	 ddl
mZmZ ddlmZmZ ddlmZ ddlZddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z' ddl(m)Z)m*Z* ddl+m,Z,m-Z-m.Z. ddl/m0Z0 ddddZ1d,ddZ2dd Z3G dd dej4Z5G dd de5Z6G dd dej4Z7G d d! d!e6Z8G d"d# d#e6Z9G d$d% d%e9Z:G d&d' d'e9Z;G d(d) d)e9Z<G d*d+ d+e6Z=dS )-ap  
wild mixture of
https://github.com/lucidrains/denoising-diffusion-pytorch/blob/7706bdfc6f527f58d33f84b7b522e61e6e3164b3/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py
https://github.com/openai/improved-diffusion/blob/e94489283bb876ac1477d5dd7709bbbd2d9902ce/improved_diffusion/gaussian_diffusion.py
https://github.com/CompVis/taming-transformers
-- merci
    N)LambdaLR)	rearrangerepeat)contextmanagernullcontext)partial)tqdm)	make_grid)rank_zero_only)
ListConfig)log_txt_as_imgexistsdefaultismapisimage	mean_flatcount_paramsinstantiate_from_config)LitEma)	normal_klDiagonalGaussianDistribution)IdentityFirstStageAutoencoderKL)make_beta_scheduleextract_into_tensor
noise_like)DDIMSamplerc_concatc_crossattny)concat	crossattnadmTc                 C   s   | S )zbOverwrite model.train with this function to make sure train/eval mode
    does not change anymore. )selfmoder#   r#   l/var/www/html/stable-diffusion-webui/repositories/stable-diffusion-stability-ai/ldm/models/diffusion/ddpm.pydisabled_train$   s    r'   c                 C   s   | | t j|d|i | S )Ndevice)torchrand)r1r2shaper(   r#   r#   r&   uniform_on_device*   s    r.   c                       sv  e Zd Zddddg ddddd	d
dddddddddddddddddddf fdd	ZdJddZedKddZe e	 dfddZ
dd Zdd Zdd Zd d! Zd"d# Zed$d%d&Ze dLd'd(Ze dMd)d*Ze dNd,d-ZdOd.d/Zd0d1 ZdPd2d3ZdQd4d5Zd6d7 Zd8d9 Zd:d; Zd<d= Ze d>d? Zd@dA ZdBdC Ze dRdFdGZ dHdI Z!  Z"S )SDDPM  linearl2NFzval/lossTimage      d   -C6?{Gz?Mb?              ?epsc                      s  t    |dv sJ d|| _t| jj d| j d d | _|| _|| _|
| _	|| _
|| _|| _t||| _t| jdd |	| _| jrt| j| _tdtt| j  d |d u| _| jr|| _|| _|| _|| _|d ur|| _|| _|rt|sJ |d urD| j|||d	 |rD| js0J td
 t| j| _|rhtd | js^J | j  | j ||||||d || _!|| _"t#j$|| j%fd| _&| j"rt'j(| j&dd| _&|pt) | _*| j*rt+j,- | _.d S )N)r<   x0vz0currently only supporting "eps" and "x0" and "v"z: Running in z-prediction modeT)verbosezKeeping EMAs of .)ignore_keys
only_model_Resetting ema to pure model weights. This is useful when restoring from an ema-only checkpoint.D +++++++++++ WARNING: RESETTING NUM_EMA UPDATES TO ZERO +++++++++++ )given_betasbeta_schedule	timestepslinear_start
linear_endcosine_s)
fill_valuesize)requires_grad)/super__init__parameterizationprint	__class____name__cond_stage_modelclip_denoisedlog_every_tfirst_stage_key
image_sizechannelsuse_positional_encodingsDiffusionWrappermodelr   use_emar   	model_emalenlistbuffersuse_schedulerscheduler_configv_posteriororiginal_elbo_weightl_simple_weightmonitormake_it_fitr   init_from_ckptreset_num_updatesregister_schedule	loss_typelearn_logvarr)   fullnum_timestepslogvarnn	Parameterdictucg_trainingnprandomRandomStateucg_prng) r$   Zunet_configrG   rF   rl   	ckpt_pathrA   Zload_only_unetrg   r]   rW   rX   rY   rV   rU   rH   rI   rJ   rE   re   rd   rf   conditioning_keyrP   rc   rZ   rm   Zlogvar_initrh   rt   	reset_emareset_num_ema_updatesrR   r#   r&   rO   0   s`    !




zDDPM.__init__c              
   C   s  t |r|}nt|||||d}d| }tj|dd}	td|	d d }
|j\}t|| _|| _|| _	|	jd | jksJ dt
tjtjd}| d|| | d	||	 | d
||
 | d|t|	 | d|td|	  | d|td|	  | d|td|	  | d|td|	 d  d| j | d|
  d|	  | j|  }| d|| | d|tt|d | d||t|
 d|	   | d|d|
 t| d|	   | jdkr| jd d| j || d| j   }nr| jdkrDdtt|	 dt|	  }nB| jdkr~t| jd d| j || d| j   }ntd|d |d< | jd|dd t| j rJ d S ) N)rH   rI   rJ   r;   r   )axisz+alphas have to be defined for each timestep)dtypebetasalphas_cumprodalphas_cumprod_prevsqrt_alphas_cumprodsqrt_one_minus_alphas_cumprodlog_one_minus_alphas_cumprodsqrt_recip_alphas_cumprodsqrt_recipm1_alphas_cumprod   posterior_varianceposterior_log_variance_clippedg#B;posterior_mean_coef1posterior_mean_coef2r<      r=         ?       @r>   zmu not supportedlvlb_weightsF)
persistent)r   r   ru   cumprodappendr-   intro   rH   rI   r   r)   tensorfloat32register_buffersqrtlogrd   maximumrP   r   r   r   Tensor	ones_likeNotImplementedErrorisnanr   all)r$   rE   rF   rG   rH   rI   rJ   r   alphasr   r   to_torchr   r   r#   r#   r&   rk      sb    

$zDDPM.register_schedulec              
   c   s   | j r<| j| j  | j| j |d ur<t| d z8d V  W | j r| j| j  |d urt| d n0| j r| j| j  |d urt| d 0 d S )Nz: Switched to EMA weightsz: Restored training weights)r]   r^   storer\   
parameterscopy_torQ   restore)r$   contextr#   r#   r&   	ema_scope   s    zDDPM.ema_scopec                 C   s  t j|dd}dt| v r&|d }t| }|D ],}|D ]"}||r>td| ||= q>q6| jrztdd t	
|  |  D }tt	
|  |  d|dD ]\}	}
|	|vrq||	 j}|
j}t|t|ksJ t|d	kr|d	d  |d	d  ksJ ||ks|
 }||	 }t|d
krdt|jd D ]}|||d   ||< qDnt|d	krpt|jd D ]@}t|jd
 D ]*}|||d  ||d
  f |||f< qqt |d
 }t|jd
 D ]}|||d
    d
7  < qt |d
 }t|jd
 D ]}|||d
   ||< q|d d d f }t|jt|k rh|d}qF|| }|||	< q|s| j|ddn| jj|dd\}}td| dt| dt| d t|dkrtd|  t|dkrtd|  d S )Ncpumap_location
state_dict Deleting key {} from state_dict.c                 S   s   g | ]\}}|qS r#   r#   ).0name_r#   r#   r&   
<listcomp>       z'DDPM.init_from_ckpt.<locals>.<listcomp>z"Fitting old weights to new weightsdesctotalr   r   r   r   FstrictRestored from  with  missing and  unexpected keyszMissing Keys:
 z
Unexpected Keys:
 )r)   loadr`   keys
startswithrQ   formatrh   r_   	itertoolschainnamed_parametersnamed_buffersr   r-   clonerangeoneszeros	unsqueezeload_state_dictr\   )r$   pathrA   rB   sdr   kikZn_paramsr   param	old_shape	new_shape	new_param	old_paramijZ
n_used_oldZ
n_used_newmissing
unexpectedr#   r#   r&   ri      st    




,

$zDDPM.init_from_ckptc                 C   sB   t | j||j| }t d| j ||j}t | j||j}|||fS )a  
        Get the distribution q(x_t | x_0).
        :param x_start: the [N x C x ...] tensor of noiseless inputs.
        :param t: the number of diffusion steps (minus 1). Here, 0 means one step.
        :return: A tuple (mean, variance, log_variance), all of x_start's shape.
        r;   )r   r   r-   r   r   )r$   x_starttmeanvarianceZlog_variancer#   r#   r&   q_mean_variance  s    zDDPM.q_mean_variancec                 C   s(   t | j||j| t | j||j|  S Nr   r   r-   r   )r$   x_tr   noiser#   r#   r&   predict_start_from_noise  s    zDDPM.predict_start_from_noisec                 C   s(   t | j||j| t | j||j|  S r   r   r   r-   r   r$   r   r   r>   r#   r#   r&   predict_start_from_z_and_v   s    zDDPM.predict_start_from_z_and_vc                 C   s(   t | j||j| t | j||j|  S r   r   r   r#   r#   r&   predict_eps_from_z_and_v(  s    zDDPM.predict_eps_from_z_and_vc                 C   sR   t | j||j| t | j||j|  }t | j||j}t | j||j}|||fS r   )r   r   r-   r   r   r   )r$   r   r   r   Zposterior_meanr   r   r#   r#   r&   q_posterior.  s    zDDPM.q_posteriorrU   c           	      C   sf   |  ||}| jdkr(| j|||d}n| jdkr6|}|rF|dd | j|||d\}}}|||fS )Nr<   r   r   r=         r;   r   r   r   )r\   rP   r   clamp_r   )	r$   xr   rU   	model_outx_recon
model_meanr   posterior_log_variancer#   r#   r&   p_mean_variance7  s    

zDDPM.p_mean_variancec                 C   s   g |j |jR ^}}}| j|||d\}}}	t|j ||}
d|dk  j|gdt|j d  R  }||d|	   |
  S )N)r   r   rU   r   r   r   r   )r-   r(   r   r   floatreshaper_   exp)r$   r   r   rU   repeat_noisebr   r(   r   model_log_variancer   nonzero_maskr#   r#   r&   p_sampleC  s
    ,zDDPM.p_samplec              	   C   s   | j j}|d }tj||d}|g}tttd| jd| jdD ]N}| j|tj	|f||tj
d| jd}|| j dks|| jd kr@|| q@|r||fS |S )Nr   r(   
Sampling tr   r(   r   r   r   )r   r(   r)   randnr   reversedr   ro   r   rn   longrU   rV   r   )r$   r-   return_intermediatesr(   r   imgintermediatesr   r#   r#   r&   p_sample_loopL  s     zDDPM.p_sample_loop   c                 C   s"   | j }| j}| j||||f|dS )N)r   )rX   rY   r  )r$   
batch_sizer   rX   rY   r#   r#   r&   sample[  s
    zDDPM.samplec                    s:   t | fdd}t| j| j  t| j| j|  S )Nc                      s
   t  S r   r)   
randn_liker#   r   r#   r&   <lambda>c  r   zDDPM.q_sample.<locals>.<lambda>)r   r   r   r-   r   )r$   r   r   r   r#   r	  r&   q_sampleb  s    zDDPM.q_samplec                 C   s(   t | j||j| t | j||j|  S r   r   )r$   r   r   r   r#   r#   r&   get_vg  s    z
DDPM.get_vc                 C   sf   | j dkr$||  }|rb| }n>| j dkrZ|rDtjj||}qbtjjj||dd}ntd|S )Nl1r2   none)	reductionzunknown loss type '{loss_type}')rl   absr   r)   rq   
functionalmse_lossr   )r$   predtargetr   lossr#   r#   r&   get_lossm  s    


zDDPM.get_lossc                    s  t | fdd}| j ||d}| ||}i }| jdkrB|}n<| jdkrR }n,| jdkrl|  ||}ntd| j d| j||d	d
jg dd}| jrdnd}	|	|	 d| i | | j
 }
| j| |  }|	|	 d|i |
| j|  }|	|	 d|i ||fS )Nc                      s
   t  S r   r  r#   r	  r#   r&   r
  }  r   zDDPM.p_losses.<locals>.<lambda>r   r   r   r<   r=   r>   zParameterization z not yet supportedFr   r   r   r5   dimtrainval/loss_simple	/loss_vlb/loss)r   r  r\   rP   r  r   r  r   trainingupdaterf   r   re   )r$   r   r   r   x_noisyr   	loss_dictr  r  Z
log_prefixloss_simpleloss_vlbr#   r	  r&   p_losses|  s(    


zDDPM.p_lossesc                 O   s<   t jd| j|jd f| jd }| j||g|R i |S )Nr   r   )r)   randintro   r-   r(   r   r'  )r$   r   argskwargsr   r#   r#   r&   forward  s    "zDDPM.forwardc                 C   s>   || }t |jdkr|d }t|d}|jtjd }|S )Nr5   ).Nb h w c -> b c h wmemory_format)r_   r-   r   tor)   contiguous_formatr   )r$   batchr   r   r#   r#   r&   	get_input  s    
zDDPM.get_inputc                 C   s"   |  || j}| |\}}||fS r   r2  rW   )r$   r1  r   r  r$  r#   r#   r&   shared_step  s    zDDPM.shared_stepc           
      C   s   | j D ]f}| j | d }| j | d }|d u r2d}tt|| D ](}| jjdd| |gdrB||| |< qBq| |\}}| j|ddddd | jd	| jdddd
d | j	r| 
 jd d }	| jd|	dddd
d |S )Npr   r   r   r5  Tprog_barloggeron_stepon_epochglobal_stepFr   lrlr_abs)rt   r   r_   rx   choicer4  log_dictr   r=  rb   
optimizersparam_groups)
r$   r1  	batch_idxr   r5  r  r   r  r$  r>  r#   r#   r&   training_step  s&    

zDDPM.training_stepc                    s   |  |\}}|  0 |  |\}  fdd D  W d    n1 sL0    Y  | j|ddddd | j ddddd d S )Nc                    s   i | ]}|d   | qS )_emar#   r   keyZloss_dict_emar#   r&   
<dictcomp>  r   z(DDPM.validation_step.<locals>.<dictcomp>FTr8  )r4  r   rA  )r$   r1  rD  r   Zloss_dict_no_emar#   rI  r&   validation_step  s    
0zDDPM.validation_stepc                 O   s   | j r| | j d S r   )r]   r^   r\   )r$   r)  r*  r#   r#   r&   on_train_batch_end  s    zDDPM.on_train_batch_endc                 C   s,   t |}t|d}t|d}t||d}|S )Nn b c h w -> b n c h wb n c h w -> (b n) c h wnrow)r_   r   r	   )r$   samplesn_imgs_per_rowdenoise_gridr#   r#   r&   _get_rows_from_list  s
    

zDDPM._get_rows_from_list   r   c                    s  t   | || j}t|jd |}t|jd |}|| jd | }| d< t }|d | }	t| j	D ]j}
|
| j
 dks|
| j	d krltt|
gd|d}
|
| j }
t|	}| j|	|
|d}|| ql| | d< |r@| d" | j|d	d
\}}W d    n1 s 0    Y  | d< | | d< |r|tt  |jd dkrj S  fdd|D S  S )Nr   inputsr   1 -> br   r  diffusion_rowPlottingT)r  r   rQ  denoise_rowc                    s   i | ]}| | qS r#   r#   rG  r   r#   r&   rJ    r   z#DDPM.log_images.<locals>.<dictcomp>)rs   r2  rW   minr-   r/  r(   r`   r   ro   rV   r   r)   r   r   r  r  r   rT  r   r  ru   intersect1dr   )r$   r1  Nn_rowr  return_keysr*  r   rY  r   r   r   r#  rQ  r[  r#   r\  r&   
log_images  s4    
2 zDDPM.log_imagesc                 C   s:   | j }t| j }| jr&|| jg }tjj||d}|S )Nr>  )	learning_rater`   r\   r   rm   rp   r)   optimAdamW)r$   r>  paramsoptr#   r#   r&   configure_optimizers   s    zDDPM.configure_optimizers)Nr1   r0   r7   r8   r9   )N)TF)F)r  F)N)T)N)rU  r   TN)#rS   
__module____qualname__rO   rk   r   r   r)   no_gradr`   ri   r   r   r   r   r   boolr   r   r  r  r  r  r  r'  r+  r2  r4  rE  rK  rL  rT  rb  ri  __classcell__r#   r#   r}   r&   r/   .   s   X  
8>	



%r/   c                       s  e Zd ZdZdT fdd	Zd	d
 Zee  fddZ	dU fdd	Z
dd Zdd ZdVddZdd Zdd Zdd  Zd!d" Zd#d$ ZdWd&d'Ze dX fd(d)	Ze dYd*d+Ze d,d- Zd.d/ Zd0d1 ZdZd2d3Zd4d5 Zd6d7 Zd[d8d9Zd\ed:d;d<Ze d]d>d?Ze d^d@dAZ e d_dBdCZ!e d`dEdFZ"e dGdH Z#e dadIdJZ$e dbdNdOZ%dPdQ Z&e dRdS Z'  Z(S )cLatentDiffusionz
main classNr3   FTr;   c                    s  || _ t|d| _|
| _| j|d ks*J |d u r>|r:dnd}|dkrP| j sPd }|dd }|dd}|d	d}|d
g }t j|d|i| || _|| _|| _	zt
|jjjd | _W n   d| _Y n0 |
s|	| _n| dt|	 | | | | || _d| _d | _d| _|d ur`| || d| _|r`| jsLJ td t| j| _|rtd | jszJ | j  d S )Nr   rG   r    r!   __is_unconditional__ry   r{   Fr|   rA   rz   r   scale_factorTrC   rD   ) force_null_conditioningr   num_timesteps_condscale_by_stdpoprN   rO   concat_modecond_stage_trainablecond_stage_keyr_   rg  ddconfigch_multZ	num_downsrq  r   r)   r   instantiate_first_stageinstantiate_cond_stagecond_stage_forwardrU   Zbbox_tokenizerrestarted_from_ckptri   r]   rQ   r   r\   r^   rj   )r$   first_stage_configZcond_stage_configrs  rx  rw  rv  r}  rz   rq  rt  rr  r)  r*  ry   r{   r|   rA   r}   r#   r&   rO     sT    


zLatentDiffusion.__init__c                 C   sR   t j| jf| jd t jd| _t t d| jd | j }|| jd | j< d S )Nr   )rL   rK   r   r   )r)   rn   ro   r   cond_idsroundlinspacers  )r$   idsr#   r#   r&   make_cond_scheduleF  s     z"LatentDiffusion.make_cond_schedulec                    s   | j r| jdkr| jdkr|dkr| js| jdks:J dtd t || j}|	| j
}| |}| | }| `| dd|    td| j  td d S )Nr   r;   z@rather not use custom rescaling and std-rescaling simultaneouslyz### USING STD-RESCALING ###rq  zsetting self.scale_factor to )rt  current_epochr=  r~  rq  rQ   rN   r2  rW   r/  r(   encode_first_stageget_first_stage_encodingdetachr   flattenstd)r$   r1  rD  dataloader_idxr   encoder_posteriorzr}   r#   r&   on_train_batch_startK  s    (
z$LatentDiffusion.on_train_batch_startr1   r0   r7   r8   r9   c                    s4   t  |||||| | jdk| _| jr0|   d S )Nr   )rN   rk   rs  shorten_cond_scheduler  )r$   rE   rF   rG   rH   rI   rJ   r}   r#   r&   rk   \  s    z!LatentDiffusion.register_schedulec                 C   s4   t |}| | _t| j_| j D ]
}d|_q$d S NF)r   evalfirst_stage_modelr'   r  r   rM   r$   configr\   r   r#   r#   r&   r{  e  s
    
z'LatentDiffusion.instantiate_first_stagec                 C   s   | j sv|dkr td | j| _q|dkrDtd| jj d d | _qt|}| | _t| j_	| j
 D ]
}d|_qhn&|dksJ |dksJ t|}|| _d S )N__is_first_stage__z%Using first stage also as cond stage.rp  z	Training z as an unconditional model.F)rw  rQ   r  rT   rR   rS   r   r  r'   r  r   rM   r  r#   r#   r&   r|  l  s     


z&LatentDiffusion.instantiate_cond_stager6  c                 C   sh   g }t ||dD ] }|| j|| j|d qt|}t|}t|d}t|d}t	||d}|S )Nr   )force_not_quantizerM  rN  rO  )
r   r   decode_first_stager/  r(   r_   r)   stackr   r	   )r$   rQ  r   Zforce_no_decoder_quantizationr[  ZzdrR  rS  r#   r#   r&   _get_denoise_row_from_list  s    



z*LatentDiffusion._get_denoise_row_from_listc                 C   sD   t |tr| }n&t |tjr&|}ntdt| d| j| S )Nzencoder_posterior of type 'z' not yet implemented)
isinstancer   r  r)   r   r   typerq  )r$   r  r  r#   r#   r&   r    s    

z(LatentDiffusion.get_first_stage_encodingc                 C   sv   | j d u rNt| jdrBt| jjrB| j|}t|trL| }qr| |}n$t| j| j s`J t| j| j |}|S )Nencode)	r}  hasattrrT   callabler  r  r   r%   getattr)r$   cr#   r#   r&   get_learned_conditioning  s    


z(LatentDiffusion.get_learned_conditioningc                 C   sV   t d||ddd|d}t d|d|d|dd}t j||gdd}|S )Nr   r   r   r  )r)   arangeviewr   cat)r$   hwr   r   arrr#   r#   r&   meshgrid  s      zLatentDiffusion.meshgridc                 C   s   t |d |d gddd}| ||| }t j|dddd }t jd| dddd }t jt j||gddddd }|S )z
        :param h: height
        :param w: width
        :return: normalized distance to image border,
         wtith min distance = 0 at border and max dist = 0.5 at image center
        r   r   r   T)r  keepdimsr   r  )r)   r   r  r  r]  r  )r$   r  r  Zlower_right_cornerr  Zdist_left_upZdist_right_downZ	edge_distr#   r#   r&   delta_border  s      zLatentDiffusion.delta_borderc                 C   s   |  ||}t|| jd | jd }|d|| ddd|| |}| jd r|  ||}t|| jd | jd }|dd|| |}|| }|S )NZclip_min_weightZclip_max_weightr   Z
tie_brakerZclip_min_tie_weightZclip_max_tie_weight)r  r)   clipZsplit_input_paramsr  r   r/  )r$   r  r  LyLxr(   	weightingZL_weightingr#   r#   r&   get_weighting  s    &
zLatentDiffusion.get_weightingr   c                 C   s  |j \}}}}	||d  |d  d }
|	|d  |d  d }|dkr|dkrt|dd|d}tjjf i |}tjjf d|j dd i|}| |d |d |
||j|j	}||
dd||	}|
dd|d |d |
| f}n"|dkr|dkrt|dd|d}tjjf i |}t|d | |d | fdd|d | |d | fd}tjjf d|j d | |j d | fi|}| |d | |d | |
||j|j	}||
dd|| |	| }|
dd|d | |d | |
| f}n|dkr
|dkr
t|dd|d}tjjf i |}t|d | |d | fdd|d | |d | fd}tjjf d|j d | |j d | fi|}| |d | |d | |
||j|j	}||
dd|| |	| }|
dd|d | |d | |
| f}nt||||fS )z
        :param x: img of size (bs, c, h, w)
        :return: n img crops of size (n, bs, c, kernel_size[0], kernel_size[1])
        r   r   )kernel_sizedilationpaddingstrideoutput_sizer   Nr5   )r-   rs   r)   rq   UnfoldFoldr  r(   r/  r   r  r   )r$   r   r  r  ufdfbsncr  r  r  r  Zfold_paramsunfoldfoldr  normalizationZfold_params2r#   r#   r&   get_fold_unfold  sD     $$.,,.,*zLatentDiffusion.get_fold_unfoldc	                    s  t  ||}	|d ur"|	d | }	|	| j}	| |	}
| |
 }| jjd ur<| j	s<|d u rj| j
}|| jkr|dv r|| }q|dv r|}qt  ||| j}n|	}| jr|rt|tst|tr| |}q| || j}n|}|d ur|d | }| jrd| |\}}t| jj }||d|d|i}n(d }d }| jrd| |\}}||d}||g}|r| |}||	|g |r||	g |r|| |S )N)captionZcoordinates_bboxtxtclass_labelclspos_xpos_y)r  r  )rN   r2  r/  r(   r  r  r  r\   rz   rr  rx  rW   rw  r  rs   r`   r  rZ   Zcompute_latent_shifts__conditioning_keys__r  extendr   )r$   r1  r   return_first_stage_outputsforce_c_encodecond_keyreturn_original_condr  Zreturn_xr   r  r  xcr  r  r  Zckeyoutxrecr}   r#   r&   r2    sR    







zLatentDiffusion.get_inputc                 C   s`   |rF|  dkr&tj| dd }| jjj|d d}t|d	 }d| j
 | }| j|S )N   r   r  )r-   r,  r;   )r  r)   argmaxr   r   r  quantizeZget_codebook_entryr   
contiguousrq  decode)r$   r  Zpredict_cidsr  r#   r#   r&   r  1  s    z"LatentDiffusion.decode_first_stagec                 C   s   | j |S r   )r  r  r$   r   r#   r#   r&   r  <  s    z"LatentDiffusion.encode_first_stagec                 K   s    |  || j\}}| ||}|S r   r3  )r$   r1  r*  r   r  r  r#   r#   r&   r4  @  s    
zLatentDiffusion.shared_stepc                 O   s   t jd| j|jd f| jd }| jjd ur||d us:J | jrJ| 	|}| j
r|| j| | j}| j||t | d}| j|||g|R i |S )Nr   r   r  )r)   r(  ro   r-   r(   r   r\   rz   rw  r  r  r  r/  r  r  r   r'  )r$   r   r  r)  r*  r   tcr#   r#   r&   r+  E  s    "
zLatentDiffusion.forwardc                 C   sj   t |trn,t |ts|g}| jjdkr,dnd}||i}| j||fi |}t |trb|sb|d S |S d S )Nr    r   r   r   )r  rs   r`   r\   rz   tuple)r$   r#  r   cond
return_idsrH  r   r#   r#   r&   apply_modelP  s    

zLatentDiffusion.apply_modelc                 C   s(   t | j||j| | t | j||j S r   r   )r$   r   r   Zpred_xstartr#   r#   r&   _predict_eps_from_xstarta  s    z(LatentDiffusion._predict_eps_from_xstartc                 C   sZ   |j d }tj| jd g| |jd}| ||\}}}t||ddd}t|t	d S )a;  
        Get the prior KL term for the variational lower-bound, measured in
        bits-per-dim.
        This term can't be optimized, as it only depends on the encoder.
        :param x_start: the [N x C x ...] tensor of inputs.
        :return: a batch of [N] KL values (in bits), one per batch element.
        r   r   r   r:   )mean1logvar1mean2logvar2r   )
r-   r)   r   ro   r(   r   r   r   ru   r   )r$   r   r  r   Zqt_meanr   Zqt_log_varianceZkl_priorr#   r#   r&   
_prior_bpde  s
    
zLatentDiffusion._prior_bpdc                    s  t | fdd}| j ||d}| |||}i }| jr>dnd}| jdkrR }	n0| jdkrb|}	n | jdkr||  ||}	nt | j||	d	d
g d}
|	| d|
 i | j
| | j}|
t| | }| jr|	| d| i |	d| j
j i | j|  }| j||	d	d
jdd}| j| |  }|	| d|i || j| 7 }|	| d|i ||fS )Nc                      s
   t  S r   r  r#   r	  r#   r&   r
  t  r   z*LatentDiffusion.p_losses.<locals>.<lambda>r  r  r  r=   r<   r>   Fr  r  r  z/loss_gammarp   r  r  r   )r   r  r  r!  rP   r  r   r  r   r"  rp   r/  r(   r)   r   rm   datarf   r   re   )r$   r   r  r   r   r#  model_outputr$  prefixr  r%  Zlogvar_tr  r&  r#   r	  r&   r'  s  s4    


zLatentDiffusion.p_lossesr   c
                 C   s   |}
| j ||
||d}|d urF| jdks,J |j| ||||fi |	}|rR|\}}| jdkrn| j|||d}n| jdkr~|}nt |r|dd |r| j|\}}\}}}| j|||d\}}}|r||||fS |r||||fS |||fS d S )N)r  r<   r   r=   r   r;   r   )	r  rP   Zmodify_scorer   r   r   r  r  r   )r$   r   r  r   rU   return_codebook_idsquantize_denoised	return_x0score_correctorcorrector_kwargsZt_inr   logitsr   r   indicesr   r   r   r#   r#   r&   r     s,    

zLatentDiffusion.p_mean_variancer:   c                 C   s*  g |j |jR ^}}}| j|||||||||d	}|rPtd|\}}}}n|rb|\}}}}n
|\}}}t|j |||	 }|
dkrtjjj||
d}d|dk	  j
|gdt|j d  R  }|r||d|   |  |jdd	fS |r||d|   |  |fS ||d|   |  S d S )
N)	r   r  r   rU   r  r  r  r  r  zSupport dropped.r:   r7  r   r   r   r   r  )r-   r(   r   DeprecationWarningr   r)   rq   r  dropoutr   r   r_   r   r  )r$   r   r  r   rU   r   r  r  r  temperaturenoise_dropoutr  r  r   r   r(   outputsr   r   r  r=   r   r   r#   r#   r&   r     s,    
,$zLatentDiffusion.p_samplec                    s"  |s
| j }| j} d ur< d ur$ n|d } gt| }n|d  } |d u rbtj|| jd}n|}g }d urttr fddD n(ttr fddD n
d   |d urt||}|rt	t
td|d|dnt
td|}t|	tkr|	g| }	|D ]}tj|f|| jtjd	}| jrr| jjd
ksJJ | j| j}| j|td| j||| j|d|	| |
||d
\}}|d ur|d usJ | ||}|| d| |  }|| dks||d kr|| |r|| |r||| q||fS )Nr   r   c                    sF   i | ]>}|t | ts(| d   ntt fdd| qS )Nc                    s   | d   S r   r#   r   r  r#   r&   r
    r   zBLatentDiffusion.progressive_denoising.<locals>.<dictcomp>.<lambda>r  r`   maprG  r  r  r#   r&   rJ    s    z9LatentDiffusion.progressive_denoising.<locals>.<dictcomp>c                    s   g | ]}|d   qS r   r#   r   r  r  r#   r&   r     r   z9LatentDiffusion.progressive_denoising.<locals>.<listcomp>Progressive Generationr   r   hybridr  T)rU   r  r  r  r  r  r  r;   r   )rV   ro   r`   r)   r   r(   r  rs   r]  r   r   r   r  r   rn   r   r  r\   rz   r  r/  r  r  r   rU   r   )r$   r  r-   r?   callbackr  img_callbackmaskr=   r  r  r  r  r  x_Tstart_TrV   rG   r   r  r  iteratorr   tsr  Z
x0_partialimg_origr#   r  r&   progressive_denoising  sd    
(






z%LatentDiffusion.progressive_denoisingc                 C   s  |s
| j }| jj}|d }|d u r2tj||d}n|}|g}|d u rJ| j}|d ur\t||}|rxttt	d|d|dntt	d|}|	d ur|
d usJ |
j
dd |	j
dd ksJ |D ]}tj|f||tjd}| jr| jjdksJ | j| |j}| j||t|d	}| j|||| j|d
}|	d urX| |
|}||	 d|	 |  }|| dkst||d kr~|| |r|| |r||| q|r||fS |S )Nr   r   r   r   r   r5   r   r  r  )rU   r  r;   r   )rV   r   r(   r)   r   ro   r]  r   r   r   r-   rn   r   r  r\   rz   r  r/  r  r  r   rU   r   )r$   r  r-   r   r  r?   r  rG   r  r  r=   r  r  rV   r(   r   r  r  r  r   r  r  r  r#   r#   r&   r    sL    
 


zLatentDiffusion.p_sample_loopr  c                    s   |
d u r | j | j| jf}
d urjttrB fddD n(ttr^ fddD n
d   | j|
|||||||	d	S )Nc                    sF   i | ]>}|t | ts(| d   ntt fdd| qS )Nc                    s   | d   S r   r#   r  r  r#   r&   r
  J  r   z3LatentDiffusion.sample.<locals>.<dictcomp>.<lambda>r  rG  r  r#   r&   rJ  I  s    z*LatentDiffusion.sample.<locals>.<dictcomp>c                    s   g | ]}|d   qS r   r#   r  r  r#   r&   r   L  r   z*LatentDiffusion.sample.<locals>.<listcomp>)r   r  r?   rG   r  r  r=   )rY   rX   r  rs   r`   r  )r$   r  r  r   r  r?   rG   r  r  r=   r-   r*  r#   r  r&   r  A  s    
(zLatentDiffusion.samplec           
      K   sb   |r>t | }| j| j| jf}|j||||fddi|\}}	n| jf ||dd|\}}	||	fS )Nr?   FT)r  r  r   )r   rY   rX   r  )
r$   r  r  ddim
ddim_stepsr*  Zddim_samplerr-   rQ  r  r#   r#   r&   
sample_logS  s     

zLatentDiffusion.sample_logc                 C   s   |d ur`|}t |trt|}t |ts2t |tr>| |}qt|drT|| j}| |}n.| jdv r| j	j
|| jd}| |S tdt |trtt|D ]"}t|| d|d| j||< qnt|d|d| j}|S )Nr/  r  r   todoz1 ... -> b ...rX  )r  r   r`   rs   r  r  r/  r(   rx  rT   get_unconditional_conditioningr   r   r_   r   )r$   r  Z
null_labelr  r  r   r#   r#   r&   r  a  s$    




"z.LatentDiffusion.get_unconditional_conditioningrU  r  2   c           -   
      s  |r
| j nt}|d u}t  | j|| jddd|d\}}}}}t|jd |}t|jd |}| d< | d< | jjd urft	| j
dr| j
|}| d< n| jdv rt|jd	 |jd
 f|| j |jd	 d d}| d< nn| jdv r<z8t|jd	 |jd
 f|d |jd	 d d}| d< W n ty8   Y n0 nt|rN| d< t|rf| | d< |r8t }|d | }t| jD ]v}|| j dks|| jd krtt|gd|d}|| j }t|}| j|||d}|| | qt |}t!|d}t!|d}t"||jd d}| d< |r*|d( | j#|||||d\}}W d    n1 sv0    Y  | |} |  d< |
r| $|}!|! d< |r*t%| j&t's*t%| j&t(s*|d* | j#|||||dd\}}W d    n1 s0    Y  | || j} |  d< |dkr| )||}"| jjd kr^|"g|d! d"}"|d#F | j#|||||||"d$\}#}$| |#}%|% d%|d&< W d    n1 s0    Y  |	r |jd |jd	 |jd
   }&}'}(t*||'|(| j})d'|)d d |'d( d
|' d( |(d( d
|( d( f< |)d d d d)f })|d*4 | j#||||||d | |)d+\}}$W d    n1 s0    Y  | || j} |  d,< |) d-< d|) })|d.4 | j#||||||d | |)d+\}}$W d    n1 s0    Y  | || j} |  d/< |r|d00 | j+|| j,| j-| j-f|d1\}*}+W d    n1 sf0    Y  | j$|+d2d3},|, d4< |rt./t 0 |jd dkr S  fd5d6|D S  S )7NTr  r  r  r  r   rV  reconstructionr  conditioningr  r  r   r5      rL   r  human_labeloriginal_conditioningr   rW  rX  r  rM  rN  rO  rY  Samplingr  r  r  r  etarQ  r[  zPlotting Quantized Denoised)r  r  r  r  r  r  Zsamples_x0_quantizedr;   crossattn-admc_admr   r  &Sampling with classifier-free guidancer  r  r  r  r  unconditional_guidance_scaleZunconditional_conditioningsamples_cfg_scale_.2fr:   r  .zPlotting Inpaint)r  r  r  r  r  r=   r  Zsamples_inpaintingr  zPlotting OutpaintZsamples_outpaintingPlotting Progressivesr-   r  r  r  progressive_rowc                    s   i | ]}| | qS r#   r#   rG  r\  r#   r&   rJ    r   z.LatentDiffusion.log_images.<locals>.<dictcomp>)1r   r   rs   r2  rW   r]  r-   r\   rz   r  rT   r  rx  r   KeyErrorr   r   to_rgbr`   r   ro   rV   r   r)   r   r/  r(   r   r  r  r   r  r  r   r	   r  r  r  r  r   r   r  r   r   rY   rX   ru   r^  r   )-r$   r1  r_  r`  r  r  ddim_etara  r  inpaintplot_denoise_rowsplot_progressive_rowsplot_diffusion_rowsr  unconditional_guidance_labeluse_ema_scoper*  r   use_ddimr  r  r   r  r  rY  z_startr   r   z_noisydiffusion_gridrQ  z_denoise_row	x_samplesrS  ucsamples_cfgr   x_samples_cfgr   r  r  r  r  progressivesprog_rowr#   r\  r&   rb  z  s    


.
,

 





*



*




0"2
*
*
* zLatentDiffusion.log_imagesc                 C   s   | j }t| j }| jr>t| jj d |t| j  }| j	rXtd |
| j tjj||d}| jrd| jv s|J t| j}td t||jdddd	g}|g|fS |S )
Nz%: Also optimizing conditioner params!z!Diffusion model optimizing logvarrc  r  z Setting up LambdaLR scheduler...)	lr_lambdastepr   )	schedulerinterval	frequency)rd  r`   r\   r   rw  rQ   rR   rS   rT   rm   r   rp   r)   re  rf  rb   rc   r   r   schedule)r$   r>  rg  rh  r3  r#   r#   r&   ri    s(    

z$LatentDiffusion.configure_optimizersc                 C   sj   |  }t| ds0td|jd dd|| _tjj	|| jd}d||
   | |
   d }|S )Ncolorizer5   r   )weightr   r;   )r   r  r)   r   r-   r/  r7  rq   r  conv2dr]  maxr  r#   r#   r&   r    s    
$zLatentDiffusion.to_rgb)	Nr3   FTNNr;   FF)Nr1   r0   r7   r8   r9   )r6  F)r   r   )FFNFNF)FF)F)N)FFFNN)	FFFFFr;   r:   NN)TNFNNNr;   r:   NNNNNN)FNTNNFNNNNN)	r  FNTNFNNN)N)rU  r  Tr  r:   NTTFTTr;   NT))rS   rj  rk  __doc__rO   r  r
   r)   rl  r  rk   r{  r|  r  r  r  r  r  r  r  r2  r  r  r4  r+  r  r  r  r'  rm  r   r   r   r  r  r  r  rb  ri  r  rn  r#   r#   r}   r&   ro  	  s            :  	
	
4  4



%         7    2   
     ro  c                       s.   e Zd Z fddZdeedddZ  ZS )r[   c                    s:   t    |dd| _t|| _|| _| jdv s6J d S )NZsequential_crossattnF)Nr    r!   r  r"   
hybrid-admr  )rN   rO   ru  sequential_cross_attnr   diffusion_modelrz   )r$   Zdiff_model_configrz   r}   r#   r&   rO     s
    

zDiffusionWrapper.__init__Nr   r   c           	      C   s  | j d u r| ||}nj| j dkrHtj|g| dd}| ||}n<| j dkr| jsft|d}n|}t| dr| |||}n| j|||d}n| j dkrtj|g| dd}t|d}| j|||d}n| j dkr|d usJ tj|g| dd}t|d}| j||||d	}nf| j d
krX|d us8J t|d}| j||||d	}n,| j dkr~|d }| j|||d}nt |S )Nr    r   r  r!   scripted_diffusion_model)r   r  r<  )r   r   r  r"   r   )r   )rz   r>  r)   r  r=  r  r@  r   )	r$   r   r   r   r   r  r  r  ccr#   r#   r&   r+  &  s<    




zDiffusionWrapper.forward)NNN)rS   rj  rk  rO   r`   r+  rn  r#   r#   r}   r&   r[     s   r[   c                       sT   e Zd Zddd fdd
Zdd Ze d fd	d
	Ze dddZ  Z	S )LatentUpscaleDiffusionLRN)low_scale_keynoise_level_keyc                   s6   t  j|i | | jrJ | | || _|| _d S r   )rN   rO   rw  instantiate_low_stagerD  rE  )r$   low_scale_configrD  rE  r)  r*  r}   r#   r&   rO   O  s
    

zLatentUpscaleDiffusion.__init__c                 C   s4   t |}| | _t| j_| j D ]
}d|_q$d S r  r   r  low_scale_modelr'   r  r   rM   r  r#   r#   r&   rF  W  s
    
z,LatentUpscaleDiffusion.instantiate_low_stageFc                    s   |st  j||d|d\}}n$t  j|| jddd|d\}}}}	}
|| j d | }t|d}|jtjd }| 	|\}}| j
d urtd|g|g|d}|r| j	|}||||	|
|||fS ||fS )NT)r  r  r  r,  r-  TODOr   r   r  )rN   r2  rW   rD  r   r/  r)   r0  r   rI  rE  r   r  )r$   r1  r   r  r  log_moder  r  r   r  r  x_lowzxnoise_level	all_conds	x_low_recr}   r#   r&   r2  ^  s     

z LatentUpscaleDiffusion.get_inputrU  r  T   r;   c           +   
      sd  |r
| j nt}|d u}t }| j|| j|dd\} }}}}}}t|jd |}t|jd |}||d< ||d< ||d< ||ddtd	d
 t	|
   < | jjd urt| jdr| j }||d< n| jdv rt|jd |jd f|| j |jd d d}||d< nT| jdv rXt|jd |jd f|d |jd d d}||d< nt|rj||d< t|r| ||d< |
rTt	 }|d | }t| jD ]v}|| j dks|| jd krtt|gd|d}|| j }t|}| j |||d}|!| "| qt#|}t$|d}t$|d}t%||jd d}||d< |r|d( | j& ||||d\}} W d    n1 s0    Y  | "|}!|!|d< |r| '| }"|"|d < |d!kr| (||}#t }$ D ]d"kr&t)  t	rt*  dksJ |#g|$< npd#krTt)  tj+sFJ   |$< nBt)  t	r fd$d%tt*  D |$< n  |$< q|d&F | j& ||||||$d'\}%}&| "|%}'|'|d(|d)< W d    n1 s0    Y  |	r`|d*0 | j, | j-| j.| j.f|d+\}(})W d    n1 s@0    Y  | j'|)d,d-}*|*|d.< |S )/NT)r  rL  r   rV  r  Zx_lrzx_lr_rec_@noise_levels-c                 S   s   t | S r   )strr  r#   r#   r&   r
    r   z3LatentUpscaleDiffusion.log_images.<locals>.<lambda>r  r	  r
  r   r5   r  r  r  r  r  r   rW  rX  r  rM  rN  rO  rY  r  r  rQ  r[  r;   r   r  c                    s   g | ]}  | qS r#   r#   )r   r   r  r   r#   r&   r     r   z5LatentUpscaleDiffusion.log_images.<locals>.<listcomp>r  r  r  r  r  r  r  r  r  )/r   r   rs   r2  rW   r]  r-   joinr  r`   r   numpyr\   rz   r  rT   r  rx  r   r   r   r  r   ro   rV   r   r)   r   r/  r(   r   r  r  r   r  r  r   r	   r  r  r  r  r_   r   r   rY   rX   )+r$   r1  r_  r`  r  r  r  ra  r!  r"  r#  r  r$  r%  r*  r   r&  r   r  r   r  r  rM  rQ  rO  rY  r'  r   r   r(  r)  rQ  r*  r+  rS  Zuc_tmpr,  r-  r   r.  r  r/  r0  r#   rU  r&   rb  t  s    *
.
,


 





*



&
&



0
*z!LatentUpscaleDiffusion.log_images)NNF)rU  r  TrR  r;   NFTTr;   NT)
rS   rj  rk  rO   rF  r)   rl  r2  rb  rn  r#   r#   r}   r&   rB  N  s      rB  c                       sJ   e Zd ZdZded fddZe dfd	d
Ze	 dddZ
  ZS )LatentFinetuneDiffusionz
         Basis for different finetunas, such as inpainting or depth2image
         To disable finetuning mode, set finetune_keys to None
    z-model.diffusion_model.input_blocks.0.0.weightz-model_ema.diffusion_modelinput_blocks00weightr  N)concat_keysc           
         s|   | dd }| dt }	t j|i | || _|| _|| _|| _|| _t	| jrdt	|sdJ dt	|rx| 
||	 d S )Nry   rA   z)can only finetune from a given checkpoint)ru  r`   rN   rO   finetune_keysrZ  	keep_dimsc_concat_log_startc_concat_log_endr   ri   )
r$   rZ  r[  Zkeep_finetune_dimsr]  r^  r)  r*  ry   rA   r}   r#   r&   rO     s    z LatentFinetuneDiffusion.__init__Fc                 C   sv  t j|dd}dt| v r&|d }t| }|D ]}|D ]"}||r>td| ||= q>t| jr6|| jv r6d }| 	 D ]4\}	}
|	| jv rtd|	 d| j
 d t |
}qt|sJ d|| |d d d | j
d	f< |||< q6|s| j|d
dn| jj|d
d\}}td| dt| dt| d t|dkrVtd|  t|dkrrtd|  d S )Nr   r   r   r   zmodifying key 'z' and keeping its original z (channels) dimensions onlyz)did not find matching parameter to modify.Fr   r   r   r   r   r   zMissing Keys: zUnexpected Keys: )r)   r   r`   r   r   rQ   r   r   r[  r   r\  
zeros_liker   r\   r_   )r$   r   rA   rB   r   r   r   r   	new_entryr   r   r   r   r#   r#   r&   ri     s8    



$z&LatentFinetuneDiffusion.init_from_ckptrU  TrR  r;   c           *   
   K   s  |r
| j nt}|d u}t }| j|| j|dd\}}}}}|d d |d d  }}t|jd |}t|jd |}||d< ||d< | jjd urbt	| j
dr| j
|}||d	< n| jd
v rt|jd |jd f|| j |jd d d}||d	< nT| jdv r8t|jd |jd f|d |jd d d}||d	< nt|rJ||d	< t|rb| ||d< | jd u rz| jd u s| |d d | j| jf |d< |rnt }|d | }t| jD ]v}|| j dks|| jd krtt|gd|d}|| j }t|}| j|||d}| | | qt!|}t"|d}t"|d}t#||jd d}||d< |r|d2 | j$|g|gd||||d\} }!W d    n1 s0    Y  | | }"|"|d< |
r| %|!}#|#|d< |d kr|| &||}$|}%|%g|$gd}&|d!P | j$|g|gd||||||&d"\}'}(| |'})|)|d#|d$< W d    n1 sr0    Y  |S )%NT)r  r  r   r   r   rV  r  r  r	  r
  r   r5   r  r  r  r  r  Zc_concat_decodedr   rW  rX  r  rM  rN  rO  rY  r  r?  r  rQ  r[  r;   r  r  r  r  )'r   r   rs   r2  rW   r]  r-   r\   rz   r  rT   r  rx  r   r   r   r  r]  r^  r  r`   r   ro   rV   r   r)   r   r/  r(   r   r  r  r   r  r   r	   r  r  r  )*r$   r1  r_  r`  r  r  r  ra  r  r   r!  r"  r#  r  r$  r%  r*  r   r&  r   r  r  r   r  r  c_catrY  r'  r   r   r(  r)  rQ  r*  r+  rS  Zuc_crossZuc_catZuc_fullr-  r   r.  r#   r#   r&   rb    s    

.
,


" 




*





0z"LatentFinetuneDiffusion.log_images)rY  r  NN)rU  r  TrR  r;   NTTFTTr;   NT)rS   rj  rk  r;  r  rO   r`   ri   r)   rl  rb  rn  r#   r#   r}   r&   rX    s           rX  c                       sL   e Zd ZdZd fdd	Ze d fdd		Ze  fd
dZ  Z	S )LatentInpaintDiffusionz
    can either run as pure inpainting model (only concat mode) or with mixed conditionings,
    e.g. mask as concat and text via cross-attn.
    To disable finetuning mode, set finetune_keys to None
     r  masked_imagerd  c                    s2   t  j|g|R i | || _| j|v s.J d S r   )rN   rO   masked_image_key)r$   rZ  re  r)  r*  r}   r#   r&   rO   m  s    zLatentInpaintDiffusion.__init__NFc                    s  | j rJ dt j|| jddd|d\}}}}	}
t| js@J t }| jD ]}t|| djt	j
d }|d ur|d | }|| j}|j}|| jkrt	jjj||dd  d}n| | |}|| qLt	j|dd	}|g|gd
}|r||||	|
fS ||fS )Nz6trainable cond stages not yet supported for inpaintingTr  r,  r-  r  r   r  r?  )rw  rN   r2  rW   r   rZ  r`   r   r/  r)   r0  r   r(   r-   re  rq   r  interpolater  r  r   r  )r$   r1  r   r  r  r  r  r  r   r  r  ra  ckrA  ZbchwrP  r}   r#   r&   r2  v  s*    

z LatentInpaintDiffusion.get_inputc                    s>   t t| j|i |}t|d d djtjd |d< |S )Nr   rd  r,  r-  )rN   rb  rb  r   r/  r)   r0  r   r$   r)  r*  r   r}   r#   r&   rb    s    z!LatentInpaintDiffusion.log_images)rc  rd  )NNF
rS   rj  rk  r;  rO   r)   rl  r2  rb  rn  r#   r#   r}   r&   rb  f  s     	rb  c                       sL   e Zd ZdZd fdd	Ze d fdd	Ze  fd	d
Z  Z	S )LatentDepth2ImageDiffusionz1
    condition on monocular depth estimation
    Zmidas_inc                    s.   t  j|d|i| t|| _|d | _d S )NrZ  r   )rN   rO   r   depth_modeldepth_stage_key)r$   Zdepth_stage_configrZ  r)  r*  r}   r#   r&   rO     s    
z#LatentDepth2ImageDiffusion.__init__NFc                    s>  | j rJ dt j|| jddd|d\}}}}	}
t| js@J t| jdksRJ t }| jD ]}|| }|d ur|d | }|| j	}| 
|}tjjj||jdd  ddd}tj|g d	dd
tj|g d	dd
 }}d||  || d  d }|| q^tj|dd}|g|gd}|r6||||	|
fS ||fS )Nz5trainable cond stages not yet supported for depth2imgTr  r   r   bicubicF)rL   r%   align_cornersr  r  keepdimr   gMbP?r;   r  r?  )rw  rN   r2  rW   r   rZ  r_   r`   r/  r(   rm  r)   rq   r  rg  r-   aminamaxr   r  )r$   r1  r   r  r  r  r  r  r   r  r  ra  rh  rA  	depth_min	depth_maxrP  r}   r#   r&   r2    s:    


z$LatentDepth2ImageDiffusion.get_inputc                    sp   t  j|i |}| |d | j }tj|g dddtj|g ddd }}d||  ||  d |d< |S )Nr   r  Trq  r   r;   depth)rN   rb  rm  rn  r)   rs  rt  )r$   r)  r*  r   rw  ru  rv  r}   r#   r&   rb    s    z%LatentDepth2ImageDiffusion.log_images)rl  )NNFrj  r#   r#   r}   r&   rk    s    rk  c                       sT   e Zd ZdZd fdd	Zdd Ze d fd	d
	Ze  fddZ	  Z
S )LatentUpscaleFinetuneDiffusionz\
        condition on low-res image (and optionally on some spatial noise augmentation)
    rc  Nc                    sR   t  j|d|i| || _d | _|d urNtd t|s>J | | || _d S )NrZ  zInitializing a low-scale model)rN   rO   reshuffle_patch_sizerI  rQ   r   rF  rD  )r$   rZ  ry  rG  rD  r)  r*  r}   r#   r&   rO     s    
z'LatentUpscaleFinetuneDiffusion.__init__c                 C   s4   t |}| | _t| j_| j D ]
}d|_q$d S r  rH  r  r#   r#   r&   rF    s
    
z4LatentUpscaleFinetuneDiffusion.instantiate_low_stageFc                    sH  | j rJ dt j|| jddd|d\}}}}	}
t| js@J t| jdksRJ t }d }| jD ]}|| }t|d}t| j	rt
| j	tsJ t|d| j	| j	d}|d ur|d | }|| j}t| jr|| jkr| |\}}|| qbtj|dd}t|r|g|g|d	}n|g|gd
}|r@||||	|
fS ||fS )Nz8trainable cond stages not yet supported for upscaling-ftTr  r   r,  z$b c (p1 h) (p2 w) -> b (p1 p2 c) h w)p1p2r  rK  r?  )rw  rN   r2  rW   r   rZ  r_   r`   r   ry  r  r   r/  r(   rI  rD  r   r)   r  )r$   r1  r   r  r  r  r  r  r   r  r  ra  rO  rh  rA  rP  r}   r#   r&   r2    s:    



z(LatentUpscaleFinetuneDiffusion.get_inputc                    s,   t  j|i |}t|d d d|d< |S )Nr   r>  r,  )rN   rb  r   ri  r}   r#   r&   rb    s    z)LatentUpscaleFinetuneDiffusion.log_images)rc  NNN)NNF)rS   rj  rk  r;  rO   rF  r)   rl  r2  rb  rn  r#   r#   r}   r&   rx    s     !rx  c                       sL   e Zd Zd fdd	ZdddZd	d
 ZdddZe dddZ	  Z
S )(ImageEmbeddingConditionedLatentDiffusionjpgr   TNc                    s8   t  j|i | || _|| _| || | | d S r   )rN   rO   	embed_keyembedding_dropout_init_embedder_init_noise_aug)r$   Zembedder_configZembedding_keyr  Zfreeze_embedderZnoise_aug_configr)  r*  r}   r#   r&   rO     s
    z1ImageEmbeddingConditionedLatentDiffusion.__init__c                 C   s8   t |}|r4| | _t| j_| j D ]
}d|_q(d S r  )r   r  embedderr'   r  r   rM   )r$   r  freezer  r   r#   r#   r&   r    s    
z7ImageEmbeddingConditionedLatentDiffusion._init_embedderc                 C   s@   |d ur6t |}t|tjs J | }t|_|| _nd | _d S r   )r   r  rq   Moduler  r'   r  noise_augmentor)r$   r  r  r#   r#   r&   r    s    z8ImageEmbeddingConditionedLatentDiffusion._init_noise_augc                 K   s   t j| ||fd|i|}|d |d  }}|| j d | }	t|	d}	| |	}
| jd urz| |
\}
}t|
|fd}
| jrt	d| j
 tj|
jd |
jdd d d f  |
 }
|g|
d}||g}||dd   |S )	Nr  r   r   r,  r;   r   r  r   )ro  r2  r~  r   r  r  r)   r  r!  	bernoullir  r   r-   r(   r  )r$   r1  r   r  r  r*  r  r  r  r  r  Znoise_level_embrP  Znoutputsr#   r#   r&   r2  (  s(    



z2ImageEmbeddingConditionedLatentDiffusion.get_inputrU  r  c              
   K   sD  t  }| j|| j|ddd\}}}}	}
||d< |	|d< | jjd usFJ | jdv sTJ t|jd |jd f|| j |jd d d	}
|
|d
< | ||	dd}|	dd}|g|d d}|	ddr| j
nt}|dV | j||d|	dd|	dd||d\}}| |}||d|d< W d    n1 s60    Y  |S )NT)r  r  r  rV  r  r
  r   r5   r  r  r	  r$  r6  r  g      @r  r  r%  r  r  r  r  r:   r  Zsamplescfg_scale_r  )rs   r2  rW   r\   rz   rx  r   r-   r  getr   r   r  r  )r$   r1  r_  r`  r*  r   r  r  r   r  r  r,  r  Zuc_r   r-  r   r.  r#   r#   r&   rb  :  s.    .



0z3ImageEmbeddingConditionedLatentDiffusion.log_images)r}  r   TN)T)NN)rU  r  )rS   rj  rk  rO   r  r  r2  r)   rl  rb  rn  r#   r#   r}   r&   r|    s     

r|  )T)>r;  r)   torch.nnrq   rW  ru   pytorch_lightningplZtorch.optim.lr_schedulerr   einopsr   r   
contextlibr   r   	functoolsr   r   r   Ztorchvision.utilsr	   'pytorch_lightning.utilities.distributedr
   	omegaconfr   ldm.utilr   r   r   r   r   r   r   r   Zldm.modules.emar   Z'ldm.modules.distributions.distributionsr   r   Zldm.models.autoencoderr   r   !ldm.modules.diffusionmodules.utilr   r   r   Zldm.models.diffusion.ddimr   r  r'   r.   LightningModuler/   ro  r[   rB  rX  rb  rk  rx  r|  r#   r#   r#   r&   <module>   sZ   (
   ^      0  25?