a
    Ad                    @   s  d dl Z d dlmZ d dlZd dlZd dlmZ d dl	m
Z
mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZmZmZmZmZmZ d d
lmZ d dl m!Z!m"Z" d dl#m$Z$m%Z%m&Z& d dl'm(Z(m)Z)m*Z* d dl+m,Z, d dl-Z.ddddZ/d ddZ0dd Z1G dd dej2Z3G dd de3Z4G dd dej2Z5G dd de4Z6e3e.j7j8j9_3e4e.j7j8j9_4e5e.j7j8j9_5e6e.j7j8j9_6dS )!    N)LambdaLR)	rearrangerepeat)contextmanager)partial)tqdm)	make_grid)rank_zero_only)log_txt_as_imgexistsdefaultismapisimage	mean_flatcount_paramsinstantiate_from_config)LitEma)	normal_klDiagonalGaussianDistribution)VQModelInterfaceIdentityFirstStageAutoencoderKL)make_beta_scheduleextract_into_tensor
noise_like)DDIMSamplerc_concatc_crossattny)concat	crossattnadmTc                 C   s   | S )zbOverwrite model.train with this function to make sure train/eval mode
    does not change anymore. )selfmoder"   r"   Q/var/www/html/stable-diffusion-webui/extensions-builtin/LDSR/sd_hijack_ddpm_v1.pydisabled_train!   s    r&   c                 C   s   | | t j|d|i | S )Ndevice)torchrand)r1r2shaper'   r"   r"   r%   uniform_on_device'   s    r-   c                       s  e Zd ZdD fdd	ZdEddZedFddZdGddZdd Zdd Z	dd Z
ed d!d"Ze dHd#d$Ze dId%d&Ze dJd(d)ZdKd*d+ZdLd,d-ZdMd.d/Zd0d1 Zd2d3 Zd4d5 Zd6d7 Ze d8d9 Zd:d; Zd<d= Ze dNd@dAZdBdC Z  ZS )ODDPMV1  linearl2NFval/lossTimage      d   -C6?{Gz?Mb?              ?epsc                    s^  t    |dv sJ d|| _t| jj d| j d d | _|| _|| _|
| _	|| _
|| _|| _t||| _t| jdd |	| _| jrt| j| _tdtt| j  d |d u| _| jr|| _|| _|| _|| _|d ur|| _|d ur
| j||pg |d	 | j||||||d
 || _|| _t j!|| j"fd| _#| jrZt$j%| j#dd| _#d S )N)r<   x0z(currently only supporting "eps" and "x0"z: Running in z-prediction modeT)verbosezKeeping EMAs of .)ignore_keys
only_model)given_betasbeta_schedule	timestepslinear_start
linear_endcosine_s)
fill_valuesize)requires_grad)&super__init__parameterizationprint	__class____name__cond_stage_modelclip_denoisedlog_every_tfirst_stage_key
image_sizechannelsuse_positional_encodingsDiffusionWrapperV1modelr   use_emar   	model_emalenlistbuffersuse_schedulerscheduler_configv_posteriororiginal_elbo_weightl_simple_weightmonitorinit_from_ckptregister_schedule	loss_typelearn_logvarr(   fullnum_timestepslogvarnn	Parameter)r#   unet_configrD   rC   rg   	ckpt_pathr@   load_only_unetrd   rZ   rT   rU   rV   rS   rR   rE   rF   rG   rB   rb   ra   rc   conditioning_keyrM   r`   rW   rh   logvar_initrO   r"   r%   rL   -   sF    



zDDPMV1.__init__c              
   C   s  t |r|}nt|||||d}d| }tj|dd}	td|	d d }
|j\}t|| _|| _|| _	|	jd | jksJ dt
tjtjd}| d|| | d	||	 | d
||
 | d|t|	 | d|td|	  | d|td|	  | d|td|	  | d|td|	 d  d| j | d|
  d|	  | j|  }| d|| | d|tt|d | d||t|
 d|	   | d|d|
 t| d|	   | jdkr| jd d| j || d| j   }n8| jdkrDdtt|	 dt|	  }ntd|d |d< | jd|dd t| j r~J d S )N)rE   rF   rG   r;   r   axisz+alphas have to be defined for each timestep)dtypebetasalphas_cumprodalphas_cumprod_prevsqrt_alphas_cumprodsqrt_one_minus_alphas_cumprodlog_one_minus_alphas_cumprodsqrt_recip_alphas_cumprodsqrt_recipm1_alphas_cumprod   posterior_varianceposterior_log_variance_clippedg#B;posterior_mean_coef1posterior_mean_coef2r<      r=         ?       @zmu not supportedlvlb_weightsF)
persistent)r   r   npcumprodappendr,   intrj   rE   rF   r   r(   tensorfloat32register_buffersqrtlogra   maximumrM   rx   r   ry   TensorNotImplementedErrorisnanr   all)r#   rB   rC   rD   rE   rF   rG   rx   alphasry   rz   to_torchr   r   r"   r"   r%   rf   t   sZ    

$zDDPMV1.register_schedulec              
   c   s   | j r<| j| j  | j| j |d ur<t| d z8d V  W | j r| j| j  |d urt| d n0| j r| j| j  |d urt| d 0 d S )Nz: Switched to EMA weightsz: Restored training weights)rZ   r[   storerY   
parameterscopy_torN   restore)r#   contextr"   r"   r%   	ema_scope   s    zDDPMV1.ema_scopec           
      C   s   t j|dd}dt| v r&|d }t| }|D ]0}|p@g D ]"}||rBtd| ||= qBq6|sz| j|ddn| jj|dd\}}	td| dt	| d	t	|	 d
 |rtd|  |	rtd|	  d S )Ncpu)map_location
state_dictz Deleting key {} from state_dict.F)strictzRestored from z with z missing and z unexpected keyszMissing Keys: zUnexpected Keys: )
r(   loadr]   keys
startswithrN   formatload_state_dictrY   r\   )
r#   pathr@   rA   sdr   kikmissing
unexpectedr"   r"   r%   re      s"    


$zDDPMV1.init_from_ckptc                 C   sB   t | j||j| }t d| j ||j}t | j||j}|||fS )a  
        Get the distribution q(x_t | x_0).
        :param x_start: the [N x C x ...] tensor of noiseless inputs.
        :param t: the number of diffusion steps (minus 1). Here, 0 means one step.
        :return: A tuple (mean, variance, log_variance), all of x_start's shape.
        r;   )r   r{   r,   ry   r}   )r#   x_starttmeanvariancelog_variancer"   r"   r%   q_mean_variance   s    zDDPMV1.q_mean_variancec                 C   s(   t | j||j| t | j||j|  S Nr   r~   r,   r   )r#   x_tr   noiser"   r"   r%   predict_start_from_noise   s    zDDPMV1.predict_start_from_noisec                 C   sR   t | j||j| t | j||j|  }t | j||j}t | j||j}|||fS r   )r   r   r,   r   r   r   )r#   r   r   r   posterior_meanr   r   r"   r"   r%   q_posterior   s    zDDPMV1.q_posteriorrR   c           	      C   sf   |  ||}| jdkr(| j|||d}n| jdkr6|}|rF|dd | j|||d\}}}|||fS )Nr<   r   r   r=         r;   r   r   r   )rY   rM   r   clamp_r   )	r#   xr   rR   	model_outx_recon
model_meanr   posterior_log_variancer"   r"   r%   p_mean_variance   s    

zDDPMV1.p_mean_variancec                 C   s   g |j |jR ^}}}| j|||d\}}}	t|j ||}
d|dk  j|gdt|j d  R  }||d|	   |
  S )N)r   r   rR   r   r   r   r   )r,   r'   r   r   floatreshaper\   exp)r#   r   r   rR   repeat_noiseb_r'   r   model_log_variancer   nonzero_maskr"   r"   r%   p_sample   s
    ,zDDPMV1.p_samplec              	   C   s   | j j}|d }tj||d}|g}tttd| jd| jdD ]N}| j|tj	|f||tj
d| jd}|| j dks|| jd kr@|| q@|r||fS |S )Nr   r'   
Sampling tdesctotalr'   rw   r   r   )rx   r'   r(   randnr   reversedrangerj   r   ri   longrR   rS   r   )r#   r,   return_intermediatesr'   r   imgintermediatesir"   r"   r%   p_sample_loop   s     zDDPMV1.p_sample_loop   c                 C   s"   | j }| j}| j||||f|dS )N)r   )rU   rV   r   )r#   
batch_sizer   rU   rV   r"   r"   r%   sample
  s
    zDDPMV1.samplec                    s:   t | fdd}t| j| j  t| j| j|  S )Nc                      s
   t  S r   r(   
randn_liker"   r   r"   r%   <lambda>      z!DDPMV1.q_sample.<locals>.<lambda>)r   r   r{   r,   r|   )r#   r   r   r   r"   r   r%   q_sample  s    zDDPMV1.q_samplec                 C   sf   | j dkr$||  }|rb| }n>| j dkrZ|rDtjj||}qbtjjj||dd}ntd|S )Nl1r1   none)	reductionzunknown loss type '{loss_type}')rg   absr   r(   rl   
functionalmse_lossr   )r#   predtargetr   lossr"   r"   r%   get_loss  s    


zDDPMV1.get_lossc                    s  t | fdd}| j ||d}| ||}i }| jdkrB|}n"| jdkrR }ntd| j d| j||dd	jg d
d}| jrdnd}	||	 d| i | | j	 }
| j
| |  }||	 d|i |
| j|  }||	 d|i ||fS )Nc                      s
   t  S r   r   r"   r   r"   r%   r   &  r   z!DDPMV1.p_losses.<locals>.<lambda>r   r   r   r<   r=   zParamterization z not yet supportedFr   r   r   r5   dimtrainval/loss_simple	/loss_vlb/loss)r   r   rY   rM   r   r   r   trainingupdaterc   r   rb   )r#   r   r   r   x_noisyr   	loss_dictr   r   
log_prefixloss_simpleloss_vlbr"   r   r%   p_losses%  s$    

zDDPMV1.p_lossesc                 O   s<   t jd| j|jd f| jd }| j||g|R i |S )Nr   r   )r(   randintrj   r,   r'   r   r  )r#   r   argskwargsr   r"   r"   r%   forwardB  s    "zDDPMV1.forwardc                 C   s>   || }t |jdkr|d }t|d}|jtjd }|S )Nr5   ).Nb h w c -> b c h w)memory_format)r\   r,   r   tor(   contiguous_formatr   )r#   batchr   r   r"   r"   r%   	get_inputH  s    
zDDPMV1.get_inputc                 C   s"   |  || j}| |\}}||fS r   r  rT   )r#   r  r   r   r   r"   r"   r%   shared_stepP  s    zDDPMV1.shared_stepc                 C   sl   |  |\}}| j|ddddd | jd| jddddd | jrh|  jd d }| jd|ddddd |S )NTprog_barloggeron_stepon_epochglobal_stepFr   lrlr_abs)r  log_dictr   r  r_   
optimizersparam_groups)r#   r  	batch_idxr   r   r  r"   r"   r%   training_stepU  s    
zDDPMV1.training_stepc                    s   |  |\}}|  0 |  |\}  fdd D  W d    n1 sL0    Y  | j|ddddd | j ddddd d S )Nc                    s   i | ]}|d   | qS )_emar"   .0keyloss_dict_emar"   r%   
<dictcomp>i  r   z*DDPMV1.validation_step.<locals>.<dictcomp>FTr  )r  r   r  )r#   r  r  r   loss_dict_no_emar"   r"  r%   validation_stepd  s    
0zDDPMV1.validation_stepc                 O   s   | j r| | j d S r   )rZ   r[   rY   )r#   r  r  r"   r"   r%   on_train_batch_endm  s    zDDPMV1.on_train_batch_endc                 C   s,   t |}t|d}t|d}t||d}|S )Nn b c h w -> b n c h wb n c h w -> (b n) c h wnrow)r\   r   r   )r#   samplesn_imgs_per_rowdenoise_gridr"   r"   r%   _get_rows_from_listq  s
    

zDDPMV1._get_rows_from_list   r   c                    s|  i  |  || j}t|jd |}t|jd |}|| jd | }| d< g }|d | }	t| jD ]j}
|
| j dks|
| jd krht	t
|
gd|d}
|
| j }
t
|	}| j|	|
|d}|| qh| | d< |r<| d" | j|d	d
\}}W d    n1 s0    Y  | d< | | d< |rxtt  |jd dkrf S  fdd|D S  S )Nr   inputsr   1 -> br   r   diffusion_rowPlottingT)r   r   r,  denoise_rowc                    s   i | ]}| | qS r"   r"   r  r   r"   r%   r$    r   z%DDPMV1.log_images.<locals>.<dictcomp>)r  rT   minr,   r  r'   r   rj   rS   r   r(   r   r   r   r   r   r/  r   r   r   intersect1dr]   r   )r#   r  Nn_rowr   return_keysr  r   r4  r   r   r   r   r,  r6  r"   r7  r%   
log_imagesx  s4    
2 zDDPMV1.log_imagesc                 C   s:   | j }t| j }| jr&|| jg }tjj||d}|S )Nr  )	learning_rater]   rY   r   rh   rk   r(   optimAdamW)r#   r  paramsoptr"   r"   r%   configure_optimizers  s    zDDPMV1.configure_optimizers)r/   r0   r1   NNFr2   Tr3   r4   r5   r6   Tr7   r8   r9   Nr:   r:   r;   Nr<   NFFr:   )Nr0   r/   r7   r8   r9   )N)NF)TF)F)r   F)N)T)N)r0  r   TN)rP   
__module____qualname__rL   rf   r   r   re   r   r   r   boolr   r(   no_gradr   r   r   r   r   r  r  r  r  r  r&  r'  r/  r=  rD  __classcell__r"   r"   rs   r%   r.   +   sr                             G  
6
	



%r.   c                       s  e Zd ZdZdT fdd	Zd	d
 Zee  fddZ	dU fdd	Z
dd Zdd ZdVddZdd Zdd Zdd  Zd!d" Zd#d$ ZdWd&d'Ze dX fd(d)	Ze dYd*d+ZdZd,d-Ze d.d/ Zd0d1 Zd2d3 Zd[d4d5Zd6d7 Zd8d9 Zd\d:d;Zd]ed<d=d>Ze d^d@dAZ e d_dBdCZ!e d`dDdEZ"e dadGdHZ#e dIdJ Z$e dbdNdOZ%dPdQ Z&e dRdS Z'  Z(S )cLatentDiffusionV1z
main classNr3   FTr;   c                    s   t |d| _|
| _| j|d ks$J |d u r8|r4dnd}|dkrDd }|dd }|dg }t j|d|i| || _|| _|| _zt	|j
jjd | _W n ty   d	| _Y n0 |
s|	| _n| d
t|	 | | | | || _d| _d | _d| _|d ur| || d| _d S )Nr   rD   r   r    __is_unconditional__ro   r@   rq   r   scale_factorFT)r   num_timesteps_condscale_by_stdpoprK   rL   concat_modecond_stage_trainablecond_stage_keyr\   rB  ddconfigch_mult	num_downs	ExceptionrL  r   r(   r   instantiate_first_stageinstantiate_cond_stagecond_stage_forwardrR   bbox_tokenizerrestarted_from_ckptre   )r#   first_stage_configcond_stage_configrM  rR  rQ  rP  rY  rq   rL  rN  r  r  ro   r@   rs   r"   r%   rL     s:    


zLatentDiffusionV1.__init__c                 C   sR   t j| jf| jd t jd| _t t d| jd | j }|| jd | j< d S )Nr   )rI   rH   rw   r   )r(   ri   rj   r   cond_idsroundlinspacerM  )r#   idsr"   r"   r%   make_cond_schedule  s     z$LatentDiffusionV1.make_cond_schedulec                    s   | j r| jdkr| jdkr|dkr| js| jdks:J dtd t || j}|	| j
}| |}| | }| `| dd|    td| j  td d S )Nr   r;   z@rather not use custom rescaling and std-rescaling simultaneouslyz### USING STD-RESCALING ###rL  zsetting self.scale_factor to )rN  current_epochr  r[  rL  rN   rK   r  rT   r  r'   encode_first_stageget_first_stage_encodingdetachr   flattenstd)r#   r  r  dataloader_idxr   encoder_posteriorzrs   r"   r%   on_train_batch_start  s    (
z&LatentDiffusionV1.on_train_batch_startr0   r/   r7   r8   r9   c                    s4   t  |||||| | jdk| _| jr0|   d S )Nr   )rK   rf   rM  shorten_cond_schedulerb  )r#   rB   rC   rD   rE   rF   rG   rs   r"   r%   rf     s    z#LatentDiffusionV1.register_schedulec                 C   s4   t |}| | _t| j_| j D ]
}d|_q$d S )NF)r   evalfirst_stage_modelr&   r   r   rJ   r#   configrY   paramr"   r"   r%   rW    s
    
z)LatentDiffusionV1.instantiate_first_stagec                 C   s   | j sv|dkr td | j| _q|dkrDtd| jj d d | _qt|}| | _t| j_	| j
 D ]
}d|_qhn&|dksJ |dksJ t|}|| _d S )N__is_first_stage__z%Using first stage also as cond stage.rK  z	Training z as an unconditional model.F)rQ  rN   ro  rQ   rO   rP   r   rn  r&   r   r   rJ   rp  r"   r"   r%   rX    s     


z(LatentDiffusionV1.instantiate_cond_stage c                 C   sh   g }t ||dD ] }|| j|| j|d qt|}t|}t|d}t|d}t	||d}|S )Nr   force_not_quantizer(  r)  r*  )
r   r   decode_first_stager  r'   r\   r(   stackr   r   )r#   r,  r   force_no_decoder_quantizationr6  zdr-  r.  r"   r"   r%   _get_denoise_row_from_list  s    



z,LatentDiffusionV1._get_denoise_row_from_listc                 C   sD   t |tr| }n&t |tjr&|}ntdt| d| j| S )Nzencoder_posterior of type 'z' not yet implemented)
isinstancer   r   r(   r   r   typerL  )r#   rj  rk  r"   r"   r%   re    s    

z*LatentDiffusionV1.get_first_stage_encodingc                 C   sv   | j d u rNt| jdrBt| jjrB| j|}t|trL| }qr| |}n$t| j| j s`J t| j| j |}|S )Nencode)	rY  hasattrrQ   callabler  r}  r   r$   getattr)r#   cr"   r"   r%   get_learned_conditioning&  s    


z*LatentDiffusionV1.get_learned_conditioningc                 C   sV   t d||ddd|d}t d|d|d|dd}t j||gdd}|S )Nr   r   rv   r   )r(   arangeviewr   cat)r#   hwr   r   arrr"   r"   r%   meshgrid3  s      zLatentDiffusionV1.meshgridc                 C   s   t |d |d gddd}| ||| }t j|dddd }t jd| dddd }t jt j||gddddd }|S )z
        :param h: height
        :param w: width
        :return: normalized distance to image border,
         wtith min distance = 0 at border and max dist = 0.5 at image center
        r   r   rv   T)r   keepdimsr   r   )r(   r   r  r  r8  r  )r#   r  r  lower_right_cornerr  dist_left_updist_right_down	edge_distr"   r"   r%   delta_border:  s      zLatentDiffusionV1.delta_borderc                 C   s   |  ||}t|| jd | jd }|d|| ddd|| |}| jd r|  ||}t|| jd | jd }|dd|| |}|| }|S )Nclip_min_weightclip_max_weightr   
tie_brakerclip_min_tie_weightclip_max_tie_weight)r  r(   clipsplit_input_paramsr  r   r  )r#   r  r  LyLxr'   	weightingL_weightingr"   r"   r%   get_weightingH  s    &
zLatentDiffusionV1.get_weightingr   c                 C   s  |j \}}}}	||d  |d  d }
|	|d  |d  d }|dkr|dkrt|dd|d}tjjf i |}tjjf d|j dd i|}| |d |d |
||j|j	}||
dd||	}|
dd|d |d |
| f}n"|dkr|dkrt|dd|d}tjjf i |}t|d | |d | fdd|d | |d | fd}tjjf d|j d | |j d | fi|}| |d | |d | |
||j|j	}||
dd|| |	| }|
dd|d | |d | |
| f}n|dkr
|dkr
t|dd|d}tjjf i |}t|d | |d | fdd|d | |d | fd}tjjf d|j d | |j d | fi|}| |d | |d | |
||j|j	}||
dd|| |	| }|
dd|d | |d | |
| f}nt||||fS )z
        :param x: img of size (bs, c, h, w)
        :return: n img crops of size (n, bs, c, kernel_size[0], kernel_size[1])
        r   r   )kernel_sizedilationpaddingstrideoutput_sizer   Nr5   )r,   dictr(   rl   UnfoldFoldr  r'   r  rw   r  r   )r#   r   r  r  ufdfbsncr  r  r  r  fold_paramsunfoldfoldr  normalizationfold_params2r"   r"   r%   get_fold_unfoldX  sD     $$.,,.,*z!LatentDiffusionV1.get_fold_unfoldc                    s  t  ||}|d ur"|d | }|| j}| |}	| |	 }
| jjd ur2|d u rb| j	}|| j
kr|dv r~|| }q|dkr|}qt  ||| j}n|}| jr|rt|tst|tr| |}q| || j}n|}|d ur|d | }| jrZ| |\}}t| jj }||d|d|i}n(d }d }| jrZ| |\}}||d}|
|g}|r| |
}|||g |r|| |S )N)captioncoordinates_bboxclass_labelpos_xpos_y)r  r  )rK   r  r  r'   rd  re  rf  rY   rq   rR  rT   rQ  r}  r  r]   r  rW   compute_latent_shifts__conditioning_keys__rx  extendr   )r#   r  r   return_first_stage_outputsforce_c_encodecond_keyreturn_original_condr  r   rj  rk  xcr  r  r  ckeyoutxrecrs   r"   r%   r    sN    






zLatentDiffusionV1.get_inputc                    sX  rF  dkr&tj dd jjjd dtd	 dj
  tdr$jd rjd	 }jd
 }jd }j\}}}	}
|d |	ks|d |
krt|d |	t|d |
f}td |d |	ks|d |
krt|d |	t|d |
f}td j|||d\}}}}|jd d|d |d jd ftjtr fddtjd D }nfddtjd D }tj|dd}|| }||jd d|jd f}||}|| }|S tjtrjjp dS jS n0tjtrHjjpB dS jS d S )N   r   r   r,   r	  r;   r  patch_distributed_vqksr  vqfr   reducing Kernelreducing strider  rv   c              	      s>   g | ]6}j jd d d d d d d d |f p4 dqS Nrv  ro  decoder   r   rw  predict_cidsr#   rk  r"   r%   
<listcomp>  s   &z8LatentDiffusionV1.decode_first_stage.<locals>.<listcomp>c              
      s6   g | ].} j d d d d d d d d |f qS r   r  r  r#   rk  r"   r%   r    s   rt   rv  r   r(   argmaxr   r   ro  quantizeget_codebook_entryr   
contiguousrL  r  r  r,   r8  rN   r  r  r}  r   r   ry  r  r#   rk  r  rw  r  r  r  r  r  r  r  r  r  r  r  output_listodecodedr"   r  r%   rx    sP    


(z$LatentDiffusionV1.decode_first_stagec                    sX  rF  dkr&tj dd jjjd dtd	 dj
  tdr$jd rjd	 }jd
 }jd }j\}}}	}
|d |	ks|d |
krt|d |	t|d |
f}td |d |	ks|d |
krt|d |	t|d |
f}td j|||d\}}}}|jd d|d |d jd ftjtr fddtjd D }nfddtjd D }tj|dd}|| }||jd d|jd f}||}|| }|S tjtrjjp dS jS n0tjtrHjjpB dS jS d S )Nr  r   r   r  r	  r;   r  r  r  r  r  r   r  r  r  rv   c              	      s>   g | ]6}j jd d d d d d d d |f p4 dqS r  r  r  r  r"   r%   r    s   &zGLatentDiffusionV1.differentiable_decode_first_stage.<locals>.<listcomp>c              
      s6   g | ].} j d d d d d d d d |f qS r   r  r  r  r"   r%   r  !  s   rt   rv  r  r  r"   r  r%   !differentiable_decode_first_stage  sP    


(z3LatentDiffusionV1.differentiable_decode_first_stagec                    s  t  dr jd r~ jd } jd } jd }|jdd   jd< |j\}}}}|d |ksp|d	 |krt|d |t|d	 |f}td
 |d |ks|d	 |krt|d |t|d	 |f}td  j||||d\}	}
}}|
|jd d|d |d	 jd f fddtjd D }tj	|dd}|| }||jd d|jd f}|	|}|| }|S  j
|S n j
|S d S )Nr  r  r  r  r  original_image_sizer   r   r  r  )r  rv   c              
      s6   g | ].} j d d d d d d d d |f qS r   )ro  r  r  r  r"   r%   r  N  s   z8LatentDiffusionV1.encode_first_stage.<locals>.<listcomp>rt   )r  r  r,   r8  rN   r  r  r   r(   ry  ro  r  )r#   r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r"   r  r%   rd  8  s6    


(z$LatentDiffusionV1.encode_first_stagec                 K   s    |  || j\}}| ||}|S r   r  )r#   r  r  r   r  r   r"   r"   r%   r  `  s    
zLatentDiffusionV1.shared_stepc                 O   s   t jd| j|jd f| jd }| jjd ur||d us:J | jrJ| 	|}| j
r|| j| | j}| j||t | d}| j|||g|R i |S )Nr   r   r   )r(   r  rj   r,   r'   r   rY   rq   rQ  r  rm  r^  r  r   r   r   r  )r#   r   r  r  r  r   tcr"   r"   r%   r  e  s    "
zLatentDiffusionV1.forwardc                    s  t trn,t tsg
jjdkr,dnd}|it
drtdksTJ |r\J 
jd 
jd |jdd  \}}
	|\}}	}
}|	|
jd	 d
d	 d jd
 ffddtjd
 D 
jdv r
jjrtt tt  t dks,J  d	  |	   
 jd	 d
d	 d  jd
 f  fddt jd
 D n
jdkrd
jv sJ dt|d	  d	  d 
jd \
jjjd }d| 		fddtjd
 D }	fdd|D }
fdd|D }t|d	 j t tsVJ dd d	 dd df 
jtj tfdd|D }t|d}t|j 
|}t|j t|djd
 d}t|j dd |D nfddtjd
 D 
fddtjd
 D }t |d	 tr@J tj|d
d}|| }|
|jd	 d
|jd
 f}|||
 }n
j|fi }t |tr|s|d	 S |S d S ) Nr   r   r   r  r   r  r  r  r   rv   c                    s.   g | ]&} d d d d d d d d |f qS r   r"   r  )rk  r"   r%   r    r   z1LatentDiffusionV1.apply_model.<locals>.<listcomp>)r3   LR_imagesegmentationZbbox_imgc              	      s4   g | ],} d d d d d d d d |f giqS r   r"   r  )r  c_keyr"   r%   r    r   r  r  z2BoudingBoxRescaling is missing original_image_sizer   c                    s<   g | ]4}d   |   d  |    fqS r   r   r"   )r   Zpatch_nr)
full_img_h
full_img_wn_patches_per_rowrescale_latentr  r"   r%   r    s   c                    s4   g | ],\}}||d    d    fqS r  r"   )r   Zx_tlZy_tl)r  r  r  r  r"   r%   r    s
   c                    s*   g | ]"}t  j|d   jqS r   )r(   
LongTensorrZ  Z_crop_encoderr  r'   )r   bbox)r#   r"   r%   r    s   z&cond must be dict to be fed into model.c                    s   g | ]}t j |gd dqS )r   r   )r(   r  )r   p)cut_condr"   r%   r    r   zl b n -> (l b) nz(l b) n d -> l b n d)lc                 S   s   g | ]}d |giqS )r   r"   )r   er"   r"   r%   r    r   c                    s   g | ]} qS r"   r"   r  )condr"   r%   r    r   c                    s(   g | ] }j | fi  | qS r"   )rY   r  )	cond_listr#   r   z_listr"   r%   r    r   rt   )r}  r  r]   rY   rq   r  r\   r  r,   r  r  r   rR  nextiterr   valuesr   ro  encodernum_resolutionsrN   r  r'   r(   ry  r   r  tuple)r#   r   r   r  
return_idsr!  r  r  r  r  r  r  rU  Ztl_patch_coordinatesZpatch_limitsZpatch_limits_tknzdZadapted_condr  r  r   r"   )r  r  r  r  r  r  r  r  r  r  r#   r  r   rk  r  r%   apply_modelp  s    



(("
 





"
zLatentDiffusionV1.apply_modelc                 C   s(   t | j||j| | t | j||j S r   r   )r#   r   r   pred_xstartr"   r"   r%   _predict_eps_from_xstart  s    z*LatentDiffusionV1._predict_eps_from_xstartc                 C   sZ   |j d }tj| jd g| |jd}| ||\}}}t||ddd}t|t	d S )a;  
        Get the prior KL term for the variational lower-bound, measured in
        bits-per-dim.
        This term can't be optimized, as it only depends on the encoder.
        :param x_start: the [N x C x ...] tensor of inputs.
        :return: a batch of [N] KL values (in bits), one per batch element.
        r   r   r   r:   )mean1logvar1mean2logvar2r   )
r,   r(   r   rj   r'   r   r   r   r   r   )r#   r   r   r   qt_meanr   qt_log_variancekl_priorr"   r"   r%   
_prior_bpd  s
    
zLatentDiffusionV1._prior_bpdc                    sh  t | fdd}| j ||d}| |||}i }| jr>dnd}| jdkrR }	n| jdkrb|}	nt | j||	dd	g d
}
|| d|
 i | j	| 
| j}|
t| | }| jr|| d| i |d| j	j i | j|  }| j||	dd	jd
d}| j| |  }|| d|i || j| 7 }|| d|i ||fS )Nc                      s
   t  S r   r   r"   r   r"   r%   r     r   z,LatentDiffusionV1.p_losses.<locals>.<lambda>r   r   r   r=   r<   Fr   r   r   z/loss_gammark   r   r   r   )r   r   r  r   rM   r   r   r   r   rk   r  r'   r(   r   rh   datarc   r   rb   )r#   r   r  r   r   r   model_outputr   prefixr   r  logvar_tr   r  r"   r   r%   r    s0    

zLatentDiffusionV1.p_lossesr   c
                 C   s   |}
| j ||
||d}|d urF| jdks,J |j| ||||fi |	}|rR|\}}| jdkrn| j|||d}n| jdkr~|}nt |r|dd |r| j|\}}\}}}| j|||d\}}}|r||||fS |r||||fS |||fS d S )N)r  r<   r   r=   r   r;   r   )	r  rM   modify_scorer   r   r   ro  r  r   )r#   r   r  r   rR   return_codebook_idsquantize_denoised	return_x0score_correctorcorrector_kwargst_inr   logitsr   r   indicesr   r   r   r"   r"   r%   r     s,    

z!LatentDiffusionV1.p_mean_variancer:   c                 C   s*  g |j |jR ^}}}| j|||||||||d	}|rPtd|\}}}}n|rb|\}}}}n
|\}}}t|j |||	 }|
dkrtjjj||
d}d|dk	  j
|gdt|j d  R  }|r||d|   |  |jdd	fS |r||d|   |  |fS ||d|   |  S d S )
N)	r   r  r   rR   r  r  r  r  r	  zSupport dropped.r:   )r  r   r   r   r   r   )r,   r'   r   DeprecationWarningr   r(   rl   r   dropoutr   r   r\   r   r  )r#   r   r  r   rR   r   r  r  r  temperaturenoise_dropoutr  r	  r   r   r'   outputsr   r   r  r=   r   r   r"   r"   r%   r   +  s,    
,$zLatentDiffusionV1.p_samplec                    s"  |s
| j }| j} d ur< d ur$ n|d } gt| }n|d  } |d u rbtj|| jd}n|}g }d urttr fddD n(ttr fddD n
d   |d urt||}|rt	t
td|d|dnt
td|}t|	tkr|	g| }	|D ]}tj|f|| jtjd	}| jrr| jjd
ksJJ | j| j}| j|td| j||| j|d|	| |
||d
\}}|d ur|d usJ | ||}|| d| |  }|| dks||d kr|| |r|| |r||| q||fS )Nr   r   c                    sB   i | ]:}|t | ts(| d   n fdd| D qS )Nc                    s   g | ]}|d   qS r   r"   r   r   r   r"   r%   r  _  r   zFLatentDiffusionV1.progressive_denoising.<locals>.<dictcomp>.<listcomp>r}  r]   r  r   r  r"   r%   r$  ^  s    z;LatentDiffusionV1.progressive_denoising.<locals>.<dictcomp>c                    s   g | ]}|d   qS r   r"   r   r  r  r"   r%   r  a  r   z;LatentDiffusionV1.progressive_denoising.<locals>.<listcomp>Progressive Generationr   r   hybridr   T)rR   r  r  r  r  r  r	  r;   r   )rS   rj   r]   r(   r   r'   r}  r  r8  r   r   r   r~  r   ri   r   rm  rY   rq   r^  r  r   r   r   rR   r   )r#   r  r,   r>   callbackr  img_callbackmaskr=   r  r  r  r	  r   x_Tstart_TrS   rD   r   r   r   iteratorr   tsr  
x0_partialimg_origr"   r  r%   progressive_denoisingJ  sh    
(






z'LatentDiffusionV1.progressive_denoisingc                 C   s  |s
| j }| jj}|d }|d u r2tj||d}n|}|g}|d u rJ| j}|d ur\t||}|rxttt	d|d|dntt	d|}|	d ur|
d usJ |
j
dd |	j
dd ksJ |D ]}tj|f||tjd}| jr| jjdksJ | j| |j}| j||t|d	}| j|||| j|d
}|	d urX| |
|}||	 d|	 |  }|| dkst||d kr~|| |r|| |r||| q|r||fS |S )Nr   r   r   r   r   r5   r   r  r   )rR   r  r;   r   )rS   rx   r'   r(   r   rj   r8  r   r   r   r,   ri   r   rm  rY   rq   r^  r  r   r   r   rR   r   )r#   r  r,   r   r  r>   r  rD   r  r  r=   r  r  rS   r'   r   r   r   r  r   r  r  r!  r"   r"   r%   r     sP    
 


zLatentDiffusionV1.p_sample_loopr   c                    s   |
d u r | j | j| jf}
d urjttrB fddD n(ttr^ fddD n
d   | j|
|||||||	d	S )Nc                    sB   i | ]:}|t | ts(| d   n fdd| D qS )Nc                    s   g | ]}|d   qS r   r"   r  r  r"   r%   r    r   z7LatentDiffusionV1.sample.<locals>.<dictcomp>.<listcomp>r  r  r  r"   r%   r$    s    z,LatentDiffusionV1.sample.<locals>.<dictcomp>c                    s   g | ]}|d   qS r   r"   r  r  r"   r%   r    r   z,LatentDiffusionV1.sample.<locals>.<listcomp>)r   r  r>   rD   r  r  r=   )rV   rU   r}  r  r]   r   )r#   r  r   r   r  r>   rD   r  r  r=   r,   r  r"   r  r%   r     s    
(zLatentDiffusionV1.samplec           
      K   sb   |r>t | }| j| j| jf}|j||||fddi|\}}	n| jf ||dd|\}}	||	fS )Nr>   FT)r  r   r   )r   rV   rU   r   )
r#   r  r   ddim
ddim_stepsr  ddim_samplerr,   r,  r   r"   r"   r%   
sample_log  s     

zLatentDiffusionV1.sample_logr0  r     c           %   
      s  |d u}i  | j || jddd|d\}}}}}t|jd |}t|jd |}| d< | d< | jjd urt| jdr| j|}| d< nv| j	dv rt
|jd	 |jd
 f|d }| d< nD| j	dkrt
|jd	 |jd
 f|d }| d< nt|r| d< t|r| | d< |rg }|d | }t| jD ]v}|| j dks`|| jd kr<tt|gd|d}|| j }t|}| j|||d}|| | q<t|}t|d}t|d}t||jd d}| d< |r4| d( | j|||||d\}}W d    n1 s,0    Y  | |}| d< |
r`|  |}| d< |rt!| j"t#st!| j"t$s| d* | j|||||dd\}}W d    n1 s0    Y  | || j}| d< |	r4|jd	 |jd
  }}t%|||| j} d| d d |d d
| d |d d
| d f< | d d d d f } | d!4 | j||||||d | | d"\}}!W d    n1 s0    Y  | || j}| d#< |  d$< | d%4 | j||||||d | | d"\}}!W d    n1 s0    Y  | || j}| d&< |r| d'0 | j&|| j'| j(| j(f|d(\}"}#W d    n1 s|0    Y  | j |#d)d*}$|$ d+< |rt)*t+ , |jd dkrƈ S  fd,d-|D S  S ).NT)r  r  r  r  r   r1  reconstructionr  conditioning)r  r   r5   r  r  human_labeloriginal_conditioningr   r2  r3  r   r(  r)  r*  r4  r5  )r  r   r#  r$  etar,  r6  zPlotting Quantized Denoised)r  r   r#  r$  r,  r  samples_x0_quantizedr:   r  .zPlotting Inpaint)r  r   r#  r,  r$  r=   r  samples_inpaintingr  zPlotting Outpaintsamples_outpaintingzPlotting Progressives)r,   r   r  ru  progressive_rowc                    s   i | ]}| | qS r"   r"   r  r7  r"   r%   r$  G  r   z0LatentDiffusionV1.log_images.<locals>.<dictcomp>)-r  rT   r8  r,   rY   rq   r  rQ   r  rR  r
   r   r   to_rgbr   rj   rS   r   r(   r   r  r'   r   r   r   r   rx  ry  r   r   r   r&  r|  r}  ro  r   r   onesr"  rV   rU   r   r9  r]   r   )%r#   r  r:  r;  r   r$  ddim_etar<  r  inpaintplot_denoise_rowsplot_progressive_rowsplot_diffusion_rowsr  use_ddimrk  r  r   r  r  r4  z_startr   r   z_noisydiffusion_gridr,  z_denoise_row	x_samplesr.  r  r  r  r   r   progressivesprog_rowr"   r7  r%   r=    s    







 




*


*2*** zLatentDiffusionV1.log_imagesc                 C   s   | j }t| j }| jr>t| jj d |t| j  }| j	rXtd |
| j tjj||d}| jrd| jv s|J t| j}td t||jdddd	g}|g|fS |S )
Nz%: Also optimizing conditioner params!z!Diffusion model optimizing logvarr>  r   z Setting up LambdaLR scheduler...)	lr_lambdastepr   )	schedulerinterval	frequency)r?  r]   rY   r   rQ  rN   rO   rP   rQ   rh   r   rk   r(   r@  rA  r_   r`   r   r   schedule)r#   r  rB  rC  rB  r"   r"   r%   rD  J  s(    

z&LatentDiffusionV1.configure_optimizersc                 C   sj   |  }t| ds0td|jd dd|| _tjj	|| jd}d||
   | |
   d }|S )Ncolorizer5   r   )weightr   r;   )r   r  r(   r   r,   r  rF  rl   r   conv2dr8  max)r#   r   r"   r"   r%   r1  b  s    
$zLatentDiffusionV1.to_rgb)Nr3   FTNNr;   F)Nr0   r/   r7   r8   r9   )rt  F)r   r   )FFNFN)FF)FF)F)N)FFFNN)	FFFFFr;   r:   NN)TNFNNNr;   r:   NNNNNN)FNTNNFNNNNN)	r   FNTNFNNN)r0  r  Tr'  r;   NTTFTT))rP   rE  rF  __doc__rL   rb  r	   r(   rH  rl  rf   rW  rX  r|  re  r  r  r  r  r  r  rx  r  rd  r  r  r  r  r  r  rG  r   r   r"  r   r   r&  r=  rD  r1  rI  r"   r"   rs   r%   rJ    s           -  	
	
4  3<
;
'
g
#         9    4   
   nrJ  c                       s.   e Zd Z fddZdeedddZ  ZS )rX   c                    s,   t    t|| _|| _| jdv s(J d S )N)Nr   r    r  r!   )rK   rL   r   diffusion_modelrq   )r#   diff_model_configrq   rs   r"   r%   rL   m  s    

zDiffusionWrapperV1.__init__N)r   r   c                 C   s   | j d u r| ||}n| j dkrDtj|g| dd}| ||}n| j dkrlt|d}| j|||d}nf| j dkrtj|g| dd}t|d}| j|||d}n*| j dkr|d }| j|||d	}nt |S )
Nr   r   r   r    )r   r  r!   r   )r   )rq   rK  r(   r  r   )r#   r   r   r   r   r  r  ccr"   r"   r%   r  s  s"    




zDiffusionWrapperV1.forward)NN)rP   rE  rF  rL   r]   r  rI  r"   r"   rs   r%   rX   l  s   rX   c                       s*   e Zd Z fddZd fdd	Z  ZS )Layout2ImgDiffusionV1c                    s*   |dksJ dt  j|d|i| d S )Nr  z>Layout2ImgDiffusion only for cond_stage_key="coordinates_bbox"rR  )rK   rL   )r#   rR  r  r  rs   r"   r%   rL     s    zLayout2ImgDiffusionV1.__init__r0  c                    s   t  j|||d|}| jr"dnd}| jjj|   j| j }g } fdd}	|| j d | D ]$}
||
	 
 |	d}|| qbtj|dd}||d	< |S )
N)r  r:  r   
validationc                    s      | S r   )Zget_textual_labelZget_category_id)Zcatnodsetr"   r%   r     r   z2Layout2ImgDiffusionV1.log_images.<locals>.<lambda>)r4   r4   r   r   Z
bbox_image)rK   r=  r   trainer
datamoduledatasetsZconditional_buildersrR  plotrf  r   r   r(   ry  )r#   r  r:  r  r  logsr!  mapperZ	bbox_imgsmap_fnZ
tknzd_bboxZbboximgZcond_imgrs   rP  r%   r=    s    z Layout2ImgDiffusionV1.log_images)r0  )rP   rE  rF  rL   r=  rI  r"   r"   rs   r%   rN    s   rN  )T):r(   torch.nnrl   numpyr   pytorch_lightningplZtorch.optim.lr_schedulerr   einopsr   r   
contextlibr   	functoolsr   r   Ztorchvision.utilsr   'pytorch_lightning.utilities.distributedr	   ldm.utilr
   r   r   r   r   r   r   r   ldm.modules.emar   'ldm.modules.distributions.distributionsr   r   ldm.models.autoencoderr   r   r   !ldm.modules.diffusionmodules.utilr   r   r   ldm.models.diffusion.ddimr   ldm.models.diffusion.ddpmldmr  r&   r-   LightningModuler.   rJ  rX   rN  models	diffusionddpmr"   r"   r"   r%   <module>   sP   (
  ~       L