a
    
dz                     @   s$  d dl mZ d dlmZ d dlmZmZmZmZm	Z	 d dl
Z
d dlZd dlZd dlZd dlmZ d dlmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZ d	d
lmZ d	dl m!Z! d	dl"m#Z# d	dl$m%Z%m&Z& d	dl'm(Z( d	dl)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/ G dd dej0Z1G dd dej0Z2G dd dej0Z3G dd de1Z4G dd de1Z5G dd de5Z6G dd de1Z7G dd de1Z8G d d! d!e1Z9G d"d# d#e1Z:G d$d% d%e1Z;G d&d' d'e1Z<G d(d) d)e1Z=G d*d+ d+ej0Z>G d,d- d-ej0Z?G d.d/ d/e1Z@G d0d1 d1e!e1ZAdS )2    )nullcontext)partial)DictListOptionalTupleUnionN)	rearrangerepeat)
ListConfig)
checkpoint)ByT5TokenizerCLIPTextModelCLIPTokenizerT5EncoderModelT5Tokenizer   )DiagonalGaussianRegularizer)Encoder)Timestep)extract_into_tensormake_beta_schedule)DiagonalGaussianDistribution)autocastcount_paramsdefaultdisabled_trainexpand_dims_likeinstantiate_from_configc                       s   e Zd Z fddZeedddZeeee	j
f dddZeeddd	Zejed
ddZejeee	j
f d
ddZejed
dd	Zejdd Zejdd Zejdd	 Z  ZS )AbstractEmbModelc                    s    t    d | _d | _d | _d S N)super__init___is_trainable	_ucg_rate
_input_keyself	__class__ c/var/www/html/stable-diffusion-webui/repositories/generative-models/sgm/modules/encoders/modules.pyr"   %   s    
zAbstractEmbModel.__init__returnc                 C   s   | j S r    r#   r&   r*   r*   r+   is_trainable+   s    zAbstractEmbModel.is_trainablec                 C   s   | j S r    r$   r&   r*   r*   r+   ucg_rate/   s    zAbstractEmbModel.ucg_ratec                 C   s   | j S r    r%   r&   r*   r*   r+   	input_key3   s    zAbstractEmbModel.input_key)valuec                 C   s
   || _ d S r    r.   r'   r4   r*   r*   r+   r/   7   s    c                 C   s
   || _ d S r    r0   r5   r*   r*   r+   r1   ;   s    c                 C   s
   || _ d S r    r2   r5   r*   r*   r+   r3   ?   s    c                 C   s   | ` d S r    r.   r&   r*   r*   r+   r/   C   s    c                 C   s   | ` d S r    r0   r&   r*   r*   r+   r1   G   s    c                 C   s   | ` d S r    r2   r&   r*   r*   r+   r3   K   s    )__name__
__module____qualname__r"   propertyboolr/   r   floattorchTensorr1   strr3   setterdeleter__classcell__r*   r*   r(   r+   r   $   s&   

r   c                       sx   e Zd ZdddddZddddZeeef d fd	d
Ze	e
e
dddZde
ee e
dddZdddZ  ZS )GeneralConditionervector	crossattnconcat)   r            rF   )rC   rD   rE   )
emb_modelsc              
      s2  t    g }t|D ]\}}t|}t|tsFJ d|jj d|dd|_	|dd|_
|j	st|_| D ]
}d|_qv|  td| d|jj d	t|d d
|j	  d|v r|d |_n&d|v r|d |_ntd|jj |dd |_|jd urtj |_|| qt|| _d S )Nzembedder model z% has to inherit from AbstractEmbModelr/   Fr1           zInitialized embedder #z: z with z params. Trainable: r3   
input_keysz5need either 'input_key' or 'input_keys' for embedder Zlegacy_ucg_value)r!   r"   	enumerater   
isinstancer   r)   r6   getr/   r1   r   train
parametersrequires_gradevalprintr   r3   rL   KeyErrorlegacy_ucg_valnprandomRandomStateucg_prngappendnn
ModuleList	embedders)r'   rJ   r^   nZ	embconfigembedderparamr(   r*   r+   r"   T   sF    
zGeneralConditioner.__init__)r`   batchr-   c                 C   s\   |j d usJ |j}|j }tt||j D ]*}|jjdd| |gdr,|||j |< q,|S )NrF   rI   )p)rV   r1   rangelenr3   rZ   choice)r'   r`   rb   rc   valir*   r*   r+   possibly_get_ucg_valx   s    z'GeneralConditioner.possibly_get_ucg_valN)rb   force_zero_embeddingsr-   c           	   
      s  t  }|d u rg }| jD ]}|jr(tntj}| l t|drp|jd urp|jd ur`| 	|  | |j }n"t|dr| fdd|j
D  }W d    n1 s0    Y  t|tjttfsJ dt| t|ttfs|g}|D ]}| j|  }|jdkrF|jd u rFttd|j tj|jd |jd	 || }t|drh|j|v rht|}||v rt|| |f| j| ||< q|||< qq|S )
Nr3   rL   c                    s   g | ]} | qS r*   r*   ).0krb   r*   r+   
<listcomp>       z.GeneralConditioner.forward.<locals>.<listcomp>z7encoder outputs must be tensors or a sequence, but got rK         ?r   device)dictr^   r/   r   r<   no_gradhasattrr3   rV   ri   rL   rN   r=   listtupletypeOUTPUT_DIM2KEYSdimr1   r   	bernoullionesshaperr   
zeros_likecat
KEY2CATDIM)	r'   rb   rj   outputr`   Zembedding_contextZemb_outembZout_keyr*   rm   r+   forward   sX    

6


zGeneralConditioner.forwardc           	      C   sr   |d u rg }t  }| jD ]}||j d|_q| |}| |d u rF|n||}t| j|D ]\}}||_qZ||fS )NrK   )rv   r^   r[   r1   zip)	r'   Zbatch_cZbatch_ucZforce_uc_zero_embeddingsZ	ucg_ratesr`   cucrater*   r*   r+   get_unconditional_conditioning   s    
z1GeneralConditioner.get_unconditional_conditioning)N)NN)r6   r7   r8   ry   r   r   r   r   r"   r   r   ri   r   r   r   rA   r*   r*   r(   r+   rB   P   s   $
 / rB   c                       s*   e Zd ZdZd fdd	Zdd Z  ZS )InceptionV3zsWrapper around the https://github.com/mseitzer/pytorch-fid inception
    port with an additional squeeze at the endFc                    s8   t    ddlm} d|d< |jf d|i|| _d S )Nr   )	inceptionTZresize_inputnormalize_input)r!   r"   Zpytorch_fidr   r   model)r'   r   kwargsr   r(   r*   r+   r"      s    
zInceptionV3.__init__c                 C   s&   |  |}t|dkr"|d  S |S )NrI   r   )r   re   squeeze)r'   inpZoutpr*   r*   r+   r      s    
zInceptionV3.forward)Fr6   r7   r8   __doc__r"   r   rA   r*   r*   r(   r+   r      s   r   c                   @   s   e Zd Zdd Zdd ZdS )IdentityEncoderc                 C   s   |S r    r*   r'   xr*   r*   r+   encode   s    zIdentityEncoder.encodec                 C   s   |S r    r*   r   r*   r*   r+   r      s    zIdentityEncoder.forwardN)r6   r7   r8   r   r   r*   r*   r*   r+   r      s   r   c                       s0   e Zd Zd
 fdd	Zdd Zddd	Z  ZS )ClassEmbedder  Fc                    s(   t    t||| _|| _|| _d S r    )r!   r"   r\   	Embedding	embedding	n_classesadd_sequence_dim)r'   	embed_dimr   r   r(   r*   r+   r"      s    
zClassEmbedder.__init__c                 C   s*   |  |}| jr&|d d d d d f }|S r    )r   r   )r'   r   r*   r*   r+   r      s    
zClassEmbedder.forwardcudac                 C   s0   | j d }tj|f|d| }| j| i}|S )NrI   rq   )r   r<   r|   keylong)r'   bsrr   Zuc_classr   r*   r*   r+   r      s
    z,ClassEmbedder.get_unconditional_conditioning)r   F)r   )r6   r7   r8   r"   r   r   rA   r*   r*   r(   r+   r      s   r   c                       s   e Zd Zd fdd	Z  ZS )ClassEmbedderForMultiCondNFc                    sX   |}t || j}t|| t}|r2|| d ||< t |||}|rL|gn|||< |S )Nr   )r   r   rN   rv   r!   r   )r'   rb   r   Zdisable_dropoutoutZislistc_outr(   r*   r+   r      s    z!ClassEmbedderForMultiCond.forward)NF)r6   r7   r8   r   rA   r*   r*   r(   r+   r      s   r   c                       s:   e Zd ZdZd fdd	Zdd	 Zd
d Zdd Z  ZS )FrozenT5Embedderz(Uses the T5 transformer encoder for textgoogle/t5-v1_1-xxlr   M   Tc                    s>   t    t|| _t|| _|| _|| _|r:| 	  d S r    )
r!   r"   r   from_pretrained	tokenizerr   transformerrr   
max_lengthfreezer'   versionrr   r   r   r(   r*   r+   r"     s    
zFrozenT5Embedder.__init__c                 C   s$   | j  | _ |  D ]
}d|_qd S NFr   rS   rQ   rR   r'   ra   r*   r*   r+   r     s    zFrozenT5Embedder.freezec              	   C   sn   | j |d| jddddd}|d | j}tjddd | j|d	}W d    n1 sZ0    Y  |j}|S 
NTFr   pt
truncationr   return_lengthreturn_overflowing_tokenspaddingreturn_tensors	input_idsr   )enabled)r   r   r   torr   r<   r   r   last_hidden_stater'   textZbatch_encodingtokensoutputszr*   r*   r+   r     s    	*zFrozenT5Embedder.forwardc                 C   s   | |S r    r*   r'   r   r*   r*   r+   r   )  s    zFrozenT5Embedder.encode)r   r   r   T	r6   r7   r8   r   r"   r   r   r   rA   r*   r*   r(   r+   r     s    r   c                       s:   e Zd ZdZd fdd	Zdd	 Zd
d Zdd Z  ZS )FrozenByT5EmbedderzI
    Uses the ByT5 transformer encoder for text. Is character-aware.
    google/byt5-baser   r   Tc                    s>   t    t|| _t|| _|| _|| _|r:| 	  d S r    )
r!   r"   r   r   r   r   r   rr   r   r   r   r(   r*   r+   r"   2  s    
zFrozenByT5Embedder.__init__c                 C   s$   | j  | _ |  D ]
}d|_qd S r   r   r   r*   r*   r+   r   =  s    zFrozenByT5Embedder.freezec              	   C   sn   | j |d| jddddd}|d | j}tjddd | j|d	}W d    n1 sZ0    Y  |j}|S r   r   r   r*   r*   r+   r   C  s    	*zFrozenByT5Embedder.forwardc                 C   s   | |S r    r*   r   r*   r*   r+   r   S  s    zFrozenByT5Embedder.encode)r   r   r   Tr   r*   r*   r(   r+   r   -  s    r   c                       sF   e Zd ZdZg dZd fd
d	Zdd Zedd Zdd Z	  Z
S )FrozenCLIPEmbedderz=Uses the CLIP transformer encoder for text (from huggingface))lastpooledhiddenopenai/clip-vit-large-patch14r   r   Tr   NFc                    s   t    || jv sJ t|| _t|| _|| _|| _	|rH| 
  || _|| _|| _|dkr|d usnJ dt|  krdksn J d S )Nr   r      )r!   r"   LAYERSr   r   r   r   r   rr   r   r   layer	layer_idxreturn_pooledabs)r'   r   rr   r   r   r   r   always_return_pooledr(   r*   r+   r"   \  s    

zFrozenCLIPEmbedder.__init__c                 C   s$   | j  | _ |  D ]
}d|_qd S r   r   r   r*   r*   r+   r   u  s    zFrozenCLIPEmbedder.freezec              	   C   s   | j |d| jddddd}|d | j}| j|| jdkd}| jd	krP|j}n0| jd
krt|jd d d d d f }n|j| j	 }| j
r||jfS |S )NTFr   r   r   r   r   )r   output_hidden_statesr   r   )r   r   r   rr   r   r   r   pooler_outputhidden_statesr   r   r   r*   r*   r+   r   {  s*    	



zFrozenCLIPEmbedder.forwardc                 C   s   | |S r    r*   r   r*   r*   r+   r     s    zFrozenCLIPEmbedder.encode)r   r   r   Tr   NF)r6   r7   r8   r   r   r"   r   r   r   r   rA   r*   r*   r(   r+   r   W  s          
r   c                       sh   e Zd ZdZg dZd fd
d	Zdd Zedd Zdd Z	dd Z
dejdddZdd Z  ZS )FrozenOpenCLIPEmbedder2z8
    Uses the OpenCLIP transformer encoder for text
    )r   r   penultimateViT-H-14laion2b_s32b_b79kr   r   Tr   Fc	                    s   t    || jv sJ tj|td|d\}	}
}
|	`|	| _|| _|| _	|| _
|r\|   || _| jdkrtd| _n| jdkrd| _nt || _d S Ncpurr   
pretrainedr   r   r   rI   )r!   r"   r   	open_clipcreate_model_and_transformsr<   rr   visualr   r   r   r   r   r   NotImplementedErrorlegacy)r'   archr   rr   r   r   r   r   r   r   _r(   r*   r+   r"     s*    


z FrozenOpenCLIPEmbedder2.__init__c                 C   s$   | j  | _ |  D ]
}d|_qd S r   r   rS   rQ   rR   r   r*   r*   r+   r     s    zFrozenOpenCLIPEmbedder2.freezec                 C   sX   t |}| || j}| js,| jr,|S | jrN| jr<J || j |d fS || j S )Nr   )r   tokenizeencode_with_transformerr   rr   r   r   r   r'   r   r   r   r*   r*   r+   r     s    

zFrozenOpenCLIPEmbedder2.forwardc                 C   s   | j |}|| j j }|ddd}| j|| j jd}| jrX|| j }| j |}|S |d }| j |}| 	||}||d< |S d S )NrI   r   rF   	attn_maskr   r   )
r   token_embeddingpositional_embeddingpermutetext_transformer_forwardr   r   r   ln_finalpool)r'   r   r   or   r*   r*   r+   r     s    
z/FrozenOpenCLIPEmbedder2.encode_with_transformerc                 C   s,   |t |jd |jddf | jj }|S )Nr   )rz   )r<   aranger}   argmaxr   Ztext_projection)r'   r   r   r*   r*   r+   r     s
    zFrozenOpenCLIPEmbedder2.poolNr   c                 C   s   i }t | jjjD ]^\}}|t| jjjd krB|ddd|d< | jjjrdtj	 sdt
|||}q|||d}q|ddd|d< |S )NrI   r   rF   r   r   r   )rM   r   r   	resblocksre   r   grad_checkpointingr<   jitis_scriptingr   )r'   r   r   r   rh   rr*   r*   r+   r     s    z0FrozenOpenCLIPEmbedder2.text_transformer_forwardc                 C   s   | |S r    r*   r   r*   r*   r+   r     s    zFrozenOpenCLIPEmbedder2.encode)r   r   r   r   Tr   FT)N)r6   r7   r8   r   r   r"   r   r   r   r   r   r<   r=   r   r   rA   r*   r*   r(   r+   r     s$           #

r   c                       sX   e Zd ZddgZd fdd		Zd
d Zdd Zdd Zdej	dddZ
dd Z  ZS )FrozenOpenCLIPEmbedderr   r   r   r   r   r   Tc           	         s   t    || jv sJ tj|td|d\}}}|`|| _|| _|| _	|rV| 
  || _| jdkrnd| _n| jdkrd| _nt d S r   )r!   r"   r   r   r   r<   rr   r   r   r   r   r   r   r   )	r'   r   r   rr   r   r   r   r   r   r(   r*   r+   r"     s"    	


zFrozenOpenCLIPEmbedder.__init__c                 C   s$   | j  | _ |  D ]
}d|_qd S r   r   r   r*   r*   r+   r   "  s    zFrozenOpenCLIPEmbedder.freezec                 C   s    t |}| || j}|S r    )r   r   r   r   rr   r   r*   r*   r+   r   '  s    
zFrozenOpenCLIPEmbedder.forwardc                 C   sV   | j |}|| j j }|ddd}| j|| j jd}|ddd}| j |}|S )NrI   r   rF   r   )r   r   r   r   r   r   r   )r'   r   r   r*   r*   r+   r   ,  s    z.FrozenOpenCLIPEmbedder.encode_with_transformerNr   c                 C   sf   t | jjjD ]R\}}|t| jjj| j kr2 qb| jjjrTtj	 sTt
|||}q|||d}q|S )Nr   )rM   r   r   r   re   r   r   r<   r   r   r   )r'   r   r   rh   r   r*   r*   r+   r   5  s    z/FrozenOpenCLIPEmbedder.text_transformer_forwardc                 C   s   | |S r    r*   r   r*   r*   r+   r   B  s    zFrozenOpenCLIPEmbedder.encode)r   r   r   r   Tr   )N)r6   r7   r8   r   r"   r   r   r   r<   r=   r   r   rA   r*   r*   r(   r+   r     s         	r   c                       sP   e Zd ZdZd fd
d	Zdd Zdd ZedddZdd Z	dd Z
  ZS )FrozenOpenCLIPImageEmbedderzA
    Uses the OpenCLIP vision transformer encoder for images
    r   r   r   r   TrK   Fr   c                    s   t    tj|td|d\}}}|`|| _|
| _| jdk| _	|	oL| j	 | _
|| _|| _|rh|   || _| jdtg ddd | jdtg d	dd || _|| _d | _|| jj_|| _d S )
Nr   r   r   mean)g3<4'?gwgM?gy{ ?F)
persistentstd)gB91?gwt.?g	U?)r!   r"   r   r   r<   rr   r   r   	max_cropspad_to_max_lenrepeat_to_max_lenr   r   	antialiasregister_bufferr=   r1   unsqueeze_dimZstored_batchr   output_tokens)r'   r   r   rr   r   r   r  r1   r  r  Znum_image_cropsr  r   r   r(   r*   r+   r"   K  s6    

z$FrozenOpenCLIPImageEmbedder.__init__c                 C   s<   t jj|ddd| jd}|d d }t j|| j| j}|S )N)   r  bicubicT)interpolationalign_cornersr  rp   g       @)korniageometryresizer  enhance	normalizer   r   r   r*   r*   r+   
preprocessw  s    z&FrozenOpenCLIPImageEmbedder.preprocessc                 C   s$   | j  | _ |  D ]
}d|_qd S r   r   r   r*   r*   r+   r     s    z"FrozenOpenCLIPImageEmbedder.freezec              	   C   s  |  |}d }| jr&|d |d  }}||j}| jdkr|s| jdkstd| j tj|j	d |j
d d d d f | }|d urttd| j tj|j	d |j
d || }| jr|d d d d d f }| jr| jrJ | jrJ ||fS | jr<| dkr$|d d d d d f }n|}t|d| jd|fS | jr| d	ksVJ t|tj|j	d | j|j	d  |j	d |j
dfd}||d d dd
f fS |S )Nr   rI   rK   rp   rq   rF   zb 1 d -> b n dr_   r   .)encode_with_vision_transformerr  r   dtyper1   r   r<   r{   r|   r}   rr   r   r  r  r  rz   r
   r   r   zeros)r'   imageZ
no_dropoutr   r   z_Zz_padr*   r*   r+   r     sh    




	z#FrozenOpenCLIPImageEmbedder.forwardc              	   C   s  |  dkr*| j|jd ks J t|d}| |}| jsZ| jjjrHJ | j|}d }n| jjjshJ | j|\}}| jdkrt|d| jd}t	d| j
 tj|jd |jd d|jd | }|d urt|d	| jd}td
| jj d | jr||fS |S )NrH   rI   zb n c h w -> (b n) c h wr   z(b n) d -> b n dr  rp   rq   z(b n) t d -> b t (n d)z2You are running very experimental token-concat in z9. Check what you are doing, and then remove this message.)rz   r   r}   r	   r  r  r   r   r<   r{   r1   r|   rr   rT   r)   r6   )r'   imgr   r   r*   r*   r+   r    s8    


z:FrozenOpenCLIPImageEmbedder.encode_with_vision_transformerc                 C   s   | |S r    r*   r   r*   r*   r+   r     s    z"FrozenOpenCLIPImageEmbedder.encode)r   r   r   r   TTrK   FFr   F)F)r6   r7   r8   r   r"   r  r   r   r   r  r   rA   r*   r*   r(   r+   r   F  s&              ,5"r   c                       s.   e Zd Zd fdd	Zdd Zd	d
 Z  ZS )FrozenCLIPT5Encoderr   google/t5-v1_1-xlr   r   c              	      sr   t    t|||d| _t|||d| _t| jjj dt	| jd dd| jjj dt	| jd dd d S )N)r   z has gư>z.2fz M parameters, z comes with z
 M params.)
r!   r"   r   clip_encoderr   
t5_encoderrT   r)   r6   r   )r'   Zclip_versionZ
t5_versionrr   Zclip_max_lengthZt5_max_lengthr(   r*   r+   r"     s    
zFrozenCLIPT5Encoder.__init__c                 C   s   | |S r    r*   r   r*   r*   r+   r     s    zFrozenCLIPT5Encoder.encodec                 C   s    | j |}| j|}||gS r    )r  r   r  )r'   r   Zclip_zZt5_zr*   r*   r+   r     s    zFrozenCLIPT5Encoder.forward)r   r  r   r   r   )r6   r7   r8   r"   r   r   rA   r*   r*   r(   r+   r    s        r  c                	       s.   e Zd Zd fdd	Zd	d
 Zdd Z  ZS )SpatialRescalerrI   bilinear      ?r   NFc
           
         s   t    || _| jdksJ |dv s*J || _ttjjj|d| _	|d upN|	| _
| j
rtd| d| d tj|||||d d| _|| _d S )	Nr   )nearestlinearr  	trilinearr  area)modezSpatial Rescaler mapping from z to z channels after resizing.rF   )kernel_sizebiasr   )r!   r"   n_stages
multiplierr   r<   r\   
functionalinterpolateinterpolatorremap_outputrT   Conv2dchannel_mapper
wrap_video)
r'   r&  methodr'  in_channelsout_channelsr%  r.  r$  r+  r(   r*   r+   r"     s&    
zSpatialRescaler.__init__c                 C   s   | j r4|jdkr4|j\}}}}}t|d}t|d}t| jD ]}| j|| jd}q>| j rvt|d|||d}t|d}| jr| 	|}|S )NrH   zb c t h w -> b t c h wzb t c h w -> (b t) c h w)scale_factorz(b t) c h w -> b t c h w)btr   zb t c h w -> b c t h w)
r.  ndimr}   r	   rd   r&  r*  r'  r+  r-  )r'   r   BCTHWstager*   r*   r+   r   *  s    



zSpatialRescaler.forwardc                 C   s   | |S r    r*   r   r*   r*   r+   r   :  s    zSpatialRescaler.encode)	rI   r  r  r   NFFrI   F)r6   r7   r8   r"   r   r   rA   r*   r*   r(   r+   r    s            'r  c                       sB   e Zd Zd fdd	ZdddZdddZdd Zdd Z  ZS )LowScaleEncoderr      @   rp   c                    s<   t    || _t|| _| j|||d| _|| _|| _d S )N)	timestepslinear_start
linear_end)	r!   r"   max_noise_levelr   r   register_scheduleZaugmentation_scheduleout_sizer2  )r'   model_configr@  rA  r?  rB  output_sizer2  r(   r*   r+   r"   ?  s    


zLowScaleEncoder.__init__r   -C6?{Gz?Mb?c                 C   s8  t |||||d}d| }tj|dd}td|d d }	|j\}t|| _|| _|| _|jd | jksrJ dt	t
jt
jd}
| d|
| | d	|
| | d
|
|	 | d|
t| | d|
td|  | d|
td|  | d|
td|  | d|
td| d  d S )N)r@  rA  cosine_srp   r   )axisr   z+alphas have to be defined for each timestep)r  betasalphas_cumprodalphas_cumprod_prevsqrt_alphas_cumprodsqrt_one_minus_alphas_cumprodZlog_one_minus_alphas_cumprodZsqrt_recip_alphas_cumprodZsqrt_recipm1_alphas_cumprodrI   )r   rW   cumprodr[   r}   intZnum_timestepsr@  rA  r   r<   tensorfloat32r  sqrtlog)r'   Zbeta_scheduler?  r@  rA  rJ  rL  alphasrM  rN  Zto_torchr*   r*   r+   rC  R  sF    
z!LowScaleEncoder.register_scheduleNc                    s:   t | fdd}t| j| j  t| j| j|  S )Nc                      s
   t  S r    )r<   
randn_liker*   x_startr*   r+   <lambda>  ro   z*LowScaleEncoder.q_sample.<locals>.<lambda>)r   r   rO  r}   rP  )r'   rZ  r4  noiser*   rY  r+   q_sample  s    zLowScaleEncoder.q_samplec                 C   s~   | j |}t|tr| }|| j }tjd| j|j	d f|j
d }| ||}| jd urvtjjj|| jdd}||fS )Nr   rq   r  )sizer#  )r   r   rN   r   sampler2  r<   randintrB  r}   rr   r   r]  rD  r\   r(  r)  )r'   r   r   Znoise_levelr*   r*   r+   r     s    



zLowScaleEncoder.forwardc                 C   s   || j  }| j|S r    )r2  r   decode)r'   r   r*   r*   r+   ra    s    
zLowScaleEncoder.decode)r   r=  r>  rp   )r   r   rG  rH  rI  )N)	r6   r7   r8   r"   rC  r]  r   ra  rA   r*   r*   r(   r+   r<  >  s            
0
r<  c                       s(   e Zd ZdZ fddZdd Z  ZS )ConcatTimestepEmbedderNDz9embeds each dimension independently and concatenates themc                    s   t    t|| _|| _d S r    )r!   r"   r   timestepoutdim)r'   rd  r(   r*   r+   r"     s    

z!ConcatTimestepEmbedderND.__init__c                 C   sn   |j dkr|d d d f }t|jdks,J |jd |jd  }}t|d}| |}t|d||| jd}|S )NrI   rF   r   zb d -> (b d)z(b d) d2 -> b (d d2))r3  dd2)r5  re   r}   r	   rc  rd  )r'   r   r3  dimsr   r*   r*   r+   r     s    


z ConcatTimestepEmbedderND.forwardr   r*   r*   r(   r+   rb    s   rb  c                       sB   e Zd Zd	eed fddZeeej	f d fddZ
  ZS )
GaussianEncoderrp   T)weightflatten_outputc                    s*   t  j|i | t | _|| _|| _d S r    )r!   r"   r   	posteriorri  rj  )r'   ri  rj  argsr   r(   r*   r+   r"     s    zGaussianEncoder.__init__r,   c                    sH   t  |}| |\}}|d |d< | j|d< | jr@t|d}||fS )NZkl_losslossri  zb c h w -> b (h w ) c)r!   r   rk  ri  rj  r	   )r'   r   r   rV  r(   r*   r+   r     s    

zGaussianEncoder.forward)rp   T)r6   r7   r8   r;   r:   r"   r   r   r<   r=   r   rA   r*   r*   r(   r+   rh    s
    rh  )B
contextlibr   	functoolsr   typingr   r   r   r   r   r  numpyrW   r   r<   torch.nnr\   Zeinopsr	   r
   	omegaconfr   torch.utils.checkpointr   transformersr   r   r   r   r   Z!modules.autoencoding.regularizersr   modules.diffusionmodules.modelr   Z$modules.diffusionmodules.openaimodelr   Zmodules.diffusionmodules.utilr   r   #modules.distributions.distributionsr   utilr   r   r   r   r   r   Moduler   rB   r   r   r   r   r   r   r   r   r   r   r  r  r<  rb  rh  r*   r*   r*   r+   <module>   sF    
,p)*AfH !<_