a
    
d1                     @   s(  d dl Z d dlmZ d dlZd dlmZ d dlmZmZm	Z	m
Z
 d dlZd dlmZmZmZ G dd dejZG dd deZG d	d
 d
ejZdddZG dd deZG dd deZG dd dejZG dd deZG dd deZG dd deZd dlmZ d dlmZ G dd deZdS )    N)
checkpoint)T5TokenizerT5EncoderModelCLIPTokenizerCLIPTextModel)defaultcount_paramsautocastc                       s$   e Zd Z fddZdd Z  ZS )AbstractEncoderc                    s   t    d S N)super__init__)self	__class__ o/var/www/html/stable-diffusion-webui/repositories/stable-diffusion-stability-ai/ldm/modules/encoders/modules.pyr      s    zAbstractEncoder.__init__c                 O   s   t d S r   )NotImplementedError)r   argskwargsr   r   r   encode   s    zAbstractEncoder.encode)__name__
__module____qualname__r   r   __classcell__r   r   r   r   r
      s   r
   c                   @   s   e Zd Zdd ZdS )IdentityEncoderc                 C   s   |S r   r   r   xr   r   r   r      s    zIdentityEncoder.encodeN)r   r   r   r   r   r   r   r   r      s   r   c                       s2   e Zd Zd fdd	Zddd	ZdddZ  ZS )ClassEmbedder  class皙?c                    s.   t    || _t||| _|| _|| _d S r   )r   r   keynn	Embedding	embedding	n_classesucg_rate)r   	embed_dimr&   r"   r'   r   r   r   r      s
    
zClassEmbedder.__init__NFc                 C   s   |d u r| j }|| d d d f }| jdkrv|svdtt|| j  }|| d| t| | jd   }| }| |}|S )N              ?   )r"   r'   torch	bernoulli	ones_liker&   longr%   )r   batchr"   disable_dropoutcmaskr   r   r   forward"   s    $
zClassEmbedder.forwardcudac                 C   s,   | j d }tj|f|d| }| j|i}|S )Nr+   device)r&   r,   onesr"   )r   bsr7   uc_classucr   r   r   get_unconditional_conditioning.   s    

z,ClassEmbedder.get_unconditional_conditioning)r   r    r!   )NF)r5   )r   r   r   r   r4   r<   r   r   r   r   r   r      s   
r   Tc                 C   s   | S )zbOverwrite model.train with this function to make sure train/eval mode
    does not change anymore.r   )r   moder   r   r   disabled_train5   s    r>   c                       s:   e Zd ZdZd fdd	Zdd	 Zd
d Zdd Z  ZS )FrozenT5Embedderz(Uses the T5 transformer encoder for textgoogle/t5-v1_1-larger5   M   Tc                    s>   t    t|| _t|| _|| _|| _|r:| 	  d S r   )
r   r   r   from_pretrained	tokenizerr   transformerr7   
max_lengthfreeze)r   versionr7   rE   rF   r   r   r   r   >   s    
zFrozenT5Embedder.__init__c                 C   s$   | j  | _ |  D ]
}d|_qd S NFrD   eval
parametersrequires_gradr   paramr   r   r   rF   H   s    zFrozenT5Embedder.freezec              	   C   s@   | j |d| jddddd}|d | j}| j|d}|j}|S )NTFrE   pt
truncationrE   return_lengthreturn_overflowing_tokenspaddingreturn_tensors	input_ids)rV   )rC   rE   tor7   rD   last_hidden_stater   textbatch_encodingtokensoutputszr   r   r   r4   N   s    zFrozenT5Embedder.forwardc                 C   s   | |S r   r   r   rZ   r   r   r   r   W   s    zFrozenT5Embedder.encode)r@   r5   rA   T)	r   r   r   __doc__r   rF   r4   r   r   r   r   r   r   r?   ;   s     
	r?   c                       sB   e Zd ZdZg dZd fd	d
	Zdd Zdd Zdd Z  Z	S )FrozenCLIPEmbedderz=Uses the CLIP transformer encoder for text (from huggingface))lastpooledhiddenopenai/clip-vit-large-patch14r5   rA   Trb   Nc                    s   t    || jv sJ t|| _t|| _|| _|| _	|rH| 
  || _|| _|dkr|d ushJ dt|  krdksn J d S )Nrd   r      )r   r   LAYERSr   rB   rC   r   rD   r7   rE   rF   layer	layer_idxabs)r   rG   r7   rE   rF   rh   ri   r   r   r   r   c   s    
zFrozenCLIPEmbedder.__init__c                 C   s$   | j  | _ |  D ]
}d|_qd S rH   rI   rM   r   r   r   rF   s   s    zFrozenCLIPEmbedder.freezec              	   C   s   | j |d| jddddd}|d | j}| j|| jdkd}| jd	krP|j}n0| jd
krt|jd d d d d f }n|j| j	 }|S )NTFrE   rO   rP   rV   rd   )rV   output_hidden_statesrb   rc   )
rC   rE   rW   r7   rD   rh   rX   pooler_outputhidden_statesri   rY   r   r   r   r4   y   s    

zFrozenCLIPEmbedder.forwardc                 C   s   | |S r   r   r_   r   r   r   r      s    zFrozenCLIPEmbedder.encode)re   r5   rA   Trb   N)
r   r   r   r`   rg   r   rF   r4   r   r   r   r   r   r   ra   [   s     ra   c                       sF   e Zd Zdej rdndddf fdd	Zdd	 Zdd
dZ  Z	S )ClipImageEmbedderFr5   cpuTr)   c                    sn   t    ddlm} ||||d\| _}|| _| jdtg ddd | jdtg d	dd || _	d S )
Nr   )load)namer7   jitmeang3<4'?gwgM?gy{ ?F
persistentstdgB91?gwt.?g	U?)
r   r   cliprp   model	antialiasregister_bufferr,   Tensorr'   )r   rz   rr   r7   r{   r'   Z	load_clip_r   r   r   r      s    
zClipImageEmbedder.__init__c                 C   s<   t jj|ddd| jd}|d d }t j|| j| j}|S N)   r   bicubicT)interpolationalign_cornersr{   r*   g       @korniageometryresizer{   enhance	normalizers   rw   r   r   r   r   
preprocess   s    
zClipImageEmbedder.preprocessc                 C   sf   | j | |}||j}| jdkrb|sbtd| j tj|j	d |j
d d d d f | }|S Nr)   r*   r   r6   )rz   encode_imager   rW   dtyper'   r,   r-   r8   shaper7   )r   r   
no_dropoutoutr   r   r   r4      s
    6zClipImageEmbedder.forward)F)
r   r   r   r,   r5   is_availabler   r   r4   r   r   r   r   r   rn      s   
rn   c                       s\   e Zd ZdZddgZd fd	d
	Zdd Zdd Zdd Zde	j
dddZdd Z  ZS )FrozenOpenCLIPEmbedderz8
    Uses the OpenCLIP transformer encoder for text
    rb   penultimateViT-H-14laion2b_s32b_b79kr5   rA   Tc           	         s   t    || jv sJ tj|td|d\}}}|`|| _|| _|| _	|rV| 
  || _| jdkrnd| _n| jdkrd| _nt d S )Nro   r7   
pretrainedrb   r   r   r+   )r   r   rg   	open_clipcreate_model_and_transformsr,   r7   visualrz   rE   rF   rh   ri   r   )	r   archrG   r7   rE   rF   rh   rz   r~   r   r   r   r      s    


zFrozenOpenCLIPEmbedder.__init__c                 C   s$   | j  | _ |  D ]
}d|_qd S rH   rz   rJ   rK   rL   rM   r   r   r   rF      s    zFrozenOpenCLIPEmbedder.freezec                 C   s    t |}| || j}|S r   )r   tokenizeencode_with_transformerrW   r7   )r   rZ   r\   r^   r   r   r   r4      s    
zFrozenOpenCLIPEmbedder.forwardc                 C   sV   | j |}|| j j }|ddd}| j|| j jd}|ddd}| j |}|S )Nr+   r      	attn_mask)rz   token_embeddingpositional_embeddingpermutetext_transformer_forwardr   ln_final)r   rZ   r   r   r   r   r      s    z.FrozenOpenCLIPEmbedder.encode_with_transformerN)r   c                 C   sf   t | jjjD ]R\}}|t| jjj| j kr2 qb| jjjrTtj	 sTt
|||}q|||d}q|S )Nr   )	enumeraterz   rD   	resblockslenri   grad_checkpointingr,   rr   is_scriptingr   )r   r   r   irr   r   r   r      s    z/FrozenOpenCLIPEmbedder.text_transformer_forwardc                 C   s   | |S r   r   r_   r   r   r   r      s    zFrozenOpenCLIPEmbedder.encode)r   r   r5   rA   Trb   )N)r   r   r   r`   rg   r   rF   r4   r   r,   r}   r   r   r   r   r   r   r   r      s     	
r   c                       sP   e Zd ZdZd fd	d
	Zdd Zdd ZedddZdd Z	dd Z
  ZS )FrozenOpenCLIPImageEmbedderzA
    Uses the OpenCLIP vision transformer encoder for images
    r   r   r5   rA   Trc   r)   c	                    s   t    tj|td|d\}	}
}
|	`|	| _|| _|| _|rH| 	  || _
| j
dkrdt d| _|| _| jdtg ddd | jd	tg d
dd || _d S )Nro   r   r   r+   rs   rt   Fru   rw   rx   )r   r   r   r   r,   r7   rD   rz   rE   rF   rh   r   ri   r{   r|   r}   r'   )r   r   rG   r7   rE   rF   rh   r{   r'   rz   r~   r   r   r   r      s$    

z$FrozenOpenCLIPImageEmbedder.__init__c                 C   s<   t jj|ddd| jd}|d d }t j|| j| j}|S r   r   r   r   r   r   r     s    
z&FrozenOpenCLIPImageEmbedder.preprocessc                 C   s$   | j  | _ |  D ]
}d|_qd S rH   r   rM   r   r   r   rF     s    z"FrozenOpenCLIPImageEmbedder.freezeFc                 C   sR   |  |}| jdkrN|sNtd| j tj|jd |jd d d d f | }|S r   )encode_with_vision_transformerr'   r,   r-   r8   r   r7   )r   imager   r^   r   r   r   r4     s    
6z#FrozenOpenCLIPImageEmbedder.forwardc                 C   s   |  |}| j|}|S r   )r   rz   r   )r   imgr   r   r   r   r   !  s    
z:FrozenOpenCLIPImageEmbedder.encode_with_vision_transformerc                 C   s   | |S r   r   r_   r   r   r   r   &  s    z"FrozenOpenCLIPImageEmbedder.encode)r   r   r5   rA   Trc   Tr)   )F)r   r   r   r`   r   r   rF   r	   r4   r   r   r   r   r   r   r   r      s     
r   c                       s.   e Zd Zd fdd	Zdd Zd	d
 Z  ZS )FrozenCLIPT5Encoderre   google/t5-v1_1-xlr5   rA   c              	      sr   t    t|||d| _t|||d| _t| jjj dt	| jd dd| jjj dt	| jd dd d S )N)rE   z has gư>z.2fz M parameters, z comes with z
 M params.)
r   r   ra   clip_encoderr?   
t5_encoderprintr   r   r   )r   clip_version
t5_versionr7   clip_max_lengtht5_max_lengthr   r   r   r   +  s    
 zFrozenCLIPT5Encoder.__init__c                 C   s   | |S r   r   r_   r   r   r   r   3  s    zFrozenCLIPT5Encoder.encodec                 C   s    | j |}| j|}||gS r   )r   r   r   )r   rZ   clip_zt5_zr   r   r   r4   6  s    zFrozenCLIPT5Encoder.forward)re   r   r5   rA   rA   )r   r   r   r   r   r4   r   r   r   r   r   r   *  s
     r   ) ImageConcatWithNoiseAugmentation)Timestepc                       s>   e Zd Zddd fdd
Zdd Zdd	 Zdd
dZ  ZS )CLIPEmbeddingNoiseAugmentationN   )clip_stats_pathtimestep_dimc                   s   t  j|i | |d u r2t|t| }}ntj|dd\}}| jd|d d d f dd | jd|d d d f dd t|| _d S )Nro   )map_location	data_meanFru   data_std)	r   r   r,   zerosr8   rp   r|   r   
time_embed)r   r   r   r   r   Z	clip_meanZclip_stdr   r   r   r   A  s    z'CLIPEmbeddingNoiseAugmentation.__init__c                 C   s   || j  d | j }|S )Nr*   )r   r   r   r   r   r   scaleK  s    z$CLIPEmbeddingNoiseAugmentation.scalec                 C   s   || j  | j }|S r   )r   r   r   r   r   r   unscaleP  s    z&CLIPEmbeddingNoiseAugmentation.unscalec                 C   sn   |d u r,t jd| j|jd f|jd }nt|t js<J | |}| 	||}| 
|}| |}||fS )Nr   r6   )r,   randintmax_noise_levelr   r7   r/   
isinstancer}   r   q_sampler   r   )r   r   noise_levelr^   r   r   r   r4   U  s    $


z&CLIPEmbeddingNoiseAugmentation.forward)N)r   r   r   r   r   r   r4   r   r   r   r   r   r   @  s   
r   )T) r,   torch.nnr#   r   torch.utils.checkpointr   transformersr   r   r   r   r   Zldm.utilr   r   r	   Moduler
   r   r   r>   r?   ra   rn   r   r   r   Z&ldm.modules.diffusionmodules.upscalingr   Z(ldm.modules.diffusionmodules.openaimodelr   r   r   r   r   r   <module>   s&   
 /&?;