a
    d`j                     @   sP  d dl mZ d dlZd dlmZmZmZmZ d dlZd dlm	Z	 d dl
mZ d dlmZ ddlmZ G d	d
 d
e	jZG dd de	jZG dd de	jZG dd de	jZG dd de	jZG dd de	jZG dd de	jZG dd de	jZG dd de	jZG dd de	jZG dd de	jZG dd  d e	jZG d!d" d"eZdS )#    )OrderedDictN)CallableOptionalSequenceTuple)nn)
functional)
checkpoint   )	to_2tuplec                   @   s    e Zd ZdZejdddZdS )LayerNormFp32zKSubclass torch's LayerNorm to handle fp16 (by casting to float32 and back).xc                 C   s2   |j }t|tj| j| j| j| j	}||S N)
dtypeF
layer_normtotorchfloat32normalized_shapeweightbiasepsselfr   	orig_type r   ^/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/open_clip/transformer.pyforward   s    "zLayerNormFp32.forwardN__name__
__module____qualname____doc__r   Tensorr   r   r   r   r   r      s   r   c                   @   s    e Zd ZdZejdddZdS )	LayerNormz;Subclass torch's LayerNorm (with cast back to input dtype).r   c                 C   s*   |j }t|| j| j| j| j}||S r   )r   r   r   r   r   r   r   r   r   r   r   r   r      s    zLayerNorm.forwardNr    r   r   r   r   r&      s   r&   c                   @   s   e Zd ZejdddZdS )	QuickGELUr   c                 C   s   |t d|  S )NgZd;?)r   Zsigmoidr   r   r   r   r   r   !   s    zQuickGELU.forwardN)r!   r"   r#   r   r%   r   r   r   r   r   r'      s   r'   c                       s&   e Zd Zd fdd	Zdd Z  ZS )
LayerScaleh㈵>Fc                    s*   t    || _t|t| | _d S r   )super__init__inplacer   	Parameterr   onesgamma)r   dimZinit_valuesr-   	__class__r   r   r,   &   s    
zLayerScale.__init__c                 C   s   | j r|| jS || j S r   )r-   Zmul_r0   r(   r   r   r   r   +   s    zLayerScale.forward)r*   F)r!   r"   r#   r,   r   __classcell__r   r   r2   r   r)   %   s   r)   c                       s*   e Zd ZdZd fdd	Zdd Z  ZS )PatchDropoutz*
    https://arxiv.org/abs/2212.00794
    Tc                    s4   t    d|  krdk s$n J || _|| _d S )Nr         ?)r+   r,   probexclude_first_token)r   r7   r8   r2   r   r   r,   4   s    
zPatchDropout.__init__c           
      C   s   | j r| jdkr|S | jrF|d d d df |d d dd f  }}n tjtj|d d d df }| d }| d }t|}|d }d| j }t	dt
|| }t||}|j|ddj}	|||	f }| jrtj||fdd}|S )N        r
   r   ).Nr1   )Ztrainingr7   r8   r   jitZannotater%   sizearangemaxintrandnZtopkindicescat)
r   r   Z
cls_tokensbatchZ
num_tokensZbatch_indicesZ	keep_probZnum_patches_keepZrandZpatch_indices_keepr   r   r   r   :   s"    , 

zPatchDropout.forward)T)r!   r"   r#   r$   r,   r   r4   r   r   r2   r   r5   /   s   r5   c                       sH   e Zd Zddddedddf fdd	Zdeej d	d
dZ	  Z
S )	Attention   TFg      Y@r9   c	           	   
      s  t    || _|| _|| dks*J d|| _|| | _| jd | _|| _t	t
|d |f| j | _|rt	t
|d | _nd | _| jrt	t
dt
|ddf | _nd | _t|| _| jrt	t
|ddf| _nd | _t||| _t|| _d S )Nr   z$dim should be divisible by num_heads         
   r
   )r+   r,   scaled_cosinescale_heads	num_headsZhead_dimscalelogit_scale_maxr   r.   r   rA   in_proj_weightzerosin_proj_biaslogr/   logit_scaleZDropout	attn_drop
head_scaleLinearout_projout_drop)	r   r1   rL   Zqkv_biasrJ   rK   rN   rT   Z	proj_dropr2   r   r   r,   X   s*    

 $zAttention.__init__N	attn_maskc                 C   s  |j \}}}t|| j| jjddd\}}}| ||| j d	dd}| ||| j d	dd}| ||| j d	dd}| j
d urttj|ddtj|dd	dd}	tj| j
| jd }
|	|| j|||
 }	|	d||}	n|| j }t||	dd}	|d ur^|jtjkrVtj||jd}||td	 |}|	|7 }	|	jdd}	| |	}	t|	|}| jd ur||| j||| j }|d||}|	dd|||}| |}| |}|S )
NrH   r:   r;   r   r
   )r?   )r   -inf)shaper   ZlinearrO   rQ   chunk
contiguousviewrL   Z	transposerS   r   Zbmm	normalizeclamprN   exprM   r   boolZ
zeros_likemasked_fill_floatZsoftmaxrT   rU   reshaperW   rX   )r   r   rZ   LNCqkvattnrS   Znew_attn_maskr   r   r   r      s8    "   
(




zAttention.forward)N)r!   r"   r#   mathrR   r,   r   r   r%   r   r4   r   r   r2   r   rE   W   s   'rE   c                       sP   e Zd Zddefeeeeed fddZejdddZ	ed	d
dZ
  ZS )AttentionalPoolerrF      )d_modelcontext_dimn_head	n_queries
norm_layerc                    sJ   t    tt||| _tj||||d| _||| _	||| _
d S )N)ZkdimZvdim)r+   r,   r   r.   r   rA   queryMultiheadAttentionrn   ln_qln_k)r   rr   rs   rt   ru   rv   r2   r   r   r,      s
    

zAttentionalPooler.__init__r   c                 C   sV   |  |ddd}|jd }| | j}| j| ||||ddd }|dddS )Nr
   r      F)need_weights)rz   permuter]   ry   rw   rn   _repeat)r   r   ri   rk   outr   r   r   r      s
    
zAttentionalPooler.forwardri   c                 C   s   | dd|dS )Nr
   )	unsqueezerepeat)r   rw   ri   r   r   r   r~      s    zAttentionalPooler._repeat)r!   r"   r#   r&   r@   r   r,   r   r%   r   r~   r4   r   r   r2   r   rp      s   rp   c                	       s   e Zd Zddejedfeeeeeee	d fddZ
dejeej eej eej ddd	Zdejeej eej eej dd
dZ  ZS )ResidualAttentionBlock      @NF)rr   rt   	mlp_ratiols_init_value	act_layerrv   is_cross_attentionc           	   
      s   t    ||| _t||| _|d ur4t||nt | _|rL||| _	||| _
t|| }ttdt||fd| fdt||fg| _|d urt||nt | _d S )Nc_fcgeluc_proj)r+   r,   ln_1r   rx   rn   r)   Identityls_1ln_1_kvln_2r@   
Sequentialr   rV   mlpls_2)	r   rr   rt   r   r   r   rv   r   	mlp_widthr2   r   r   r,      s    





zResidualAttentionBlock.__init__q_xk_xv_xrZ   c                 C   sP   |d ur|n|}|d ur|n|}|d ur4| |jnd }| j|||d|dd S )NF)r|   rZ   r   )r   r   rn   )r   r   r   r   rZ   r   r   r   	attention   s    
z ResidualAttentionBlock.attentionc              	   C   s   t | dr|d ur| |nd }t | dr<|d ur<| |nd }|| | j| ||||d }|| | | | }|S )Nr   r   )hasattrr   r   r   r   r   r   r   )r   r   r   r   rZ   r   r   r   r   r      s
      "zResidualAttentionBlock.forward)NNN)NNN)r!   r"   r#   r   GELUr&   r@   rf   r   rd   r,   r   r%   r   r   r   r4   r   r   r2   r   r      s@         r   c                       sb   e Zd Zddejeddddfeeeeeee	e	e	e	d
 fddZ
d
ejeej ddd	Z  ZS )CustomResidualAttentionBlockr   NF)
rr   rt   r   r   r   rv   scale_cosine_attnrK   
scale_attnscale_fcc                    s   t    ||| _t||||d| _|	r2||nt | _|d urNt||nt | _	||| _
t|| }ttdt||fd|
r||nt fd| fdt||fg| _|d urt||nt | _d S )N)rJ   rK   r   lnr   r   )r+   r,   r   rE   rn   r   r   ln_attnr)   r   r   r@   r   r   rV   r   r   )r   rr   rt   r   r   r   rv   r   rK   r   r   r   r2   r   r   r,      s$    



z%CustomResidualAttentionBlock.__init__r   rZ   c              	   C   sB   ||  | | j| ||d }|| | | | }|S NrY   )r   r   rn   r   r   r   r   )r   r   rZ   r   r   r   r     s    $z$CustomResidualAttentionBlock.forward)N)r!   r"   r#   r   r   r&   r@   rf   r   rd   r,   r   r%   r   r   r4   r   r   r2   r   r      s*   "r   c                	       sd   e Zd Zddejefeeeeeeed fddZ	e
jdddZde
jee
j d	d
dZ  ZS )Transformerr   Nwidthlayersheadsr   r   r   rv   c                    sH   t    | _|| _d| _t fddt|D | _d S )NFc              
      s    g | ]}t  d qS )r   r   rv   r   .0_r   r   r   r   rv   r   r   r   
<listcomp>0  s   z(Transformer.__init__.<locals>.<listcomp>)	r+   r,   r   r   grad_checkpointingr   
ModuleListrange	resblocks)r   r   r   r   r   r   r   rv   r2   r   r   r,   !  s    

zTransformer.__init__)returnc                 C   s6   t | jd jjdr$| jd jjjS | jd jjjjS )Nr   int8_original_dtype)r   r   r   r   r   r   r   r   r   r   r   get_cast_dtype6  s    zTransformer.get_cast_dtyper   c                 C   s>   | j D ]2}| jr,tj s,t||d d |}q|||d}q|S r   )r   r   r   r<   is_scriptingr	   )r   r   rZ   rr   r   r   r   ;  s
    
zTransformer.forward)N)r!   r"   r#   r   r   r&   r@   rf   r   r,   r   r   r   r%   r   r   r4   r   r   r2   r   r      s   r   c                       s   e Zd ZU ejje ed< ddddddddej	e
dfeeeeeeeeeeeeeeeeed fd	d
ZdddZdd ZejjdddZejeejejf dddZejdddZ  ZS )VisionTransformeroutput_tokensNFrq   rF      r9   )
image_size
patch_sizer   r   r   r   r   global_average_poolattentional_poolru   attn_pooler_heads
output_dimpatch_dropoutinput_patchnormr   rv   r   c              	      s  t    || _t| \}}| _t| \}}| _|| || f| _|| _|| _|r||| d }t	|| _
t||| _n t | _
tjd|||dd| _|d }t|t| | _t|t| jd | jd  d | | _|dkrt|nt | _||| _t|||||||d| _|| _|	rdt||||
d	| _||| _t|t|| | _n(d | _||| _t|t|| | _|   d S )
NrH   F)Zin_channelsZout_channelsZkernel_sizeZstrider   rG   r   r
   r9   r   )rt   ru   ) r+   r,   r   r   r   r   	grid_sizer   r   r&   patchnorm_pre_lnr   rV   conv1r   ZConv2dr.   r   rA   class_embeddingpositional_embeddingr5   r   ln_prer   transformerr   rp   	attn_poolln_postprojinit_parameters)r   r   r   r   r   r   r   r   r   r   ru   r   r   r   r   r   rv   r   Zimage_heightZimage_widthZpatch_heightZpatch_widthZpatch_input_dimrM   r2   r   r   r,   H  sH    


,



zVisionTransformer.__init__r   c                    s|   |   D ]
}d|_q|dkrx| j| j| j| jgg| jjd d | jjd | jg| j	} fdd  || d   d S )NFr   r:   c                    sL   t | tr| D ]} | qn*t | tjjr4d| _n|  D ]
}d|_q<d S )NT)
isinstancer   r   r   r.   requires_grad
parameters)r   gp_unlockr   r   r     s    
z'VisionTransformer.lock.<locals>._unlock)
r   r   r   r   r   r   r   r   r   r   )r   Zunlocked_groupsZfreeze_bn_statsparamgroupsr   r   r   lock  s&    	
zVisionTransformer.lockc                 C   s   d S r   r   r   r   r   r   r     s    z!VisionTransformer.init_parametersTc                 C   s   || j _d S r   r   r   r   enabler   r   r   set_grad_checkpointing  s    z(VisionTransformer.set_grad_checkpointing)r   r   c                 C   s>   | j r|jdd|fS |d d df |d d dd f fS d S )Nr
   r;   r   )r   meanr(   r   r   r   _global_pool  s    zVisionTransformer._global_poolr   c              	   C   s  | j r||jd |jd | jd | jd | jd | jd }|dddddd}||jd | jd | jd  d}| |}| |}n2| |}||jd |jd d}|ddd}tj	| j
|jtj|jd d|jd |j|jd |gdd	}|| j|j }| |}| |}|ddd}| |}|ddd}| jd urz| |}| |}| |\}}n| |\}}| |}| jd ur|| j }| jr||fS |S )
Nr   r
   r{      rH      r:   r   devicer;   )r   rg   r]   r   r   r}   r   r   r   rC   r   r   r   rP   r   r   r   r   r   r   r   r   r   r   )r   r   pooledtokensr   r   r   r     s@    8$

0






zVisionTransformer.forward)r   F)T)r!   r"   r#   r   r<   Finalrd   __annotations__r   r   r&   r@   rf   r   r,   r   r   ignorer   r%   r   r   r   r4   r   r   r2   r   r   E  sJ   

E
!r   c                       s   e Zd ZU ejje ed< dddddddej	e
dd	dfeeeeeeeeeeeed
 fddZdd ZejjdddZdd ZejdddZedddZdd Z  ZS )TextTransformerr   M   i   r   rF      NFr   )context_length
vocab_sizer   r   r   r   r   r   rv   	embed_clspad_idr   c                    s   t    || _| | _| _|| _|| _|| _|| _|| _	t
t||| _|
rtt
t|| _|  jd7  _nd | _t
||| _t
t| j|| _t||||||	d| _|	|| _| jd|  dd |   d S )Nr
   )r   r   r   r   r   rv   rZ   F
persistent)r+   r,   r   num_posr   r   r   r   r   r   r   r.   r   emptytext_projectioncls_embZ	Embeddingtoken_embeddingr   r   r   ln_finalregister_bufferbuild_attention_maskr   )r   r   r   r   r   r   r   r   r   rv   r   r   r   r2   r   r   r,     s4    

zTextTransformer.__init__c                 C   s  t jj| jjdd t jj| jdd | jd urBt jj| jdd | jjd d| jj	 d  }| jjd }d| jj d }| jj
D ]Z}t jj|jj|d t jj|jjj|d t jj|jjj|d t jj|jjj|d q| jd urt jj| j| jjd d d S )Ng{Gz?Zstdg{Gz?rG   r{   )r   initnormal_r   r   r   r   r   r   r   r   rn   rO   rW   r   r   r   r   r   Zproj_stdZattn_stdZfc_stdblockr   r   r   r   -  s    
zTextTransformer.init_parametersTc                 C   s   || j _d S r   r   r   r   r   r   r   ?  s    z&TextTransformer.set_grad_checkpointingc                 C   s,   t | j| j}|td |d |S Nr\   r
   )r   r   r   fill_rf   triu_r   maskr   r   r   r   C  s    
z$TextTransformer.build_attention_mask)
cast_dtypec                 C   sr   || j kd}tj|dd|jd dfdd}tj|j||jd}|d |	| t
d t|| jd}|S )Nr
   r   r{   r6   )valuer   r\   )r   r   r   padr]   r   r   r   r   re   rf   Zrepeat_interleaver   )r   textr  cls_maskZadditive_maskr   r   r   build_cls_maskK  s    
zTextTransformer.build_cls_maskr   c                 C   s   | ddd|ddS )Nr
   r:   )rg   r   )r   tri   r   r   r   r~   T  s    zTextTransformer._repeatc           	      C   st  | j  }|jd }| ||}| j}| jd ur|d7 }tj|| 	| j|jd gdd}| 
||}|d d |d |f |d d d |d |f  }|| jd | | }|ddd}| j ||d}|ddd}| jd ur|d d df |d d d df  }}| |}n0| |}|t|jd |jddf | }}| jd ur`|| j }| jrp||fS |S )Nr
   r   r;   r{   rY   r:   )r   r   r]   r   r   rZ   r   r   rC   r~   r	  r   r}   r   r>   Zargmaxr   r   )	r   r  r  seq_lenr   rZ   r  r   r   r   r   r   r   W  s.    


"0&
&
zTextTransformer.forward)T)r!   r"   r#   r   r<   r   rd   r   r   r   r&   r@   rf   r   r,   r   r   r   r   r   r	  r~   r   r4   r   r   r2   r   r     sD   
0	r   c                       sl   e Zd Zdddejedfeeeeeeeeed	 fddZ	dd	 Z
d
d Zdd ZejjdddZ  ZS )MultimodalTransformerr   r   Nr   )	r   r   r   r   r   r   r   rv   r   c
           
   	      s~   t  j| d || _t fddt|D | _| jd|  dd | _	t
t|	| _d S )Nr   c                    s"   g | ]}t  d dqS )T)r   r   rv   r   r   r   r   r   r   r     s   
z2MultimodalTransformer.__init__.<locals>.<listcomp>rZ   Fr   )r+   r,   r   r   r   r   
cross_attnr   r   r   r.   r   r   r   )
r   r   r   r   r   r   r   r   rv   r   r2   r   r   r,   {  s     	


zMultimodalTransformer.__init__c                 C   s*  | j jd d| j j d  }| j jd }d| j j d }| j jD ]Z}tjj|jj|d tjj|jj	j
|d tjj|jjj
|d tjj|jjj
|d q@| j jD ]Z}tjj|jj|d tjj|jj	j
|d tjj|jjj
|d tjj|jjj
|d q| jd ur&tjj| j| j jd d d S )NrG   r{   r   )r   r   r   r   r   r   r   rn   rO   rW   r   r   r   r   r  r   r   r   r   r   r     s    z%MultimodalTransformer.init_parametersc                 C   s,   t | j| j}|td |d |S r   )r   r   r   r   rf   r  r  r   r   r   r     s    
z*MultimodalTransformer.build_attention_maskc              
   C   s   | ddd}| ddd}|jd }t| j| jD ]x\}}| jrtj st	||d d | j
d |d |f }t	||||d }q4||| j
d |d |f d}||||d}q4| ddd}| |}| jd ur|| j }|S )Nr
   r   r{   rY   )r   r   )r}   r]   zipr   r  r   r   r<   r   r	   rZ   r   r   )r   Z
image_embsZ	text_embsr  Zresblockr  r   r   r   r   r     s    
"


zMultimodalTransformer.forwardTc                 C   s
   || _ d S r   )r   r   r   r   r   r     s    z,MultimodalTransformer.set_grad_checkpointing)T)r!   r"   r#   r   r   r&   r@   rf   r   r,   r   r   r   r   r<   r   r   r4   r   r   r2   r   r  z  s,   )r  )collectionsr   ro   typingr   r   r   r   r   r   Ztorch.nnr   r   Ztorch.utils.checkpointr	   utilsr   r&   r   Moduler'   r)   r5   rE   rp   r   r   r   r   r   r  r   r   r   r   <module>   s,   		
(L:)% 6 