a
    ³
þdÞC  ã                   @   s  d dl mZ d dlmZmZ d dlZd dlZd dlm	  m
Z d dlm	Z	 G dd„ de	jƒZG dd„ de	jƒZG d	d
„ d
e	jƒZG dd„ de	jƒZG dd„ de	jƒZG dd„ de	jƒZG dd„ de	jƒZG dd„ de	jƒZG dd„ de	jƒZe	jdœdd„Zedœdd„ZdS )é    )ÚOrderedDict)ÚTupleÚUnionN)Únnc                       s2   e Zd ZdZd‡ fdd„	Zejdœdd„Z‡  ZS )	Ú
Bottlenecké   é   c                    s0  t ƒ  ¡  tj||ddd| _t |¡| _tjdd| _tj||dddd| _	t |¡| _
tjdd| _|dkrzt |¡nt ¡ | _tj||| j ddd| _t || j ¡| _tjdd| _d | _|| _|dksâ||tj kr,t tdt |¡fd	tj||| j dddd
fdt || j ¡fgƒ¡| _d S )Nr   F)ÚbiasT©Zinplaceé   )Úpaddingr	   z-1Ú0)Ústrider	   Ú1)ÚsuperÚ__init__r   ÚConv2dÚconv1ÚBatchNorm2dÚbn1ÚReLUÚrelu1Úconv2Úbn2Úrelu2Ú	AvgPool2dZIdentityÚavgpoolÚ	expansionÚconv3Úbn3Úrelu3Ú
downsampler   r   Ú
Sequentialr   )ÚselfZinplanesÚplanesr   ©Ú	__class__© úS/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/clip/model.pyr      s&    
ýzBottleneck.__init__©Úxc                 C   st   |}|   |  |  |¡¡¡}|  |  |  |¡¡¡}|  |¡}|  |  |¡¡}| j	d ur^|  	|¡}||7 }|  
|¡}|S ©N)r   r   r   r   r   r   r   r   r   r!   r    )r#   r*   ZidentityÚoutr'   r'   r(   Úforward*   s    



zBottleneck.forward)r   )	Ú__name__Ú
__module__Ú__qualname__r   r   ÚtorchÚTensorr-   Ú__classcell__r'   r'   r%   r(   r   
   s   r   c                       s2   e Zd Zdeeeedœ‡ fdd„Zdd„ Z‡  ZS )ÚAttentionPool2dN)Úspacial_dimÚ	embed_dimÚ	num_headsÚ
output_dimc                    st   t ƒ  ¡  t t |d d |¡|d  ¡| _t ||¡| _t ||¡| _	t ||¡| _
t ||pd|¡| _|| _d S )Né   r   ç      à?)r   r   r   Ú	Parameterr1   ÚrandnÚpositional_embeddingÚLinearÚk_projÚq_projÚv_projÚc_projr7   )r#   r5   r6   r7   r8   r%   r'   r(   r   ;   s    
$zAttentionPool2d.__init__c                 C   sÒ   |j dd ddd¡}tj|jddd|gdd}|| jd d …d d d …f  |j¡ }tj	|d d… |||j
d | j| jj| jj| jjd t | jj| jj| jjg¡d d d	d| jj| jjd| jd	d
\}}| d¡S )Nr9   )Z	start_dimr   r   T©ÚdimZkeepdim©rD   éÿÿÿÿF)ÚqueryÚkeyÚvalueZembed_dim_to_checkr7   Zq_proj_weightZk_proj_weightZv_proj_weightÚin_proj_weightÚin_proj_biasÚbias_kÚbias_vZadd_zero_attnZ	dropout_pZout_proj_weightZout_proj_biasZuse_separate_proj_weightÚtrainingÚneed_weights)ÚflattenÚpermuter1   ÚcatÚmeanr=   ÚtoÚdtypeÚFZmulti_head_attention_forwardÚshaper7   r@   Úweightr?   rA   r	   rB   rN   Zsqueeze)r#   r*   Ú_r'   r'   r(   r-   D   s.    $ï
zAttentionPool2d.forward)N)r.   r/   r0   Úintr   r-   r3   r'   r'   r%   r(   r4   :   s   	r4   c                       s4   e Zd ZdZd‡ fdd„	Zddd„Zd	d
„ Z‡  ZS )ÚModifiedResNeta  
    A ResNet class that is similar to torchvision's but contains the following changes:
    - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool.
    - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1
    - The final pooling layer is a QKV attention instead of an average pool
    éà   é@   c                    sR  t ƒ  ¡  || _|| _tjd|d ddddd| _t |d ¡| _tj	dd| _
tj|d |d dddd| _t |d ¡| _tj	dd| _tj|d |dddd| _t |¡| _tj	dd| _t d¡| _|| _|  ||d	 ¡| _| j|d |d dd
| _| j|d |d dd
| _| j|d |d dd
| _|d }t|d |||ƒ| _d S )Nr   r9   r   F)Úkernel_sizer   r   r	   Tr
   )r^   r   r	   r   )r   r   é   é    )r   r   r8   Úinput_resolutionr   r   r   r   r   r   r   r   r   r   r   r   r    r   r   Ú	_inplanesÚ_make_layerÚlayer1Úlayer2Úlayer3Úlayer4r4   Úattnpool)r#   Úlayersr8   Úheadsra   Úwidthr6   r%   r'   r(   r   f   s(    
zModifiedResNet.__init__r   c                 C   sH   t | j||ƒg}|t j | _td|ƒD ]}| t | j|ƒ¡ q&tj|Ž S )Nr   )r   rb   r   ÚrangeÚappendr   r"   )r#   r$   Zblocksr   ri   rY   r'   r'   r(   rc      s
    zModifiedResNet._make_layerc                    sZ   ‡ fdd„}|  ˆ jjj¡}||ƒ}ˆ  |¡}ˆ  |¡}ˆ  |¡}ˆ  |¡}ˆ  |¡}|S )Nc                    sP   ˆ   ˆ  ˆ  | ¡¡¡} ˆ  ˆ  ˆ  | ¡¡¡} ˆ  ˆ  ˆ  | ¡¡¡} ˆ  	| ¡} | S r+   )
r   r   r   r   r   r   r    r   r   r   r)   ©r#   r'   r(   Ústem‹   s
    
z$ModifiedResNet.forward.<locals>.stem)	Útyper   rX   rU   rd   re   rf   rg   rh   )r#   r*   ro   r'   rn   r(   r-   Š   s    




zModifiedResNet.forward)r\   r]   )r   )r.   r/   r0   Ú__doc__r   rc   r-   r3   r'   r'   r%   r(   r[   ^   s   
	r[   c                       s(   e Zd ZdZejdœ‡ fdd„Z‡  ZS )Ú	LayerNormz*Subclass torch's LayerNorm to handle fp16.r)   c                    s$   |j }tƒ  | tj¡¡}| |¡S r+   )rU   r   r-   rp   r1   Zfloat32)r#   r*   Z	orig_typeÚretr%   r'   r(   r-       s    zLayerNorm.forward)r.   r/   r0   rq   r1   r2   r-   r3   r'   r'   r%   r(   rr      s   rr   c                   @   s   e Zd Zejdœdd„ZdS )Ú	QuickGELUr)   c                 C   s   |t  d| ¡ S )Ng¬Zd;û?)r1   Zsigmoid©r#   r*   r'   r'   r(   r-   §   s    zQuickGELU.forwardN)r.   r/   r0   r1   r2   r-   r'   r'   r'   r(   rt   ¦   s   rt   c                       sJ   e Zd Zd
eeejdœ‡ fdd„Zejdœdd„Zejdœdd	„Z‡  Z	S )ÚResidualAttentionBlockN)Úd_modelÚn_headÚ	attn_maskc              
      sr   t ƒ  ¡  t ||¡| _t|ƒ| _t tdt 	||d ¡fdt
ƒ fdt 	|d |¡fgƒ¡| _t|ƒ| _|| _d S )NÚc_fcr   ZgelurB   )r   r   r   ÚMultiheadAttentionÚattnrr   Úln_1r"   r   r>   rt   ÚmlpÚln_2ry   )r#   rw   rx   ry   r%   r'   r(   r   ¬   s    

ý

zResidualAttentionBlock.__init__r)   c                 C   s>   | j d ur| j j|j|jdnd | _ | j|||d| j dd S )N©rU   ÚdeviceF)rO   ry   r   )ry   rT   rU   r   r|   ru   r'   r'   r(   Ú	attention¹   s    $z ResidualAttentionBlock.attentionc                 C   s,   ||   |  |¡¡ }||  |  |¡¡ }|S r+   )r‚   r}   r~   r   ru   r'   r'   r(   r-   ½   s    zResidualAttentionBlock.forward)N)
r.   r/   r0   rZ   r1   r2   r   r‚   r-   r3   r'   r'   r%   r(   rv   «   s   rv   c                       s<   e Zd Zdeeeejdœ‡ fdd„Zejdœdd„Z‡  ZS )	ÚTransformerN©rk   ri   rj   ry   c                    s<   t ƒ  ¡  ˆ| _|| _tj‡ ‡‡fdd„t|ƒD ƒŽ | _d S )Nc                    s   g | ]}t ˆˆˆ ƒ‘qS r'   )rv   )Ú.0rY   ©ry   rj   rk   r'   r(   Ú
<listcomp>È   ó    z(Transformer.__init__.<locals>.<listcomp>)r   r   rk   ri   r   r"   rl   Ú	resblocks)r#   rk   ri   rj   ry   r%   r†   r(   r   Ä   s    
zTransformer.__init__r)   c                 C   s
   |   |¡S r+   )r‰   ru   r'   r'   r(   r-   Ê   s    zTransformer.forward)N)	r.   r/   r0   rZ   r1   r2   r   r-   r3   r'   r'   r%   r(   rƒ   Ã   s   rƒ   c                       s<   e Zd Zeeeeeedœ‡ fdd„Zejdœdd„Z‡  ZS )ÚVisionTransformer©ra   Ú
patch_sizerk   ri   rj   r8   c                    s¬   t ƒ  ¡  || _|| _tjd|||dd| _|d }t |t 	|¡ ¡| _
t |t 	|| d d |¡ ¡| _t|ƒ| _t|||ƒ| _t|ƒ| _t |t 	||¡ ¡| _d S )Nr   F)Zin_channelsZout_channelsr^   r   r	   ç      à¿r9   r   )r   r   ra   r8   r   r   r   r;   r1   r<   Úclass_embeddingr=   rr   Úln_prerƒ   ÚtransformerÚln_postÚproj)r#   ra   rŒ   rk   ri   rj   r8   Zscaler%   r'   r(   r   Ï   s    
$

zVisionTransformer.__init__r)   c              	   C   sè   |   |¡}| |jd |jd d¡}| ddd¡}tj| j |j¡tj	|jd d|jd |j|j
d |gdd}|| j |j¡ }|  |¡}| ddd¡}|  |¡}| ddd¡}|  |d d …dd d …f ¡}| jd urä|| j }|S )Nr   r   rF   r9   r€   rE   )r   ZreshaperW   rQ   r1   rR   rŽ   rT   rU   Úzerosr   r=   r   r   r‘   r’   ru   r'   r'   r(   r-   ß   s    
@



zVisionTransformer.forward)	r.   r/   r0   rZ   r   r1   r2   r-   r3   r'   r'   r%   r(   rŠ   Î   s   rŠ   c                       s|   e Zd Zeeeeeeeef ef eeeeeeedœ
‡ fdd„Zdd„ Zdd„ Ze	dd	„ ƒZ
d
d„ Zdd„ Zdd„ Z‡  ZS )ÚCLIP)
r6   Úimage_resolutionÚvision_layersÚvision_widthÚvision_patch_sizeÚcontext_lengthÚ
vocab_sizeÚtransformer_widthÚtransformer_headsÚtransformer_layersc                    sä   t ƒ  ¡  || _t|ttfƒr@|d d }t|||||d| _n|d }t||||||d| _t	||
|	|  
¡ d| _|| _t ||¡| _t t | j|¡¡| _t|ƒ| _t t ||¡¡| _t t g ¡t d¡ ¡| _|  ¡  d S )Nr`   r]   )ri   r8   rj   ra   rk   r‹   r„   g$I’$I’,@)r   r   r™   Ú
isinstanceÚtupleÚlistr[   ÚvisualrŠ   rƒ   Úbuild_attention_maskr   rš   r   Z	EmbeddingÚtoken_embeddingr;   r1   Úemptyr=   rr   Úln_finalÚtext_projectionZonesÚnpÚlogÚlogit_scaleÚinitialize_parameters)r#   r6   r•   r–   r—   r˜   r™   rš   r›   rœ   r   Zvision_headsr%   r'   r(   r   ô   sB    
û
ú	ü
zCLIP.__init__c           	      C   sÀ  t jj| jjdd t jj| jdd t| jtƒrø| jj	d ur®| jj	j
jd }t jj| jj	jj|d t jj| jj	jj|d t jj| jj	jj|d t jj| jj	j
j|d | jj| jj| jj| jjfD ],}| ¡ D ]\}}| d¡rÖt j |¡ qÖqÊ| jjd d| jj d  }| jjd }d| jj d }| jjD ]\}t jj|jj|d t jj|jjj|d t jj|jjj|d t jj|jj
j|d q8| jd ur¼t jj| j| jjd d d S )Ng{®Gáz”?)Ústdg{®Gáz„?r   z
bn3.weightr9   ) r   ÚinitZnormal_r£   rX   r=   rž   r¡   r[   rh   rB   Zin_featuresr@   r?   rA   rd   re   rf   rg   Znamed_parametersÚendswithZzeros_r   rk   ri   r‰   r|   rJ   Zout_projr~   rz   r¦   )	r#   r«   Zresnet_blockÚnameÚparamZproj_stdZattn_stdZfc_stdÚblockr'   r'   r(   rª   +  s.     
zCLIP.initialize_parametersc                 C   s,   t  | j| j¡}| tdƒ¡ | d¡ |S )Nz-infr   )r1   r¤   r™   Zfill_ÚfloatZtriu_)r#   Úmaskr'   r'   r(   r¢   H  s    
zCLIP.build_attention_maskc                 C   s   | j jjjS r+   )r¡   r   rX   rU   rn   r'   r'   r(   rU   P  s    z
CLIP.dtypec                 C   s   |   | | j¡¡S r+   )r¡   rp   rU   )r#   Úimager'   r'   r(   Úencode_imageT  s    zCLIP.encode_imagec                 C   s†   |   |¡ | j¡}|| j | j¡ }| ddd¡}|  |¡}| ddd¡}|  |¡ | j¡}|t |j	d ¡|j
ddf | j }|S )Nr   r   r9   rF   rE   )r£   rp   rU   r=   rQ   r   r¥   r1   ZarangerW   Zargmaxr¦   )r#   Útextr*   r'   r'   r(   Úencode_textW  s    
&zCLIP.encode_textc                 C   sb   |   |¡}|  |¡}||jddd }||jddd }| j ¡ }|| | ¡  }| ¡ }||fS )Nr   TrC   )r´   r¶   Znormr©   ÚexpÚt)r#   r³   rµ   Zimage_featuresZtext_featuresr©   Zlogits_per_imageZlogits_per_textr'   r'   r(   r-   f  s    


zCLIP.forward)r.   r/   r0   rZ   r   r   r   rª   r¢   ÚpropertyrU   r´   r¶   r-   r3   r'   r'   r%   r(   r”   ó   s$   ô7
r”   )Úmodelc                 C   s   dd„ }|   |¡ dS )z+Convert applicable model parameters to fp16c                 S   sÄ   t | tjtjtjfƒr@| jj ¡ | j_| jd ur@| jj ¡ | j_t | tj	ƒrŽg dd„ dD ƒ¢d‘d‘d‘D ]"}t
| |ƒ}|d urj|j ¡ |_qjdD ],}t| |ƒr’t
| |ƒ}|d ur’|j ¡ |_q’d S )Nc                 S   s   g | ]}|› d ‘qS )Z_proj_weightr'   )r…   Úsr'   r'   r(   r‡     rˆ   zEconvert_weights.<locals>._convert_weights_to_fp16.<locals>.<listcomp>)ÚinÚqÚkÚvrK   rL   rM   )r¦   r’   )rž   r   ZConv1dr   r>   rX   ÚdataZhalfr	   r{   ÚgetattrÚhasattr)ÚlÚattrZtensorr®   r'   r'   r(   Ú_convert_weights_to_fp16z  s    
"


z1convert_weights.<locals>._convert_weights_to_fp16N)Zapply)rº   rÅ   r'   r'   r(   Úconvert_weightsw  s    rÆ   ©Ú
state_dictc                    st  dˆ v }|rbˆ d j d }tdd„ ˆ  ¡ D ƒƒ}ˆ d j d }tˆ d j d d d	 ƒ}|| }nl‡ fd
d„dD ƒ}t|ƒ}ˆ d j d }tˆ d j d d d	 ƒ}d }|d d ˆ d j d ksÆJ ‚|d }ˆ d j d }	ˆ d j d }
ˆ d j d }ˆ d j d }|d }ttdd„ ˆ D ƒƒƒ}t|	|||||
||||ƒ
}dD ]}|ˆ v rBˆ |= qBt|ƒ | ˆ ¡ | 	¡ S )Nzvisual.projzvisual.conv1.weightr   c                 S   s$   g | ]}|  d ¡r| d¡r|‘qS )zvisual.z.attn.in_proj_weight)Ú
startswithr­   ©r…   r¾   r'   r'   r(   r‡   ”  rˆ   zbuild_model.<locals>.<listcomp>rF   zvisual.positional_embeddingr   r:   c                    s&   g | ]‰ t t‡ fd d„ˆD ƒƒƒ‘qS )c                 3   s,   | ]$}|  d ˆ › ¡r| d¡d V  qdS )zvisual.layerÚ.r9   N©rÉ   ÚsplitrÊ   ©Úbr'   r(   Ú	<genexpr>™  rˆ   z)build_model.<locals>.<listcomp>.<genexpr>)ÚlenÚset)r…   rÇ   rÎ   r(   r‡   ™  rˆ   )r   r9   r   r   zvisual.layer1.0.conv1.weightz$visual.attnpool.positional_embeddingr9   r`   r¦   r=   ztoken_embedding.weightzln_final.weightr]   c                 s   s&   | ]}|  d ¡r| d¡d V  qdS )ztransformer.resblocksrË   r9   NrÌ   rÊ   r'   r'   r(   rÐ   ¦  rˆ   zbuild_model.<locals>.<genexpr>)ra   r™   rš   )
rW   rÑ   ÚkeysÚroundrŸ   rÒ   r”   rÆ   Zload_state_dictÚeval)rÈ   Zvitr—   r–   r˜   Z	grid_sizer•   ÚcountsZoutput_widthr6   r™   rš   r›   rœ   r   rº   rH   r'   rÇ   r(   Úbuild_model  s>    

ý


r×   )Úcollectionsr   Útypingr   r   Únumpyr§   r1   Ztorch.nn.functionalr   Z
functionalrV   ÚModuler   r4   r[   rr   rt   rv   rƒ   rŠ   r”   rÆ   Údictr×   r'   r'   r'   r(   Ú<module>   s"   0$?	% 