a
    þd{  ã                   @   sr   d dl mZ d dlZd dlmZ d dlmZ d dlmZ G dd„ dej	ƒZ
G dd	„ d	ej	ƒZG d
d„ dej	ƒZdS )é    )ÚOrderedDictN)Únn)Ú
functional)Úfreeze_batch_norm_2dc                       s2   e Zd ZdZd‡ fdd„	Zejdœdd„Z‡  ZS )	Ú
Bottlenecké   é   c                    s0  t ƒ  ¡  tj||ddd| _t |¡| _tjdd| _tj||dddd| _	t |¡| _
tjdd| _|dkrzt |¡nt ¡ | _tj||| j ddd| _t || j ¡| _tjdd| _d | _|| _|dksâ||tj kr,t tdt |¡fd	tj||| j dddd
fdt || j ¡fgƒ¡| _d S )Nr   F)ÚbiasT©Zinplaceé   )Úpaddingr	   z-1Ú0)Ústrider	   Ú1)ÚsuperÚ__init__r   ÚConv2dÚconv1ÚBatchNorm2dÚbn1ÚReLUÚact1Úconv2Úbn2Úact2Ú	AvgPool2dZIdentityÚavgpoolÚ	expansionÚconv3Úbn3Úact3Ú
downsampler   r   Ú
Sequentialr   )ÚselfZinplanesÚplanesr   ©Ú	__class__© úb/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/open_clip/modified_resnet.pyr      s&    
ýzBottleneck.__init__)Úxc                 C   st   |}|   |  |  |¡¡¡}|  |  |  |¡¡¡}|  |¡}|  |  |¡¡}| j	d ur^|  	|¡}||7 }|  
|¡}|S ©N)r   r   r   r   r   r   r   r   r   r!   r    )r#   r)   ÚidentityÚoutr'   r'   r(   Úforward*   s    



zBottleneck.forward)r   )	Ú__name__Ú
__module__Ú__qualname__r   r   ÚtorchZTensorr-   Ú__classcell__r'   r'   r%   r(   r   
   s   r   c                       s2   e Zd Zdeeeedœ‡ fdd„Zdd„ Z‡  ZS )ÚAttentionPool2dN)Úspacial_dimÚ	embed_dimÚ	num_headsÚ
output_dimc                    st   t ƒ  ¡  t t |d d |¡|d  ¡| _t ||¡| _t ||¡| _	t ||¡| _
t ||pd|¡| _|| _d S )Né   r   g      à?)r   r   r   Ú	Parameterr1   ZrandnÚpositional_embeddingZLinearÚk_projÚq_projÚv_projÚc_projr6   )r#   r4   r5   r6   r7   r%   r'   r(   r   ;   s    
$zAttentionPool2d.__init__c                 C   sæ   |  |jd |jd |jd |jd  ¡ ddd¡}tj|jddd|gdd}|| jd d …d d d …f  |j¡ }t	j
||||jd | j| jj| jj| jjd t | jj| jj| jjg¡d d d	d
| jj| jjd| jd	d\}}|d S )Nr   r   r8   r   T)ÚdimZkeepdim)r?   éÿÿÿÿFg        )ÚqueryÚkeyÚvalueZembed_dim_to_checkr6   Zq_proj_weightZk_proj_weightZv_proj_weightZin_proj_weightZin_proj_biasZbias_kZbias_vZadd_zero_attnZ	dropout_pZout_proj_weightZout_proj_biasZuse_separate_proj_weightÚtrainingZneed_weights)ZreshapeÚshapeZpermuter1   ÚcatÚmeanr:   ÚtoZdtypeÚFZmulti_head_attention_forwardr6   r<   Úweightr;   r=   r	   r>   rD   )r#   r)   Ú_r'   r'   r(   r-   D   s.    4$ï
zAttentionPool2d.forward)N)r.   r/   r0   Úintr   r-   r2   r'   r'   r%   r(   r3   :   s   	r3   c                       s`   e Zd ZdZd‡ fdd„	Zddd„Zd	d
„ Zddd„Zej	j
ddd„ƒZdd„ Zdd„ Z‡  ZS )ÚModifiedResNeta  
    A ResNet class that is similar to torchvision's but contains the following changes:
    - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool.
    - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1
    - The final pooling layer is a QKV attention instead of an average pool
    éà   é@   c                    sZ  t ƒ  ¡  || _|| _tjd|d ddddd| _t |d ¡| _tj	dd| _
tj|d |d dddd| _t |d ¡| _tj	dd| _tj|d |dddd| _t |¡| _tj	dd| _t d¡| _|| _|  ||d	 ¡| _| j|d |d dd
| _| j|d |d dd
| _| j|d |d dd
| _|d }t|d |||ƒ| _|  ¡  d S )Nr   r8   r   F)Úkernel_sizer   r   r	   Tr
   )rP   r   r	   r   )r   r   é   é    )r   r   r7   Ú
image_sizer   r   r   r   r   r   r   r   r   r   r   r   r    r   r   Ú	_inplanesÚ_make_layerÚlayer1Úlayer2Úlayer3Úlayer4r3   ÚattnpoolÚinit_parameters)r#   Úlayersr7   ZheadsrS   Úwidthr5   r%   r'   r(   r   g   s*    
zModifiedResNet.__init__r   c                 C   sH   t | j||ƒg}|t j | _td|ƒD ]}| t | j|ƒ¡ q&tj|Ž S )Nr   )r   rT   r   ÚrangeÚappendr   r"   )r#   r$   Úblocksr   r\   rK   r'   r'   r(   rU   „   s
    zModifiedResNet._make_layerc                 C   s¶   | j d urp| j jjd }tjj| j jj|d tjj| j jj|d tjj| j j	j|d tjj| j jj|d | j
| j| j| jfD ],}| ¡ D ]\}}| d¡rtj |¡ qq„d S )Ng      à¿)Ústdz
bn3.weight)rZ   r>   Zin_featuresr   ÚinitZnormal_r<   rJ   r;   r=   rV   rW   rX   rY   Znamed_parametersÚendswithZzeros_)r#   ra   Zresnet_blockÚnameÚparamr'   r'   r(   r[      s    

zModifiedResNet.init_parametersr   Fc                 C   s4   |dksJ dƒ‚|   ¡ D ]
}d|_q|r0t| ƒ d S )Nr   z6partial locking not currently supported for this modelF)Ú
parametersZrequires_gradr   )r#   Zunlocked_groupsZfreeze_bn_statsre   r'   r'   r(   Úlockš   s
    zModifiedResNet.lockTc                 C   s   d S r*   r'   )r#   Úenabler'   r'   r(   Úset_grad_checkpointing¡   s    z%ModifiedResNet.set_grad_checkpointingc                 C   sP   |   |  |  |¡¡¡}|  |  |  |¡¡¡}|  |  |  |¡¡¡}|  	|¡}|S r*   )
r   r   r   r   r   r   r    r   r   r   ©r#   r)   r'   r'   r(   Ústem¦   s
    
zModifiedResNet.stemc                 C   s@   |   |¡}|  |¡}|  |¡}|  |¡}|  |¡}|  |¡}|S r*   )rk   rV   rW   rX   rY   rZ   rj   r'   r'   r(   r-   ­   s    





zModifiedResNet.forward)rN   rO   )r   )r   F)T)r.   r/   r0   Ú__doc__r   rU   r[   rg   r1   ZjitÚignoreri   rk   r-   r2   r'   r'   r%   r(   rM   _   s   
	
rM   )Úcollectionsr   r1   r   Ztorch.nnr   rI   Zopen_clip.utilsr   ÚModuler   r3   rM   r'   r'   r'   r(   Ú<module>   s   0%