a
    þd,J  ã                	   @   sö  d Z ddlZddlmZ ddlmZ ddlZddlmZ ddl	m  m
Z ddlmZmZ ddlmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ dgZeeef Z eG dd„ dej!ƒƒZ"G dd„ dej!ƒZ#G dd„ dej!ƒZ$G dd„ dej!ƒZ%G dd„ dej!ƒZ&G dd„ dej!ƒZ'd.dd„Z(d/dd„Z)ee)dde)dde)dde)dde)dde)ddd œƒZ*ed0e'd!œd"d#„ƒZ+ed1e'd!œd$d%„ƒZ,ed2e'd!œd&d'„ƒZ-ed3e'd!œd(d)„ƒZ.ed4e'd!œd*d+„ƒZ/ed5e'd!œd,d-„ƒZ0dS )6zô Twins
A PyTorch impl of : `Twins: Revisiting the Design of Spatial Attention in Vision Transformers`
    - https://arxiv.org/pdf/2104.13840.pdf

Code/weights from https://github.com/Meituan-AutoML/Twins, original copyright/license info below

é    N)Úpartial)ÚTuple©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚMlpÚDropPathÚ	to_2tupleÚtrunc_normal_Úuse_fused_attné   )Úbuild_model_with_cfg)Úregister_notrace_module)Úregister_modelÚgenerate_default_cfgs)Ú	AttentionÚTwinsc                       sB   e Zd ZU dZejje ed< d‡ fdd„	Z	e
dœd	d
„Z‡  ZS )ÚLocallyGroupedAttnz( LSA: self attention within a group
    Ú
fused_attné   ç        r   c                    s¨   |dksJ ‚t t| ƒ ¡  || dks<J d|› d|› dƒ‚|| _|| _|| }|d | _tƒ | _tj	||d dd	| _
t |¡| _t 	||¡| _t |¡| _|| _d S )
Nr   r   údim ú  should be divided by num_heads Ú.ç      à¿é   T©Úbias)Úsuperr   Ú__init__ÚdimÚ	num_headsÚscaler   r   ÚnnÚLinearÚqkvÚDropoutÚ	attn_dropÚprojÚ	proj_dropÚws)Úselfr    r!   r'   r)   r*   Úhead_dim©Ú	__class__© úZ/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/twins.pyr   (   s    "
zLocallyGroupedAttn.__init__©Úsizec              	   C   sü  |j \}}}|\}}| ||||¡}d }}	| j|| j  | j }
| j|| j  | j }t |dd||
|	|f¡}|j \}}}}|| j || j  }}| ||| j|| j|¡ dd¡}|  |¡ ||| | j| j d| j|| j ¡ 	dddddd¡}| 
d¡\}}}| jr"tj|||| jjd}n8|| j }|| dd	¡ }|jd	d
}|  |¡}|| }| dd¡ |||| j| j|¡}| dd¡ ||| j || j |¡}|
dks²|dkrÖ|d d …d |…d |…d d …f  ¡ }| |||¡}|  |¡}|  |¡}|S )Nr   é   r   r   é   é   ©Z	dropout_péþÿÿÿéÿÿÿÿ©r    )ÚshapeÚviewr*   ÚFÚpadÚreshapeÚ	transposer%   r!   ÚpermuteÚunbindr   Úscaled_dot_product_attentionr'   Úpr"   ÚsoftmaxÚ
contiguousr(   r)   )r+   Úxr2   ÚBÚNÚCÚHÚWZpad_lZpad_tZpad_rZpad_bÚ_ZHpZWpZ_hÚ_wr%   ÚqÚkÚvÚattnr/   r/   r0   Úforward9   sD     
 ÿÿþ

 $$

zLocallyGroupedAttn.forward)r   r   r   r   ©Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚtorchÚjitÚFinalÚboolÚ__annotations__r   ÚSize_rR   Ú__classcell__r/   r/   r-   r0   r   "   s   
r   c                       sB   e Zd ZU dZejje ed< d‡ fdd„	Z	e
dœd	d
„Z‡  ZS )ÚGlobalSubSampleAttnzQ GSA: using a  key to summarize the information for a group to be efficient.
    r   r   r   r   c                    sà   t ƒ  ¡  || dks,J d|› d|› dƒ‚|| _|| _|| }|d | _tƒ | _tj||dd| _	tj||d dd| _
t |¡| _t ||¡| _t |¡| _|| _|d	krÐtj||||d
| _t |¡| _nd | _d | _d S )Nr   r   r   r   r   Tr   r3   r   ©Úkernel_sizeÚstride)r   r   r    r!   r"   r   r   r#   r$   rN   Úkvr&   r'   r(   r)   Úsr_ratioÚConv2dÚsrÚ	LayerNormÚnorm)r+   r    r!   r'   r)   rd   r,   r-   r/   r0   r   ‰   s$    
"
zGlobalSubSampleAttn.__init__r1   c                 C   sJ  |j \}}}|  |¡ ||| j|| j ¡ dddd¡}| jd ur†| ddd¡j||g|¢R Ž }|  |¡ ||d¡ ddd¡}|  |¡}|  |¡ |dd| j|| j ¡ ddddd¡}| d¡\}}	| j	rät
jjj|||	| jjd}n8|| j }|| dd¡ }
|
jdd	}
|  |
¡}
|
|	 }| dd¡ |||¡}|  |¡}|  |¡}|S )
Nr   r3   r   r   r8   r4   r6   r7   r9   )r:   rN   r>   r!   r@   rf   rh   rc   rA   r   rX   r#   Ú
functionalrB   r'   rC   r"   r?   rD   r(   r)   )r+   rF   r2   rG   rH   rI   rN   rc   rO   rP   rQ   r/   r/   r0   rR   ¡   s,    *

.þ



zGlobalSubSampleAttn.forward)r   r   r   r   rS   r/   r/   r-   r0   r_   „   s   
r_   c                       s@   e Zd Zddddejejddf‡ fdd„	Zedœdd	„Z‡  Z	S )
ÚBlockg      @r   r   Nc                    s¼   t ƒ  ¡  ||ƒ| _|
d u r2t||dd ||ƒ| _n.|
dkrNt|||||	ƒ| _nt|||||
ƒ| _|dkrpt|ƒnt 	¡ | _
||ƒ| _t|t|| ƒ||d| _|dkr®t|ƒnt 	¡ | _d S )NFr   r   )Zin_featuresZhidden_featuresÚ	act_layerÚdrop)r   r   Únorm1r   rQ   r_   r   r   r#   ÚIdentityÚ
drop_path1Únorm2r   ÚintÚmlpÚ
drop_path2)r+   r    r!   Ú	mlp_ratior)   r'   Ú	drop_pathrk   Ú
norm_layerrd   r*   r-   r/   r0   r   Á   s     



üzBlock.__init__r1   c                 C   s:   ||   |  |  |¡|¡¡ }||  |  |  |¡¡¡ }|S ©N)ro   rQ   rm   rs   rr   rp   )r+   rF   r2   r/   r/   r0   rR   á   s    zBlock.forward)
rT   rU   rV   r#   ZGELUrg   r   r]   rR   r^   r/   r/   r-   r0   rj   ¿   s   õ rj   c                       s4   e Zd Zd
‡ fdd„	Zedœdd„Zdd	„ Z‡  ZS )ÚPosConvé   r   c                    s8   t t| ƒ ¡  t tj||d|dd|d¡| _|| _d S )Nr   r   T)r   Úgroups)r   rx   r   r#   Z
Sequentialre   r(   rb   )r+   Úin_chansÚ	embed_dimrb   r-   r/   r0   r   é   s
    ÿzPosConv.__init__r1   c                 C   sZ   |j \}}}| dd¡j||g|¢R Ž }|  |¡}| jdkrD||7 }| d¡ dd¡}|S )Nr   r3   )r:   r?   r;   r(   rb   Úflatten)r+   rF   r2   rG   rH   rI   Zcnn_feat_tokenr/   r/   r0   rR   ð   s    

zPosConv.forwardc                 C   s   dd„ t dƒD ƒS )Nc                 S   s   g | ]}d | ‘qS )zproj.%d.weightr/   ©Ú.0Úir/   r/   r0   Ú
<listcomp>ú   ó    z+PosConv.no_weight_decay.<locals>.<listcomp>r4   )Úrange©r+   r/   r/   r0   Úno_weight_decayù   s    zPosConv.no_weight_decay)ry   r   )rT   rU   rV   r   r]   rR   r…   r^   r/   r/   r-   r0   rx   ç   s   	rx   c                       s:   e Zd ZdZd‡ fdd„	Zeejef dœd	d
„Z	‡  Z
S )Ú
PatchEmbedz Image to Patch Embedding
    éà   é   r   ry   c                    s¼   t ƒ  ¡  t|ƒ}t|ƒ}|| _|| _|d |d  dkrN|d |d  dksdJ d|› d|› dƒ‚|d |d  |d |d   | _| _| j| j | _tj	||||d| _
t |¡| _d S )Nr   r   z	img_size z! should be divided by patch_size r   r`   )r   r   r	   Úimg_sizeÚ
patch_sizerJ   rK   Znum_patchesr#   re   r(   rg   rh   )r+   r‰   rŠ   r{   r|   r-   r/   r0   r     s    
*ÿ&zPatchEmbed.__init__©Úreturnc                 C   sT   |j \}}}}|  |¡ d¡ dd¡}|  |¡}|| jd  || jd  f}||fS )Nr3   r   r   )r:   r(   r}   r?   rh   rŠ   )r+   rF   rG   rI   rJ   rK   Zout_sizer/   r/   r0   rR     s
    
zPatchEmbed.forward)r‡   rˆ   r   ry   )rT   rU   rV   rW   r   r   rX   ZTensorr]   rR   r^   r/   r/   r-   r0   r†   ý   s   r†   c                       sÆ   e Zd ZdZdddddddd	d
dddddddeejddef‡ fdd„	Ze	j
jdd„ ƒZe	j
jd'dd„ƒZe	j
jd(dd„ƒZe	j
jdd„ ƒZd)dd„Zdd„ Zd d!„ Zd*ed"œd#d$„Zd%d&„ Z‡  ZS )+r   z™ Twins Vision Transfomer (Revisiting Spatial Attention)

    Adapted from PVT (PyramidVisionTransformer) class at https://github.com/whai362/PVT.git
    r‡   r4   r   éè  Úavg©é@   é€   é   é   )r   r3   r4   r   ©r4   r4   r4   r4   ©r   r4   é   r   ©r   r4   r3   r   Nr   gíµ ÷Æ°>)Zepsc                    sž  t ƒ  ¡  || _|| _|	| _ˆ| _ˆd | _d| _t|ƒ}|}t	 
¡ | _t	 
¡ | _tt|	ƒƒD ]T}| j t|ˆ	|ˆ| ƒ¡ | j t	j|d¡ ˆ| }t‡	fdd„|D ƒƒ}d‰	q^t	 
¡ | _dd„ t d	|t|	ƒ¡D ƒ‰d	‰tt|	ƒƒD ]R‰t	 
‡ ‡‡‡‡‡‡‡‡‡
‡‡fd
d„t|	ˆ ƒD ƒ¡}| j |¡ ˆ|	ˆ 7 ‰qêt	 
dd„ ˆD ƒ¡| _ˆ| jƒ| _t	 |¡| _|d	kr„t	 | j|¡nt	 ¡ | _|  | j¡ d S )Nr8   F)rC   c                 3   s   | ]}|ˆ  V  qd S rw   r/   )r   Út)rŠ   r/   r0   Ú	<genexpr>C  r‚   z!Twins.__init__.<locals>.<genexpr>r3   c                 S   s   g | ]}|  ¡ ‘qS r/   )Úitem)r   rF   r/   r/   r0   r   G  r‚   z"Twins.__init__.<locals>.<listcomp>r   c                    sZ   g | ]R}ˆˆˆ ˆˆ ˆˆ ˆ	ˆ ˆˆ|  ˆˆ
ˆ ˆd u sF|d dkrJdnˆˆ d	‘qS )Nr3   r   )	r    r!   rt   r)   r'   ru   rv   rd   r*   r/   r~   )Úattn_drop_rateÚ	block_clsÚcurÚdprÚ
embed_dimsrO   Ú
mlp_ratiosrv   r!   Úproj_drop_rateÚ	sr_ratiosÚwssr/   r0   r   J  s   	÷
÷c                 S   s   g | ]}t ||ƒ‘qS r/   )rx   )r   r|   r/   r/   r0   r   X  r‚   )r   r   Únum_classesÚglobal_poolÚdepthsrŸ   Únum_featuresZgrad_checkpointingr	   r#   Z
ModuleListÚpatch_embedsÚ	pos_dropsrƒ   ÚlenÚappendr†   r&   ÚtupleÚblocksrX   ZlinspaceÚsumÚ	pos_blockrh   Ú	head_dropr$   rn   ÚheadÚapplyÚ_init_weights)r+   r‰   rŠ   r{   r¤   r¥   rŸ   r!   r    r¦   r¢   r£   Z	drop_rateZpos_drop_rater¡   r›   Zdrop_path_raterv   rœ   Zprev_chsr€   Ú_blockr-   )r›   rœ   r   rž   rŸ   rO   r    rv   r!   rŠ   r¡   r¢   r£   r0   r     s>    




$	
÷"zTwins.__init__c                 C   s   t dd„ | j ¡ D ƒƒS )Nc                 S   s   g | ]\}}d | ‘qS )z
pos_block.r/   )r   ÚnrC   r/   r/   r0   r   e  r‚   z)Twins.no_weight_decay.<locals>.<listcomp>)Úsetr¯   Znamed_parametersr„   r/   r/   r0   r…   c  s    zTwins.no_weight_decayFc                 C   s    t d|rddgng d¢d}|S )Nz^patch_embeds.0)z)^(?:blocks|patch_embeds|pos_block)\.(\d+)N©z^norm)iŸ† ))z^blocks\.(\d+)\.(\d+)N)z"^(?:patch_embeds|pos_block)\.(\d+))r   r·   )Ústemr­   )Údict)r+   ZcoarseZmatcherr/   r/   r0   Úgroup_matcherg  s    ýþûzTwins.group_matcherTc                 C   s   |rJ dƒ‚d S )Nz$gradient checkpointing not supportedr/   )r+   Úenabler/   r/   r0   Úset_grad_checkpointingv  s    zTwins.set_grad_checkpointingc                 C   s   | j S rw   )r±   r„   r/   r/   r0   Úget_classifierz  s    zTwins.get_classifierc                 C   sD   || _ |d ur |dv sJ ‚|| _|dkr6t | j|¡nt ¡ | _d S )N)Ú rŽ   r   )r¤   r¥   r#   r$   r§   rn   r±   )r+   r¤   r¥   r/   r/   r0   Úreset_classifier~  s
    zTwins.reset_classifierc                 C   sÔ   t |tjƒrBt|jdd t |tjƒrÐ|jd urÐtj |jd¡ nŽt |tjƒrptj |jd¡ tj |jd¡ n`t |tj	ƒrÐ|j
d |j
d  |j }||j }|jj dt d| ¡¡ |jd urÐ|jj ¡  d S )Ng{®Gáz”?)Ústdr   g      ð?r   g       @)Ú
isinstancer#   r$   r
   Zweightr   ÚinitZ	constant_rg   re   ra   Zout_channelsrz   ÚdataZnormal_ÚmathÚsqrtZzero_)r+   ÚmZfan_outr/   r/   r0   r³   …  s    

zTwins._init_weightsc                 C   s¾   |j d }tt| j| j| j| jƒƒD ]Š\}\}}}}||ƒ\}}||ƒ}t|ƒD ]$\}	}
|
||ƒ}|	dkrP|||ƒ}qP|t| jƒd k r$|j	|g|¢d‘R Ž  
dddd¡ ¡ }q$|  |¡}|S )Nr   r   r8   r   r3   )r:   Ú	enumerateÚzipr¨   r©   r­   r¯   rª   r¦   r>   r@   rE   rh   )r+   rF   rG   r€   Úembedrl   r­   Zpos_blkr2   ÚjZblkr/   r/   r0   Úforward_features”  s    
ÿ
(
zTwins.forward_features)Ú
pre_logitsc                 C   s2   | j dkr|jdd}|  |¡}|r(|S |  |¡S )NrŽ   r   r9   )r¥   Úmeanr°   r±   )r+   rF   rÌ   r/   r/   r0   Úforward_head£  s    

zTwins.forward_headc                 C   s   |   |¡}|  |¡}|S rw   )rË   rÎ   )r+   rF   r/   r/   r0   rR   ©  s    

zTwins.forward)F)T)N)F)rT   rU   rV   rW   r   r#   rg   rj   r   rX   rY   Úignorer…   rº   r¼   r½   r¿   r³   rË   r[   rÎ   rR   r^   r/   r/   r-   r0   r     sB   íE


Fc                 K   s,   |  dd ¡rtdƒ‚tt| |fi |¤Ž}|S )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.)ÚgetÚRuntimeErrorr   r   )ÚvariantÚ
pretrainedÚkwargsÚmodelr/   r/   r0   Ú_create_twins¯  s    rÖ   r¾   c                 K   s    | ddd dddt tdddœ|¥S )	Nr   )r   r‡   r‡   gÍÌÌÌÌÌì?ZbicubicTzpatch_embeds.0.projr±   )Úurlr¤   Z
input_sizeZ	pool_sizeZcrop_pctÚinterpolationZfixed_input_sizerÍ   rÀ   Z
first_convÚ
classifierr   )r×   rÔ   r/   r/   r0   Ú_cfg·  s    ûúrÚ   ztimm/)Z	hf_hub_id)ztwins_pcpvt_small.in1kztwins_pcpvt_base.in1kztwins_pcpvt_large.in1kztwins_svt_small.in1kztwins_svt_base.in1kztwins_svt_large.in1kr‹   c                 K   sF   t dg d¢g d¢g d¢g d¢g d¢d}td
d	| it |fi |¤Ž¤ŽS )Nr4   ©r   r‘   i@  r“   ©r   r3   r5   r   ©r   r   r4   r4   r•   r—   ©rŠ   rŸ   r!   r    r¦   r¢   Útwins_pcpvt_smallrÓ   )rß   ©r¹   rÖ   ©rÓ   rÔ   Z
model_argsr/   r/   r0   rß   Ì  s
    þrß   c                 K   sF   t dg d¢g d¢g d¢g d¢g d¢d}td
d	| it |fi |¤Ž¤ŽS )Nr4   rÛ   rÜ   rÝ   )r   r4   é   r   r—   rÞ   Útwins_pcpvt_baserÓ   )rã   rà   rá   r/   r/   r0   rã   Ô  s
    þrã   c                 K   sF   t dg d¢g d¢g d¢g d¢g d¢d}td
d	| it |fi |¤Ž¤ŽS )Nr4   rÛ   rÜ   rÝ   )r   r   é   r   r—   rÞ   Útwins_pcpvt_largerÓ   )rå   rà   rá   r/   r/   r0   rå   Ü  s
    þrå   c              	   K   sL   t dg d¢g d¢g d¢g d¢g d¢g d¢d}tdd
| it |fi |¤Ž¤ŽS )Nr4   r   )r3   r4   r   rˆ   r”   )r3   r3   é
   r4   ©é   rè   rè   rè   r—   ©rŠ   rŸ   r!   r    r¦   r£   r¢   Útwins_svt_smallrÓ   )rê   rà   rá   r/   r/   r0   rê   ä  s
    þrê   c              	   K   sL   t dg d¢g d¢g d¢g d¢g d¢g d¢d}tdd
| it |fi |¤Ž¤ŽS )Nr4   )é`   éÀ   i€  ry   )r   r–   é   é   r”   ©r3   r3   râ   r3   rç   r—   ré   Útwins_svt_baserÓ   )rð   rà   rá   r/   r/   r0   rð   ì  s
    þrð   c              	   K   sL   t dg d¢g d¢g d¢g d¢g d¢g d¢d}tdd
| it |fi |¤Ž¤ŽS )Nr4   )r‘   r’   r“   i   )r4   r   rˆ   é    r”   rï   rç   r—   ré   Útwins_svt_largerÓ   )rò   rà   rá   r/   r/   r0   rò   ô  s
    þrò   )F)r¾   )F)F)F)F)F)F)1rW   rÄ   Ú	functoolsr   Útypingr   rX   Ztorch.nnr#   Ztorch.nn.functionalri   r<   Z	timm.datar   r   Ztimm.layersr   r   r	   r
   r   Z_builderr   Z_features_fxr   Ú	_registryr   r   Zvision_transformerr   Ú__all__rq   r]   ÚModuler   r_   rj   rx   r†   r   rÖ   rÚ   Zdefault_cfgsrß   rã   rå   rê   rð   rò   r/   r/   r/   r0   Ú<module>   sX   a;( 

ú
