a
    þdmC  ã                
   @   s  d Z ddlZddlmZmZmZmZ ddlZddlm	Z	 ddl
m	  mZ ddlm  mZ ddlmZmZ ddlmZmZmZmZmZmZ ddlmZ ddlmZmZ d	gZG d
d„ de	j ƒZ!G dd„ de	j ƒZ"G dd„ de	j ƒZ#G dd„ de	j ƒZ$G dd„ de	j ƒZ%G dd	„ d	e	j ƒZ&dd„ Z'd/dd„Z(d0dd„Z)ee)dde)dde)dde)dde)dde)dde)dddœƒZ*ed1e&d œd!d"„ƒZ+ed2e&d œd#d$„ƒZ,ed3e&d œd%d&„ƒZ-ed4e&d œd'd(„ƒZ.ed5e&d œd)d*„ƒZ/ed6e&d œd+d,„ƒZ0ed7e&d œd-d.„ƒZ1dS )8a   Pyramid Vision Transformer v2

@misc{wang2021pvtv2,
      title={PVTv2: Improved Baselines with Pyramid Vision Transformer},
      author={Wenhai Wang and Enze Xie and Xiang Li and Deng-Ping Fan and Kaitao Song and Ding Liang and
        Tong Lu and Ping Luo and Ling Shao},
      year={2021},
      eprint={2106.13797},
      archivePrefix={arXiv},
      primaryClass={cs.CV}
}

Based on Apache 2.0 licensed code at https://github.com/whai362/PVT

Modifications and timm support by / Copyright 2022, Ross Wightman
é    N)ÚTupleÚListÚCallableÚUnion©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚ	to_2tupleÚ	to_ntupleÚtrunc_normal_Ú	LayerNormÚuse_fused_attné   )Úbuild_model_with_cfg)Úregister_modelÚgenerate_default_cfgsÚPyramidVisionTransformerV2c                       s<   e Zd Zddejddf‡ fdd„	Zee dœdd„Z‡  Z	S )	ÚMlpWithDepthwiseConvNç        Fc              	      s~   t ƒ  ¡  |p|}|p|}t ||¡| _|r4t ¡ nt ¡ | _tj||dddd|d| _	|ƒ | _
t ||¡| _t |¡| _d S )Né   r   T)ÚbiasÚgroups)ÚsuperÚ__init__ÚnnÚLinearÚfc1ZReLUÚIdentityÚreluÚConv2dÚdwconvÚactÚfc2ÚDropoutÚdrop)ÚselfÚin_featuresÚhidden_featuresZout_featuresÚ	act_layerr%   Ú
extra_relu©Ú	__class__© ú[/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/pvt_v2.pyr   #   s    	
zMlpWithDepthwiseConv.__init__©Ú	feat_sizec                 C   sˆ   |   |¡}|j\}}}| dd¡ |||d |d ¡}|  |¡}|  |¡}| d¡ dd¡}|  |¡}|  |¡}|  	|¡}|  |¡}|S )Nr   é   r   )
r   ÚshapeÚ	transposeÚviewr   r!   Úflattenr"   r%   r#   )r&   Úxr0   ÚBÚNÚCr-   r-   r.   Úforward6   s    
 





zMlpWithDepthwiseConv.forward)
Ú__name__Ú
__module__Ú__qualname__r   ÚGELUr   r   Úintr:   Ú__classcell__r-   r-   r+   r.   r   "   s   ùr   c                       sB   e Zd ZU ejje ed< d‡ fdd„	Ze	e
 d	œd
d„Z‡  ZS )Ú	AttentionÚ
fused_attné   r   FTr   c                    s&  t ƒ  ¡  || dks,J d|› d|› dƒ‚|| _|| _|| | _| jd | _tƒ | _tj	|||d| _
tj	||d |d| _t |¡| _t 	||¡| _t |¡| _|sìd | _|dkrØtj||||d	| _t |¡| _nd | _d | _d | _n6t d
¡| _tj||ddd	| _t |¡| _t ¡ | _d S )Nr   zdim z  should be divided by num_heads Ú.g      à¿)r   r1   r   )Úkernel_sizeÚstrideé   )r   r   ÚdimÚ	num_headsÚhead_dimÚscaler   rB   r   r   ÚqÚkvr$   Ú	attn_dropÚprojÚ	proj_dropÚpoolr    Úsrr   Únormr"   ZAdaptiveAvgPool2dr>   )r&   rH   rI   Úsr_ratioÚlinear_attnÚqkv_biasrN   rP   r+   r-   r.   r   G   s0    

"
zAttention.__init__r/   c                 C   sø  |j \}}}|\}}|  |¡ ||| jd¡ dddd¡}| jd urÀ| ddd¡ ||||¡}|  |  |¡¡ ||d¡ ddd¡}|  |¡}|  |¡}|  	|¡ |dd| j| j
¡ ddddd¡}	n¤| jd ur:| ddd¡ ||||¡}|  |¡ ||d¡ ddd¡}|  |¡}|  	|¡ |dd| j| j
¡ ddddd¡}	n*|  	|¡ |dd| j| j
¡ ddddd¡}	|	 d¡\}
}| jr’tj||
|| jjd}n8|| j }||
 dd¡ }|jdd	}|  |¡}|| }| dd¡ |||¡}|  |¡}|  |¡}|S )
Néÿÿÿÿr   r1   r   r   é   )Z	dropout_péþÿÿÿ©rH   )r2   rL   ÚreshaperI   ÚpermuterQ   rR   rS   r"   rM   rJ   ZunbindrB   ÚFZscaled_dot_product_attentionrN   ÚprK   r3   ZsoftmaxrO   rP   )r&   r6   r0   r7   r8   r9   ÚHÚWrL   rM   ÚkÚvÚattnr-   r-   r.   r:   o   s6    $
$

,
,*



zAttention.forward)rC   r   FTr   r   )r;   r<   r=   ÚtorchÚjitÚFinalÚboolÚ__annotations__r   r   r?   r:   r@   r-   r-   r+   r.   rA   D   s   
      ø(rA   c                	       sD   e Zd Zdddddddejef	‡ fdd„	Zee dœdd	„Z	‡  Z
S )
ÚBlockç      @r   Fr   c              	      sŠ   t ƒ  ¡  ||ƒ| _t|||||||d| _|	dkr<t|	ƒnt ¡ | _||ƒ| _	t
|t|| ƒ|
||d| _|	dkr|t|	ƒnt ¡ | _d S )N)rI   rT   rU   rV   rN   rP   r   )r'   r(   r)   r%   r*   )r   r   Únorm1rA   rc   r	   r   r   Ú
drop_path1Únorm2r   r?   ÚmlpÚ
drop_path2)r&   rH   rI   Ú	mlp_ratiorT   rU   rV   rP   rN   Ú	drop_pathr)   Ú
norm_layerr+   r-   r.   r   •   s*    

ù	

ûzBlock.__init__r/   c                 C   s<   ||   |  |  |¡|¡¡ }||  |  |  |¡|¡¡ }|S ©N)rl   rc   rk   ro   rn   rm   )r&   r6   r0   r-   r-   r.   r:   º   s    zBlock.forward)r;   r<   r=   r   r>   r   r   r   r?   r:   r@   r-   r-   r+   r.   ri   “   s   ô%ri   c                       s*   e Zd ZdZd
‡ fdd„	Zdd	„ Z‡  ZS )ÚOverlapPatchEmbedz Image to Patch Embedding
    rG   rX   r   é   c                    sf   t ƒ  ¡  t|ƒ}t|ƒ|ks&J dƒ‚|| _tj|||||d d |d d fd| _t |¡| _	d S )Nz!Set larger patch_size than strider   r1   r   )rF   Úpadding)
r   r   r
   ÚmaxÚ
patch_sizer   r    rO   r   rS   )r&   rx   rF   Úin_chansÚ	embed_dimr+   r-   r.   r   Ä   s    
þzOverlapPatchEmbed.__init__c                 C   s(   |   |¡}| dddd¡}|  |¡}|S )Nr   r1   r   r   )rO   r\   rS   ©r&   r6   r-   r-   r.   r:   Î   s    

zOverlapPatchEmbed.forward)rG   rX   r   ru   )r;   r<   r=   Ú__doc__r   r:   r@   r-   r-   r+   r.   rt   Á   s   
rt   c                       sd   e Zd Zdddddddddef
eeeeeeeeeeeeee ef e	dœ‡ fdd	„Z
d
d„ Z‡  ZS )ÚPyramidVisionTransformerStageTrC   r   Frj   r   )rH   Údim_outÚdepthÚ
downsamplerI   rT   rU   rp   rV   rP   rN   rq   rr   c                    sx   t ƒ  ¡  d| _|r(tdd|ˆd| _n|ˆks4J ‚d | _t ‡ ‡‡‡‡‡‡‡‡‡	f
dd„t|ƒD ƒ¡| _ˆˆƒ| _	d S )NFr   r1   ©rx   rF   ry   rz   c                    s:   g | ]2}t ˆˆˆ	ˆˆˆˆˆ tˆtƒr,ˆ| nˆˆd 
‘qS ))
rH   rI   rT   rU   rp   rV   rP   rN   rq   rr   )ri   Ú
isinstanceÚlist)Ú.0Úi©
rN   r~   rq   rU   rp   rr   rI   rP   rV   rT   r-   r.   Ú
<listcomp>ô   s   õöz:PyramidVisionTransformerStage.__init__.<locals>.<listcomp>)
r   r   Úgrad_checkpointingrt   r€   r   Z
ModuleListÚrangeÚblocksrS   )r&   rH   r~   r   r€   rI   rT   rU   rp   rV   rP   rN   rq   rr   r+   r†   r.   r   Ö   s    
ü
 õ
z&PyramidVisionTransformerStage.__init__c                 C   s¤   | j d ur|   |¡}|j\}}}}||f}| |d|¡}| jD ].}| jrbtj ¡ sbt |||¡}q>|||ƒ}q>|  	|¡}| ||d |d d¡ 
dddd¡ ¡ }|S )NrW   r   r   r   r1   )r€   r2   r[   rŠ   rˆ   rd   re   Zis_scriptingÚ
checkpointrS   r\   Ú
contiguous)r&   r6   r7   r_   r`   r9   r0   Zblkr-   r-   r.   r:     s    



(z%PyramidVisionTransformerStage.forward)r;   r<   r=   r   r?   rg   Úfloatr   r   r   r   r:   r@   r-   r-   r+   r.   r}   Õ   s4   òò-r}   c                       s²   e Zd Zddddddddd	d
ddddef‡ fdd„	Zdd„ Zdd„ Zejj	dd„ ƒZ
ejj	d$dd„ƒZejj	d%dd„ƒZdd„ Zd&dd„Zdd„ Zd'edœd d!„Zd"d#„ Z‡  ZS )(r   r   éè  Úavg©r   rX   é   r   )é@   é€   é   é   )r   r1   rX   rC   )rC   rX   r1   r   )ç       @r–   rj   rj   TFr   c                    s–  t ƒ  ¡  || _|dv sJ ‚|| _|| _t|ƒ}t|ƒ|ƒ}t|ƒ|ƒ}t|ƒ|ƒ}t|ƒ|ksdJ ‚g | _tdd||d d| _	dd„ t
 d|t|ƒ¡ |¡D ƒ}d}|d }g }t|ƒD ]„}|t||| || |dk|| || || |
|	|||| |dg7 }|| }||| 7 }|  jt|dd	|  d
|› dg7  _qºtj|Ž | _|d | _t |¡| _|dkr|t |d |¡nt ¡ | _|  | j¡ d S )N©r   Ú rG   rX   r   r   c                 S   s   g | ]}|  ¡ ‘qS r-   )Útolist)r„   r6   r-   r-   r.   r‡   ;  ó    z7PyramidVisionTransformerV2.__init__.<locals>.<listcomp>)rH   r~   r   r€   rI   rT   rp   rU   rV   rP   rN   rq   rr   r1   ústages.)Znum_chsZ	reductionÚmodulerW   )r   r   Únum_classesÚglobal_poolÚdepthsÚlenr   Zfeature_infort   Úpatch_embedrd   ZlinspaceÚsumÚsplitr‰   r}   Údictr   Z
SequentialÚstagesZnum_featuresr$   Ú	head_dropr   r   ÚheadÚapplyÚ_init_weights)r&   ry   r   rž   rŸ   Ú
embed_dimsrI   Z	sr_ratiosÚ
mlp_ratiosrV   ÚlinearZ	drop_rateZproj_drop_rateZattn_drop_rateZdrop_path_raterr   Z
num_stagesZdprÚcurZprev_dimr¥   r…   r+   r-   r.   r     sZ    
ü"ó
*
$z#PyramidVisionTransformerV2.__init__c                 C   s¦   t |tjƒrBt|jdd t |tjƒr¢|jd ur¢tj |jd¡ n`t |tjƒr¢|j	d |j	d  |j
 }||j }|jj dt d| ¡¡ |jd ur¢|jj ¡  d S )Ng{®Gáz”?)Ústdr   r   g       @)r‚   r   r   r   Zweightr   ÚinitZ	constant_r    rE   Zout_channelsr   ÚdataZnormal_ÚmathÚsqrtZzero_)r&   ÚmZfan_outr-   r-   r.   r©   [  s    

z(PyramidVisionTransformerV2._init_weightsc                 C   s   d| j _d S )NF)r¡   Zrequires_grad©r&   r-   r-   r.   Úfreeze_patch_embg  s    z+PyramidVisionTransformerV2.freeze_patch_embc                 C   s   i S rs   r-   r´   r-   r-   r.   Úno_weight_decayj  s    z*PyramidVisionTransformerV2.no_weight_decayc                 C   s   t ddd}|S )Nz^patch_embedz^stages\.(\d+))ÚstemrŠ   )r¤   )r&   ZcoarseZmatcherr-   r-   r.   Úgroup_matchern  s
    þz(PyramidVisionTransformerV2.group_matcherc                 C   s   | j D ]
}||_qd S rs   )r¥   rˆ   )r&   ÚenableÚsr-   r-   r.   Úset_grad_checkpointingv  s    
z1PyramidVisionTransformerV2.set_grad_checkpointingc                 C   s   | j S rs   )r§   r´   r-   r-   r.   Úget_classifier{  s    z)PyramidVisionTransformerV2.get_classifierNc                 C   sD   || _ |d ur |dv sJ ‚|| _|dkr6t | j|¡nt ¡ | _d S )Nr—   r   )r   rž   r   r   rz   r   r§   )r&   r   rž   r-   r-   r.   Úreset_classifier~  s
    z+PyramidVisionTransformerV2.reset_classifierc                 C   s   |   |¡}|  |¡}|S rs   )r¡   r¥   r{   r-   r-   r.   Úforward_features…  s    

z+PyramidVisionTransformerV2.forward_features)Ú
pre_logitsc                 C   s.   | j r|jdd}|  |¡}|r$|S |  |¡S )N)rW   rY   rZ   )rž   Úmeanr¦   r§   )r&   r6   r¿   r-   r-   r.   Úforward_headŠ  s    
z'PyramidVisionTransformerV2.forward_headc                 C   s   |   |¡}|  |¡}|S rs   )r¾   rÁ   r{   r-   r-   r.   r:     s    

z"PyramidVisionTransformerV2.forward)F)T)N)F)r;   r<   r=   r   r   r©   rµ   rd   re   Úignorer¶   r¸   r»   r¼   r½   r¾   rg   rÁ   r:   r@   r-   r-   r+   r.   r     s:   ðE

c                 C   s    d| v r| S i }ddl }|  ¡ D ]z\}}| d¡rb| dd¡}| dd¡}| dd	¡}| d
d¡}| dd¡}| ddd„ |¡}| ddd„ |¡}|||< q |S )z$ Remap original checkpoints -> timm zpatch_embed.proj.weightr   Nr¡   Zpatch_embed1Zpatch_embed2zstages.1.downsampleZpatch_embed3zstages.2.downsampleZpatch_embed4zstages.3.downsamplezdwconv.dwconvr!   zblock(\d+).(\d+)c                 S   s$   dt |  d¡ƒd › d|  d¡› S )Nr›   r   z.blocks.r1   ©r?   Úgroup©r6   r-   r-   r.   Ú<lambda>¤  rš   z'_checkpoint_filter_fn.<locals>.<lambda>z
^norm(\d+)c                 S   s   dt |  d¡ƒd › dS )Nr›   r   z.normrÃ   rÅ   r-   r-   r.   rÆ   ¥  rš   )ÚreÚitemsÚ
startswithÚreplaceÚsub)Z
state_dictÚmodelZout_dictrÇ   ra   rb   r-   r-   r.   Ú_checkpoint_filter_fn–  s    

rÍ   Fc                 K   s>   t tdƒƒ}| d|¡}tt| |fttd|ddœ|¤Ž}|S )NrX   Úout_indicesT)Zflatten_sequentialrÎ   )Zpretrained_filter_fnZfeature_cfg)Útupler‰   Úpopr   r   rÍ   r¤   )ÚvariantÚ
pretrainedÚkwargsZdefault_out_indicesrÎ   rÌ   r-   r-   r.   Ú_create_pvt2ª  s    ý
ûúrÔ   r˜   c                 K   s    | dddddt tdddd	œ|¥S )
NrŽ   )r   éà   rÕ   )rG   rG   gÍÌÌÌÌÌì?Zbicubiczpatch_embed.projr§   F)Úurlr   Z
input_sizeZ	pool_sizeZcrop_pctÚinterpolationrÀ   r®   Z
first_convÚ
classifierZfixed_input_sizer   )rÖ   rÓ   r-   r-   r.   Ú_cfg¸  s    üûrÙ   ztimm/)Z	hf_hub_id)zpvt_v2_b0.in1kzpvt_v2_b1.in1kzpvt_v2_b2.in1kzpvt_v2_b3.in1kzpvt_v2_b4.in1kzpvt_v2_b5.in1kzpvt_v2_b2_li.in1k)Úreturnc                 K   s,   t dddd}tdd| it |fi |¤Ž¤ŽS )N©r1   r1   r1   r1   )é    r’   é    r”   ©r   r1   é   rC   ©rŸ   rª   rI   Ú	pvt_v2_b0rÒ   )rá   ©r¤   rÔ   ©rÒ   rÓ   Z
model_argsr-   r-   r.   rá   Í  s    rá   c                 K   s,   t dddd}tdd| it |fi |¤Ž¤ŽS )NrÛ   ©r’   r“   i@  r•   rÞ   rà   Ú	pvt_v2_b1rÒ   )rå   râ   rã   r-   r-   r.   rå   Ó  s    rå   c                 K   s,   t dddd}tdd| it |fi |¤Ž¤ŽS )Nr   rä   rÞ   rà   Ú	pvt_v2_b2rÒ   )ræ   râ   rã   r-   r-   r.   ræ   Ù  s    ræ   c                 K   s,   t dddd}tdd| it |fi |¤Ž¤ŽS )N)r   rX   é   r   rä   rÞ   rà   Ú	pvt_v2_b3rÒ   )rè   râ   rã   r-   r-   r.   rè   ß  s    rè   c                 K   s,   t dddd}tdd| it |fi |¤Ž¤ŽS )N)r   rC   é   r   rä   rÞ   rà   Ú	pvt_v2_b4rÒ   )rê   râ   rã   r-   r-   r.   rê   å  s    rê   c                 K   s.   t ddddd}tdd| it |fi |¤Ž¤ŽS )	N)r   r‘   é(   r   rä   rÞ   )rX   rX   rX   rX   )rŸ   rª   rI   r«   Ú	pvt_v2_b5rÒ   )rì   râ   rã   r-   r-   r.   rì   ë  s    ÿrì   c                 K   s.   t ddddd}tdd| it |fi |¤Ž¤ŽS )	Nr   rä   rÞ   T)rŸ   rª   rI   r¬   Úpvt_v2_b2_lirÒ   )rí   râ   rã   r-   r-   r.   rí   ò  s    ÿrí   )F)r˜   )F)F)F)F)F)F)F)2r|   r±   Útypingr   r   r   r   rd   Ztorch.nnr   Ztorch.nn.functionalZ
functionalr]   Ztorch.utils.checkpointÚutilsr‹   Z	timm.datar   r   Ztimm.layersr	   r
   r   r   r   r   Z_builderr   Ú	_registryr   r   Ú__all__ÚModuler   rA   ri   rt   r}   r   rÍ   rÔ   rÙ   Zdefault_cfgsrá   rå   ræ   rè   rê   rì   rí   r-   r-   r-   r.   Ú<module>   sX    "O.@ 


ù