a
    þdßI  ã                   @   sê  d Z ddlmZ ddlZddlmZ ddlmZmZ ddl	m
Z
mZmZmZ ddlmZ ddlmZ dd	lmZmZ d
gZddddœZddddœZG dd„ dejjƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZ G d d!„ d!ejƒZ!G d"d#„ d#ejƒZ"G d$d%„ d%ejƒZ#G d&d'„ d'ejƒZ$G d(d
„ d
ejƒZ%d)d*„ Z&d;d,d-„Z'ee'd.d/e'd.d/e'd.d/d0œƒZ(d<d2d3„Z)ed=e%d4œd5d6„ƒZ*ed>e%d4œd7d8„ƒZ+ed?e%d4œd9d:„ƒZ,dS )@aû   EfficientFormer

@article{li2022efficientformer,
  title={EfficientFormer: Vision Transformers at MobileNet Speed},
  author={Li, Yanyu and Yuan, Geng and Wen, Yang and Hu, Eric and Evangelidis, Georgios and Tulyakov,
   Sergey and Wang, Yanzhi and Ren, Jian},
  journal={arXiv preprint arXiv:2206.01191},
  year={2022}
}

Based on Apache 2.0 licensed code at https://github.com/snap-research/EfficientFormer, Copyright (c) 2022 Snap Inc.

Modifications and timm support by / Copyright 2022, Ross Wightman
é    )ÚDictN©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚtrunc_normal_Ú	to_2tupleÚMlpé   )Úbuild_model_with_cfg)Úcheckpoint_seq)Úgenerate_default_cfgsÚregister_modelÚEfficientFormer)é0   é`   éà   iÀ  )é@   é€   i@  i   )r   éÀ   é€  i   )Úl1Úl3Úl7)é   é   é   é   )r   r   é   r   )r   r   é   é   c                       sd   e Zd ZU eeejf ed< d‡ fdd„	Ze 	¡ d‡ fd
d„	ƒZ
ejejdœdd„Zdd„ Z‡  ZS )Ú	AttentionÚattention_bias_cacher   é    r    r   é   c              	      s&  t ƒ  ¡  || _|d | _|| _|| | _t|| ƒ| _| j| | _|| _	t
 || jd | j ¡| _t
 | j|¡| _t|ƒ}t t t |d ¡t |d ¡¡¡ d¡}|dd d …d f |dd d d …f   ¡ }|d |d  |d  }tj
 t ||d |d  ¡¡| _|  dt |¡¡ i | _d S )Ng      à¿r   r   r
   .Úattention_bias_idxs)ÚsuperÚ__init__Ú	num_headsÚscaleÚkey_dimZkey_attn_dimÚintÚval_dimÚval_attn_dimÚ
attn_ratioÚnnÚLinearÚqkvÚprojr   ÚtorchÚstackZmeshgridZarangeÚflattenÚabsÚ	ParameterÚzerosÚattention_biasesZregister_bufferZ
LongTensorr"   )ÚselfÚdimr*   r(   r.   Ú
resolutionÚposZrel_pos©Ú	__class__© úd/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/efficientformer.pyr'   -   s"    


,("zAttention.__init__Tc                    s    t ƒ  |¡ |r| jri | _d S ©N)r&   Útrainr"   )r:   Úmoder>   r@   rA   rC   I   s    
zAttention.train)ÚdeviceÚreturnc                 C   s^   t j ¡ s| jr$| jd d …| jf S t|ƒ}|| jvrP| jd d …| jf | j|< | j| S d S rB   )r3   ÚjitÚ
is_tracingÚtrainingr9   r%   Ústrr"   )r:   rE   Z
device_keyr@   r@   rA   Úget_attention_biasesO   s    
zAttention.get_attention_biasesc           
      C   s°   |j \}}}|  |¡}| ||| jd¡ dddd¡}|j| j| j| jgdd\}}}|| dd¡ | j	 }	|	|  
|j¡ }	|	jdd}	|	|  dd¡ ||| j¡}|  |¡}|S )Néÿÿÿÿr   r   r
   r   ©r;   éþÿÿÿ)Úshaper1   Zreshaper(   ZpermuteÚsplitr*   r,   Ú	transposer)   rK   rE   Zsoftmaxr-   r2   )
r:   ÚxÚBÚNÚCr1   ÚqÚkÚvZattnr@   r@   rA   ÚforwardX   s    
 
zAttention.forward)r   r#   r    r   r$   )T)Ú__name__Ú
__module__Ú__qualname__r   rJ   r3   ZTensorÚ__annotations__r'   Zno_gradrC   rE   rK   rY   Ú__classcell__r@   r@   r>   rA   r!   *   s   
     ú	r!   c                       s&   e Zd Zejejf‡ fdd„	Z‡  ZS )ÚStem4c              
      s”   t ƒ  ¡  d| _|  dtj||d dddd¡ |  d||d ƒ¡ |  d|ƒ ¡ |  d	tj|d |dddd¡ |  d
||ƒ¡ |  d|ƒ ¡ d S )Nr   Zconv1r   r   r
   ©Úkernel_sizeÚstrideÚpaddingÚnorm1Zact1Zconv2Únorm2Zact2)r&   r'   rb   Z
add_moduler/   ÚConv2d)r:   Úin_chsÚout_chsÚ	act_layerÚ
norm_layerr>   r@   rA   r'   h   s    
  zStem4.__init__)rZ   r[   r\   r/   ZReLUÚBatchNorm2dr'   r^   r@   r@   r>   rA   r_   g   s   r_   c                       s4   e Zd ZdZdddejf‡ fdd„	Zdd„ Z‡  ZS )	Ú
DownsamplezŽ
    Downsampling via strided conv w/ norm
    Input: tensor in shape [B, C, H, W]
    Output: tensor in shape [B, C, H/stride, W/stride]
    r   r   Nc                    s>   t ƒ  ¡  |d u r|d }tj|||||d| _||ƒ| _d S )Nr   r`   )r&   r'   r/   rf   ÚconvÚnorm)r:   rg   rh   ra   rb   rc   rj   r>   r@   rA   r'   {   s
    
zDownsample.__init__c                 C   s   |   |¡}|  |¡}|S rB   )rm   rn   ©r:   rR   r@   r@   rA   rY   ‚   s    

zDownsample.forward)	rZ   r[   r\   Ú__doc__r/   rk   r'   rY   r^   r@   r@   r>   rA   rl   t   s   rl   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚFlatc                    s   t ƒ  ¡  d S rB   )r&   r'   ©r:   r>   r@   rA   r'   Š   s    zFlat.__init__c                 C   s   |  d¡ dd¡}|S )Nr   r
   )r5   rQ   ro   r@   r@   rA   rY      s    zFlat.forward©rZ   r[   r\   r'   rY   r^   r@   r@   r>   rA   rq   ˆ   s   rq   c                       s*   e Zd ZdZd‡ fdd„	Zdd„ Z‡  ZS )ÚPoolingzP
    Implementation of pooling for PoolFormer
    --pool_size: pooling size
    r   c                    s&   t ƒ  ¡  tj|d|d dd| _d S )Nr
   r   F)rb   rc   Zcount_include_pad)r&   r'   r/   Z	AvgPool2dÚpool)r:   Ú	pool_sizer>   r@   rA   r'   ˜   s    
zPooling.__init__c                 C   s   |   |¡| S rB   )ru   ro   r@   r@   rA   rY   œ   s    zPooling.forward)r   )rZ   r[   r\   rp   r'   rY   r^   r@   r@   r>   rA   rt   ’   s   rt   c                       s8   e Zd ZdZddejejdf‡ fdd„	Zdd„ Z‡  Z	S )ÚConvMlpWithNormz`
    Implementation of MLP with 1*1 convolutions.
    Input: tensor with shape [B, C, H, W]
    Nç        c                    s†   t ƒ  ¡  |p|}|p|}t ||d¡| _|d ur:||ƒnt ¡ | _|ƒ | _t ||d¡| _|d url||ƒnt ¡ | _	t 
|¡| _d S )Nr
   )r&   r'   r/   rf   Úfc1ÚIdentityrd   ÚactÚfc2re   ÚDropoutÚdrop)r:   Úin_featuresÚhidden_featuresZout_featuresri   rj   r~   r>   r@   rA   r'   ¦   s    	
zConvMlpWithNorm.__init__c                 C   sJ   |   |¡}|  |¡}|  |¡}|  |¡}|  |¡}|  |¡}|  |¡}|S rB   )ry   rd   r{   r~   r|   re   ro   r@   r@   rA   rY   ¹   s    






zConvMlpWithNorm.forward)
rZ   r[   r\   rp   r/   ÚGELUrk   r'   rY   r^   r@   r@   r>   rA   rw       s   ùrw   c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )Ú
LayerScaleçñhãˆµøä>Fc                    s*   t ƒ  ¡  || _t |t |¡ ¡| _d S rB   ©r&   r'   Úinplacer/   r7   r3   ZonesÚgamma©r:   r;   Zinit_valuesr…   r>   r@   rA   r'   Å   s    
zLayerScale.__init__c                 C   s   | j r| | j¡S || j S rB   )r…   Úmul_r†   ro   r@   r@   rA   rY   Ê   s    zLayerScale.forward)rƒ   Frs   r@   r@   r>   rA   r‚   Ä   s   r‚   c                       s6   e Zd Zdejejdddf‡ fdd„	Zdd„ Z‡  ZS )ÚMetaBlock1dç      @rx   rƒ   c                    sx   t ƒ  ¡  ||ƒ| _t|ƒ| _||ƒ| _t|t|| ƒ||d| _|dkrRt	|ƒnt
 ¡ | _t||ƒ| _t||ƒ| _d S )N)r   r€   ri   r~   rx   )r&   r'   rd   r!   Útoken_mixerre   r	   r+   Úmlpr   r/   rz   Ú	drop_pathr‚   Úls1Úls2)r:   r;   Ú	mlp_ratiori   rj   Ú	proj_dropr   Úlayer_scale_init_valuer>   r@   rA   r'   Ð   s    





üzMetaBlock1d.__init__c              
   C   sD   ||   |  |  |  |¡¡¡¡ }||   |  |  |  |¡¡¡¡ }|S rB   )r   rŽ   r‹   rd   r   rŒ   re   ro   r@   r@   rA   rY   é   s      zMetaBlock1d.forward)	rZ   r[   r\   r/   r   Ú	LayerNormr'   rY   r^   r@   r@   r>   rA   r‰   Î   s   ør‰   c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )ÚLayerScale2drƒ   Fc                    s*   t ƒ  ¡  || _t |t |¡ ¡| _d S rB   r„   r‡   r>   r@   rA   r'   ð   s    
zLayerScale2d.__init__c                 C   s*   | j  dddd¡}| jr"| |¡S || S )Nr
   rL   )r†   Úviewr…   rˆ   )r:   rR   r†   r@   r@   rA   rY   õ   s    zLayerScale2d.forward)rƒ   Frs   r@   r@   r>   rA   r”   ï   s   r”   c                       s8   e Zd Zddejejdddf‡ fdd„	Zdd„ Z‡  ZS )	ÚMetaBlock2dr   rŠ   rx   rƒ   c	           	         s‚   t ƒ  ¡  t|d| _t||ƒ| _|dkr2t|ƒnt ¡ | _	t
|t|| ƒ|||d| _t||ƒ| _|dkrtt|ƒnt ¡ | _d S )N)rv   rx   )r€   ri   rj   r~   )r&   r'   rt   r‹   r”   rŽ   r   r/   rz   Ú
drop_path1rw   r+   rŒ   r   Ú
drop_path2)	r:   r;   rv   r   ri   rj   r‘   r   r’   r>   r@   rA   r'   ü   s    

ûzMetaBlock2d.__init__c                 C   s8   ||   |  |  |¡¡¡ }||  |  |  |¡¡¡ }|S rB   )r—   rŽ   r‹   r˜   r   rŒ   ro   r@   r@   rA   rY     s    zMetaBlock2d.forward)	rZ   r[   r\   r/   r   rk   r'   rY   r^   r@   r@   r>   rA   r–   ú   s   ÷r–   c                
       s@   e Zd Zddddejejejdddf
‡ fdd„	Zd	d
„ Z‡  Z	S )ÚEfficientFormerStageTr
   r   rŠ   rx   rƒ   c                    sî   t ƒ  ¡  d| _|r*t|||	d| _|}n||ks6J ‚t ¡ | _g }|r\||kr\| tƒ ¡ t	|ƒD ]x}|| d }|r¢||kr¢| t
||||
||| |d¡ qd| t|||||	||| |d¡ |rd||krd| tƒ ¡ qdtj|Ž | _d S )NF)rg   rh   rj   r
   )r   ri   rj   r‘   r   r’   )rv   r   ri   rj   r‘   r   r’   )r&   r'   Úgrad_checkpointingrl   Ú
downsampler/   rz   Úappendrq   Úranger‰   r–   Ú
SequentialÚblocks)r:   r;   Zdim_outÚdepthr›   Únum_vitrv   r   ri   rj   Únorm_layer_clr‘   r   r’   rŸ   Z	block_idxZ
remain_idxr>   r@   rA   r'     sN    

ùÿøÿzEfficientFormerStage.__init__c                 C   s6   |   |¡}| jr(tj ¡ s(t| j|ƒ}n
|  |¡}|S rB   )r›   rš   r3   rG   Úis_scriptingr   rŸ   ro   r@   r@   rA   rY   Z  s
    

zEfficientFormerStage.forward)
rZ   r[   r\   r/   r   rk   r“   r'   rY   r^   r@   r@   r>   rA   r™     s   ò<r™   c                       sÊ   e Zd Zdddddddddejejejdddf‡ fd	d
„	Zdd„ Ze	j
jdd„ ƒZe	j
jd"dd„ƒZe	j
jd#dd„ƒZe	j
jdd„ ƒZd$dd„Ze	j
jd%dd„ƒZdd„ Zd&edœdd„Zd d!„ Z‡  ZS )'r   Nr   éè  Úavgr   r   rƒ   rx   c                    s\  t ƒ  ¡  || _|| _t||d |d| _|d }dd„ t d|t|ƒ¡ 	|¡D ƒ}|pjddt
|ƒd   }g }tt
|ƒƒD ]T}t||| || || |dkr¢|nd|	||||||| |
d	}|| }| |¡ q|tj|Ž | _|d
 | _|| jƒ| _t |¡| _|dkrt | j|¡nt ¡ | _|dkr<t |d
 |¡nt ¡ | _d| _|  | j¡ d S )Nr   )rj   c                 S   s   g | ]}|  ¡ ‘qS r@   )Útolist)Ú.0rR   r@   r@   rA   Ú
<listcomp>  ó    z,EfficientFormer.__init__.<locals>.<listcomp>)F)Tr
   r   )
r›   r¡   rv   r   ri   r¢   rj   r‘   r   r’   rL   F)r&   r'   Únum_classesÚglobal_poolr_   Ústemr3   ZlinspaceÚsumrP   Úlenr   r™   rœ   r/   rž   ÚstagesÚnum_featuresrn   r}   Ú	head_dropr0   rz   ÚheadÚ	head_distÚdistilled_trainingÚapplyÚ_init_weights)r:   ÚdepthsÚ
embed_dimsZin_chansrª   r«   Zdownsamplesr¡   Z
mlp_ratiosrv   r’   ri   rj   r¢   Z	drop_rateZproj_drop_rateZdrop_path_rateÚkwargsZprev_dimZdprr¯   ÚiZstager>   r@   rA   r'   e  sD    
"ó
"$zEfficientFormer.__init__c                 C   sD   t |tjƒr@t|jdd t |tjƒr@|jd ur@tj |jd¡ d S )Ng{®Gáz”?)Ústdr   )Ú
isinstancer/   r0   r   ZweightZbiasÚinitZ	constant_)r:   Úmr@   r@   rA   r¶   ¥  s    zEfficientFormer._init_weightsc                 C   s   dd„ |   ¡ D ƒS )Nc                 S   s   h | ]\}}d |v r|’qS )r9   r@   )r§   rW   Ú_r@   r@   rA   Ú	<setcomp>­  r©   z2EfficientFormer.no_weight_decay.<locals>.<setcomp>)Znamed_parametersrr   r@   r@   rA   Úno_weight_decay«  s    zEfficientFormer.no_weight_decayFc                 C   s   t dddgd}|S )Nz^stem)z^stages\.(\d+)N)z^norm)iŸ† )r¬   rŸ   )Údict)r:   ZcoarseZmatcherr@   r@   rA   Úgroup_matcher¯  s
    þzEfficientFormer.group_matcherTc                 C   s   | j D ]
}||_qd S rB   )r¯   rš   )r:   ÚenableÚsr@   r@   rA   Úset_grad_checkpointing·  s    
z&EfficientFormer.set_grad_checkpointingc                 C   s   | j | jfS rB   ©r²   r³   rr   r@   r@   rA   Úget_classifier¼  s    zEfficientFormer.get_classifierc                 C   sX   || _ |d ur|| _|dkr*t | j|¡nt ¡ | _|dkrJt | j|¡nt ¡ | _d S )Nr   )rª   r«   r/   r0   r°   rz   r²   r³   )r:   rª   r«   r@   r@   rA   Úreset_classifierÀ  s
     z EfficientFormer.reset_classifierc                 C   s
   || _ d S rB   )r´   )r:   rÄ   r@   r@   rA   Úset_distilled_trainingÇ  s    z&EfficientFormer.set_distilled_trainingc                 C   s"   |   |¡}|  |¡}|  |¡}|S rB   )r¬   r¯   rn   ro   r@   r@   rA   Úforward_featuresË  s    


z EfficientFormer.forward_features)Ú
pre_logitsc                 C   sl   | j dkr|jdd}|  |¡}|r(|S |  |¡|  |¡ }}| jr\| jr\tj 	¡ s\||fS || d S d S )Nr¥   r
   rM   r   )
r«   Úmeanr±   r²   r³   r´   rI   r3   rG   r£   )r:   rR   rÌ   Zx_distr@   r@   rA   Úforward_headÑ  s    

zEfficientFormer.forward_headc                 C   s   |   |¡}|  |¡}|S rB   )rË   rÎ   ro   r@   r@   rA   rY   ß  s    

zEfficientFormer.forward)F)T)N)T)F)rZ   r[   r\   r/   r   rk   r“   r'   r¶   r3   rG   ÚignorerÁ   rÃ   rÆ   rÈ   rÉ   rÊ   rË   ÚboolrÎ   rY   r^   r@   r@   r>   rA   r   c  s>   ï@


c                 C   sä   d| v r| S i }ddl }d}|  ¡ D ]º\}}| d¡rf| dd¡}| dd¡}| d	d
¡}| dd¡}| d|¡rz|d7 }| dd|› d|¡}| dd|› d|¡}| dd|› d|¡}| dd|¡}| dd¡}|||< q$|S )z$ Remap original checkpoints -> timm zstem.0.weightr   NZpatch_embedzpatch_embed.0ú
stem.conv1zpatch_embed.1z
stem.norm1zpatch_embed.3z
stem.conv2zpatch_embed.4z
stem.norm2znetwork\.(\d+)\.proj\.weightr
   znetwork.(\d+).(\d+)zstages.z
.blocks.\2znetwork.(\d+).projz.downsample.convznetwork.(\d+).normz.downsample.normzlayer_scale_([0-9])z
ls\1.gammaZ	dist_headr³   )ÚreÚitemsÚ
startswithÚreplaceÚmatchÚsub)Z
state_dictÚmodelZout_dictrÒ   Z	stage_idxrW   rX   r@   r@   rA   Ú_checkpoint_filter_fnå  s(    

rÙ   Ú c                 K   s    | ddd dddt tdddœ|¥S )	Nr¤   )r   r   r   Tgffffffî?ZbicubicrÑ   rÇ   )Úurlrª   Z
input_sizerv   Zfixed_input_sizeZcrop_pctÚinterpolationrÍ   r»   Z
first_convÚ
classifierr   )rÛ   r¹   r@   r@   rA   Ú_cfg   s    ûúrÞ   ztimm/)Z	hf_hub_id)z!efficientformer_l1.snap_dist_in1kz!efficientformer_l3.snap_dist_in1kz!efficientformer_l7.snap_dist_in1kFc                 K   s   t t| |fdti|¤Ž}|S )NZpretrained_filter_fn)r   r   rÙ   )ÚvariantÚ
pretrainedr¹   rØ   r@   r@   rA   Ú_create_efficientformer  s    ÿþýrá   )rF   c                 K   s4   t td td dd}tdd| it |fi |¤Ž¤ŽS )Nr   r
   ©r·   r¸   r¡   Úefficientformer_l1rà   )rã   ©rÂ   ÚEfficientFormer_depthÚEfficientFormer_widthrá   ©rà   r¹   Z
model_argsr@   r@   rA   rã      s    ýrã   c                 K   s4   t td td dd}tdd| it |fi |¤Ž¤ŽS )Nr   r   râ   Úefficientformer_l3rà   )rè   rä   rç   r@   r@   rA   rè   *  s    ýrè   c                 K   s4   t td td dd}tdd| it |fi |¤Ž¤ŽS )Nr   r    râ   Úefficientformer_l7rà   )ré   rä   rç   r@   r@   rA   ré   4  s    ýré   )rÚ   )F)F)F)F)-rp   Útypingr   r3   Ztorch.nnr/   Z	timm.datar   r   Ztimm.layersr   r   r   r	   Z_builderr   Z_manipulater   Ú	_registryr   r   Ú__all__ræ   rå   ÚModuler!   rž   r_   rl   rq   rt   rw   r‚   r‰   r”   r–   r™   r   rÙ   rÞ   Zdefault_cfgsrá   rã   rè   ré   r@   r@   r@   rA   Ú<module>   sd   ýý=
$
!"G 
ÿÿÿù
		