a
    d)a                     @   s,  d Z ddlZddlmZ ddlmZ ddlZddlmZ ddl	m
Z
mZ ddlmZmZmZmZmZ ddlmZmZmZmZ dd	lmZ dd
lmZ ddlmZmZ dddddZdddddZdddddZG dd dej Z!G dd dejj Z"G dd dejj Z#G dd  d ejj Z$G d!d" d"ej Z%G d#d$ d$ej Z&G d%d& d&ej Z'G d'd( d(ej Z(G d)d* d*ej)Z*G d+d, d,ej Z+G d-d. d.ej Z,dAd0d1Z-ee-d2d3e-d2d3e-d2d3e-d2d3d4Z.dBd6d7Z/edCe,d8d9d:Z0edDe,d8d;d<Z1edEe,d8d=d>Z2edFe,d8d?d@Z3dS )GaJ   EfficientFormer-V2

@article{
    li2022rethinking,
    title={Rethinking Vision Transformers for MobileNet Size and Speed},
    author={Li, Yanyu and Hu, Ju and Wen, Yang and Evangelidis, Georgios and Salahi, Kamyar and Wang, Yanzhi and Tulyakov, Sergey and Ren, Jian},
    journal={arXiv preprint arXiv:2212.08059},
    year={2022}
}

Significantly refactored and cleaned up for timm from original at: https://github.com/snap-research/EfficientFormer

Original code licensed Apache 2.0, Copyright (c) 2022 Snap Inc.

Modifications and timm support by / Copyright 2023, Ross Wightman
    N)partial)DictIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)create_conv2dcreate_norm_layerget_act_layerget_norm_layerConvNormAct)DropPathtrunc_normal_	to_2tuple	to_ntuple   )build_model_with_cfg)checkpoint_seq)generate_default_cfgsregister_model)(   P        )    @      i   )r   0   x      )r   r   `      )LS2S1S0)   r%      
   )   r(         )   r+   	      )   r.   r-   r(   )r(   r(   )r(   r(   r(   r(   r+   r+   r+   r+   r+   r+   r+   r(   r(   r(   r(   )
r(   r(   r(   r+   r+   r+   r+   r(   r(   r(   )r(   r(   )r(   r(   r+   r+   r+   r+   r+   r+   r(   r(   r(   r(   )r(   r(   r+   r+   r+   r+   r(   r(   )r(   r(   )	r(   r(   r+   r+   r+   r+   r(   r(   r(   )r(   r(   r+   r+   r(   r(   )r(   r(   )r(   r+   r+   r+   r(   r(   )r(   r+   r+   r(   c                       s&   e Zd Zd
 fdd	Zdd	 Z  ZS )ConvNormr    Tbatchnorm2dNc              
      sH   |
pi }
t t|   t||||||||d| _t|	|fi |
| _d S )N)stridepaddingdilationgroupsbias)superr/   __init__r   convr   bn)selfZin_channelsZout_channelskernel_sizer2   r3   r4   r5   r6   
norm_layerZnorm_kwargs	__class__ g/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/efficientformer_v2.pyr8   7   s    
zConvNorm.__init__c                 C   s   |  |}| |}|S N)r9   r:   r;   xr@   r@   rA   forwardR   s    

zConvNorm.forward)r   r   r0   r   r   Tr1   N__name__
__module____qualname__r8   rE   __classcell__r@   r@   r>   rA   r/   6   s           r/   c                       st   e Zd ZU eeejf ed< dddddej	df fdd		Z
e d fdd	ZejejdddZdd Z  ZS )Attention2dattention_bias_cacher   r   r*   r(      Nc              	      s  t    || _|d | _|| _t|} d urlt fdd|D }t||d |d| _t	j
 dd| _nd | _d | _|| _| jd | jd	  | _t|| | _t|| | | _|| _| j| j }t||| _t||| _t|| j| _t| j| jd| jd
| _t	j| j| jd	d| _t	j| j| jd	d| _| | _t| j|d	| _ttt| jd t| jd	 d	}	|	dd d d f |	dd d d f    }
|
d | jd	  |
d	  }
tj	!t"|| j| _#| j$dt%|
dd i | _&d S )N      c                    s   g | ]}t |  qS r@   mathceil.0rr2   r@   rA   
<listcomp>l       z(Attention2d.__init__.<locals>.<listcomp>r+   r<   r2   r5   Zbilinear)Zscale_factormoder   r   )r<   r5   )r<   .attention_bias_idxsF
persistent)'r7   r8   	num_headsscalekey_dimr   tupler/   stride_convnnZUpsampleupsample
resolutionNintddh
attn_ratioqkvv_localConv2dtalking_head1talking_head2actprojtorchstackmeshgridarangeflattenabs	Parameterzerosattention_biasesregister_bufferZ
LongTensorrL   )r;   dimr_   r]   ri   rd   	act_layerr2   khposrel_posr>   rU   rA   r8   [   s>    


0(zAttention2d.__init__Tc                    s    t  | |r| jri | _d S rB   r7   trainrL   r;   rY   r>   r@   rA   r      s    
zAttention2d.traindevicereturnc                 C   s^   t j s| jr$| jd d | jf S t|}|| jvrP| jd d | jf | j|< | j| S d S rB   rs   jit
is_tracingtrainingr{   rZ   strrL   r;   r   Z
device_keyr@   r@   rA   get_attention_biases   s    
z Attention2d.get_attention_biasesc                 C   s@  |j \}}}}| jd ur"| |}| ||| jd| jdddd}| ||| jd| jdddd}| |}| 	|}	||| jd| jdddd}|| | j
 }
|
| |j }
| |
}
|
jdd}
| |
}
|
| dd}||| j| jd | jd |	 }| jd ur(| |}| |}| |}|S Nr   r   r+   r.   r}   )shapera   rj   reshaper]   re   permuterk   rl   rm   r^   r   r   ro   softmaxrp   	transposerh   rd   rc   rq   rr   r;   rD   BCHWrj   rk   rl   rm   attnr@   r@   rA   rE      s(    

&&

 

"


zAttention2d.forward)TrG   rH   rI   r   r   rs   ZTensor__annotations__rb   GELUr8   Zno_gradr   r   r   rE   rJ   r@   r@   r>   rA   rK   X   s   
0	rK   c                       s$   e Zd Z fddZdd Z  ZS )LocalGlobalQueryc                    sD   t    tddd| _tj||ddd|d| _t||d| _d S )Nr   r.   r   r+   )r<   r2   r3   r5   )	r7   r8   rb   Z	AvgPool2dpoolrn   localr/   rr   )r;   Zin_dimout_dimr>   r@   rA   r8      s    
zLocalGlobalQuery.__init__c                 C   s*   |  |}| |}|| }| |}|S rB   )r   r   rr   )r;   rD   Zlocal_qZpool_qrj   r@   r@   rA   rE      s
    


zLocalGlobalQuery.forwardrF   r@   r@   r>   rA   r      s   r   c                       st   e Zd ZU eeejf ed< ddddddej	f fdd		Z
e d fdd	ZejejdddZdd Z  ZS )Attention2dDownsamplerL   r      r*   r(   rM   Nc              
      s  t    || _|d | _|| _t|| _tdd | jD | _| jd | jd  | _	| jd | jd  | _
t|| | _t|| | | _|| _|p|| _| j| j }t||| _t||d| _t|| jd| _t| j| jdd| jd| _| | _t| j| jd| _tt|| j	| _ttt| jd t| jd  d}	tttjd| jd dd	tjd| jd dd	 d}
|
d
d d d f |	d
d d d f  ! }|d | jd  |d  }| j"d|dd i | _#d S )NrN   c                 S   s   g | ]}t |d  qS r.   rO   rR   r@   r@   rA   rV      rW   z2Attention2dDownsample.__init__.<locals>.<listcomp>r   r   r+   r.   rX   )step.rZ   Fr[   )$r7   r8   r]   r^   r_   r   rd   r`   resolution2re   N2rf   rg   rh   ri   r   r   rj   r/   rk   rl   rm   rq   rr   rb   ry   rs   rz   r{   rt   ru   rv   rw   rx   r|   rL   )r;   r}   r_   r]   ri   rd   r   r~   r   Zk_posZq_posr   r>   r@   rA   r8      sJ    




(zAttention2dDownsample.__init__Tc                    s    t  | |r| jri | _d S rB   r   r   r>   r@   rA   r      s    
zAttention2dDownsample.trainr   c                 C   s^   t j s| jr$| jd d | jf S t|}|| jvrP| jd d | jf | j|< | j| S d S rB   r   r   r@   r@   rA   r      s    
z*Attention2dDownsample.get_attention_biasesc                 C   s  |j \}}}}| ||| jd| jdddd}| ||| jd| jdddd}| |}| 	|}	||| jd| jdddd}|| | j
 }
|
| |j }
|
jdd}
|
| dd}||| j| jd | jd |	 }| |}| |}|S r   )r   rj   r   r]   r   r   rk   re   rl   rm   r^   r   r   r   r   rh   r   rq   rr   r   r@   r@   rA   rE     s    &&

 "

zAttention2dDownsample.forward)Tr   r@   r@   r>   rA   r      s   
.	r   c                       s8   e Zd Zdddddejejf fdd	Zdd	 Z  ZS )

Downsampler+   r.   r   rM   Fc
           
         sf   t    t|}t|}t|}|	p,t }	t||||||	d| _|r\t||||d| _nd | _d S )N)r<   r2   r3   r=   )r}   r   rd   r~   )	r7   r8   r   rb   Identityr/   r9   r   r   )
r;   in_chsout_chsr<   r2   r3   rd   use_attnr~   r=   r>   r@   rA   r8     s*    
	
zDownsample.__init__c                 C   s&   |  |}| jd ur"| || S |S rB   )r9   r   )r;   rD   outr@   r@   rA   rE   >  s    

zDownsample.forward	rG   rH   rI   rb   r   BatchNorm2dr8   rE   rJ   r@   r@   r>   rA   r     s   %r   c                       s:   e Zd ZdZddejejddf fdd	Zdd Z  Z	S )	ConvMlpWithNormz`
    Implementation of MLP with 1*1 convolutions.
    Input: tensor with shape [B, C, H, W]
    N        Fc              	      s   t    |p|}|p|}t||dd||d| _|rNt||d|d||d| _n
t | _t|| _t	||d|d| _
t|| _d S )Nr   T)r6   r=   r~   r+   )r5   r6   r=   r~   )r=   )r7   r8   r   fc1midrb   r   Dropoutdrop1r/   fc2drop2)r;   in_featureshidden_featuresZout_featuresr~   r=   dropmid_convr>   r@   rA   r8   K  s     



zConvMlpWithNorm.__init__c                 C   s6   |  |}| |}| |}| |}| |}|S rB   )r   r   r   r   r   rC   r@   r@   rA   rE   e  s    




zConvMlpWithNorm.forward)
rG   rH   rI   __doc__rb   r   r   r8   rE   rJ   r@   r@   r>   rA   r   E  s   r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )LayerScale2dh㈵>Fc                    s*   t    || _t|t| | _d S rB   )r7   r8   inplacerb   ry   rs   Zonesgamma)r;   r}   Zinit_valuesr   r>   r@   rA   r8   o  s    
zLayerScale2d.__init__c                 C   s*   | j dddd}| jr"||S || S )Nr   r   )r   viewr   Zmul_)r;   rD   r   r@   r@   rA   rE   t  s    zLayerScale2d.forward)r   FrF   r@   r@   r>   rA   r   n  s   r   c                	       s<   e Zd Zdejejddddddf	 fdd	Zd	d
 Z  ZS )EfficientFormerV2Block      @r   r   rM   NTc                    s   t    |
rXt||||	d| _|d ur2t||nt | _|dkrLt|nt | _	nd | _d | _d | _	t
|t|| |||dd| _|d urt||nt | _|dkrt|nt | _d S )N)rd   r~   r2   r   T)r   r   r~   r=   r   r   )r7   r8   rK   token_mixerr   rb   r   ls1r   
drop_path1r   rf   mlpls2
drop_path2)r;   r}   	mlp_ratior~   r=   	proj_drop	drop_pathlayer_scale_init_valuerd   r2   r   r>   r@   rA   r8   z  sB    

	zEfficientFormerV2Block.__init__c                 C   sB   | j d ur$|| | |  | }|| | | | }|S rB   )r   r   r   r   r   r   rC   r@   r@   rA   rE     s    
zEfficientFormerV2Block.forwardr   r@   r@   r>   rA   r   y  s   *r   c                       s&   e Zd Zejejf fdd	Z  ZS )Stem4c              
      sP   t    d| _t||d dddd||d| _t|d |dddd||d| _d S )Nr(   r.   r+   r   T)r<   r2   r3   r6   r=   r~   )r7   r8   r2   r   Zconv1Zconv2)r;   r   r   r~   r=   r>   r@   rA   r8     s    
zStem4.__init__)rG   rH   rI   rb   r   r   r8   rJ   r@   r@   r>   rA   r     s   r   c                       sB   e Zd Zddddddddddejejf fd	d
	Zdd Z  ZS )EfficientFormerV2StagerM   TNFr   r   r   r   c                    s   t    d| _t||
}
t|}|rVt||||||d| _|}tdd |D }n||ksbJ t	 | _g }t
|D ]F}||	 d }t||||
| |o||k||| |||d
}||g7 }qxtj| | _d S )NF)r   rd   r=   r~   c                 S   s   g | ]}t |d  qS r   rO   rR   r@   r@   rA   rV     rW   z3EfficientFormerV2Stage.__init__.<locals>.<listcomp>r   )	rd   r2   r   r   r   r   r   r~   r=   )r7   r8   grad_checkpointingr   r   r   
downsampler`   rb   r   ranger   
Sequentialblocks)r;   r}   Zdim_outdepthrd   r   block_stridedownsample_use_attnblock_use_attnnum_vitr   r   r   r   r~   r=   r   Z	block_idxZ
remain_idxbr>   r@   rA   r8     sD    


zEfficientFormerV2Stage.__init__c                 C   s6   |  |}| jr(tj s(t| j|}n
| |}|S rB   )r   r   rs   r   is_scriptingr   r   rC   r@   r@   rA   rE     s
    

zEfficientFormerV2Stage.forwardr   r@   r@   r>   rA   r     s   9r   c                       s   e Zd Zd% fdd	Zdd Zejjdd Zejjd&ddZ	ejjd'ddZ
ejjdd Zd(ddZejjd)ddZdd Zd*ed d!d"Zd#d$ Z  ZS )+EfficientFormerV2r+   r   avgNr(   r1   r   gelu  r   r   Tc                    s  t    |dv sJ || _|| _g | _t|}tt||	d}t|
}
t	||d |
|d| _
|d }d t|}dd td|t||D }|pdd	t|d
   }t||}g }t|D ]}t fdd|D }t||| || ||| |dkrdnd |dk|dk||| ||| ||
|d}|| r@ d9  || }|  jt| d| dg7  _|| qtj| | _|d | _||d | _t|| _|dkrt|d |nt | _|| _ | j r|dkrt|d |nt | _!nd | _!| "| j# d| _$d S )N)r   r0   )Zepsr   )r~   r=   r(   c                 S   s   g | ]}|  qS r@   )tolist)rS   rD   r@   r@   rA   rV      rW   z.EfficientFormerV2.__init__.<locals>.<listcomp>)F)Tr   c                    s   g | ]}t |  qS r@   rO   )rS   srU   r@   rA   rV   %  rW   r.   r+   )r   rd   r   r   r   r   r   r   r   r   r   r~   r=   zstages.)Znum_chsZ	reductionmoduler   F)%r7   r8   num_classesglobal_poolZfeature_infor   r   r
   r	   r   stemlenrs   Zlinspacesumsplitr   r   r`   r   dictappendrb   r   stagesnum_featuresnormr   	head_dropLinearr   headdist	head_distapplyinit_weightsdistilled_training)r;   depthsZin_chansZimg_sizer   
embed_dimsZdownsamples
mlp_ratiosr=   Znorm_epsr~   r   Z	drop_rateZproj_drop_ratedrop_path_rater   r   ZdistillationZprev_dimZ
num_stagesZdprr   iZcurr_resolutionZstager>   rU   rA   r8     sf    
"
 
$&zEfficientFormerV2.__init__c                 C   s8   t |tjr4t|jdd |jd ur4tj|jd d S )N{Gz?)stdr   )
isinstancerb   r   r   Zweightr6   initZ	constant_)r;   mr@   r@   rA   r   M  s    
zEfficientFormerV2.init_weightsc                 C   s   dd |   D S )Nc                 S   s   h | ]\}}d |v r|qS )r{   r@   )rS   rk   _r@   r@   rA   	<setcomp>U  rW   z4EfficientFormerV2.no_weight_decay.<locals>.<setcomp>)Znamed_parametersr;   r@   r@   rA   no_weight_decayS  s    z!EfficientFormerV2.no_weight_decayFc                 C   s   t dddgd}|S )Nz^stem)z^stages\.(\d+)N)z^norm)i )r   r   )r   )r;   ZcoarseZmatcherr@   r@   rA   group_matcherW  s
    zEfficientFormerV2.group_matcherc                 C   s   | j D ]
}||_qd S rB   )r   r   )r;   enabler   r@   r@   rA   set_grad_checkpointing_  s    
z(EfficientFormerV2.set_grad_checkpointingc                 C   s   | j | jfS rB   r   r   r  r@   r@   rA   get_classifierd  s    z EfficientFormerV2.get_classifierc                 C   sX   || _ |d ur|| _|dkr*t| j|nt | _|dkrJt| j|nt | _d S )Nr   )r   r   rb   r   r   r   r   r   )r;   r   r   r@   r@   rA   reset_classifierh  s
     z"EfficientFormerV2.reset_classifierc                 C   s
   || _ d S rB   )r   )r;   r  r@   r@   rA   set_distilled_trainingo  s    z(EfficientFormerV2.set_distilled_trainingc                 C   s"   |  |}| |}| |}|S rB   )r   r   r   rC   r@   r@   rA   forward_featuress  s    


z"EfficientFormerV2.forward_features)
pre_logitsc                 C   sl   | j dkr|jdd}| |}|r(|S | || | }}| jr\| jr\tj	 s\||fS || d S d S )Nr   )r.   r+   r   r.   )
r   meanr   r   r   r   r   rs   r   r   )r;   rD   r  Zx_distr@   r@   rA   forward_heady  s    

zEfficientFormerV2.forward_headc                 C   s   |  |}| |}|S rB   )r  r  rC   r@   r@   rA   rE     s    

zEfficientFormerV2.forward)r+   r   r   NNr(   r1   r   r   r   r   r   r   r   r   T)F)T)N)T)F)rG   rH   rI   r8   r   rs   r   ignorer  r  r  r
  r  r  r  boolr  rE   rJ   r@   r@   r>   rA   r     s@                   O


r   r0   c                 K   s    | ddd dddt tddd|S )	Nr   )r+   r   r   Tgffffff?Zbicubicr	  zstem.conv1.conv)urlr   Z
input_sizeZ	pool_sizeZfixed_input_sizeZcrop_pctinterpolationr  r   
classifierZ
first_convr   )r  kwargsr@   r@   rA   _cfg  s    r  ztimm/)Z	hf_hub_id)z#efficientformerv2_s0.snap_dist_in1kz#efficientformerv2_s1.snap_dist_in1kz#efficientformerv2_s2.snap_dist_in1kz"efficientformerv2_l.snap_dist_in1kFc                 K   s0   | dd}tt| |fdtd|di|}|S )Nout_indices)r   r   r.   r+   Zfeature_cfgT)Zflatten_sequentialr  )popr   r   r   )variant
pretrainedr  r  modelr@   r@   rA   _create_efficientformerv2  s    
r  )r   c                 K   s<   t td td ddtd d}tdd| it |fi |S )Nr$   r.   r   r   r   r   r   r   efficientformerv2_s0r  )r  r   EfficientFormer_depthEfficientFormer_width EfficientFormer_expansion_ratiosr  r  r  Z
model_argsr@   r@   rA   r    s    r  c                 K   s<   t td td ddtd d}tdd| it |fi |S )Nr#   r.   r   r  efficientformerv2_s1r  )r%  r   r$  r@   r@   rA   r%    s    r%  c                 K   s<   t td td ddtd d}tdd| it |fi |S )Nr"   r(   r   r  efficientformerv2_s2r  )r&  r   r$  r@   r@   rA   r&    s    r&  c                 K   s<   t td td ddtd d}tdd| it |fi |S )Nr!   r-   g?r  efficientformerv2_lr  )r'  r   r$  r@   r@   rA   r'    s    r'  )r0   )F)F)F)F)F)4r   rP   	functoolsr   typingr   rs   Ztorch.nnrb   Z	timm.datar   r   Ztimm.layersr   r   r	   r
   r   r   r   r   r   Z_builderr   Z_manipulater   	_registryr   r   r"  r!  r#  Moduler/   rK   r   r   r   r   r   r   r   r   r   r   r  Zdefault_cfgsr  r  r%  r&  r'  r@   r@   r@   rA   <module>   s|   "]T-)2D 

	