a
    dI                 K   @   s  d Z ddlZddlmZ ddlmZmZmZ ddlm	Z	 ddl
mZmZmZmZmZ ddlZddlmZ ddlmZ dd	lmZmZ dd
lmZmZmZmZmZmZ ddlmZmZm Z m!Z!m"Z"m#Z# ddlm$Z$m%Z%m&Z&m'Z'm(Z( ddlm)Z)m*Z*m+Z+m,Z, ddl-m.Z. ddl/m0Z0 ddl1m2Z2m3Z3 ddl4m5Z5m6Z6 g dZ7eG dd dZ8eG dd dZ9eG dd dZ:G dd dej;Z<G dd dej;Z=G dd dej;Z>G d d! d!ej;Z?G d"d# d#ej;Z@dVd%d&ZAG d'd( d(ej;ZBdWd)d*ZCd+d, ZDG d-d. d.ej;ZEG d/d0 d0ej;ZFeeG d1d2d3ZHe0eeG eeG d4d5d6ZIeeG d7d8d9ZJe0eeG eeG d:d;d<ZKe8d=d>d?ZLG d@dA dAej;ZMG dBdC dCej;ZNeeG d1dDdEZOe0eeG eeG d4dFdGZPeeG d7dHdIZQe0eeG eeG d:dJdKZRG dLdM dMej;ZSG dNdO dOej;ZTG dPdQ dQej;ZUG dRdS dSej;ZVG dTdU dUej;ZWe8eeGeGf dVdWdXZXe:d=dYdZZYG d[d\ d\ej;ZZdXdfdgZ[dYdkdlZ\dZdodpZ]dqdr Z^e_e:f dsdtdudve\dadwdxe:f dsdydudve\d]dadwdze:f d{d|dudve[dad_d}e:f d{d~dudve[dhdad_de:f dd~ddve[dhdde:f dd~ddve[dhddmde:f d{d|dudve[dhdad_dde:f dsdydudve\dadwdndde:f d{d|dudve[dhdnde:f d{d~dudve[ddad_dndde:f d{d~dudve[dhdndede:f dd~ddve[dhddmdnde:f dd~ddve[dhddmdnde:f dsdydudde[ e:f dsdydudde]dddde:d{dtddde:d{d~ddde:dd~ddde:dd~ddde:ddddde:ddddde:f ddddde\ e:f dsdddude\ e:f dsdddude\ e:f dsdddude\ e:f ddddde\dnde:f dsdddude\dnde:f dsdddude\dnde:f d{dddude\dndmde:f d{d~ddudde\dnde:f dsdddudde] e:f dsdddude] e:f d{dddde] e:f d{ddddde]daddde:f ddddde]dade:f dddddde]dade:f dsddddadede^ e:f d{ddddadde^ e:f d{d~dddadde^ e:f dd~dddadde^ e:f dd~dddadde^ d)Z`ej;dddZad[ddZbd\ddZce5ecd$decddddecdddecdddecddÍecddÍecddÍecddddddȍecddeeddˍecddddecd$decdddecdddecd$decd$decddddecdddэecdddэecdddэecdddэecd$decd$decd$decd$decd$decd$decd$dddԍecddddd֍ecdddecd$dddԍecd$dddԍecddddd֍ecddddd֍ecddddd֍ecddddecd$dddԍecddÍecddddddȍecdddэecddddd֍ecd$dddԍecddddd֍ecddddލecddÍecddddddȍecd$decdddэecdeedߍecddddddȍecddddddȍecdeedߍecddddddȍecddddddȍecdeedߍecddddddȍecddddddȍecdeedߍecddddddȍecddddddȍecdddэecddddddȍecddddddȍecdddэecddddddȍecdddddecdddэecddddddȍecddddddȍdDZde6d]eZdddZee6d^eZdddZfe6d_eZdddZge6d`eZdddZhe6daeZdddZie6dbeZdddZje6dceZdddZke6ddeZdddZle6deeZdddZme6dfeZdddZne6dgeZdddZoe6dheZdddZpe6dieZdddZqe6djeZdd dZre6dkeZdddZse6dleZdddZte6dmeZdddZue6dneZddd	Zve6doeZdd
dZwe6dpeZdddZxe6dqeZdddZye6dreZdddZze6dseZdddZ{e6dteZdddZ|e6dueZdddZ}e6dveZdddZ~e6dweZdddZe6dxeZdddZe6dyeZdddZe6dzeZdd d!Ze6d{eZdd"d#Ze6d|eZdd$d%Ze6d}eZdd&d'Ze6d~eZdd(d)Ze6deZdd*d+Ze6deZdd,d-Ze6deZdd.d/Ze6deZdd0d1Ze6deZdd2d3Ze6deZdd4d5Ze6deZdd6d7Ze6deZdd8d9Ze6deZdd:d;Ze6deZdd<d=Ze6deZdd>d?Ze6deZdd@dAZe6deZddBdCZe6deZddDdEZe6deZddFdGZe6deZddHdIZe6deZddJdKZe6deZddLdMZe6deZddNdOZe6deZddPdQZe6deZddRdSZe6deZddTdUZdS (  a   MaxVit and CoAtNet Vision Transformer - CNN Hybrids in PyTorch

This is a from-scratch implementation of both CoAtNet and MaxVit in PyTorch.

99% of the implementation was done from papers, however last minute some adjustments were made
based on the (as yet unfinished?) public code release https://github.com/google-research/maxvit

There are multiple sets of models defined for both architectures. Typically, names with a
 `_rw` suffix are my own original configs prior to referencing https://github.com/google-research/maxvit.
These configs work well and appear to be a bit faster / lower resource than the paper.

The models without extra prefix / suffix' (coatnet_0_224, maxvit_tiny_224, etc), are intended to
match paper, BUT, without any official pretrained weights it's difficult to confirm a 100% match.

Papers:

MaxViT: Multi-Axis Vision Transformer - https://arxiv.org/abs/2204.01697
@article{tu2022maxvit,
  title={MaxViT: Multi-Axis Vision Transformer},
  author={Tu, Zhengzhong and Talebi, Hossein and Zhang, Han and Yang, Feng and Milanfar, Peyman and Bovik, Alan and Li, Yinxiao},
  journal={ECCV},
  year={2022},
}

CoAtNet: Marrying Convolution and Attention for All Data Sizes - https://arxiv.org/abs/2106.04803
@article{DBLP:journals/corr/abs-2106-04803,
  author    = {Zihang Dai and Hanxiao Liu and Quoc V. Le and Mingxing Tan},
  title     = {CoAtNet: Marrying Convolution and Attention for All Data Sizes},
  journal   = {CoRR},
  volume    = {abs/2106.04803},
  year      = {2021}
}

Hacked together by / Copyright 2022, Ross Wightman
    N)OrderedDict)	dataclassreplacefield)partial)CallableOptionalUnionTupleList)nn)Final)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)MlpConvMlpDropPath	LayerNormClassifierHeadNormMlpClassifierHead)create_attnget_act_layerget_norm_layerget_norm_act_layercreate_conv2dcreate_pool2d)trunc_normal_tf_	to_2tupleextend_tuplemake_divisible_assert)	RelPosMlp
RelPosBiasRelPosBiasTfuse_fused_attn   )build_model_with_cfg)register_notrace_function)named_applycheckpoint_seq)generate_default_cfgsregister_model)
MaxxVitCfgMaxxVitConvCfgMaxxVitTransformerCfgMaxxVitc                   @   s.  e Zd ZU dZeed< dZeed< dZe	ed< dZ
eed< dZeed< dZeed	< d
Ze	ed< d
Ze	ed< dZeed< dZeed< dZeed< dZeed< dZeeeef  ed< dZeeeef  ed< dZeed< dZeed< dZee	 ed< dZeed< dZeed< dZeed < d!Ze	ed"< d#d$ ZdS )%r.       dim_headT
head_first      @expand_ratioexpand_firstshortcut_bias	attn_bias        	attn_drop	proj_dropavg2	pool_typebiasrel_pos_type   rel_pos_dimpartition_ratioNwindow_size	grid_sizeFno_block_attnuse_nchw_attninit_valuesgelu	act_layerlayernorm2d
norm_layer	layernormnorm_layer_clư>norm_epsc                 C   sB   | j d urt| j | _ | jd ur>t| j| _| j d u r>| j| _ d S N)rC   r   rB   self rR   \/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/maxxvit.py__post_init__T   s    


z#MaxxVitTransformerCfg.__post_init__) __name__
__module____qualname__r1   int__annotations__r2   boolr4   floatr5   r6   r7   r9   r:   r<   strr>   r@   rA   rB   r   r
   rC   rD   rE   rF   rH   rJ   rL   rN   rT   rR   rR   rR   rS   r.   <   s,   
r.   c                   @   s  e Zd ZU dZeed< dZeed< dZe	ed< dZ
eed< d	Zeed
< dZe	ed< dZe	ed< dZeed< dZeed< dZeed< dZeed< dZe	ed< dZeed< dZeed< dZeed< dZee ed< dZeed< dZeed < dZeed!< d"Zee ed#< d$d% Zd"S )&r-   mbconv
block_typer3   r4   Texpand_output   kernel_sizer%   
group_sizeFpre_norm_actoutput_biasdwstride_moder;   r<   downsample_pool_type padding
attn_earlyse
attn_layersiluattn_act_layer      ?
attn_ratiorM   rF   rG   rH   rJ   rL   NrN   c                 C   sf   | j dv sJ | j dk}| js,|r&dnd| _| js<|s<d| _| jd u rT|rNdnd| _| jp^| j| _d S )N)r]   convnextr]   batchnorm2drI   rK   h㈵>rM   )r^   rJ   rL   rN   rg   r<   )rQ   Z
use_mbconvrR   rR   rS   rT   t   s    


zMaxxVitConvCfg.__post_init__)rU   rV   rW   r^   r\   rY   r4   r[   r_   rZ   ra   rX   rb   rc   rd   rf   r<   rg   ri   rj   rl   rn   rp   rF   r   rH   rJ   rL   rN   rT   rR   rR   rR   rS   r-   ]   s*   
r-   c                   @   s   e Zd ZU dZeedf ed< dZeedf ed< dZee	e
ee
df f df ed< dZe	eeeef f ed	< d
Zeed< eedZeed< eedZeed< dZeed< dZe
ed< dS )r,   `           .	embed_dim   r`      r{   depths)Cr~   Tr   r^   @   
stem_widthF	stem_bias)default_factoryconv_cfgtransformer_cfgNhead_hidden_sizevit_effweight_init)rU   rV   rW   ry   r
   rX   rY   r}   r^   r	   r\   r   r   rZ   r   r-   r   r.   r   r   r   rR   rR   rR   rS   r,      s   
$r,   c                       s\   e Zd ZU ee ed< deee eeeeee	e	d	 fddZ
deej d	d
dZ  ZS )Attention2d
fused_attnNr0   Tr8   	dimdim_outr1   r=   r5   r2   rel_pos_clsr9   r:   c
                    s   t    |p|}|r|n|}
|
| | _|| _|| _|d | _t | _tj	||
d d|d| _
|rn|| jdnd | _t|| _tj	|
|d|d| _t|	| _d S )N      r`   r%   r=   	num_heads)super__init__r   r1   r2   scaler$   r   r   Conv2dqkvrel_posDropoutr9   projr:   rQ   r   r   r1   r=   r5   r2   r   r9   r:   Zdim_attn	__class__rR   rS   r      s    


zAttention2d.__init__shared_rel_posc                 C   sp  |j \}}}}| jrD| ||| j| jd djddd\}}}	n(| ||d| j| jdd\}}}	| j	rd }
| j
d ur| j
 }
n|d ur|}
tjjj|dd|dd|	dd|
| jjddd|d||}nv|| j }|dd| }| j
d ur| 
|}n|d ur&|| }|jdd}| |}|	|dd |d||}| |}| |}|S )Nr`   r{   r   r%   Z	attn_maskZ	dropout_p)shaper2   r   viewr   r1   chunkreshapeunbindr   r   get_biastorchr   
functionalscaled_dot_product_attention	transposer9   pr   softmaxr   r:   )rQ   xr   Br~   HWqkvr7   attnrR   rR   rS   forward   sB    0(








zAttention2d.forward)Nr0   TTTNr8   r8   )N)rU   rV   rW   r   rZ   rY   rX   r   r   r[   r   r   Tensorr   __classcell__rR   rR   r   rS   r      s*   
        r   c                       s`   e Zd ZU dZee ed< deee eeeee	e
e
d	 fdd	Zdeej d
ddZ  ZS )AttentionClz0 Channels-last multi-head attention (B, ..., C) r   Nr0   Tr8   r   c
                    s   t    |p|}|r"||kr"|n|}
|
| dks:J d|
| | _|| _|| _|d | _t | _tj	||
d |d| _
|r|| jdnd | _t|| _tj	|
||d| _t|	| _d S )Nr   z(attn dim should be divisible by head_dimr   r`   r   r   )r   r   r   r1   r2   r   r$   r   r   Linearr   r   r   r9   r   r:   r   r   rR   rS   r      s    


zAttentionCl.__init__r   c           
      C   sd  |j d }|j d d }| jrV| ||d| j| jd ddjddd\}}}n0| ||dd| j| jdd	d\}}}| j
rd }| jd ur| j }n|d ur|}tjjj||||| jjd}nf|| j }||dd }	| jd ur| j|	|d	}	n|d ur|	| }	|	jdd}	| |	}	|	| }|dd|d
 }| |}| |}|S )Nr   r   r`   r%   r{   r   r   r   r   )r   )r   r2   r   r   r   r1   r   r   r   r   r   r   r   r   r   r   r   r9   r   r   r   r   r:   )
rQ   r   r   r   Zrestore_shaper   r   r   r7   r   rR   rR   rS   r      s:    
80





zAttentionCl.forward)Nr0   TTTNr8   r8   )N)rU   rV   rW   __doc__r   rZ   rY   rX   r   r   r[   r   r   r   r   r   rR   rR   r   rS   r      s,   
        r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )
LayerScalers   Fc                    s*   t    || _t|t| | _d S rO   r   r   inplacer   	Parameterr   ZonesgammarQ   r   rF   r   r   rR   rS   r     s    
zLayerScale.__init__c                 C   s   | j }| jr||S || S rO   )r   r   mul_rQ   r   r   rR   rR   rS   r     s    zLayerScale.forward)rs   FrU   rV   rW   r   r   r   rR   rR   r   rS   r     s   r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )LayerScale2drs   Fc                    s*   t    || _t|t| | _d S rO   r   r   r   rR   rS   r   %  s    
zLayerScale2d.__init__c                 C   s*   | j dddd}| jr"||S || S )Nr%   r   )r   r   r   r   r   rR   rR   rS   r   *  s    zLayerScale2d.forward)rs   Fr   rR   rR   r   rS   r   $  s   r   c                       s8   e Zd ZdZd
eeeeed fddZdd	 Z  Z	S )Downsample2da4   A downsample pooling module supporting several maxpool and avgpool modes
    * 'max' - MaxPool2d w/ kernel_size 3, stride 2, padding 1
    * 'max2' - MaxPool2d w/ kernel_size = stride = 2
    * 'avg' - AvgPool2d w/ kernel_size 3, stride 2, padding 1
    * 'avg2' - AvgPool2d w/ kernel_size = stride = 2
    r;   rh   T)r   r   r<   ri   r=   c                    s   t    |dv sJ |dkr6tddd|p,dd| _nT|dkrTtdd|pJdd	| _n6|d
krvtd
ddd|pldd| _ntd
d|pdd	| _||krtj||d|d| _n
t | _d S )N)maxmax2avgr;   r   r`   r{   r%   )ra   strideri   r   r   )ri   r   F)ra   r   Zcount_include_padri   r   )r   r   r   poolr   r   expandIdentity)rQ   r   r   r<   ri   r=   r   rR   rS   r   7  s    

zDownsample2d.__init__c                 C   s   |  |}| |}|S rO   )r   r   rQ   r   rR   rR   rS   r   P  s    

zDownsample2d.forward)r;   rh   T)
rU   rV   rW   r   rX   r\   rZ   r   r   r   rR   rR   r   rS   r   /  s      r   rh   c                 C   s   t | tjtjfr|dkrFtjj| jdd | jd urtj| j n|dkrvt	| jdd | jd urtj| j nr|dkrtj
| j | jd urtj| j nBtj| j | jd urd|v rtjj| jdd ntj| j d S )Nnormal{Gz?stdtrunc_normalxavier_normalmlprM   )
isinstancer   r   r   initnormal_weightr=   zeros_r   xavier_normal_Zxavier_uniform_)modulenameschemerR   rR   rS   _init_transformerV  s$    



r   c                       s\   e Zd ZdZdde dfeeeeeed fddZdd	d
Z	de
ej dddZ  ZS )TransformerBlock2daX   Transformer block with 2D downsampling
    '2D' NCHW tensor layout

    Some gains can be seen on GPU using a 1D / CL block, BUT w/ the need to switch back/forth to NCHW
    for spatial pooling, the benefit is minimal so ended up using just this variant for CoAt configs.

    This impl was faster on TPU w/ PT XLA than the 1D experiment.
    r%   Nr8   )r   r   r   r   cfg	drop_pathc           	   
      sX  t    tt|j|jd}t|j}|dkrtt|||j	|j
d| _ttd||fdt|||j	dfg| _n ||ksJ t | _||| _t|||j|j|j||j|jd| _|jrt||jdnt | _|d	krt|nt | _||| _t|t||j  ||jd
| _!|jr.t||jdnt | _"|d	krJt|nt | _#d S )NZepsr{   )r<   r=   normdownr<   )r1   r5   r=   r   r9   r:   rF   r8   in_featureshidden_featuresrH   drop)$r   r   r   r   rJ   rN   r   rH   r   r<   r6   shortcutr   
Sequentialr   norm1r   r   r1   r5   r7   r9   r:   r   rF   r   ls1r   
drop_path1norm2r   rX   r4   r   ls2
drop_path2)	rQ   r   r   r   r   r   r   rJ   rH   r   rR   rS   r   x  sB    	






 zTransformerBlock2d.__init__rh   c                 C   s   t tt|d|  d S Nr   )r(   r   r   rQ   r   rR   rR   rS   init_weights  s    zTransformerBlock2d.init_weightsr   c              
   C   sN   |  || | | j| ||d }|| | | | | }|S )Nr   )	r   r   r   r   r   r   r   r   r   )rQ   r   r   rR   rR   rS   r     s    * zTransformerBlock2d.forward)rh   )N)rU   rV   rW   r   r.   rX   r   r[   r   r   r   r   r   r   r   rR   rR   r   rS   r   n  s   .
r   c                 C   s   t | tjr|dkr@tjj| jdd | jd urtj| j n|dkrpt| jdd | jd urtj| j n|dkrtj	| j | jd urtj| j nX| j
d | j
d  | j }|| j }tj| jdtd|  | jd urtj| j d S )	Nr   r   r   r   r   r   r%   g       @)r   r   r   r   r   r   r=   r   r   r   ra   Zout_channelsgroupsmathsqrt)r   r   r   Zfan_outrR   rR   rS   
_init_conv  s$    




r   c                 C   s$   | sdS ||  dksJ ||  S d S )Nr%   r   rR   )rb   ZchannelsrR   rR   rS   
num_groups  s    r   c                       sV   e Zd ZdZdde dfeeeeeef eed fddZdd	d
Z	dd Z
  ZS )MbConvBlockzL Pre-Norm Conv Block - 1x1 - kxk - 1x1, w/ inverted bottleneck (expand)
    r%   r%   r%   r8   )in_chsout_chsr   dilationr   r   c              	      s"  t t|   tt|j|j|jd}t|j	r2|n||j
 }t|j|}	|dkrnt|||j|j|jd| _n
t | _|jdv sJ d\}
}}|jdkr||d  }
}n(|jdkr||d  }}n||d	  }}|||jd
| _|
dkrt|||j|jd| _n
t | _t||d|d| _||| _t|||j|||	|jd| _i }t|jt r|jdksp|jdkr|j!|d< t"|j#|j	r|n| |d< |j$rt%|j|fi || _&||| _'d | _(n&d | _&||| _'t%|j|fi || _(t||d|jd| _)|dkrt*|nt | _+d S )Nr   r{   )r<   r=   ri   )r   1x1re   )r%   r%   r%   r   r%   r  r   )Z	apply_act)r<   ri   )r   )r   r  r   ri   rk   ZecarH   Zrd_channelsr   r8   ),r   r   r   r   r   rJ   rH   rN   r   r_   r4   r   rb   r   r<   rd   ri   r   r   r   rf   rc   pre_normrg   r   r   	conv1_1x1r   ra   	conv2_kxkr   rl   r\   rn   rX   rp   rj   r   se_earlyr   rk   	conv3_1x1r   r   )rQ   r  r  r   r  r   r   norm_act_layerZmid_chsr   stride_poolZstride_1Zstride_2Z
dilation_2attn_kwargsr   rR   rS   r     sP    	











zMbConvBlock.__init__rh   c                 C   s   t tt|d|  d S r   r(   r   r   r   rR   rR   rS   r     s    zMbConvBlock.init_weightsc                 C   s   |  |}| |}| |}| |}| |}| |}| jd urP| |}| |}| jd urn| |}| 	|}| 
|| }|S rO   )r   r  r   r  r   r  r  r   rk   r	  r   rQ   r   r   rR   rR   rS   r     s    











zMbConvBlock.forward)rh   )rU   rV   rW   r   r-   rX   r
   r[   r   r   r   r   rR   rR   r   rS   r     s   
?
r   c                
       sZ   e Zd ZdZdddde ddfeee eeeeef eee	d fd	d
Z
dd Z  ZS )ConvNeXtBlockz ConvNeXt Block
    N   r%   r   Tr8   )r  r  ra   r   r  r   conv_mlpr   c	              	      s  t    |p|}t|j}	|r:tt|j|jd}
t}nd|jv sHJ t	}
t
}|| _|dkrlt||| _n*||krtj||d|jd| _n
t | _|jdv sJ d\}}|jdkr|}n|}|dkrt|||jd	| _n
t | _t|||||d d
|jd| _|
|| _||t|j| |j|	d| _|rR|jrFt||jnt | _n|jrft||jnt | _|dkrt|nt | _ d S )Nr   rK   r{   r%   )ra   r=   )r   re   r   r   r   T)ra   r   r  Z	depthwiser=   )r=   rH   r8   )!r   r   r   rH   r   r   rJ   rN   r   r   r   use_conv_mlpr   r   r   r   rd   r   rf   rg   r   r   conv_dwr   rX   r4   r   rF   r   lsr   r   r   )rQ   r  r  ra   r   r  r   r  r   rH   rJ   Z	mlp_layerr  Z	stride_dwr   rR   rS   r   /  sB    





 zConvNeXtBlock.__init__c                 C   s   |  |}| |}| |}| jrD| |}| |}| |}n>|dddd}| |}| |}| |}|dddd}| || }|S Nr   r{   r`   r%   )	r   r   r  r  r   r   r  permuter   r  rR   rR   rS   r   e  s    







zConvNeXtBlock.forward)rU   rV   rW   r   r-   rX   r   r
   rZ   r[   r   r   r   rR   rR   r   rS   r  +  s&   
6r  rB   c                 C   s   | j \}}}}t||d  dkd| d|d  d t||d  dkd | |||d  |d ||d  |d |} | ddddd	d
 d|d |d |}|S )Nr   height () must be divisible by window ()r%   rh   r`   r{      r|   r   r   r    r   r  
contiguous)r   rB   r   r   r   r~   windowsrR   rR   rS   window_partitionx  s    (,,r  )rB   img_sizec                 C   sf   |\}}| j d }| d||d  ||d  |d |d |}|dddddd d|||}|S )Nr   r   r%   r`   r{   r  r|   r   r   r  r  r  rB   r   r   r   r~   r   rR   rR   rS   window_reverse  s
    
,$r#  )rC   c              	   C   s   | j \}}}}t||d  dkd| d|d   t||d  dkd | ||d ||d  |d ||d  |} | dddddd	 d
|d |d |}|S )Nr   height  must be divisible by grid r%   rh   r{   r  r`   r|   r   r  )r   rC   r   r   r   r~   r  rR   rR   rS   grid_partition  s    &,,r&  )rC   r   c                 C   sf   |\}}| j d }| d||d  ||d  |d |d |}|dddddd d|||}|S )Nr   r   r%   r`   r  r{   r|   r!  r  rC   r   r   r   r~   r   rR   rR   rS   grid_reverse  s
    
,$r(  )r   c                 C   sR   d }| j dkr tt|| jd}n.| j dkr8tt|d}n| j dkrNtt|d}|S )Nr   )rB   Z
hidden_dimr=   r  bias_tf)r>   r   r!   r@   r"   r#   )r   rB   r   rR   rR   rS   get_rel_pos_cls  s    


r*  c                       sF   e Zd ZdZde dfeeeed fddZdd Z	d	d
 Z
  ZS )PartitionAttentionClzR Grid or Block partition + Attn + FFN.
    NxC 'channels last' tensor layout.
    blockr8   r   partition_typer   r   c              
      s"  t    tt|j|jd}t|j}|dk| _t	| jr@|j
n|j| _t|| j}||| _t|||j|j|j||j|jd| _|jrt||jdnt | _|dkrt|nt | _||| _t|t||j  ||jd| _!|jrt||jdnt | _"|dkrt|nt | _#d S Nr   r,  r1   r=   r2   r   r9   r:   r   r8   r   )$r   r   r   r   rL   rN   r   rH   partition_blockr   rB   rC   partition_sizer*  r   r   r1   r7   r2   r9   r:   r   rF   r   r   r   r   r   r   r   r   rX   r4   r   r   r   rQ   r   r.  r   r   rJ   rH   r   r   rR   rS   r     s8    





zPartitionAttentionCl.__init__c                 C   s`   |j dd }| jr"t|| j}nt|| j}| |}| jrNt|| j|}nt|| j|}|S )Nr%   r`   )r   r1  r  r2  r&  r   r#  r(  rQ   r   r   ZpartitionedrR   rR   rS   _partition_attn  s    
z$PartitionAttentionCl._partition_attnc              
   C   sD   ||  | | | | }|| | | | | }|S rO   r   r   r5  r   r   r   r   r   r   rR   rR   rS   r     s      zPartitionAttentionCl.forwardrU   rV   rW   r   r.   rX   r\   r[   r   r5  r   r   rR   rR   r   rS   r+    s   &r+  c                       sB   e Zd ZdZe dfeeed fddZdd Zdd	 Z	  Z
S )
ParallelPartitionAttentionzP Experimental. Grid and Block partition + single FFN
    NxC tensor layout.
    r8   r   r   r   c              
      s\  t    |d dksJ tt|j|jd}t|j}|j|j	ksHJ t
|j| _t|| j}||| _t||d |j|j|j||j|jd| _t||d |j|j|j||j|jd| _|jrt||jdnt | _|dkrt|nt | _||| _t|t||j  |||jd| _!|jr2t||jdnt | _"|dkrNt|nt | _#d S )Nr{   r   r   r0  r   r8   )r   r   Zout_featuresrH   r   )$r   r   r   r   rL   rN   r   rH   rB   rC   r   r2  r*  r   r   r1   r7   r2   r9   r:   
attn_block	attn_gridrF   r   r   r   r   r   r   r   r   rX   r4   r   r   r   )rQ   r   r   r   rJ   rH   r   r   rR   rS   r     sP    





 z#ParallelPartitionAttention.__init__c                 C   sh   |j dd }t|| j}| |}t|| j|}t|| j}| |}t|| j|}tj	||gddS )Nr%   r`   r   r   )
r   r  r2  r:  r#  r&  r;  r(  r   cat)rQ   r   r   Zpartitioned_blockZx_windowZpartitioned_gridZx_gridrR   rR   rS   r5    s    

z*ParallelPartitionAttention._partition_attnc              
   C   sD   ||  | | | | }|| | | | | }|S rO   r6  r   rR   rR   rS   r   *  s      z"ParallelPartitionAttention.forward)rU   rV   rW   r   r.   rX   r[   r   r5  r   r   rR   rR   r   rS   r8    s   1r8  c              	   C   s   | j \}}}}t||d  dkd| d|d  d t||d  dkd | ||||d  |d ||d  |d } | ddddd	d
 d||d |d }|S )Nr   r  r  r  r%   rh   r{   r  r`   r|   r   r  )r   rB   r   r~   r   r   r  rR   rR   rS   window_partition_nchw0  s    (,,r=  c              	   C   sf   |\}}| j d }| d||d  ||d  ||d |d }|dddddd d|||}|S )Nr%   r   r   r`   r  r{   r|   r!  r"  rR   rR   rS   window_reverse_nchw9  s
    
,$r>  c              
   C   s   | j \}}}}t||d  dkd| d|d   t||d  dkd | |||d ||d  |d ||d  } | dddddd	 d
||d |d }|S )Nr   r$  r%  r%   rh   r`   r|   r{   r  r   r  )r   rC   r   r~   r   r   r  rR   rR   rS   grid_partition_nchwB  s    &,,r?  c              	   C   sf   |\}}| j d }| d||d  ||d  ||d |d }|dddddd d|||}|S )Nr%   r   r   r`   r  r|   r{   r!  r'  rR   rR   rS   grid_reverse_nchwK  s
    
,$r@  c                       sF   e Zd ZdZde dfeeeed fddZdd Z	d	d
 Z
  ZS )PartitionAttention2dzH Grid or Block partition + Attn + FFN

    '2D' NCHW tensor layout.
    r,  r8   r-  c              
      s"  t    tt|j|jd}t|j}|dk| _t	| jr@|j
n|j| _t|| j}||| _t|||j|j|j||j|jd| _|jrt||jdnt | _|dkrt|nt | _||| _t|t||j  ||jd| _!|jrt||jdnt | _"|dkrt|nt | _#d S r/  )$r   r   r   r   rJ   rN   r   rH   r1  r   rB   rC   r2  r*  r   r   r1   r7   r2   r9   r:   r   rF   r   r   r   r   r   r   r   r   rX   r4   r   r   r   r3  r   rR   rS   r   Z  s8    





zPartitionAttention2d.__init__c                 C   s`   |j dd  }| jr"t|| j}nt|| j}| |}| jrNt|| j|}nt|| j|}|S )Nr   )r   r1  r=  r2  r?  r   r>  r@  r4  rR   rR   rS   r5    s    
z$PartitionAttention2d._partition_attnc              
   C   sD   ||  | | | | }|| | | | | }|S rO   r6  r   rR   rR   rS   r     s      zPartitionAttention2d.forwardr7  rR   rR   r   rS   rA  T  s   &rA  c                       sP   e Zd ZdZde e dfeeeeeed fddZddd	Z	d
d Z
  ZS )MaxxVitBlockz@ MaxVit conv, window partition + FFN , grid partition + FFN
    r%   r8   )r   r   r   r   r   r   c           
         s   t    |j| _|jdkr tnt}||||||d| _t|||d}| jrPt	nt
}	|jr^d n|	f i || _|	f ddi|| _d S )Nrq   r   r   r   r9  r.  Zgrid)r   r   rE   	nchw_attnr^   r  r   convdictrA  r+  rD   r:  r;  )
rQ   r   r   r   r   r   r   conv_clsr  Zpartition_layerr   rR   rS   r     s    	
zMaxxVitBlock.__init__rh   c                 C   sJ   | j d urttt|d| j  ttt|d| j ttt|d| j d S r   )r:  r(   r   r   r;  r   rE  r   rR   rR   rS   r     s    
zMaxxVitBlock.init_weightsc                 C   sX   |  |}| js |dddd}| jd ur4| |}| |}| jsT|dddd}|S r  )rE  rD  r  r:  r;  r   rR   rR   rS   r     s    



zMaxxVitBlock.forward)rh   )rU   rV   rW   r   r-   r.   rX   r[   r   r   r   r   rR   rR   r   rS   rB    s   
rB  c                       sJ   e Zd ZdZdde e dfeed fddZdd	d
Zdd Z  Z	S )ParallelMaxxVitBlockzX MaxVit block with parallel cat(window + grid), one FF
    Experimental timm block.
    r%   r{   r8   r   r   c           
         s   t    |jdkrtnt}|dkrd||||||dg}	|	|||||dg|d  7 }	tj|	 | _n||||||d| _t|||d| _	d S )Nrq   r%   rC  )r   r   r9  )
r   r   r^   r  r   r   r   rE  r8  r   )
rQ   r   r   r   Znum_convr   r   r   rG  Zconvsr   rR   rS   r     s    

zParallelMaxxVitBlock.__init__rh   c                 C   s,   t tt|d| j t tt|d| j d S r   )r(   r   r   r   r   rE  r   rR   rR   rS   r     s    z!ParallelMaxxVitBlock.init_weightsc                 C   s8   |  |}|dddd}| |}|dddd}|S r  )rE  r  r   r   rR   rR   rS   r     s
    

zParallelMaxxVitBlock.forward)rh   )
rU   rV   rW   r   r-   r.   r   r   r   r   rR   rR   r   rS   rH    s   
rH  c                       sn   e Zd Zdddde e dfeeeeeeef eeee f eeee	e
e	 f d	 fddZd	d
 Z  ZS )MaxxVitStager{   r  )   rK  r~   r8   )	r  r  r   depth	feat_sizeblock_typesr   r   r   c
              
      s  t    d| _t||}g }
t|D ]\}}|dkr:|nd}|dv sJJ |dkr|jdkr`tnt}|
||||||	| dg7 }
n|dkrt||}|
t	||||||	| d	g7 }
nP|d
kr|
t
||||||	| dg7 }
n(|dkr|
t||||||	| dg7 }
|}q&tj|
 | _d S )NFr   r%   )r~   r   MPMr~   rq   rC  r   )r   r   r   r   rO  )r   r   r   r   rP  )r   r   grad_checkpointingr   	enumerater^   r  r   r*  r   rB  rH  r   r   blocks)rQ   r  r  r   rL  rM  rN  r   r   r   rS  itZblock_striderG  r   r   rR   rS   r     s\    




zMaxxVitStage.__init__c                 C   s,   | j rtj st| j|}n
| |}|S rO   )rQ  r   jitZis_scriptingr)   rS  r   rR   rR   rS   r   $  s    
zMaxxVitStage.forward)rU   rV   rW   r.   r-   rX   r
   r	   r\   r[   r   r   r   r   rR   rR   r   rS   rJ    s&   
<rJ  c                
       sD   e Zd Zdeeeeeeeed fdd	Zdd
dZdd Z	  Z
S )Stemr`   rh   FrG   rr   rs   )r  r  ra   ri   r=   rH   rJ   rN   c	           
         s   t    t|ttfs t|}tt|||d}	|d | _d| _	t
||d |d||d| _|	|d | _t
|d |d |d||d| _d S )Nr   r   r{   r   )r   ri   r=   r%   )r   r   r   listtupler   r   r   r  r   r   conv1r   conv2)
rQ   r  r  ra   ri   r=   rH   rJ   rN   r
  r   rR   rS   r   .  s    

zStem.__init__c                 C   s   t tt|d|  d S r   r  r   rR   rR   rS   r   E  s    zStem.init_weightsc                 C   s"   |  |}| |}| |}|S rO   )rZ  r   r[  r   rR   rR   rS   r   H  s    


zStem.forward)r`   rh   FrG   rr   rs   )rh   )rU   rV   rW   rX   r\   rZ   r[   r   r   r   r   rR   rR   r   rS   rW  ,  s$         
rW  )r   r   c                 C   sF   | j d ur| jsJ | S |d | j |d | j f}t| ||d} | S )Nr   r%   )rB   rC   )rB   rC   rA   r   )r   r   r2  rR   rR   rS   cfg_window_sizeO  s    

r\  c                 K   s   i }i }i }|  D ]H\}}|dr8|||dd< q|drT|||dd< q|||< qt| ft| jfi |t| jfi |d|} | S )NZtransformer_rh   Zconv_)r   r   )items
startswithr   r   r   )r   kwargsZtransformer_kwargsZconv_kwargsZbase_kwargsr   r   rR   rR   rS   _overlay_kwargsX  s$    


r`  c                	       s   e Zd ZdZd!eeeeeef f eeee	e	d fdd	Z
d"ddZejjdd Zejjd#ddZejjd$ddZejjdd Zd%ddZdd Zd&edddZdd  Z  ZS )'r/   z| CoaTNet + MaxVit base model.

    Highly configurable for different block compositions, tensor layouts, pooling types.
       r`     r   r8   )r   r   in_chansnum_classesglobal_pool	drop_ratedrop_path_ratec                    sZ  t    t|}|r&t|fi |}t|j|}	|| _|| _|jd  | _	| _|| _
d| _g | _t||j|jj|j|jj|jj|jjd| _| jj}
|  jt| jjdddg7  _tdd t|t|
D }t|j}t|j|ksJ d	d td
|t|j |jD }| jj}g }t!|D ]}d |j| }t fdd|D }|t"|||j| |j#| |j|	||| dg7 }|
 9 }
|}|  jt||
d| dg7  _q&t$j%| | _&t't(|jj|jjd}|j)| _)| j)rt$* | _+t,| j	|| j)|||d| _-n || j	| _+t.| j	|||d| _-|j/dv s8J |j/rVt0t'| j1|j/d|  d S )Nr   F)r  r  ri   r=   rH   rJ   rN   r{   stem)Znum_chsZ	reductionr   c                 S   s   g | ]\}}|| qS rR   rR   ).0rT  srR   rR   rS   
<listcomp>      z$MaxxVit.__init__.<locals>.<listcomp>c                 S   s   g | ]}|  qS rR   )tolist)ri  r   rR   rR   rS   rk    rl  r   c                    s   g | ]}|d    d  qS )r%   rR   )ri  rZstage_striderR   rS   rk    rl  )rL  rN  r   r   rM  r   zstages.r   )Zhidden_sizer<   rf  rJ   )r<   rf  )rh   r   r   r   r   r   )2r   r   r   r`  r\  r   rd  re  ry   Znum_featuresrf  rQ  Zfeature_inforW  r   r   ri   r   rH   rJ   rN   rh  r   rF  r  rY  ziplenr}   r   ZlinspacesumsplitrangerJ  r^   r   r   stagesr   r   r   r   r   r   headr   r   r(   _init_weights)rQ   r   r   rc  rd  re  rf  rg  r_  r   r   rM  Z
num_stagesZdprr  ru  rT  r  Zfinal_norm_layerr   ro  rS   r   r  s~    
	
&


$


zMaxxVit.__init__rh   c                 C   s:   t |dr6z|j|d W n ty4   |  Y n0 d S )Nr   r   )hasattrr   	TypeError)rQ   r   r   r   rR   rR   rS   rw    s
    
zMaxxVit._init_weightsc                 C   s   dd |   D S )Nc                    s*   h | ]"\ }t  fd ddD r qS )c                 3   s   | ]}| v V  qd S rO   rR   )ri  nr   rR   rS   	<genexpr>  rl  z4MaxxVit.no_weight_decay.<locals>.<setcomp>.<genexpr>)Zrelative_position_bias_tablezrel_pos.mlp)any)ri  _rR   r{  rS   	<setcomp>  s   z*MaxxVit.no_weight_decay.<locals>.<setcomp>)Znamed_parametersrP   rR   rR   rS   no_weight_decay  s    zMaxxVit.no_weight_decayFc                 C   s   t dddgd}|S )Nz^stem)z^stages\.(\d+)N)z^norm)i )rh  rS  )rF  )rQ   ZcoarseZmatcherrR   rR   rS   group_matcher  s
    zMaxxVit.group_matcherTc                 C   s   | j D ]
}||_qd S rO   )ru  rQ  )rQ   enablerj  rR   rR   rS   set_grad_checkpointing  s    
zMaxxVit.set_grad_checkpointingc                 C   s   | j jS rO   )rv  ZfcrP   rR   rR   rS   get_classifier  s    zMaxxVit.get_classifierNc                 C   s   || _ | j|| d S rO   )rd  rv  reset)rQ   rd  re  rR   rR   rS   reset_classifier  s    zMaxxVit.reset_classifierc                 C   s"   |  |}| |}| |}|S rO   )rh  ru  r   r   rR   rR   rS   forward_features  s    


zMaxxVit.forward_features
pre_logitsc                 C   s   | j ||dS )Nr  )rv  )rQ   r   r  rR   rR   rS   forward_head  s    zMaxxVit.forward_headc                 C   s   |  |}| |}|S rO   )r  r  r   rR   rR   rS   r     s    

zMaxxVit.forward)ra  r`   rb  r   r8   r8   )rh   )F)T)N)F)rU   rV   rW   r   r,   r	   rX   r
   r\   r[   r   rw  r   rV  ignorer  r  r  r  r  r  rZ   r  r   r   rR   rR   r   rS   r/   l  s:         R



r/   r   r;   FreluTrI   rK   r=   r?   c                 C   s6   t t| |dd|||d|d	td|||	|||
|ddS )NTFrm   )	rf   r<   rc   r_   rd   rj   rn   rH   rJ   )r5   r6   r<   rF   rJ   rL   r>   r@   rI  rF  r-   r.   )rf   r<   conv_output_biasconv_attn_earlyconv_attn_act_layerconv_norm_layertransformer_shortcut_biastransformer_norm_layertransformer_norm_layer_clrF   r>   r@   rR   rR   rS   _rw_coat_cfg  s.    r  re         ?r0   c                 C   s4   t t| |d||d|dtd||||	|||
|d	dS )NFrm   )rf   r<   r_   rd   rp   rH   rJ   )	r5   r<   r1   rB   rF   rJ   rL   r>   r@   rI  r  )rf   r<   r  conv_attn_ratior  r  r  rB   r1   rF   r>   r@   rR   rR   rS   _rw_max_cfg)  s,    	r  rM   r   c                 C   sD   t |}ttd| |d|d ||dtd||||d |||	|
d	dS )Nrq   Fr   )r^   rf   r<   r_   rF   rJ   rL   r%   )	r5   r<   rB   rD   rF   rJ   rL   r>   r@   rI  )r   rF  r-   r.   )rf   r<   r  Zconv_norm_layer_clr  r  rB   rD   rF   r>   r@   rR   rR   rS   	_next_cfgU  s.    	r  c                   C   s"   t tddddtdddddd	S )
NgMbP?Z	gelu_tanhZsame)rN   rH   ri   rs   Fr)  )rN   rH   r2   r>   rI  r  rR   rR   rR   rS   _tf_cfg|  s    r  )r         r?   rz   )r0   r   )ry   r}   r   ro   )r  r  )r`   r     r`   )rf   r  r  rt   )r{   r`   r  r{   )r  r  )r{   r  rK  r{   )rf   r  r  )r  r  r?      )r   r  rm   )rf   r  )rv   rw   rx      )ru   rv   )rf   r  rF   rr   )rf   r  r  r  rw   )r  r  r>   r@   )rf   r>   r   )r<   r  r  r>   r@   )rf   r>   r@   )rf   r  rF   r>   )r~   r~   r~   r   r  )ry   r}   r   r^   r   )ry   r}   r   r   )rs   N)r>   rF   r   rx   )ry   r}   r   r   r  r  rv   r  )r{         r{   )r  r?         r  )r0   r   r  r  )r{   r{   r|   r{   )rO  rO  rO  rO  )   r0   )ry   r}   r^   r   )r%   r{   r`   r%   )rP  rP  rP  rP  )r>   )ry   r}   r^   r   r   )ry   r}   r^   r   r   )0   ru   )rD   r>   )r{   r  r  r{   )rD   )   i@  i  r  )r{   r     r{   )P   r  r  )ry   r}   r^   r   r   r   ))Zcoatnet_pico_rwZcoatnet_nano_rwZcoatnet_0_rwZcoatnet_1_rwZcoatnet_2_rwZcoatnet_3_rwZcoatnet_bn_0_rwZcoatnet_rmlp_nano_rwZcoatnet_rmlp_0_rwZcoatnet_rmlp_1_rwZcoatnet_rmlp_1_rw2Zcoatnet_rmlp_2_rwZcoatnet_rmlp_3_rwZcoatnet_nano_ccZcoatnext_nano_rwZ	coatnet_0Z	coatnet_1Z	coatnet_2Z	coatnet_3Z	coatnet_4Z	coatnet_5Zmaxvit_pico_rwZmaxvit_nano_rwZmaxvit_tiny_rwZmaxvit_tiny_pmZmaxvit_rmlp_pico_rwZmaxvit_rmlp_nano_rwZmaxvit_rmlp_tiny_rwZmaxvit_rmlp_small_rwZmaxvit_rmlp_base_rwZmaxxvit_rmlp_nano_rwZmaxxvit_rmlp_tiny_rwZmaxxvit_rmlp_small_rwZmaxxvitv2_nano_rwZmaxxvitv2_rmlp_base_rwZmaxxvitv2_rmlp_large_rwmaxvit_tiny_tfmaxvit_small_tfmaxvit_base_tfmaxvit_large_tfmaxvit_xlarge_tf)modelc                 C   st   |  }i }|  D ]Z\}}||v rf|j|| jkrf| ||  krf|jdv sVJ ||| j}|||< q|S )N)r{   r  )
state_dictr]  ndimZnumelr   r   )r  r  Zmodel_state_dictZout_dictr   r   rR   rR   rS   checkpoint_filter_fn  s    ,
r  c                 K   sT   |d u r.| t v r| }nd| dd d }tt| |ft | tddtd|S )Nr~  r   T)Zflatten_sequential)Z	model_cfgZfeature_cfgZpretrained_filter_fn)
model_cfgsjoinrs  r&   r/   rF  r  )variantZcfg_variant
pretrainedr_  rR   rR   rS   _create_maxxvit	  s    r  c                 K   s    | dddddddddd	d
|S )Nrb  )r`   ra  ra  )r  r  ffffff?Zbicubic)      ?r  r  z
stem.conv1zhead.fcT)urlrd  
input_size	pool_sizecrop_pctinterpolationmeanr   Z
first_conv
classifierZfixed_input_sizerR   )r  r_  rR   rR   rS   _cfg  s    r  )r  ztimm/zyhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_nano_rw_224_sw-f53093b4.pthg?)	hf_hub_idr  r  zvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_0_rw_224_sw-a6439706.pth)r  r  zvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_1_rw_224_sw-5cae1ea8.pth)r  )r`   rw   rw   )r  r  g      ?Zsquash)r  r  r  r  	crop_modezyhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_bn_0_rw_224_sw-c228e218.pthr  )r  r  r  r   r  z~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_nano_rw_224_sw-bd1d51b3.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_1_rw_224_sw-9051e6c3.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_2_rw_224_sw-5ccfac55.pthzzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnext_nano_rw_224_ad-22cb71c2.pthi-.  )r  rd  )r`   r  r  )   r  )r  r  r  zxhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_nano_rw_256_sw-fb127241.pth)r  r  r  r  zxhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_tiny_rw_224_sw-7d0dffeb.pthz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_pico_rw_256_sw-8d82f2c6.pthz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_nano_rw_256_sw-c17bb0d6.pthz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_tiny_rw_256_sw-bbef0ff5.pthz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_small_rw_224_sw-6ef0ae4f.pthz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxxvit_rmlp_nano_rw_256_sw-0325d459.pthzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxxvit_rmlp_small_rw_256_sw-37e217ff.pth)r  r  r  )r  r  r   )r`   r?   r?   )r  r  iSU  )r  r  r  r  )Dzcoatnet_pico_rw_224.untrainedzcoatnet_nano_rw_224.sw_in1kzcoatnet_0_rw_224.sw_in1kzcoatnet_1_rw_224.sw_in1kz!coatnet_2_rw_224.sw_in12k_ft_in1kz'coatnet_rmlp_1_rw2_224.sw_in12k_ft_in1kz&coatnet_rmlp_2_rw_224.sw_in12k_ft_in1kz&coatnet_rmlp_2_rw_384.sw_in12k_ft_in1kzcoatnet_bn_0_rw_224.sw_in1kz coatnet_rmlp_nano_rw_224.sw_in1kzcoatnet_rmlp_0_rw_224.untrainedzcoatnet_rmlp_1_rw_224.sw_in1kzcoatnet_rmlp_2_rw_224.sw_in1kzcoatnet_rmlp_3_rw_224.untrainedzcoatnet_nano_cc_224.untrainedzcoatnext_nano_rw_224.sw_in1kzcoatnet_2_rw_224.sw_in12kzcoatnet_3_rw_224.sw_in12kzcoatnet_rmlp_1_rw2_224.sw_in12kzcoatnet_rmlp_2_rw_224.sw_in12kzcoatnet_0_224.untrainedzcoatnet_1_224.untrainedzcoatnet_2_224.untrainedzcoatnet_3_224.untrainedzcoatnet_4_224.untrainedzcoatnet_5_224.untrainedzmaxvit_pico_rw_256.untrainedzmaxvit_nano_rw_256.sw_in1kzmaxvit_tiny_rw_224.sw_in1kzmaxvit_tiny_rw_256.untrainedzmaxvit_tiny_pm_256.untrainedzmaxvit_rmlp_pico_rw_256.sw_in1kzmaxvit_rmlp_nano_rw_256.sw_in1kzmaxvit_rmlp_tiny_rw_256.sw_in1kz maxvit_rmlp_small_rw_224.sw_in1kz"maxvit_rmlp_small_rw_256.untrainedz(maxvit_rmlp_base_rw_224.sw_in12k_ft_in1kz(maxvit_rmlp_base_rw_384.sw_in12k_ft_in1kz maxvit_rmlp_base_rw_224.sw_in12kz maxxvit_rmlp_nano_rw_256.sw_in1kz"maxxvit_rmlp_tiny_rw_256.untrainedz!maxxvit_rmlp_small_rw_256.sw_in1kzmaxxvitv2_nano_rw_256.sw_in1kz+maxxvitv2_rmlp_base_rw_224.sw_in12k_ft_in1kz+maxxvitv2_rmlp_base_rw_384.sw_in12k_ft_in1kz%maxxvitv2_rmlp_large_rw_224.untrainedz#maxxvitv2_rmlp_base_rw_224.sw_in12kzmaxvit_tiny_tf_224.in1kzmaxvit_tiny_tf_384.in1kzmaxvit_tiny_tf_512.in1kzmaxvit_small_tf_224.in1kzmaxvit_small_tf_384.in1kzmaxvit_small_tf_512.in1kzmaxvit_base_tf_224.in1kzmaxvit_base_tf_384.in1kzmaxvit_base_tf_512.in1kzmaxvit_large_tf_224.in1kzmaxvit_large_tf_384.in1kzmaxvit_large_tf_512.in1kzmaxvit_base_tf_224.in21kz maxvit_base_tf_384.in21k_ft_in1kz maxvit_base_tf_512.in21k_ft_in1kzmaxvit_large_tf_224.in21kz!maxvit_large_tf_384.in21k_ft_in1kz!maxvit_large_tf_512.in21k_ft_in1kzmaxvit_xlarge_tf_224.in21kz"maxvit_xlarge_tf_384.in21k_ft_in1kz"maxvit_xlarge_tf_512.in21k_ft_in1k)returnc                 K   s   t dd| i|S )Ncoatnet_pico_rw_224r  )r  r  r  r_  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Ncoatnet_nano_rw_224r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Ncoatnet_0_rw_224r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Ncoatnet_1_rw_224r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Ncoatnet_2_rw_224r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Ncoatnet_3_rw_224r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Ncoatnet_bn_0_rw_224r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Ncoatnet_rmlp_nano_rw_224r  )r  r  r  rR   rR   rS   r     s    r  c                 K   s   t dd| i|S )Ncoatnet_rmlp_0_rw_224r  )r  r  r  rR   rR   rS   r  %  s    r  c                 K   s   t dd| i|S )Ncoatnet_rmlp_1_rw_224r  )r  r  r  rR   rR   rS   r  *  s    r  c                 K   s   t dd| i|S )Ncoatnet_rmlp_1_rw2_224r  )r  r  r  rR   rR   rS   r  /  s    r  c                 K   s   t dd| i|S )Ncoatnet_rmlp_2_rw_224r  )r  r  r  rR   rR   rS   r  4  s    r  c                 K   s   t dd| i|S )Ncoatnet_rmlp_2_rw_384r  )r  r  r  rR   rR   rS   r  9  s    r  c                 K   s   t dd| i|S )Ncoatnet_rmlp_3_rw_224r  )r  r  r  rR   rR   rS   r  >  s    r  c                 K   s   t dd| i|S )Ncoatnet_nano_cc_224r  )r  r  r  rR   rR   rS   r  C  s    r  c                 K   s   t dd| i|S )Ncoatnext_nano_rw_224r  )r  r  r  rR   rR   rS   r  H  s    r  c                 K   s   t dd| i|S )Ncoatnet_0_224r  )r  r  r  rR   rR   rS   r  M  s    r  c                 K   s   t dd| i|S )Ncoatnet_1_224r  )r  r  r  rR   rR   rS   r  R  s    r  c                 K   s   t dd| i|S )Ncoatnet_2_224r  )r  r  r  rR   rR   rS   r  W  s    r  c                 K   s   t dd| i|S )Ncoatnet_3_224r  )r  r  r  rR   rR   rS   r  \  s    r  c                 K   s   t dd| i|S )Ncoatnet_4_224r  )r  r  r  rR   rR   rS   r  a  s    r  c                 K   s   t dd| i|S )Ncoatnet_5_224r  )r  r  r  rR   rR   rS   r  f  s    r  c                 K   s   t dd| i|S )Nmaxvit_pico_rw_256r  )r  r  r  rR   rR   rS   r  k  s    r  c                 K   s   t dd| i|S )Nmaxvit_nano_rw_256r  )r  r  r  rR   rR   rS   r  p  s    r  c                 K   s   t dd| i|S )Nmaxvit_tiny_rw_224r  )r  r  r  rR   rR   rS   r  u  s    r  c                 K   s   t dd| i|S )Nmaxvit_tiny_rw_256r  )r  r  r  rR   rR   rS   r  z  s    r  c                 K   s   t dd| i|S )Nmaxvit_rmlp_pico_rw_256r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_rmlp_nano_rw_256r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_rmlp_tiny_rw_256r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_rmlp_small_rw_224r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_rmlp_small_rw_256r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_rmlp_base_rw_224r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_rmlp_base_rw_384r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_tiny_pm_256r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxxvit_rmlp_nano_rw_256r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxxvit_rmlp_tiny_rw_256r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxxvit_rmlp_small_rw_256r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxxvitv2_nano_rw_256r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxxvitv2_rmlp_base_rw_224r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxxvitv2_rmlp_base_rw_384r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxxvitv2_rmlp_large_rw_224r  )r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_tiny_tf_224r  r  )r  r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_tiny_tf_384r  r  )r  r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_tiny_tf_512r  r  )r  r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_small_tf_224r  r  )r  r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_small_tf_384r  r  )r  r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_small_tf_512r  r  )r  r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_base_tf_224r  r  )r  r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_base_tf_384r  r  )r  r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_base_tf_512r  r  )r  r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_large_tf_224r  r  )r  r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_large_tf_384r  r  )r  r  r  r  rR   rR   rS   r    s    r  c                 K   s   t dd| i|S )Nmaxvit_large_tf_512r  r  )r  r  r  r  rR   rR   rS   r  	  s    r  c                 K   s   t dd| i|S )Nmaxvit_xlarge_tf_224r  r  )r  r  r  r  rR   rR   rS   r  	  s    r  c                 K   s   t dd| i|S )Nmaxvit_xlarge_tf_384r  r  )r  r  r  r  rR   rR   rS   r  	  s    r  c                 K   s   t dd| i|S )Nmaxvit_xlarge_tf_512r  r  )r  r  r  r  rR   rR   rS   r  	  s    r  )rh   )rh   )r   r;   FFr  rh   TrI   rK   Nr=   r?   )re   r;   Fr  rh   rI   rK   Nr0   Nr=   r?   )re   r;   rI   rK   rI   rK   NFrM   r   r?   )NF)rh   )F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)r   r   collectionsr   Zdataclassesr   r   r   	functoolsr   typingr   r   r	   r
   r   r   r   Z	torch.jitr   Z	timm.datar   r   Ztimm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   Z_builderr&   Z_features_fxr'   Z_manipulater(   r)   	_registryr*   r+   __all__r.   r-   r,   Moduler   r   r   r   r   r   r   r   r   r   r  rX   r  r#  r&  r(  r*  r+  r8  r=  r>  r?  r@  rA  rB  rH  rJ  rW  r\  r`  r/   r  r  r  r  rF  r  r  r  r  Zdefault_cfgsr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rR   rR   rR   rS   <module>   s  $   #EF'A	\M		@I		A,&E#	            
5           
-          
'	
	

	

					    s  \