a
    d                  /   @   s  d Z ddlZddlmZ ddlZddlmZ ddlmZ ddl	m
Z
mZ ddlmZmZmZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZ ddlmZ dgZeG dd dejZdddZG dd dejZ G dd dejZ!G dd dejZ"G dd dejZ#G dd dejZ$G dd dejZ%dd Z&dd d!Z'dd#d$Z(ee(d%d&d'e(d%d(d'e(d%d)d*d+e(d%d,d'e(d%d-d'e(d%d.d*d+e(d%d/d'e(d%d0d'e(d%d1d*d+e(d%d2d'e(d%d3d'e(d%d4d*d+e(d%d5d'e(d%d6d'e(d%d7d*d+e(d%d8d'e(d%d9d'e(d%d:d*d+e(d%d;d'e(d%d<d'e(d%d=d*d+e(d%d>d'e(d%d?d'e(d%d@d*d+e(d%dAd'e(d%dBd'e(d%dCd*d+e(d%dDd'e(d%dEd'e(d%dFd*d+e(d%dGd'e(d%dHd'e(d%dId*d+e(d%dJd'e(d%dKd'e(d%dLd*d+e(d%dMd'e(d%dNd'e(d%dOd*d+e(d%dPd'e(d%dQd'e(d%dRd*d+dS*Z)ede%dTdUdVZ*ede%dTdWdXZ+ede%dTdYdZZ,ede%dTd[d\Z-ede%dTd]d^Z.ede%dTd_d`Z/ede%dTdadbZ0ede%dTdcddZ1ede%dTdedfZ2ede%dTdgdhZ3ede%dTdidjZ4ede%dTdkdlZ5ede%dTdmdnZ6ede%dTdodpZ7ede%dTdqdrZ8ede%dTdsdtZ9ede%dTdudvZ:ede%dTdwdxZ;ede%dTdydzZ<ede%dTd{d|Z=ede%dTd}d~Z>ede%dTddZ?ede%dTddZ@ede%dTddZAede%dTddZBede%dTddZCede%dTddZDede%dTddZEeeFdddddddddddddddddddddddddddd dS )a]   Cross-Covariance Image Transformer (XCiT) in PyTorch

Paper:
    - https://arxiv.org/abs/2106.09681

Same as the official implementation, with some minor adaptations, original copyright below
    - https://github.com/facebookresearch/xcit/blob/master/xcit.py

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    N)partial)
checkpointIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPathtrunc_normal_	to_2tuple   )build_model_with_cfg)register_notrace_module)register_modelgenerate_default_cfgsregister_model_deprecations)	ClassAttn)MlpXcitc                       s4   e Zd ZdZd
 fdd	Zeeeddd	Z  ZS )PositionalEncodingFourierz
    Positional encoding relying on a fourier kernel matching the one used in the "Attention is all you Need" paper.
    Based on the official XCiT code
        - https://github.com/facebookresearch/xcit/blob/master/xcit.py
           '  c                    sH   t    tj|d |dd| _dtj | _|| _|| _	|| _
d| _d S )N   r
   )kernel_sizeư>)super__init__nnConv2dtoken_projectionmathpiscaletemperature
hidden_dimdimeps)selfr#   r$   r"   	__class__ Y/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/xcit.pyr   (   s    
z"PositionalEncodingFourier.__init__)BHWc              	   C   s  | j jj}tjd|d tj|dddd|}tjd|d tj|dd|d}||d d dd d d f | j  | j	 }||d d d d dd f | j  | j	 }tj| j
tj|d}| jdtj|ddd | j
  }|d d d d d d d f | }|d d d d d d d f | }	tj|d d d d d d dd df  |d d d d d d dd df  gdd	d
}tj|	d d d d d d dd df  |	d d d d d d dd df  gdd	d
}	tj|	|fd
d	dd
dd}
|  |
}
|
|dddS )Nr
   )Zdtypedevicer   floor)Zrounding_moder      r$      )r   weightr.   torchZarangefloat32Z	unsqueezerepeatr%   r!   r#   r"   divstacksincosflattencatpermute)r&   r+   r,   r-   r.   Zy_embedZx_embedZdim_tZpos_xZpos_yposr)   r)   r*   forward1   s    
("**   \\
z!PositionalEncodingFourier.forward)r   r   r   )__name__
__module____qualname____doc__r   intr@   __classcell__r)   r)   r'   r*   r       s   	r   c              
   C   s&   t jtj| |d|dddt|S )z3x3 convolution + batch normr3   r
   F)r   stridepaddingbias)r5   r   
Sequentialr   BatchNorm2d)Z	in_planesZ
out_planesrG   r)   r)   r*   conv3x3B   s    rL   c                       s6   e Zd ZdZddddejf fdd	Zdd	 Z  ZS )
ConvPatchEmbedz<Image to Patch Embedding using multiple convolutional layers      r3   r   c                    s   t    t|}|d | |d |  }|| _|| _|| _|dkrtjt	||d d| t	|d |d d| t	|d |d d| t	|d |d| _
nP|dkrtjt	||d d| t	|d |d d| t	|d |d| _
ndd S )Nr
   r   rO      r   r1   z=For convolutional projection, patch size has to be in [8, 16])r   r   r	   img_size
patch_sizenum_patchesr5   r   rJ   rL   proj)r&   rQ   rR   in_chans	embed_dim	act_layerrS   r'   r)   r*   r   M   s2    
	zConvPatchEmbed.__init__c                 C   s>   |  |}|jd |jd  }}|ddd}|||ffS )Nr   r3   r
   )rT   shaper<   	transpose)r&   xHpWpr)   r)   r*   r@   j   s    
zConvPatchEmbed.forward)	rA   rB   rC   rD   r   GELUr   r@   rF   r)   r)   r'   r*   rM   J   s   rM   c                       s:   e Zd ZdZdejdf fdd	ZeedddZ  Z	S )	LPIa  
    Local Patch Interaction module that allows explicit communication between tokens in 3x3 windows to augment the
    implicit communication performed by the block diagonal scatter attention. Implemented using 2 layers of separable
    3x3 convolutions with GeLU and BatchNorm2d
    Nr3   c                    sb   t    |p|}|d }tjj|||||d| _| | _t|| _tjj|||||d| _	d S )Nr   )r   rH   groups)
r   r   r5   r   r   conv1actrK   bnconv2)r&   in_featuresZout_featuresrW   r   rH   r'   r)   r*   r   x   s    


zLPI.__init__r,   r-   c                 C   sj   |j \}}}|ddd||||}| |}| |}| |}| |}||||ddd}|S )Nr   r   r
   )rX   r>   reshaper`   ra   rb   rc   )r&   rZ   r,   r-   r+   NCr)   r)   r*   r@      s    



zLPI.forward)
rA   rB   rC   rD   r   r]   r   rE   r@   rF   r)   r)   r'   r*   r^   q   s   r^   c                	       s@   e Zd ZdZdddddejejddf	 fdd	Zdd	 Z  Z	S )
ClassAttentionBlockzAClass Attention Layer as in CaiT https://arxiv.org/abs/2103.17239      @F              ?c                    s   t    |	|| _t|||||d| _|dkr8t|nt | _|	|| _	t
|t|| ||d| _|
d urt|
t| | _t|
t| | _nd\| _| _|| _d S )N	num_headsqkv_bias	attn_drop	proj_droprk   rd   Zhidden_featuresrW   Zdrop)rl   rl   )r   r   norm1r   attnr   r   Identity	drop_pathnorm2r   rE   mlp	Parameterr5   onesgamma1gamma2tokens_norm)r&   r$   rn   	mlp_ratioro   rq   rp   rv   rW   
norm_layeretar}   r'   r)   r*   r      s    



zClassAttentionBlock.__init__c                 C   s   |  |}tj| ||d d dd f gdd}|| | j|  }| jrX| |}n8tj| |d d ddf |d d dd f gdd}|}|d d ddf }| j| 	| }tj||d d dd f gdd}|| | }|S )Nr
   r2   r   )
rs   r5   r=   rt   rv   r{   r}   rw   r|   rx   )r&   rZ   Zx_norm1Zx_attnZx_res	cls_tokenr)   r)   r*   r@      s    
(8"zClassAttentionBlock.forward)
rA   rB   rC   rD   r   r]   	LayerNormr   r@   rF   r)   r)   r'   r*   ri      s   !ri   c                       s:   e Zd ZdZd fdd	Zdd Zejjd	d
 Z	  Z
S )XCAz Cross-Covariance Attention (XCA)
    Operation where the channels are updated using a weighted sum. The weights are obtained from the (softmax
    normalized) Cross-covariance matrix (Q^T \cdot K \in d_h \times d_h)
    rP   Frk   c                    sf   t    || _tt|dd| _tj||d |d| _	t
|| _t||| _t
|| _d S )Nr
   r3   )rI   )r   r   rn   r   ry   r5   rz   r"   LinearqkvDropoutrp   rT   rq   )r&   r$   rn   ro   rp   rq   r'   r)   r*   r      s    
zXCA.__init__c           
      C   s   |j \}}}| |||d| j|| j ddddd}|d\}}}tjjj	|dd}tjjj	|dd}||
dd | j }	|	jdd}	| |	}	|	| dddd|||}| |}| |}|S )	Nr3   r   r   r1   r
   r/   r2   )rX   r   rf   rn   r>   Zunbindr5   r   Z
functional	normalizerY   r"   Zsoftmaxrp   rT   rq   )
r&   rZ   r+   rg   rh   r   qkvrt   r)   r)   r*   r@      s    .


zXCA.forwardc                 C   s   dhS )Nr"   r)   r&   r)   r)   r*   no_weight_decay   s    zXCA.no_weight_decay)rP   Frk   rk   )rA   rB   rC   rD   r   r@   r5   jitignorer   rF   r)   r)   r'   r*   r      s
   	r   c                       sB   e Zd Zdddddejejdf fdd	Zeeddd	Z  Z	S )
XCABlockrj   Frk   rl   c                    s   t    |	|| _t|||||d| _|dkr8t|nt | _|	|| _	t
||d| _|	|| _t|t|| ||d| _t|
t| | _t|
t| | _t|
t| | _d S )Nrm   rk   )rd   rW   rr   )r   r   rs   r   rt   r   r   ru   rv   norm3r^   local_mprw   r   rE   rx   ry   r5   rz   r{   gamma3r|   )r&   r$   rn   r~   ro   rq   rp   rv   rW   r   r   r'   r)   r*   r      s    



zXCABlock.__init__re   c              	   C   sh   ||  | j| | |  }||  | j| | |||  }||  | j| | 	|  }|S N)
rv   r{   rt   rs   r   r   r   r|   rx   rw   )r&   rZ   r,   r-   r)   r)   r*   r@     s     $ zXCABlock.forward)
rA   rB   rC   r   r]   r   r   rE   r@   rF   r)   r)   r'   r*   r      s   r   c                       s   e Zd ZdZd& fdd	Zdd Zejjdd Z	ejjd'ddZ
ejjd(ddZejjdd Zd)ddZdd  Zd*ed!d"d#Zd$d% Z  ZS )+r   z
    Based on timm and DeiT code bases
    https://github.com/rwightman/pytorch-image-models/tree/master/timm
    https://github.com/facebookresearch/deit/
    rN   rO   r3     tokenr      rj   Trk   Nr   rl   Fc                    s  t    |dv sJ t|}|d | dkr>|d | dksFJ dpVttjdd p`tj || _ | _| _	|| _
d| _t||| d| _ttdd| _|rtd	| _nd
| _tj|d| _t 	
f
ddt|D | _t 
f
ddt|D | _| _t| _|dkr^t| j|nt | _t| jdd |  | j! d
S )a  
        Args:
            img_size (int, tuple): input image size
            patch_size (int): patch size
            in_chans (int): number of input channels
            num_classes (int): number of classes for classification head
            embed_dim (int): embedding dimension
            depth (int): depth of transformer
            num_heads (int): number of attention heads
            mlp_ratio (int): ratio of mlp hidden dim to embedding dim
            qkv_bias (bool): enable bias for qkv if True
            drop_rate (float): dropout rate after positional embedding, and in XCA/CA projection + MLP
            pos_drop_rate: position embedding dropout rate
            proj_drop_rate (float): projection dropout rate
            attn_drop_rate (float): attention dropout rate
            drop_path_rate (float): stochastic depth rate (constant across all layers)
            norm_layer: (nn.Module): normalization layer
            cls_attn_layers: (int) Depth of Class attention layers
            use_pos_embed: (bool) whether to use positional encoding
            eta: (float) layerscale initialization value
            tokens_norm: (bool) Whether to normalize all tokens or just the cls_token in the CA

        Notes:
            - Although `layer_norm` is user specifiable, there are hard-coded `BatchNorm2d`s in the local patch
              interaction (class LPI) and the patch embedding (class ConvPatchEmbed)
         avgr   r   z2`patch_size` should divide image dimensions evenlyr   )r%   F)rQ   rR   rU   rV   rW   r
   r2   N)pc                    s(   g | ] }t 	 d 
qS ))
r$   rn   r~   ro   rq   rp   rv   rW   r   r   )r   .0_)
rW   attn_drop_ratedrop_path_raterV   r   r~   r   rn   proj_drop_ratero   r)   r*   
<listcomp>f  s   z!Xcit.__init__.<locals>.<listcomp>c                    s(   g | ] }t  	d 
qS ))
r$   rn   r~   ro   rq   rp   rW   r   r   r}   )ri   r   )
rW   r   	drop_raterV   r   r~   r   rn   ro   r}   r)   r*   r   u  s   {Gz?std)"r   r   r	   r   r   r   r]   num_classesnum_featuresrV   global_poolgrad_checkpointingrM   patch_embedry   r5   zerosr   r   	pos_embedr   pos_dropZ
ModuleListrangeblockscls_attn_blocksnorm	head_dropr   ru   headr   apply_init_weights)r&   rQ   rR   rU   r   r   rV   depthrn   r~   ro   r   Zpos_drop_rater   r   r   rW   r   Zcls_attn_layersuse_pos_embedr   r}   r'   )rW   r   r   r   rV   r   r~   r   rn   r   ro   r}   r*   r     sF    2
"
 
 

"zXcit.__init__c                 C   sD   t |tjr@t|jdd t |tjr@|jd ur@tj|jd d S )Nr   r   r   )
isinstancer   r   r   r4   rI   initZ	constant_)r&   mr)   r)   r*   r     s    zXcit._init_weightsc                 C   s   ddhS )Nr   r   r)   r   r)   r)   r*   r     s    zXcit.no_weight_decayc                 C   s   t ddddgdS )Nz ^cls_token|pos_embed|patch_embedz^blocks\.(\d+))z^cls_attn_blocks\.(\d+)N)z^norm)i )stemr   r   )dict)r&   Zcoarser)   r)   r*   group_matcher  s
    zXcit.group_matcherc                 C   s
   || _ d S r   )r   )r&   enabler)   r)   r*   set_grad_checkpointing  s    zXcit.set_grad_checkpointingc                 C   s   | j S r   )r   r   r)   r)   r*   get_classifier  s    zXcit.get_classifierr   c                 C   sD   || _ |d ur |dv sJ || _|dkr6t| j|nt | _d S )Nr   r   )r   r   r   r   r   ru   r   )r&   r   r   r)   r)   r*   reset_classifier  s
    zXcit.reset_classifierc                 C   s   |j d }| |\}\}}| jd urV| ||||d|j d ddd}|| }| |}| jD ]0}| jrtj	
 st||||}qf||||}qftj| j|dd|fdd}| jD ](}| jrtj	
 st||}q||}q| |}|S )Nr   r/   r
   r   r2   )rX   r   r   rf   r>   r   r   r   r5   r   Zis_scriptingr   r=   r   expandr   r   )r&   rZ   r+   r[   r\   Zpos_encodingZblkr)   r)   r*   forward_features  s"    

(




zXcit.forward_features)
pre_logitsc                 C   sX   | j r<| j dkr,|d d dd f jddn|d d df }| |}|rN|S | |S )Nr   r
   r2   r   )r   meanr   r   )r&   rZ   r   r)   r)   r*   forward_head  s    6
zXcit.forward_headc                 C   s   |  |}| |}|S r   )r   r   )r&   rZ   r)   r)   r*   r@     s    

zXcit.forward)rN   rO   r3   r   r   r   r   r   rj   Trk   rk   rk   rk   rk   NNr   Trl   F)F)T)r   )F)rA   rB   rC   rD   r   r   r5   r   r   r   r   r   r   r   r   boolr   r@   rF   r)   r)   r'   r*   r     sH                        u


c              	   C   s6  d| v r| d } t |dd d u}dd | D }|D ]&}|rR| || |dd< q2| |= q2d| v r2d| v r2t|j}t|D ]}| d	| d
}|dd|jd }t	dD ]"\}}	|| | d	| d|	 d< q| d	| dd }
|
d ur|
dd}
t	dD ]$\}}	|
| | d	| d|	 d< q
q| S )Nmodelr   c                 S   s   g | ]}| d r|qS )r   )
startswith)r   r   r)   r)   r*   r         z(checkpoint_filter_fn.<locals>.<listcomp>zpos_embeder.z
pos_embed.z!cls_attn_blocks.0.attn.qkv.weightzcls_attn_blocks.0.attn.q.weightzcls_attn_blocks.z.attn.qkv.weightr3   r/   r   z.attn.z.weightz.attn.qkv.biasz.bias)
getattrpopreplace
state_dictlenr   r   rf   rX   	enumerate)r   r   r   Zpos_embed_keysr   Znum_ca_blocksiZ
qkv_weightjZ	subscriptro   r)   r)   r*   checkpoint_filter_fn  s*    
 r   Fc                 K   s   t t| |fdti|}|S )NZpretrained_filter_fn)r   r   r   )variant
pretrainedZdefault_cfgkwargsr   r)   r)   r*   _create_xcit  s    r   r   c                 K   s    | ddd dddt tddd|S )	Nr   )r3   rN   rN   rl   ZbicubicTzpatch_embed.proj.0.0r   )urlr   
input_sizeZ	pool_sizeZcrop_pctinterpolationZfixed_input_sizer   r   Z
first_conv
classifierr   )r   r   r)   r)   r*   _cfg  s    r   ztimm/z<https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p16_224.pth)	hf_hub_idr   zAhttps://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p16_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p16_384_dist.pth)r3     r   )r   r   r   z<https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p16_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p16_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p16_384_dist.pthz<https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p16_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p16_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p16_384_dist.pthz=https://dl.fbaipublicfiles.com/xcit/xcit_small_12_p16_224.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p16_224_dist.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p16_384_dist.pthz=https://dl.fbaipublicfiles.com/xcit/xcit_small_24_p16_224.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p16_224_dist.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p16_384_dist.pthz>https://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_224.pthzChttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_224_dist.pthzChttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_384_dist.pthz=https://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_224.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_224_dist.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_384_dist.pthz;https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_224.pthz@https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_224_dist.pthz@https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_384_dist.pthz;https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_224.pthz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_224_dist.pthz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_384_dist.pthz;https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_224.pthz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_224_dist.pthz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_384_dist.pthz<https://dl.fbaipublicfiles.com/xcit/xcit_small_12_p8_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p8_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p8_384_dist.pthz<https://dl.fbaipublicfiles.com/xcit/xcit_small_24_p8_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p8_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p8_384_dist.pthz=https://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p8_224.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p8_224_dist.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p8_384_dist.pthz<https://dl.fbaipublicfiles.com/xcit/xcit_large_24_p8_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p8_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p8_384_dist.pth)*zxcit_nano_12_p16_224.fb_in1k!xcit_nano_12_p16_224.fb_dist_in1k!xcit_nano_12_p16_384.fb_dist_in1kzxcit_tiny_12_p16_224.fb_in1k!xcit_tiny_12_p16_224.fb_dist_in1k!xcit_tiny_12_p16_384.fb_dist_in1kzxcit_tiny_24_p16_224.fb_in1k!xcit_tiny_24_p16_224.fb_dist_in1k!xcit_tiny_24_p16_384.fb_dist_in1kzxcit_small_12_p16_224.fb_in1k"xcit_small_12_p16_224.fb_dist_in1k"xcit_small_12_p16_384.fb_dist_in1kzxcit_small_24_p16_224.fb_in1k"xcit_small_24_p16_224.fb_dist_in1kz"xcit_small_24_p16_384.fb_dist_in1kzxcit_medium_24_p16_224.fb_in1k#xcit_medium_24_p16_224.fb_dist_in1k#xcit_medium_24_p16_384.fb_dist_in1kzxcit_large_24_p16_224.fb_in1k"xcit_large_24_p16_224.fb_dist_in1k"xcit_large_24_p16_384.fb_dist_in1kzxcit_nano_12_p8_224.fb_in1k xcit_nano_12_p8_224.fb_dist_in1k xcit_nano_12_p8_384.fb_dist_in1kzxcit_tiny_12_p8_224.fb_in1k xcit_tiny_12_p8_224.fb_dist_in1k xcit_tiny_12_p8_384.fb_dist_in1kzxcit_tiny_24_p8_224.fb_in1k xcit_tiny_24_p8_224.fb_dist_in1k xcit_tiny_24_p8_384.fb_dist_in1kzxcit_small_12_p8_224.fb_in1k!xcit_small_12_p8_224.fb_dist_in1k!xcit_small_12_p8_384.fb_dist_in1kzxcit_small_24_p8_224.fb_in1k!xcit_small_24_p8_224.fb_dist_in1k!xcit_small_24_p8_384.fb_dist_in1kzxcit_medium_24_p8_224.fb_in1k"xcit_medium_24_p8_224.fb_dist_in1k"xcit_medium_24_p8_384.fb_dist_in1kzxcit_large_24_p8_224.fb_in1k!xcit_large_24_p8_224.fb_dist_in1k!xcit_large_24_p8_384.fb_dist_in1k)returnc                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrO      r   r1   rl   FrR   rV   r   rn   r   r}   xcit_nano_12_p16_224r   )r   r   r   r   r   Z
model_argsr   r)   r)   r*   r     s
    r   c              	   K   s8   t dddddddd}tdd
| it |fi |}|S )NrO   r   r   r1   rl   Fr   )rR   rV   r   rn   r   r}   rQ   xcit_nano_12_p16_384r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrO      r   r1   rl   Tr   xcit_tiny_12_p16_224r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrO   r  r   r1   rl   Tr   xcit_tiny_12_p16_384r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrO   r   r   rP   rl   Tr   xcit_small_12_p16_224r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrO   r   r   rP   rl   Tr   xcit_small_12_p16_384r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrO   r     r1   h㈵>Tr   xcit_tiny_24_p16_224r   )r	  r   r   r)   r)   r*   r	    s
    r	  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrO   r  r  r1   r  Tr   xcit_tiny_24_p16_384r   )r
  r   r   r)   r)   r*   r
    s
    r
  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrO   r   r  rP   r  Tr   xcit_small_24_p16_224r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrO   r   r  rP   r  Tr   xcit_small_24_p16_384r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrO      r  rP   r  Tr   xcit_medium_24_p16_224r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrO   r  r  rP   r  Tr   xcit_medium_24_p16_384r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td	d| it |fi |}|S )
NrO   r   r  r  Tr   xcit_large_24_p16_224r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td	d| it |fi |}|S )
NrO   r   r  r  Tr   xcit_large_24_p16_384r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrP   r   r   r1   rl   Fr   xcit_nano_12_p8_224r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrP   r   r   r1   rl   Fr   xcit_nano_12_p8_384r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrP   r  r   r1   rl   Tr   xcit_tiny_12_p8_224r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrP   r  r   r1   rl   Tr   xcit_tiny_12_p8_384r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td	d| it |fi |}|S )
NrP   r   r   rl   Tr   xcit_small_12_p8_224r   )r  r   r   r)   r)   r*   r    s
    r  c                 K   s6   t ddddddd}td	d| it |fi |}|S )
NrP   r   r   rl   Tr   xcit_small_12_p8_384r   )r  r   r   r)   r)   r*   r  '  s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrP   r  r  r1   r  Tr   xcit_tiny_24_p8_224r   )r  r   r   r)   r)   r*   r  /  s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrP   r  r  r1   r  Tr   xcit_tiny_24_p8_384r   )r  r   r   r)   r)   r*   r  7  s
    r  c                 K   s6   t ddddddd}td	d| it |fi |}|S )
NrP   r   r  r  Tr   xcit_small_24_p8_224r   )r  r   r   r)   r)   r*   r  ?  s
    r  c                 K   s6   t ddddddd}td	d| it |fi |}|S )
NrP   r   r  r  Tr   xcit_small_24_p8_384r   )r  r   r   r)   r)   r*   r  G  s
    r  c                 K   s6   t ddddddd}td	d| it |fi |}|S )
NrP   r  r  r  Tr   xcit_medium_24_p8_224r   )r  r   r   r)   r)   r*   r  O  s
    r  c                 K   s6   t ddddddd}td	d| it |fi |}|S )
NrP   r  r  r  Tr   xcit_medium_24_p8_384r   )r  r   r   r)   r)   r*   r  W  s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrP   r   r  rO   r  Tr   xcit_large_24_p8_224r   )r  r   r   r)   r)   r*   r  _  s
    r  c                 K   s6   t ddddddd}td
d	| it |fi |}|S )NrP   r   r  rO   r  Tr   xcit_large_24_p8_384r   )r  r   r   r)   r)   r*   r  g  s
    r  r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )Zxcit_nano_12_p16_224_distZxcit_nano_12_p16_384_distZxcit_tiny_12_p16_224_distZxcit_tiny_12_p16_384_distZxcit_tiny_24_p16_224_distZxcit_tiny_24_p16_384_distZxcit_small_12_p16_224_distZxcit_small_12_p16_384_distZxcit_small_24_p16_224_distZxcit_medium_24_p16_224_distZxcit_medium_24_p16_384_distZxcit_large_24_p16_224_distZxcit_large_24_p16_384_distZxcit_nano_12_p8_224_distZxcit_nano_12_p8_384_distZxcit_tiny_12_p8_224_distZxcit_tiny_12_p8_384_distZxcit_tiny_24_p8_224_distZxcit_tiny_24_p8_384_distZxcit_small_12_p8_224_distZxcit_small_12_p8_384_distZxcit_small_24_p8_224_distZxcit_small_24_p8_384_distZxcit_medium_24_p8_224_distZxcit_medium_24_p8_384_distZxcit_large_24_p8_224_distZxcit_large_24_p8_384_dist)r
   )FN)r   )F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)GrD   r   	functoolsr   r5   Ztorch.nnr   Ztorch.utils.checkpointr   Z	timm.datar   r   Ztimm.layersr   r   r	   Z_builderr   Z_features_fxr   	_registryr   r   r   Zcaitr   Zvision_transformerr   __all__Moduler   rL   rM   r^   ri   r   r   r   r   r   r   Zdefault_cfgsr   r  r  r  r  r  r	  r
  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rA   r)   r)   r)   r*   <module>   s4  !
'4'& F

 