a
    d                      @   s  d Z ddlZddlmZmZmZmZ ddlZddlm	Z	 ddl
m	  mZ ddlmZ ddlmZmZmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z% dd	l&m'Z'm(Z( d
gZ)G dd de	j*Z+G dd de	j*Z,G dd de	j*Z-G dd
 d
e	j*Z.dYddZ/dZddZ0d[ddZ1e'e1dde1dddd d!e1deeddd d"e1deed#dd d"e1dd$dd d!e1dd$dd d!e1dd$dd d!e1dddd%e1dddd%e1dd$dd%e1dd$dd%e1dd$dd%e1dd$dd d&d'e1dd$dd d&d'e1dd$dd d&d'e1ddd(e1ddd(e1ddd(e1ddd(e1ddd(e1d)d*d+d,e1d-d*d+d,e1d.d*d/d,e1d0d*d1d,e1d2d*ddd1d3e1d4d*d+d,e1d5d*d+d,e1dd6d7Z2e(d\e.d8d9d:Z3e(d]e.d8d;d<Z4e(d^e.d8d=d>Z5e(d_e.d8d?d@Z6e(d`e.d8dAdBZ7e(dae.d8dCdDZ8e(dbe.d8dEdFZ9e(dce.d8dGdHZ:e(dde.d8dIdJZ;e(dee.d8dKdLZ<e(dfe.d8dMdNZ=e(dge.d8dOdPZ>e(dhe.d8dQdRZ?e(die.d8dSdTZ@e(dje.d8dUdVZAe(dke.d8dWdXZBdS )la   EVA

EVA from https://github.com/baaivision/EVA , paper: https://arxiv.org/abs/2211.07636

@article{EVA,
  title={EVA: Exploring the Limits of Masked Visual Representation Learning at Scale},
  author={Fang, Yuxin and Wang, Wen and Xie, Binhui and Sun, Quan and Wu, Ledell and Wang, Xinggang and Huang,
  Tiejun and Wang, Xinlong and Cao, Yue},
  journal={arXiv preprint arXiv:2211.07636},
  year={2022}
}

EVA-02: A Visual Representation for Neon Genesis - https://arxiv.org/abs/2303.11331
@article{EVA02,
  title={EVA-02: A Visual Representation for Neon Genesis},
  author={Fang, Yuxin and Sun, Quan and Wang, Xinggang and Huang, Tiejun and Wang, Xinlong and Cao, Yue},
  journal={arXiv preprint arXiv:2303.11331},
  year={2023}
}

This file contains EVA & EVA02 model implementations evolved from BEiT, additional models in vision_transformer.py.

Modifications by / Copyright 2023 Ross Wightman, original copyrights below
    N)CallableOptionalTupleUnion)
checkpoint)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STDOPENAI_CLIP_MEANOPENAI_CLIP_STD)
PatchEmbedMlpGluMlpSwiGLU	LayerNormDropPathPatchDropoutRotaryEmbeddingCatapply_rot_embed_catapply_keep_indices_nlctrunc_normal_resample_patch_embedresample_abs_pos_embed	to_2tupleuse_fused_attn   )build_model_with_cfg)generate_default_cfgsregister_modelEvac                
       sj   e Zd ZU ejje ed< deeeee	e	e
e e
e d fddZde
ej e
ej d	d
dZ  ZS )EvaAttention
fused_attn   T        N)dim	num_headsqkv_bias	qkv_fused	attn_drop	proj_dropattn_head_dim
norm_layerc	                    sT  t    || _|| }	|dur$|}	|	| j }
|	d | _t | _|rtj||
d dd| _d | _	 | _
| _|rtt|
| _| jdt|
dd tt|
| _nd | _ | _| _nNtj||
|d| _	tj||
dd| _
tj||
|d| _d| _d | _ | _| _t|| _|dur,||
nt | _t|
|| _t|| _dS )z

        Args:
            dim:
            num_heads:
            qkv_bias:
            qkv_fused:
            attn_drop:
            proj_drop:
            attn_head_dim:
            norm_layer:
        Ng         F)biask_bias)
persistent)super__init__r$   scaler   r    nnLinearqkvq_projk_projv_proj	Parametertorchzerosq_biasZregister_bufferv_biasr-   Dropoutr'   Identitynormprojr(   )selfr#   r$   r%   r&   r'   r(   r)   r*   Zhead_dimZall_head_dim	__class__ X/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/eva.pyr0   2   s2    


zEvaAttention.__init__rope	attn_maskc              
   C   sb  |j \}}}| jd ur| jd ur6t| j| j| jfnd }tj|| jj	|d}|
||d| jdddddd}|d\}	}
}n`| |
||| jddd}	| |
||| jddd}
| |
||| jddd}|d urt|	d d d d d dd d f t|	d d d d dd d d f |gd|}	t|
d d d d d dd d f t|
d d d d dd d d f |gd|}
| jrtj|	|
||| jjd}nt|	| j }	|	|
d	d }|jdd
}|d ur|tj}||d d d d d d f  td}| |}|| }|dd
|||}| |}| |}|  |}|S )N)inputweightr,   r+      r   r      )rH   Z	dropout_pr#   z-inf)!shaper4   r;   r9   catr-   r<   FZlinearrJ   Zreshaper$   ZpermuteZunbindr5   Z	transposer6   r7   r   Ztype_asr    Zscaled_dot_product_attentionr'   pr1   ZsoftmaxtoboolZmasked_fillfloatr?   r@   r(   )rA   xrG   rH   BNCr%   r4   qkvattnrD   rD   rE   forwardg   s>    
$"   
TT

&



zEvaAttention.forward)r!   TTr"   r"   NN)NN)__name__
__module____qualname__r9   jitFinalrU   __annotations__intrV   r   r   r0   Tensorr_   __classcell__rD   rD   rB   rE   r   /   s0   
       8  r   c                       s   e Zd Zddddddddddejedfeeeeeeeeeeee	e e
e
e	e d fddZde	ej e	ej d	d
dZ  ZS )EvaBlockT      @Fr"   Nr#   r$   r%   r&   	mlp_ratio
swiglu_mlp	scale_mlpscale_attn_innerr(   r'   	drop_pathinit_values	act_layerr*   r)   c              
      s2  t    ||| _t|||||
|	||r,|ndd| _|durRt|t| nd| _	|dkrht
|nt | _||| _t|| }|r|rt|||r|nd|	d| _qt||d |r|ndtjd|	d| _nt||||r|nd|	d| _|durt|t| nd| _|dkr$t
|nt | _dS 	ay  

        Args:
            dim:
            num_heads:
            qkv_bias:
            qkv_fused:
            mlp_ratio:
            swiglu_mlp:
            scale_mlp:
            scale_attn_inner:
            proj_drop:
            attn_drop:
            drop_path:
            init_values:
            act_layer:
            norm_layer:
            attn_head_dim:
        N)r$   r%   r&   r'   r(   r)   r*   r"   )in_featureshidden_featuresr*   droprL   F)rt   ru   r*   rr   Z	gate_lastrv   )rt   ru   rr   r*   rv   )r/   r0   norm1r   r^   r2   r8   r9   Zonesgamma_1r   r>   
drop_path1norm2rf   r   mlpr   SiLUr   gamma_2
drop_path2rA   r#   r$   r%   r&   rl   rm   rn   ro   r(   r'   rp   rq   rr   r*   r)   ru   rB   rD   rE   r0      sR    %



"




	
$zEvaBlock.__init__rF   c              	   C   s   | j d u rF|| | j| |||d }|| | | | }nF|| | j | j| |||d  }|| | j| | |  }|S NrF   )rx   ry   r^   rw   r~   r{   rz   r}   rA   rW   rG   rH   rD   rD   rE   r_      s    
 & zEvaBlock.forward)NN)r`   ra   rb   r2   GELUr   rf   rU   rV   r   r   r0   r9   rg   r_   rh   rD   rD   rB   rE   ri      s>   Tri   c                       s   e Zd ZdZddddddddddejejdfeeeee	eeee	e	e	e
e	 eee
e d fdd	Zde
ej e
ej d
ddZ  ZS )EvaBlockPostNormzF EVA block w/ post-norm and support for swiglu, MLP norm scale, ROPE. Trj   Fr"   Nrk   c              
      s   t    t|||||
|	||r"|ndd| _||| _|dkrFt|nt | _t	|| }|r|rt
|||rr|nd|	d| _qt||d |r|ndtjd|	d| _nt||||r|nd|	d| _||| _|dkrt|nt | _dS rs   )r/   r0   r   r^   rw   r   r2   r>   ry   rf   r   r{   r   r|   r   rz   r~   r   rB   rD   rE   r0      sN    %







	

zEvaBlockPostNorm.__init__rF   c              
   C   s>   ||  | | j|||d }|| | | | }|S r   )ry   rw   r^   r~   rz   r{   r   rD   rD   rE   r_   H  s     zEvaBlockPostNorm.forward)NN)r`   ra   rb   __doc__r2   r   r   rf   rU   rV   r   r   r0   r9   rg   r_   rh   rD   rD   rB   rE   r      s@   Rr   c                        sL  e Zd ZdZddddddddd	d	d
dddddddddedd	d	ddddfeeeeef f eeeeef f eeeeeee	e	e
e	e	e	e
e
e
e
e
e
eee
 e	e	e	e	eeeeef ef  e
d fddZdd Zdd Zejjdd Zejjd'ddZejjd(ddZejjdd Zd)ddZd d! Zd*e	d"d#d$Zd%d& Z  ZS )+r   a!   Eva Vision Transformer w/ Abs & Rotary Pos Embed

    This class implements the EVA and EVA02 models that were based on the BEiT ViT variant
      * EVA - abs pos embed, global avg pool
      * EVA02 - abs + rope pos embed, global avg pool, SwiGLU, scale Norm in MLP (ala normformer)
          r+     avg      Trj   Fr"   NgMbP?)img_size
patch_sizein_chansnum_classesglobal_pool	embed_dimdepthr$   r%   r&   rl   rm   rn   ro   	drop_ratepos_drop_ratepatch_drop_rateproj_drop_rateattn_drop_ratedrop_path_rater*   rq   class_tokenuse_abs_pos_embuse_rot_pos_embuse_post_normref_feat_shapehead_init_scalec                    s@  t    || _|| _ | _| _|r*dnd| _d| _t|||d| _	| j	j
}|rhttddnd| _|rttd|| j nd| _tj|d| _|dkrt|| jdd| _nd| _|r|durt|nd}t d| j	j|d	| _nd| _d
d td||D |rtntt 	
fddt|D | _| jdk}|rrt n| _ |rnt | _!t|| _"|dkrt#|nt | _$| %| j& | jdurt'| jdd t'| jdd | (  t)| j$tj#r<t'| j$j*dd | j$j*j+,| | j$j-j+,| dS )a  

        Args:
            img_size:
            patch_size:
            in_chans:
            num_classes:
            global_pool:
            embed_dim:
            depth:
            num_heads:
            qkv_bias:
            qkv_fused:
            mlp_ratio:
            swiglu_mlp:
            scale_mlp:
            scale_attn_inner:
            drop_rate:
            pos_drop_rate:
            proj_drop_rate:
            attn_drop_rate:
            drop_path_rate:
            norm_layer:
            init_values:
            class_token:
            use_abs_pos_emb:
            use_rot_pos_emb:
            use_post_norm:
            ref_feat_shape:
            head_init_scale:
        r   r   F)r   r   r   r   N)rS   T)num_prefix_tokensZreturn_indices)Z	in_pixelsZ
feat_shaper   c                 S   s   g | ]}|  qS rD   )item).0rW   rD   rD   rE   
<listcomp>      z Eva.__init__.<locals>.<listcomp>c                    s2   g | ]*}	
 | d qS ))r#   r$   r%   r&   rl   rm   rn   ro   r(   r'   rp   r*   rq   rD   )r   ir   Zblock_fnZdprr   rq   rl   r*   r$   r   r%   r&   ro   rn   rm   rD   rE   r     s    r   {Gz?std).r/   r0   r   r   Znum_featuresr   r   grad_checkpointingr   patch_embednum_patchesr2   r8   r9   r:   	cls_token	pos_embedr=   pos_dropr   
patch_dropr   r   	grid_sizerG   Zlinspacer   ri   Z
ModuleListrangeblocksr>   r?   fc_norm	head_dropr3   headapply_init_weightsr   fix_init_weight
isinstancerJ   dataZmul_r,   )rA   r   r   r   r   r   r   r   r$   r%   r&   rl   rm   rn   ro   r   r   r   r   r   r   r*   rq   r   r   r   r   r   r   r   Zuse_fc_normrB   r   rE   r0   V  sp    >


(

 zEva.__init__c                 C   sL   dd }t | jD ]4\}}||jjjj|d  ||jjjj|d  qd S )Nc                 S   s   |  td|  d S )Ng       @)Zdiv_mathsqrt)paramlayer_idrD   rD   rE   rescale  s    z$Eva.fix_init_weight.<locals>.rescaler   )	enumerater   r^   r@   rJ   r   r{   Zfc2)rA   r   r   ZlayerrD   rD   rE   r     s    zEva.fix_init_weightc                 C   s6   t |tjr2t|jdd |jd ur2tj|j d S )Nr   r   )r   r2   r3   r   rJ   r,   initZzeros_)rA   mrD   rD   rE   r     s    
zEva._init_weightsc                 C   s   ddh}|S )Nr   r   rD   )rA   ZnwdrD   rD   rE   no_weight_decay  s    zEva.no_weight_decayc                 C   s
   || _ d S N)r   )rA   enablerD   rD   rE   set_grad_checkpointing  s    zEva.set_grad_checkpointingc                 C   s   t dddgd}|S )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)i )stemr   )dict)rA   ZcoarseZmatcherrD   rD   rE   group_matcher  s
    zEva.group_matcherc                 C   s   | j S r   )r   )rA   rD   rD   rE   get_classifier   s    zEva.get_classifierc                 C   s8   || _ |d ur|| _|dkr*t| j|nt | _d S )Nr   )r   r   r2   r3   r   r>   r   )rA   r   r   rD   rD   rE   reset_classifier  s    zEva.reset_classifierc                 C   s   |  |}| jd ur8tj| j|jd dd|fdd}| jd urL|| j }| |}| jd urj| j	 nd }| j
d ur| 
|\}}|d ur|d urt|||}| jD ]0}| jrtj st|||d}q|||d}q| |}|S )Nr   rK   r   rO   )rG   )r   r   r9   rQ   expandrP   r   r   rG   Z	get_embedr   r   r   r   rc   Zis_scriptingr   r?   )rA   rW   Zrot_pos_embedZkeep_indicesZblkrD   rD   rE   forward_features
  s"    

$





zEva.forward_features)
pre_logitsc                 C   sd   | j r>| j dkr.|d d | jd f jddn|d d df }| |}| |}|rZ|S | |S )Nr   r   rO   r   )r   r   meanr   r   r   )rA   rW   r   rD   rD   rE   forward_head%  s
    8

zEva.forward_headc                 C   s   |  |}| |}|S r   )r   r   )rA   rW   rD   rD   rE   r_   ,  s    

zEva.forward)T)F)N)F)r`   ra   rb   r   r   r   rf   r   strrU   rV   r   r   r0   r   r   r9   rc   ignorer   r   r   r   r   r   r   r_   rh   rD   rD   rB   rE   r   N  s   	 


bicubicTc              	   C   s  i }|  d| } |  d| } |  d| } |  d| } d| v rBd}nd| v rPd}nd	}|d
 | v }|d | v }t|}|  D ]x\}	}
|r|	|r||	|d }	nq|d|	v rq|d|	v r|jjjj\}}}}|
jd |ks|
jd |krt|
||f||dd}
nX|	dkrV|
jd |j	jd krVt
|ddr2dn
t
|dd}t|
|jj|||dd}
|	dd}	|	dd}	|	dd}	|	d d!}	|	d"d#}	|	d$d%}	|r|	d&d'}	|	d(d)}	|r|	d*v r|	d+ks|	d,kr||	d-d.}	nq||
||	< q||S )/zJ convert patch embedding weight from manual patchify + linear proj to convZ	model_emamodelmodule
state_dictzvisual.trunk.pos_embedzvisual.trunk.zvisual.pos_embedzvisual. 
mask_tokenzblocks.0.attn.q_proj.weightNrG   zpatch_embed.proj.weightrK   rN   T)interpolation	antialiasverboser   r   Zno_embed_classFr   r   )Znew_sizer   r   r   r   z
mlp.ffn_lnzmlp.normzattn.inner_attn_lnz	attn.normzmlp.w12zmlp.fc1zmlp.w1z	mlp.fc1_gzmlp.w2z	mlp.fc1_xzmlp.w3zmlp.fc2r;   zq_proj.biasr<   zv_proj.bias)r   zlm_head.weightzlm_head.biasnorm.weight	norm.biasr   r   r?   r   )getlenitems
startswithr   r@   rJ   rP   r   r   getattrr   r   replace)r   r   r   r   Zout_dictprefixZmim_weightsZno_qkvZ
len_prefixr\   r]   _HWr   rD   rD   rE   checkpoint_filter_fn2  sn    
"	
r   Fc                 K   s0   | dd rtdtt| |fdti|}|S )NZfeatures_onlyz-features_only not implemented for Eva models.Zpretrained_filter_fn)r   RuntimeErrorr   r   r   )variant
pretrainedkwargsr   rD   rD   rE   _create_eva  s    r   r   c                 K   s"   | ddd dddt tdddd	|S )
Nr   )r+   r   r   g?r   Tzpatch_embed.projr   Zmit)urlr   
input_sizeZ	pool_sizecrop_pctr   Zfixed_input_sizer   r   Z
first_conv
classifierlicense)r	   r
   )r   r   rD   rD   rE   _cfg  s    r   ztimm/)	hf_hub_id)r+   P  r   g      ?Zsquash)r   r   r   	crop_mode)r   r   r   r   r   r   )r+   0  r   )r+     r   )r   r   r   iQU  )r   r   r   r   r   )r   r   z3timm/eva_giant_patch14_clip_224.laion400m_s11b_b41kzopen_clip_pytorch_model.bin   )r   hf_hub_filenamer   z8timm/eva_giant_patch14_plus_clip_224.merged2b_s11b_b114kz3timm/eva02_base_patch16_clip_224.merged2b_s8b_b131ki   z4timm/eva02_large_patch14_clip_224.merged2b_s4b_b131kr   z3timm/eva02_large_patch14_clip_336.merged2b_s6b_b61k)r   r   r   r   r   z6timm/eva02_enormous_patch14_clip_224.laion2b_s4b_b115kz;timm/eva02_enormous_patch14_plus_clip_224.laion2b_s9b_b144k)r   )z"eva_giant_patch14_224.clip_ft_in1kz"eva_giant_patch14_336.clip_ft_in1kz(eva_giant_patch14_336.m30m_ft_in22k_in1kz(eva_giant_patch14_560.m30m_ft_in22k_in1kz.eva02_base_patch14_448.mim_in22k_ft_in22k_in1kz/eva02_large_patch14_448.mim_in22k_ft_in22k_in1kz.eva02_large_patch14_448.mim_m38m_ft_in22k_in1kz(eva02_tiny_patch14_336.mim_in22k_ft_in1kz)eva02_small_patch14_336.mim_in22k_ft_in1kz(eva02_base_patch14_448.mim_in22k_ft_in1kz)eva02_large_patch14_448.mim_in22k_ft_in1kz(eva02_large_patch14_448.mim_m38m_ft_in1kz)eva02_base_patch14_448.mim_in22k_ft_in22kz*eva02_large_patch14_448.mim_in22k_ft_in22kz)eva02_large_patch14_448.mim_m38m_ft_in22kz eva02_tiny_patch14_224.mim_in22kz!eva02_small_patch14_224.mim_in22kz eva02_base_patch14_224.mim_in22kz!eva02_large_patch14_224.mim_in22kz eva02_large_patch14_224.mim_m38mz$eva_giant_patch14_clip_224.laion400mz#eva_giant_patch14_clip_224.merged2bz$eva02_base_patch16_clip_224.merged2bz%eva02_large_patch14_clip_224.merged2bz%eva02_large_patch14_clip_336.merged2bz'eva02_enormous_patch14_clip_224.laion2bz,eva02_enormous_patch14_clip_224.laion2b_plusz(eva02_enormous_patch14_clip_224.pretrain)returnc                 K   s4   t dddddd}td	d| it |fi |}|S )
. EVA-g model https://arxiv.org/abs/2211.07636      (   r   tE]t@r   r   r   r$   rl   eva_giant_patch14_224r   )r  r   r   r   r   Z
model_argsr   rD   rD   rE   r  ;  s    r  c                 K   s4   t dddddd}td	d| it |fi |}|S )
r   r   r   r   r   r   r   eva_giant_patch14_336r   )r  r  r  rD   rD   rE   r  C  s    r  c                 K   s4   t dddddd}td	d| it |fi |}|S )
r   r   r   r   r   r   r   eva_giant_patch14_560r   )r  r  r  rD   rD   rE   r  K  s    r  c                 K   s<   t dddddddddd		}tdd| it |fi |}|S )Nr   r      r   r+   UUUUUU@Tr   r   	r   r   r   r   r$   rl   rm   r   r   eva02_tiny_patch14_224r   )r
  r  r  rD   rD   rE   r
  S  s    r
  c                 K   s<   t dddddddddd		}tdd| it |fi |}|S )Nr   r     r      r  Tr  r	  eva02_small_patch14_224r   )r  r  r  rD   rD   rE   r  d  s    r  c                 K   s@   t dddddddddddd	}tdd| it |fi |}|S )Nr   r   r   r   Fr  Tr  r   r   r   r   r$   r&   rl   rm   rn   r   r   eva02_base_patch14_224r   )r  r  r  rD   rD   rE   r  u  s    r  c                 K   s@   t ddddddddddd	d
}tdd| it |fi |}|S )Nr   r   r      r   r  FTr  r   r   r   r   r$   rl   r&   rm   rn   r   r   eva02_large_patch14_224r   )r  r  r  rD   rD   rE   r    s    r  c                 K   s<   t dddddddddd		}tdd| it |fi |}|S )Nr   r   r  r   r+   r  Tr  r	  eva02_tiny_patch14_336r   )r  r  r  rD   rD   rE   r    s    r  c                 K   s<   t dddddddddd		}tdd| it |fi |}|S )Nr   r   r  r   r  r  Tr  r	  eva02_small_patch14_336r   )r  r  r  rD   rD   rE   r    s    r  c                 K   s@   t dddddddddddd	}tdd| it |fi |}|S )Nr   r   r   r   Fr  Tr  r  eva02_base_patch14_448r   )r  r  r  rD   rD   rE   r    s    r  c                 K   s@   t ddddddddddd	d
}tdd| it |fi |}|S )Nr   r   r   r  r   r  FTr  r  eva02_large_patch14_448r   )r  r  r  rD   rD   rE   r    s    r  c              
   K   s>   t ddddd|ddd}tdd
| it |fi |}|S )zB EVA-g CLIP model (only difference from non-CLIP is the pooling)  r   r   r   r   r   r   token)r   r   r   r$   rl   r   eva_giant_patch14_clip_224r   )r  r   popr   r  rD   rD   rE   r    s    

r  c                 K   sL   t dddddddddddd|d	d
d}tdd| it |fi |}|S )zU A EVA-CLIP specific variant that adds additional attn scale layernorm to eva02_base r   r   r   r   Fr  Tr  r   r  )r   r   r   r   r$   r&   rl   rm   rn   ro   r   r   r   eva02_base_patch16_clip_224r   )r  r  r  rD   rD   rE   r    s"    
r  c                 K   sL   t dddddddddddd	|d
dd}tdd| it |fi |}|S )V A EVA-CLIP specific variant that adds additional attn scale layernorm to eva02_large r   r   r   r  r   r  FTr  r   r  r   r   r   r   r$   rl   r&   rm   rn   ro   r   r   r   eva02_large_patch14_clip_224r   )r  r  r  rD   rD   rE   r    s"    
r  c                 K   sL   t dddddddddddd	|d
dd}tdd| it |fi |}|S )r  r   r   r   r  r   r  FTr  r   r  r  eva02_large_patch14_clip_336r   )r  r  r  rD   rD   rE   r    s"    
r  c                 K   sB   t ddddddd|dd	d
}tdd| it |fi |}|S )zD A EVA-CLIP specific variant that uses residual post-norm in blocks r   r   i   @   r   gI$I$!@Tr   r  )r   r   r   r   r$   rl   r   r   eva02_enormous_patch14_clip_224r   )r!  r  r  rD   rD   rE   r!  /  s    

r!  )r   T)F)r   )F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F)Cr   r   typingr   r   r   r   r9   Ztorch.nnr2   Ztorch.nn.functionalZ
functionalrR   Ztorch.utils.checkpointr   Z	timm.datar   r   r	   r
   Ztimm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Z_builderr   	_registryr   r   __all__Moduler   ri   r   r   r   r   r   Zdefault_cfgsr  r  r  r
  r  r  r  r  r  r  r  r  r  r  r  r!  rD   rD   rD   rE   <module>   sb  De`Z h  
O

  %	