a
    dF                     @   s:  d Z ddlmZ ddlZddlmZ ddlmZmZ ddl	m
Z
mZmZmZmZ ddlmZ ddlmZ dd	lmZmZ g d
ZG dd dejZG dd dejZG dd dejZG dd dejZG dd dejZdBddZdCddZdDddZeedddd edd!d"edd#dd edd$d"edd%d"edd&dd edd'd"edd(d"edd)d"edd*d+d d,
Z edEed-d.d/Z!edFed-d0d1Z"edGed-d2d3Z#edHed-d4d5Z$edIed-d6d7Z%edJed-d8d9Z&edKed-d:d;Z'edLed-d<d=Z(edMed-d>d?Z)edNed-d@dAZ*dS )Oa3   Class-Attention in Image Transformers (CaiT)

Paper: 'Going deeper with Image Transformers' - https://arxiv.org/abs/2103.17239

Original code and weights from https://github.com/facebookresearch/deit, copyright below

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    )partialNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)
PatchEmbedMlpDropPathtrunc_normal_use_fused_attn   )build_model_with_cfg)checkpoint_seq)register_modelgenerate_default_cfgs)Cait	ClassAttnLayerScaleBlockClassAttnLayerScaleBlockTalkingHeadAttnc                       s8   e Zd ZU ejje ed< d	 fdd	Zdd Z	  Z
S )
r   
fused_attn   F        c                    s   t    || _|| }|d | _t | _tj|||d| _tj|||d| _	tj|||d| _
t|| _t||| _t|| _d S )N      ࿩bias)super__init__	num_headsscaler
   r   nnLinearqkvDropout	attn_dropproj	proj_dropselfdimr   qkv_biasr%   r'   Zhead_dim	__class__ Y/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/cait.pyr      s    

zClassAttn.__init__c           
      C   s$  |j \}}}| |d d df d|d| j|| j dddd}| |||| j|| j dddd}| |||| j|| j dddd}| jrt	j
jj|||| jjd}n8|| j }||dd }	|	jdd}	| |	}	|	| }|dd|d|}| |}| |}|S )	Nr   r         )Z	dropout_pr*   )shaper!   Z	unsqueezereshaper   permuter"   r#   r   torchr   Z
functionalZscaled_dot_product_attentionr%   pr   	transposesoftmaxr&   r'   )
r)   xBNCr!   r"   r#   x_clsattnr.   r.   r/   forward,   s$    <**



zClassAttn.forward)r   Fr   r   )__name__
__module____qualname__r8   jitFinalbool__annotations__r   rB   __classcell__r.   r.   r,   r/   r      s   
r   c                
       s>   e Zd Zdddddejejeedf
 fdd	Zdd Z	  Z
S )	r         @Fr   -C6?c                    s   t    |	|| _|
|||||d| _|dkr8t|nt | _|	|| _t	|| }|||||d| _
t|t| | _t|t| | _d S N)r   r+   r%   r'   r   )Zin_featuresZhidden_features	act_layerZdropr   r   norm1rA   r   r   Identity	drop_pathnorm2intmlp	Parameterr8   Zonesgamma_1gamma_2r)   r*   r   	mlp_ratior+   r'   r%   rR   rN   
norm_layer
attn_block	mlp_blockinit_valuesZmlp_hidden_dimr,   r.   r/   r   H   s(    


z!LayerScaleBlockClassAttn.__init__c              	   C   sV   t j||fdd}|| | j| | |  }|| | j| | |  }|S )Nr   r4   )	r8   catrR   rW   rA   rP   rX   rU   rS   )r)   r<   r@   ur.   r.   r/   rB   l   s      z LayerScaleBlockClassAttn.forward)rC   rD   rE   r   GELU	LayerNormr   r   r   rB   rJ   r.   r.   r,   r/   r   E   s   $r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )	r   r   Fr   c                    s~   t    || _|| }|d | _tj||d |d| _t|| _t||| _	t||| _
t||| _t|| _d S )Nr   r1   r   )r   r   r   r   r   r    qkvr$   r%   r&   proj_lproj_wr'   r(   r,   r.   r/   r   v   s    

zTalkingHeadAttn.__init__c           
      C   s   |j \}}}| |||d| j|| j ddddd}|d | j |d |d   }}}||dd }	| |	dddddddd}	|	jdd}	| 	|	dddddddd}	| 
|	}	|	| dd|||}| |}| |}|S )	Nr1   r0   r   r      r2   r3   r4   )r5   rc   r6   r   r7   r   r:   rd   r;   re   r%   r&   r'   )
r)   r<   r=   r>   r?   rc   r!   r"   r#   rA   r.   r.   r/   rB      s    ."""


zTalkingHeadAttn.forward)r   Fr   r   )rC   rD   rE   r   rB   rJ   r.   r.   r,   r/   r   s   s   r   c                
       s>   e Zd Zdddddejejeedf
 fdd	Zdd Z	  Z
S )	r   rK   Fr   rL   c                    s   t    |	|| _|
|||||d| _|dkr8t|nt | _|	|| _t	|| }|||||d| _
t|t| | _t|t| | _d S rM   rO   rY   r,   r.   r/   r      s(    


zLayerScaleBlock.__init__c              	   C   sD   ||  | j| | |  }||  | j| | |  }|S N)rR   rW   rA   rP   rX   rU   rS   r)   r<   r.   r.   r/   rB      s      zLayerScaleBlock.forward)rC   rD   rE   r   ra   rb   r   r   r   rB   rJ   r.   r.   r,   r/   r      s   $r   c                       s   e Zd Zdddddddddd	d
d
d
d
d
eeeeejddej	e
edeeddf fdd	Zdd Zejjdd Zejjd&ddZejjd'ddZejjdd Zd(ddZdd  Zd)ed!d"d#Zd$d% Z  ZS )*r         r1     token      rK   Tr   ư>)ZepsrL   r0   c                    sx  t    |dv sJ || _|| _ | _| _d| _||||d| _| jj}t	
tdd| _t	
td|| _t	j|d| _fddt|D t	j 	
fddt|D  | _t	 	f
d	dt|D | _| _td
ddg| _t	|| _|d
krBt	|nt	 | _t| jdd t| jdd | | j d S )N rl   avgF)img_size
patch_sizein_chans	embed_dimr   )r9   c                    s   g | ]} qS r.   r.   .0i)drop_path_rater.   r/   
<listcomp>       z!Cait.__init__.<locals>.<listcomp>c                    s0   g | ](}
| 	 d qS ))r*   r   rZ   r+   r'   r%   rR   r[   rN   r\   r]   r^   r.   rw   )rN   r\   attn_drop_rateblock_layersdprrv   r^   r]   rZ   r[   r   proj_drop_rater+   r.   r/   r{     s   c                    s&   g | ]}	 d 	qS ))	r*   r   rZ   r+   r[   rN   r\   r]   r^   r.   )rx   _)
rN   attn_block_token_onlyblock_layers_tokenrv   r^   mlp_block_token_onlymlp_ratio_token_onlyr[   r   r+   r.   r/   r{     s   
r   head)Znum_chsZ	reductionmodule{Gz?std) r   r   num_classesglobal_poolnum_featuresrv   grad_checkpointingpatch_embednum_patchesr   rV   r8   zeros	cls_token	pos_embedr$   pos_droprangeZ
SequentialblocksZ
ModuleListblocks_token_onlynormdictZfeature_info	head_dropr    rQ   r   r	   apply_init_weights)r)   rs   rt   ru   r   r   rv   depthr   rZ   r+   Z	drop_rateZpos_drop_rater   r}   rz   r~   r   Zpatch_layerr[   rN   r\   r]   r^   r   r   Zdepth_token_onlyr   r   r,   )rN   r\   r   r}   r~   r   r   rz   rv   r^   r]   r   rZ   r   r[   r   r   r+   r/   r      s<    
&
 


 zCait.__init__c                 C   sr   t |tjrBt|jdd t |tjrn|jd urntj|jd n,t |tjrntj|jd tj|jd d S )Nr   r   r         ?)	
isinstancer   r    r	   Zweightr   initZ	constant_rb   )r)   mr.   r.   r/   r   &  s    zCait._init_weightsc                 C   s   ddhS )Nr   r   r.   r)   r.   r.   r/   no_weight_decay/  s    zCait.no_weight_decayc                 C   s
   || _ d S rg   )r   )r)   enabler.   r.   r/   set_grad_checkpointing3  s    zCait.set_grad_checkpointingFc                    s    fdd}|S )Nc                    s   t  fdddD rdS  dr:t dd d S  drrtjtj d }t dd | S  d	rtjS td
S d S )Nc                    s   g | ]}  |qS r.   )
startswith)rx   nnamer.   r/   r{   :  r|   z8Cait.group_matcher.<locals>._matcher.<locals>.<listcomp>)r   r   r   r   zblocks..r   zblocks_token_only.znorm.inf)anyr   rT   splitlenr   r   float)r   Z	to_offsetr   r   r/   _matcher9  s    



z$Cait.group_matcher.<locals>._matcherr.   )r)   Zcoarser   r.   r   r/   group_matcher7  s    zCait.group_matcherc                 C   s   | j S rg   )r   r   r.   r.   r/   get_classifierH  s    zCait.get_classifierNc                 C   sD   || _ |d ur |dv sJ || _|dkr6t| j|nt | _d S )Nrp   r   )r   r   r   r    r   rQ   r   )r)   r   r   r.   r.   r/   reset_classifierL  s
    zCait.reset_classifierc                 C   s   |  |}|| j }| |}| jr<tj s<t| j|}n
| |}| j	
|jd dd}t| jD ]\}}|||}qftj||fdd}| |}|S )Nr   r3   r   r4   )r   r   r   r   r8   rF   Zis_scriptingr   r   r   expandr5   	enumerater   r_   r   )r)   r<   Z
cls_tokensry   Zblkr.   r.   r/   forward_featuresS  s    




zCait.forward_features)
pre_logitsc                 C   sX   | j r<| j dkr,|d d dd f jddn|d d df }| |}|rN|S | |S )Nrr   r   r4   r   )r   meanr   r   )r)   r<   r   r.   r.   r/   forward_headb  s    6
zCait.forward_headc                 C   s   |  |}| |}|S rg   )r   r   rh   r.   r.   r/   rB   h  s    

zCait.forward)T)F)N)F)rC   rD   rE   r   r   r   r   r   rb   ra   r   r   r   r   r   r8   rF   ignorer   r   r   r   r   r   rH   r   rB   rJ   r.   r.   r,   r/   r      sR   Y	


r   c                 C   s:   d| v r| d } i }|   D ]\}}|||dd< q|S )Nmodelzmodule.rq   )itemsreplace)Z
state_dictr   Zcheckpoint_no_moduler"   r#   r.   r.   r/   checkpoint_filter_fnn  s    r   Fc                 K   s0   | dd rtdtt| |fdti|}|S )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.Zpretrained_filter_fn)getRuntimeErrorr   r   r   )variant
pretrainedkwargsr   r.   r.   r/   _create_caitw  s    r   rq   c                 K   s    | ddd dddt tddd|S )	Nrk   )r1     r   r   ZbicubicTzpatch_embed.projr   )urlr   
input_sizeZ	pool_sizeZcrop_pctinterpolationZfixed_input_sizer   r   Z
first_conv
classifierr   )r   r   r.   r.   r/   _cfg  s    r   ztimm/z1https://dl.fbaipublicfiles.com/deit/XXS24_224.pth)r1   ri   ri   )	hf_hub_idr   r   z1https://dl.fbaipublicfiles.com/deit/XXS24_384.pth)r   r   z1https://dl.fbaipublicfiles.com/deit/XXS36_224.pthz1https://dl.fbaipublicfiles.com/deit/XXS36_384.pthz0https://dl.fbaipublicfiles.com/deit/XS24_384.pthz/https://dl.fbaipublicfiles.com/deit/S24_224.pthz/https://dl.fbaipublicfiles.com/deit/S24_384.pthz/https://dl.fbaipublicfiles.com/deit/S36_384.pthz/https://dl.fbaipublicfiles.com/deit/M36_384.pthz/https://dl.fbaipublicfiles.com/deit/M48_448.pth)r1     r   )
zcait_xxs24_224.fb_dist_in1kzcait_xxs24_384.fb_dist_in1kzcait_xxs36_224.fb_dist_in1kzcait_xxs36_384.fb_dist_in1kzcait_xs24_384.fb_dist_in1kzcait_s24_224.fb_dist_in1kzcait_s24_384.fb_dist_in1kzcait_s36_384.fb_dist_in1kzcait_m36_384.fb_dist_in1kzcait_m48_448.fb_dist_in1k)returnc                 K   s4   t dddddd}td	d| it |fi |}|S )
Nrj         rf   h㈵>rt   rv   r   r   r^   cait_xxs24_224r   )r   r   r   r   r   Z
model_argsr   r.   r.   r/   r     s    r   c                 K   s4   t dddddd}td	d| it |fi |}|S )
Nrj   r   r   rf   r   r   cait_xxs24_384r   )r   r   r   r.   r.   r/   r     s    r   c                 K   s4   t dddddd}td	d| it |fi |}|S )
Nrj   r   $   rf   r   r   cait_xxs36_224r   )r   r   r   r.   r.   r/   r     s    r   c                 K   s4   t dddddd}td	d| it |fi |}|S )
Nrj   r   r   rf   r   r   cait_xxs36_384r   )r   r   r   r.   r.   r/   r     s    r   c                 K   s4   t dddddd}td	d| it |fi |}|S )
Nrj   i   r      r   r   cait_xs24_384r   )r   r   r   r.   r.   r/   r     s    r   c                 K   s4   t dddddd}td	d| it |fi |}|S )
Nrj   r   r   r   r   r   cait_s24_224r   )r   r   r   r.   r.   r/   r     s    r   c                 K   s4   t dddddd}td	d| it |fi |}|S )
Nrj   r   r   r   r   r   cait_s24_384r   )r   r   r   r.   r.   r/   r     s    r   c                 K   s4   t dddddd}td	d| it |fi |}|S )
Nrj   r   r   r   ro   r   cait_s36_384r   )r   r   r   r.   r.   r/   r     s    r   c                 K   s4   t dddddd}tdd| it |fi |}|S )	Nrj   rm   r   ro   r   cait_m36_384r   )r   r   r   r.   r.   r/   r     s    r   c                 K   s4   t dddddd}tdd| it |fi |}|S )	Nrj   rm   0   ro   r   cait_m48_448r   )r   r   r   r.   r.   r/   r     s    r   )N)F)rq   )F)F)F)F)F)F)F)F)F)F)+__doc__	functoolsr   r8   Ztorch.nnr   Z	timm.datar   r   Ztimm.layersr   r   r   r	   r
   Z_builderr   Z_manipulater   	_registryr   r   __all__Moduler   r   r   r   r   r   r   r   Zdefault_cfgsr   r   r   r   r   r   r   r   r   r   r.   r.   r.   r/   <module>   s   
,.*- %
	

0