a
    d24                     @   s.  d Z ddlZddlZddlmZ ddlmZ ddlmZm	Z	 ddl
mZmZmZmZmZ ddlmZ ddlmZ dd	lmZ d
gZd$ddZeddddeddddZG dd dejZG dd dejZG dd dejZG dd
 d
ejZdd Zd%ddZed&edd d!Z ed'edd"d#Z!dS )(a   Transformer in Transformer (TNT) in PyTorch

A PyTorch implement of TNT as described in
'Transformer in Transformer' - https://arxiv.org/abs/2103.00112

The official mindspore code is released and available at
https://gitee.com/mindspore/mindspore/tree/master/model_zoo/research/cv/TNT
    N)
checkpointIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)MlpDropPathtrunc_normal__assert	to_2tuple   )build_model_with_cfg)register_model)resize_pos_embedTNT c                 K   s    | ddd dddt tddd|S )	N  )      r   g?ZbicubicTzpixel_embed.projhead)urlnum_classesZ
input_sizeZ	pool_sizeZcrop_pctinterpolationZfixed_input_sizemeanstdZ
first_conv
classifierr   )r   kwargs r   X/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/tnt.py_cfg   s    r   zchttps://github.com/contrastive/pytorch-image-models/releases/download/TNT/tnt_s_patch16_224.pth.tar)      ?r   r   )r   r   r   )r   r   )tnt_s_patch16_224tnt_b_patch16_224c                       s*   e Zd ZdZd	 fdd	Zdd Z  ZS )
	Attentionz Multi-Head Attention
       F        c                    s   t    || _|| _|| }|| _|d | _tj||d |d| _tj|||d| _	tj
|dd| _t||| _tj
|dd| _d S )Ng         biasT)Zinplace)super__init__
hidden_dim	num_headshead_dimscalennLinearqkvDropout	attn_dropproj	proj_drop)selfdimr*   r+   qkv_biasr3   r5   r,   	__class__r   r   r)   1   s    

zAttention.__init__c           
      C   s   |j \}}}| |||d| j| jddddd}|d\}}| |||| jddddd}||dd | j	 }	|	j
dd}	| |	}	|	| dd||d}| |}| |}|S )	Nr%   r   r   r      r7   )shaper0   reshaper+   r,   ZpermuteZunbindr1   	transposer-   Zsoftmaxr3   r4   r5   )
r6   xBNCr0   qkr1   Zattnr   r   r   forward?   s    *$


zAttention.forward)r#   Fr$   r$   __name__
__module____qualname____doc__r)   rH   __classcell__r   r   r9   r   r"   .   s   r"   c                	       s@   e Zd ZdZdddddddejejf	 fdd	Zd	d
 Z  Z	S )Blockz TNT Block
    r;            @Fr$   c                    s   t    ||| _t|||||	|d| _||| _t|t|d |||d| _||| _	t
j|| |dd| _||| _t|||||	|d| _|
dkrt|
nt
 | _||| _t|t|| |||d| _d S )N)r+   r8   r3   r5   r;   )Zin_featuresZhidden_featuresZout_features	act_layerZdropTr&   r$   )r(   r)   norm_inr"   attn_innorm_mlp_inr   intmlp_in
norm1_projr.   r/   r4   norm_outattn_outr   Identity	drop_pathnorm_mlpmlp)r6   r7   dim_out	num_pixelnum_heads_innum_heads_out	mlp_ratior8   r5   r3   r\   rR   
norm_layerr9   r   r   r)   R   sL    

	





zBlock.__init__c              
   C   s   ||  | | | }||  | | | }| \}}}tj|d d ddf |d d dd f | | 	|
||d d gdd}||  | | | }||  | | | }||fS )Nr   r   r<   r>   )r\   rT   rS   rW   rU   sizetorchcatr4   rX   r@   rZ   rY   r^   r]   )r6   pixel_embedpatch_embedrC   rD   rE   r   r   r   rH      s    DzBlock.forward)
rJ   rK   rL   rM   r.   ZGELU	LayerNormr)   rH   rN   r   r   r9   r   rO   O   s   <rO   c                       s*   e Zd ZdZd fdd	Zd	d
 Z  ZS )
PixelEmbedz Image to Pixel Embedding
    r      r   0   r;   c                    s   t    t|}t|}|d |d  |d |d  f| _| jd | jd  }|| _|| _|| _ fdd|D }|| _tj	|| jdd d| _
tj||d| _d S )	Nr   r   c                    s   g | ]}t |  qS r   )mathceil).0Zpsstrider   r   
<listcomp>       z'PixelEmbed.__init__.<locals>.<listcomp>   r   )kernel_sizepaddingrr   )rv   rr   )r(   r)   r
   	grid_sizeimg_sizenum_patchesin_dimnew_patch_sizer.   ZConv2dr4   ZUnfoldunfold)r6   ry   
patch_sizein_chansr{   rr   rz   r|   r9   rq   r   r)      s    
"zPixelEmbed.__init__c                 C   s   |j \}}}}t|| jd kd| d| d| jd  d| jd  d	 t|| jd kd| d| d| jd  d| jd  d	 | |}| |}|dd|| j | j| j	d | j	d }|| }||| j | jddd}|S )	Nr   zInput image size (*z) doesn't match model (r   z).r%   r<   )
r?   r	   ry   r4   r}   rA   r@   rz   r{   r|   )r6   rB   	pixel_posrC   rE   HWr   r   r   rH      s    ((

,zPixelEmbed.forward)r   rl   r   rm   r;   rI   r   r   r9   r   rk      s   rk   c                       s   e Zd ZdZdddddddd	d
d	dddddddejd
f fdd	Zdd Zej	j
dd Zej	j
d%ddZej	j
d&ddZej	j
dd Zd'ddZdd Zd(ed d!d"Zd#d$ Z  ZS ))r   zC Transformer in Transformer - https://arxiv.org/abs/2103.00112
    r   rl   r   r   tokeni   rm   rP   r;   rQ   Fr$   c                    s  t    |dv sJ || _|| _| | _| _d| _t|||||d| _| jj	}|| _	| jj
}|d |d  }||| | _t|| || _||| _ttdd|| _ttd|d || _ttd||d |d | _tj|d| _dd td||D }g }t|D ].}|t||||	|
|||||| |d	 qt|| _||| _t|| _|dkrt||nt  | _!t"| jd
d t"| jd
d t"| jd
d | #| j$ d S )Nr   r   avgF)ry   r~   r   r{   rr   r   r   )pc                 S   s   g | ]}|  qS r   )item)rp   rB   r   r   r   rs      rt   z TNT.__init__.<locals>.<listcomp>)r7   r_   r`   ra   rb   rc   r8   r5   r3   r\   rd   {Gz?r   )%r(   r)   r   global_poolZnum_features	embed_dimgrad_checkpointingrk   rh   rz   r|   rX   r.   r/   r4   
norm2_proj	Parameterrf   zeros	cls_token	patch_posr   r2   pos_dropZlinspacerangeappendrO   Z
ModuleListblocksnorm	head_dropr[   r   r   apply_init_weights)r6   ry   r~   r   r   r   r   	inner_dimdepthZnum_heads_innernum_heads_outerrc   r8   Z	drop_rateZpos_drop_rateZproj_drop_rateZattn_drop_rateZdrop_path_raterd   Zfirst_striderz   r|   r`   Zdprr   ir9   r   r   r)      s`    

 
 zTNT.__init__c                 C   sr   t |tjrBt|jdd t |tjrn|jd urntj|jd n,t |tjrntj|jd tj|jd d S )Nr   r   r   g      ?)	
isinstancer.   r/   r   Zweightr'   initZ	constant_rj   )r6   mr   r   r   r     s    zTNT._init_weightsc                 C   s   h dS )N>   r   r   r   r   r6   r   r   r   no_weight_decay  s    zTNT.no_weight_decayc                 C   s   t dddgd}|S )Nz=^cls_token|patch_pos|pixel_pos|pixel_embed|norm[12]_proj|proj)z^blocks\.(\d+)N)z^norm)i )stemr   )dict)r6   ZcoarseZmatcherr   r   r   group_matcher  s    zTNT.group_matcherTc                 C   s
   || _ d S N)r   )r6   enabler   r   r   set_grad_checkpointing%  s    zTNT.set_grad_checkpointingc                 C   s   | j S r   )r   r   r   r   r   get_classifier)  s    zTNT.get_classifierNc                 C   s>   || _ |d ur|dv sJ |dkr0t| j|nt | _d S )Nr   r   )r   r.   r/   r   r[   r   )r6   r   r   r   r   r   reset_classifier-  s    zTNT.reset_classifierc                 C   s   |j d }| || j}| | | ||| jd}tj	| j
|dd|fdd}|| j }| |}| jrtj s| jD ]}t|||\}}qn| jD ]}|||\}}q| |}|S )Nr   r<   r   r>   )r?   rh   r   r   r4   rX   r@   rz   rf   rg   r   expandr   r   r   jitZis_scriptingr   r   r   )r6   rB   rC   rh   ri   Zblkr   r   r   forward_features3  s    
"




zTNT.forward_features)
pre_logitsc                 C   sX   | j r<| j dkr,|d d dd f jddn|d d df }| |}|rN|S | |S )Nr   r   r>   r   )r   r   r   r   )r6   rB   r   r   r   r   forward_headF  s    6
zTNT.forward_headc                 C   s   |  |}| |}|S r   )r   r   )r6   rB   r   r   r   rH   L  s    

zTNT.forward)F)T)N)F)rJ   rK   rL   rM   r.   rj   r)   r   rf   r   ignorer   r   r   r   r   r   boolr   rH   rN   r   r   r9   r   r      sD   M	



c                 C   s:   | d j |jj kr6t| d |jt|dd|jj| d< | S )zJ convert patch embedding weight from manual patchify + linear proj to convr   Z
num_tokensr   )r?   r   r   getattrrh   rx   )Z
state_dictmodelr   r   r   checkpoint_filter_fnR  s
    r   Fc                 K   s0   | dd rtdtt| |fdti|}|S )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.Zpretrained_filter_fn)getRuntimeErrorr   r   r   )variant
pretrainedr   r   r   r   r   _create_tntZ  s    r   )returnc                 K   s6   t ddddddd}td
d	| it |fi |}|S )Nrl   i     rP      Fr~   r   r   r   r   r8   r    r   )r    r   r   r   r   Z	model_cfgr   r   r   r   r    e  s    
r    c                 K   s6   t ddddddd}td
d	| it |fi |}|S )Nrl   i  (   rP   
   Fr   r!   r   )r!   r   r   r   r   r   r!   n  s    
r!   )r   )F)F)F)"rM   rn   rf   Ztorch.nnr.   Ztorch.utils.checkpointr   Z	timm.datar   r   Ztimm.layersr   r   r   r	   r
   Z_builderr   	_registryr   Zvision_transformerr   __all__r   Zdefault_cfgsModuler"   rO   rk   r   r   r   r    r!   r   r   r   r   <module>   s<   
!M! 
