a
    dt                     @   s,  d Z ddlmZ ddlmZmZmZ ddlZddlm	Z	 ddl
m	  mZ ddlmZmZ ddlmZmZmZmZmZmZmZ ddlmZ dd	lmZmZ d
gZG dd de	jZG dd de	jZ G dd de	jZ!G dd de	jZ"G dd de	jZ#G dd
 d
e	jZ$dd Z%dd Z&dd Z'd:ddZ(d;d d!Z)ee)d"d#e)d"d#e)d"d#e)d"d#e)d"d#e)d"d#e)d"d#e)d"d$d%d&d'd(Z*ed<e$d)d*d+Z+ed=e$d)d,d-Z,ed>e$d)d.d/Z-ed?e$d)d0d1Z.ed@e$d)d2d3Z/edAe$d)d4d5Z0edBe$d)d6d7Z1edCe$d)d8d9Z2dS )Dz 
CoaT architecture.

Paper: Co-Scale Conv-Attentional Image Transformers - https://arxiv.org/abs/2104.06399

Official CoaT code at: https://github.com/mlpc-ucsd/CoaT

Modified from timm/models/vision_transformer.py
    )partial)TupleListUnionNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)
PatchEmbedMlpDropPath	to_2tupletrunc_normal__assert	LayerNorm   )build_model_with_cfg)register_modelgenerate_default_cfgsCoaTc                       s6   e Zd ZdZ fddZeeef dddZ  ZS )ConvRelPosEncz+ Convolutional relative position encoding. c           	   	      s   t    t|tr$||i}|| _nt|tr6|| _nt t | _	g | _
| D ]j\}}d}||d |d   d }tj|  |  ||f||f||f|  d}| j	| | j
| qT fdd| j
D | _dS )aj  
        Initialization.
            Ch: Channels per head.
            h: Number of heads.
            window: Window size(s) in convolutional relative positional encoding. It can have two forms:
                1. An integer of window size, which assigns all attention heads with the same window s
                    size in ConvRelPosEnc.
                2. A dict mapping window size to #attention head splits (
                    e.g. {window size 1: #attention head split 1, window size 2: #attention head split 2})
                    It will apply different window size to the attention head splits.
        r      )kernel_sizepaddingdilationgroupsc                    s   g | ]}|  qS  r   .0xhead_chsr   Y/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/coat.py
<listcomp>C       z*ConvRelPosEnc.__init__.<locals>.<listcomp>N)super__init__
isinstanceintwindowdict
ValueErrornn
ModuleList	conv_listZhead_splitsitemsConv2dappendchannel_splits)	selfr    	num_headsr(   Z
cur_windowZcur_head_splitr   Zpadding_sizeZcur_conv	__class__r   r!   r%      s.    



zConvRelPosEnc.__init__sizec                 C   s  |j \}}}}|\}}	t|d||	  kd |d d d d dd d d f }
|d d d d dd d d f }|dd||| ||	}tj|| jdd}g }t| jD ]\}}|	|||  qtj
|dd}||||||	 dd}|
| }t|d}|S )Nr    dim)r   r   r   r   r   r   )shaper   	transposereshapetorchsplitr1   	enumerater-   r0   catFpad)r2   qvr7   Br3   NCHWZq_imgZv_imgZ
v_img_listZconv_v_img_listiconvZ
conv_v_imgZEV_hatr   r   r!   forwardE   s      zConvRelPosEnc.forward	__name__
__module____qualname____doc__r%   r   r'   rO   __classcell__r   r   r4   r!   r      s   *r   c                       s8   e Zd ZdZd fdd	Zeeef dd	d
Z  ZS )FactorAttnConvRelPosEnczK Factorized attention with convolutional relative position encoding class.    F        Nc                    sh   t    || _|| }|d | _tj||d |d| _t|| _t||| _	t|| _
|| _d S )Ng         )bias)r$   r%   r3   scaler+   LinearqkvDropout	attn_dropproj	proj_dropcrpe)r2   r<   r3   qkv_biasr_   ra   shared_crpeZhead_dimr4   r   r!   r%   ]   s    	

z FactorAttnConvRelPosEnc.__init__r6   c                 C   s   |j \}}}| |||d| j|| j ddddd}|d\}}}	|jdd}
|
dd|	 }|| }| j||	|d	}| j	| | }|dd|||}| 
|}| |}|S )
NrY   r   r   r      r;   r9   r:   r6   )r=   r]   r?   r3   permuteZunbindZsoftmaxr>   rb   r[   r`   ra   )r2   r   r7   rH   rI   rJ   r]   rF   krG   Z	k_softmaxZ
factor_attrb   r   r   r!   rO   s   s    .

zFactorAttnConvRelPosEnc.forward)rW   FrX   rX   NrP   r   r   r4   r!   rV   [   s        rV   c                       s8   e Zd ZdZd fdd	Zeeef dddZ  ZS )	
ConvPosEnczz Convolutional Position Encoding. 
        Note: This module is similar to the conditional position encoding in CPVT.
    rY   c                    s.   t t|   tj|||d|d |d| _d S )Nr   r   )r   )r$   rh   r%   r+   r/   r`   )r2   r<   rg   r4   r   r!   r%      s    zConvPosEnc.__init__r6   c                 C   s   |j \}}}|\}}t|d||  kd |d d d df |d d dd f  }}	|	dd||||}
| |
|
 }|ddd}tj||fdd}|S )Nr   r8   r   r;   )r=   r   r>   viewr`   flattenr@   rC   )r2   r   r7   rH   rI   rJ   rK   rL   	cls_token
img_tokensZfeatr   r   r!   rO      s    *zConvPosEnc.forward)rY   rP   r   r   r4   r!   rh      s   rh   c                	       sN   e Zd ZdZdddddejejddf	 fdd	Zee	e	f dd	d
Z
  ZS )SerialBlockz Serial block class.
        Note: In this implementation, each serial block only contains a conv-attention and a FFN (MLP) module.       @FrX   Nc                    sv   t    |
| _|	|| _t||||||d| _|dkr@t|nt | _	|	|| _
t|| }t||||d| _d S )Nr3   rc   r_   ra   rd   rX   Zin_featuresZhidden_features	act_layerZdrop)r$   r%   cpenorm1rV   factoratt_crper   r+   Identity	drop_pathnorm2r'   r
   mlp)r2   r<   r3   	mlp_ratiorc   ra   r_   rv   rq   
norm_layer
shared_cperd   mlp_hidden_dimr4   r   r!   r%      s(    


zSerialBlock.__init__r6   c                 C   sV   |  ||}| |}| ||}|| | }| |}| |}|| | }|S N)rr   rs   rt   rv   rw   rx   )r2   r   r7   curr   r   r!   rO      s    


zSerialBlock.forward)rQ   rR   rS   rT   r+   GELUr   r%   r   r'   rO   rU   r   r   r4   r!   rm      s   (rm   c                       s   e Zd ZdZg ddddejejdf fdd	Zee	e
e
f ddd	Zee	e
e
f dd
dZee	e
e
f dddZee	e
e
f  dddZ  ZS )ParallelBlockz Parallel block class. FrX   Nc                    s\  t    |	|d | _|	|d | _|	|d | _t|d |||||
d d| _t|d |||||
d d| _t|d |||||
d d| _|dkrt	|nt
 | _|	|d | _|	|d | _|	|d | _|d |d   kr|d ksn J |d |d   kr|d ks"n J t|d |d  }t|d |||d | _ | _| _d S )Nr   r   rY   ro   rX   rp   )r$   r%   norm12norm13norm14rV   factoratt_crpe2factoratt_crpe3factoratt_crpe4r   r+   ru   rv   norm22norm23norm24r'   r
   mlp2mlp3mlp4)r2   dimsr3   
mlp_ratiosrc   ra   r_   rv   rq   rz   shared_crpesr|   r4   r   r!   r%      sR    
&*zParallelBlock.__init__factorr7   c                 C   s   | j |||dS )z Feature map up-sampling. scale_factorr7   interpolater2   r   r   r7   r   r   r!   upsample   s    zParallelBlock.upsamplec                 C   s   | j |d| |dS )z Feature map down-sampling.       ?r   r   r   r   r   r!   
downsample$  s    zParallelBlock.downsampler   c                 C   s   |j \}}}|\}}t|d||  kd |ddddddf }	|ddddddf }
|
dd||||}
tj|
|dddd}
|
||ddd}
tj|	|
fdd	}|S )
z Feature map interpolation. r   r8   Nr   FZbilinear)r   Zrecompute_scale_factormodeZalign_cornersr9   r;   )r=   r   r>   r?   rD   r   r@   rC   )r2   r   r   r7   rH   rI   rJ   rK   rL   rk   rl   outr   r   r!   r   (  s     zParallelBlock.interpolatesizesc                 C   st  |\}}}}	|  |}
| |}| |}| j|
|d}
| j||d}| j||	d}| j|d|d}| j|d|	d}| j|d|	d}| j|
d|d}| j|d|d}| j|
d|d}|
| | }
|| | }|| | }|| |
 }|| | }|| | }| 	|}
| 
|}| |}| |
}
| |}| |}|| |
 }|| | }|| | }||||fS )Nr6   g       @r   rn   )r   r   r   r   r   r   r   r   rv   r   r   r   r   r   r   )r2   x1x2x3x4r   _ZS2ZS3ZS4Zcur2Zcur3Zcur4Zupsample3_2Zupsample4_3Zupsample4_2Zdownsample2_3Zdownsample3_4Zdownsample2_4r   r   r!   rO   ?  s:    








zParallelBlock.forward)rQ   rR   rS   rT   r+   r   r   r%   floatr   r'   r   r   r   r   rO   rU   r   r   r4   r!   r      s   <r   c                       s   e Zd ZdZdddddddd	d
dddddeddddf fdd	Zdd Zejj	dd Z
ejj	d&ddZejj	d'ddZejj	dd Zd(ddZdd Zd)eejeej f ed d!d"Zejd#d$d%Z  ZS )*r   z CoaT class.       rY     @      @     rY   re      rY   r   rW   )re   re   re   re   TrX   FNtokenc                    s>  t    |dv sJ |p$dddd}|_|_ _ d _|_|_t|}t	||| d t
jd_t	dd	 |D d d  d
 t
jd_t	dd	 |D d d
  d t
jd_t	dd	 |D d d  d t
jd_t
td
d
 d _t
td
d
 d
 _t
td
d
 d _t
td
d
 d _t d dd_t d
 dd_t d dd_t d dd_t d | ||d_t d
 | ||d_t d | ||d_t d | ||d_ |}|dks
J t!||
||||dt
" fdd	t#|d D _$t
" fdd	t#|d
 D _%t
" fdd	t#|d D _&t
" fdd	t#|d D _'|_(j(dkrt
" fdd	t#|D _)nd _)jsj)d ur.| d
 _*| d _+nd  _*_+| d _,j(dkr d
  d   krx d ks~n J tj
j-dd
d
d_.t
/|_0|dkrt
1j|nt
2 _3n4d _.t
/|_0|dkrt
1j|nt
2 _3t4jdd t4jdd t4jdd t4jdd 5j6 d S )Nr   avgr   rY   )rY         r9   r   )img_size
patch_sizein_chansZ	embed_dimrz   c                 S   s   g | ]}|d  qS )re   r   r   r   r   r!   r"     r#   z!CoaT.__init__.<locals>.<listcomp>r   c                 S   s   g | ]}|d  qS )rW   r   r   r   r   r!   r"     r#   c                 S   s   g | ]}|d  qS )r   r   r   r   r   r!   r"     r#   )r<   rg   )r    r3   r(   rX   )r3   rc   ra   r_   rv   rz   c              	      s0   g | ](}t f  d  d  jjdqS )r   r<   ry   r{   rd   )rm   cpe1crpe1r   r   
embed_dimsr   r2   Zskwargsr   r!   r"     s   c              	      s0   g | ](}t f  d  d  jjdqS )r   r   )rm   cpe2crpe2r   r   r   r!   r"     s   c              	      s0   g | ](}t f  d  d  jjdqS )r   r   )rm   cpe3crpe3r   r   r   r!   r"     s   c              	      s0   g | ](}t f  d  d  jjdqS )rY   r   )rm   cpe4crpe4r   r   r   r!   r"     s   c              
      s2   g | ]*}t f  jjjjfd qS ))r   r   r   )r   r   r   r   r   r   r   r   r!   r"     s   )Zin_channelsZout_channelsr   {Gz?std)7r$   r%   return_interm_layersout_featuresr   num_featuresnum_classesglobal_poolr   r	   r+   r   patch_embed1patch_embed2patch_embed3patch_embed4	Parameterr@   zeros
cls_token1
cls_token2
cls_token3
cls_token4rh   r   r   r   r   r   r   r   r   r   r)   r,   rangeserial_blocks1serial_blocks2serial_blocks3serial_blocks4parallel_depthparallel_blocksrw   norm3norm4ZConv1d	aggregater^   	head_dropr\   ru   headr   apply_init_weights)r2   r   r   r   r   r   serial_depthsr   r3   r   rc   Z	drop_rateZproj_drop_rateZattn_drop_rateZdrop_path_raterz   r   r   Zcrpe_windowr   Zdprr4   r   r!   r%   d  s    















*$"zCoaT.__init__c                 C   sr   t |tjrBt|jdd t |tjrn|jd urntj|jd n,t |tjrntj|jd tj|jd d S )Nr   r   r   r   )	r&   r+   r\   r   ZweightrZ   initZ	constant_r   )r2   mr   r   r!   r     s    zCoaT._init_weightsc                 C   s   h dS )N>   r   r   r   r   r   r2   r   r   r!   no_weight_decay  s    zCoaT.no_weight_decayc                 C   s   |rJ dd S )Nz$gradient checkpointing not supportedr   )r2   enabler   r   r!   set_grad_checkpointing  s    zCoaT.set_grad_checkpointingc                 C   s"   t ddddddddd	d
gd	}|S )Nz#^cls_token1|patch_embed1|crpe1|cpe1z^serial_blocks1\.(\d+)z#^cls_token2|patch_embed2|crpe2|cpe2z^serial_blocks2\.(\d+)z#^cls_token3|patch_embed3|crpe3|cpe3z^serial_blocks3\.(\d+)z#^cls_token4|patch_embed4|crpe4|cpe4z^serial_blocks4\.(\d+))z^parallel_blocks\.(\d+)N)z^norm|aggregate)i )	Zstem1r   Zstem2r   Zstem3r   Zstem4r   r   )r)   )r2   ZcoarseZmatcherr   r   r!   group_matcher  s    zCoaT.group_matcherc                 C   s   | j S r}   )r   r   r   r   r!   get_classifier/  s    zCoaT.get_classifierc                 C   sD   || _ |d ur |dv sJ || _|dkr6t| j|nt | _d S )Nr   r   )r   r   r+   r\   r   ru   r   )r2   r   r   r   r   r!   reset_classifier3  s
    zCoaT.reset_classifierc                 C   s  |j d }| |}| jj\}}t|| j}| jD ]}||||fd}q2t||||ddddd	 }| 
|}| j
j\}	}
t|| j}| jD ]}|||	|
fd}qt|||	|
ddddd	 }| |}| jj\}}t|| j}| jD ]}||||fd}qt||||ddddd	 }| |}| jj\}}t|| j}| jD ]}||||fd}qXt||||ddddd	 }| jd u rtj s| jri }d| jv r||d< d| jv r||d< d	| jv r||d	< d
| jv r||d
< |S | |}|S | jD ]l}| ||	|
f| |||f| |||f  }}}|||||||f|	|
f||f||fgd\}}}}q tj s| jri }d| jv rt||||ddddd	 }||d< d| jv rt|||	|
ddddd	 }||d< d	| jv rNt||||ddddd	 }||d	< d
| jv rt||||ddddd	 }||d
< |S | |}| |}| |}|||gS d S )Nr   r6   r9   rY   r   r   x1_noclsx2_noclsx3_noclsx4_noclsr   )r=   r   Z	grid_size
insert_clsr   r   
remove_clsr?   rf   
contiguousr   r   r   r   r   r   r   r   r   r   r@   jitis_scriptingr   r   r   r   r   r   rw   r   )r2   Zx0rH   r   ZH1ZW1Zblkr   r   ZH2ZW2r   r   ZH3ZW3r   r   ZH4ZW4r   Zfeat_outr   r   r!   forward_features:  sz    


$

$

$

$

46$$$$


zCoaT.forward_features)x_feat
pre_logitsc                 C   s   t |trh| jd usJ | jdkr<tjdd |D dd}ntjdd |D dd}| |jdd}n6| jdkr|d d dd f jddn|d d df }| 	|}|r|S | 
|S )Nr   c                 S   s*   g | ]"}|d d dd f j dddqS )Nr   T)r<   Zkeepdim)meanr   Zxlr   r   r!   r"     r#   z%CoaT.forward_head.<locals>.<listcomp>r   r;   c                 S   s   g | ]}|d d df qS )Nr   r   r   r   r   r!   r"     r#   r   )r&   listr   r   r@   rC   stackZsqueezer   r   r   )r2   r   r   r   r   r   r!   forward_head  s    

6
zCoaT.forward_headreturnc                 C   s6   t j s| jr| |S | |}| |}|S d S r}   )r@   r   r   r   r   r   )r2   r   r   r   r   r!   rO     s
    


zCoaT.forward)T)F)N)F)rQ   rR   rS   rT   r   r%   r   r@   r   ignorer   r   r   r   r   r   r   ZTensorr   boolr   rO   rU   r   r   r4   r!   r   b  sF    )	


Q"c                 C   s*   | | jd dd}tj|| fdd} | S )z Insert CLS token. r   r9   r   r;   )expandr=   r@   rC   )r   rk   Z
cls_tokensr   r   r!   r     s    r   c                 C   s   | ddddddf S )z Remove CLS token. Nr   r   )r   r   r   r!   r     s    r   c                 C   sb   i }|  d| } |  D ]D\}}|ds|jd u r>|ds|jd u rT|drTq|||< q|S )Nmodelrs   rw   r   )getr.   
startswithrw   r   )Z
state_dictr  Zout_dictrg   rG   r   r   r!   checkpoint_filter_fn  s    

r  Fc                 K   s0   | dd rtdtt| |fdti|}|S )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.Zpretrained_filter_fn)r  RuntimeErrorr   r   r  )variant
pretrainedZdefault_cfgkwargsr  r   r   r!   _create_coat  s    r
  r8   c                 K   s    | ddd dddt tddd|S )	Nr   )rY   r   r   g?ZbicubicTzpatch_embed1.projr   )urlr   
input_sizeZ	pool_sizecrop_pctinterpolationZfixed_input_sizer   r   Z
first_conv
classifierr   )r  r	  r   r   r!   	_cfg_coat  s    r  ztimm/)	hf_hub_id)rY     r  r   Zsquash)r  r  r  Z	crop_mode)zcoat_tiny.in1kzcoat_mini.in1kzcoat_small.in1kzcoat_lite_tiny.in1kzcoat_lite_mini.in1kzcoat_lite_small.in1kzcoat_lite_medium.in1kzcoat_lite_medium_384.in1kr   c                 K   s:   t dg dg ddd}tdd| it |fi |}|S )	Nre   )   r  r  r  r   r   r   r   r   r   r   r   r   	coat_tinyr  )r  r)   r
  r  r	  Z	model_cfgr  r   r   r!   r    s
    r  c                 K   s:   t dg dg ddd}tdd| it |fi |}|S )	Nre   )r     r  r  r  r   r  	coat_minir  )r  r  r  r   r   r!   r    s
    r  c                 K   sB   t f dg dg ddd|}tdd| it |fi |}|S )	Nre   )r  r   r   r   r  r   r  
coat_smallr  )r  r  r  r   r   r!   r    s    r  c                 K   s>   t dg dg dg dd}tdd| it |fi |}|S )	Nre   )r   r      r   r  rW   rW   re   re   r   r   r   r   coat_lite_tinyr  )r  r  r  r   r   r!   r    s
    r  c                 K   s>   t dg dg dg dd}tdd| it |fi |}|S )	Nre   r   r  r  r  coat_lite_minir  )r   r  r  r   r   r!   r     s
    r   c                 K   s>   t dg dg dg dd}tdd| it |fi |}|S )	Nre   r   r   r  r  coat_lite_smallr  )r!  r  r  r   r   r!   r!    s
    r!  c                 K   s8   t dg dg dd}tdd| it |fi |}|S )Nre   r   r  r   r   rY   r   
   rW   )r   r   r   coat_lite_mediumr  )r%  r  r  r   r   r!   r%    s
    r%  c                 K   s:   t ddg dg dd}tdd| it |fi |}|S )	Nr  re   r"  r#  )r   r   r   r   coat_lite_medium_384r  )r&  r  r  r   r   r!   r&    s
    r&  )FN)r8   )F)F)F)F)F)F)F)F)3rT   	functoolsr   typingr   r   r   r@   Ztorch.nnr+   Ztorch.nn.functionalZ
functionalrD   Z	timm.datar   r   Ztimm.layersr	   r
   r   r   r   r   r   Z_builderr   	_registryr   r   __all__Moduler   rV   rh   rm   r   r   r   r   r  r
  r  Zdefault_cfgsr  r  r  r  r   r!  r%  r&  r   r   r   r!   <module>   sj   	$B2:   C

