a
    þdÅ;  ã                   @   sL  d Z ddlmZ ddlZddlmZ ddlmZmZ ddl	m
Z
mZmZmZmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ dgZeG dd„ dejƒƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZd#dd„Zd$dd„ZeeddeddedddœƒZ ed%edœdd„ƒZ!ed&edœdd „ƒZ"ed'edœd!d"„ƒZ#dS )(a   ConViT Model

@article{d2021convit,
  title={ConViT: Improving Vision Transformers with Soft Convolutional Inductive Biases},
  author={d'Ascoli, St{'e}phane and Touvron, Hugo and Leavitt, Matthew and Morcos, Ari and Biroli, Giulio and Sagun, Levent},
  journal={arXiv preprint arXiv:2103.10697},
  year={2021}
}

Paper link: https://arxiv.org/abs/2103.10697
Original code: https://github.com/facebookresearch/convit, original copyright below

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
é    )ÚpartialN©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚtrunc_normal_Ú
PatchEmbedÚMlpÚ	LayerNormé   )Úbuild_model_with_cfg)Úregister_notrace_module)Úregister_modelÚgenerate_default_cfgs)ÚHybridEmbedÚConVitc                       sR   e Zd Zd‡ fdd„	Zdd„ Zd	d
„ Zddd„Zdd„ Zee	j
dœdd„Z‡  ZS )ÚGPSAé   Fç        ç      ð?c                    s´   t ƒ  ¡  || _|| _|| }|d | _|| _tj||d |d| _tj|||d| _	t 
|¡| _t ||¡| _t d|¡| _t 
|¡| _t t | j¡¡| _t dddd¡| _d S )Nç      à¿é   ©Úbiasé   r   )ÚsuperÚ__init__Ú	num_headsÚdimÚscaleÚlocality_strengthÚnnÚLinearÚqkÚvÚDropoutÚ	attn_dropÚprojÚpos_projÚ	proj_dropÚ	ParameterÚtorchZonesÚgating_paramÚzerosÚrel_indices)Úselfr   r   Úqkv_biasr&   r)   r    Úhead_dim©Ú	__class__© ú[/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/convit.pyr   +   s    	

zGPSA.__init__c                 C   s˜   |j \}}}| jd u s&| jj d |kr2|  |¡| _|  |¡}|  |¡ ||| j|| j ¡ dddd¡}||  dd¡ |||¡}|  	|¡}|  
|¡}|S )Nr   r   r   r   )Úshaper.   Úget_rel_indicesÚget_attentionr$   Úreshaper   ÚpermuteÚ	transposer'   r)   )r/   ÚxÚBÚNÚCÚattnr$   r4   r4   r5   ÚforwardE   s    
*

zGPSA.forwardc                 C   sø   |j \}}}|  |¡ ||d| j|| j ¡ ddddd¡}|d |d  }}| j |ddd¡}|  |¡ dddd¡}|| dd¡ | j	 }	|	j
dd}	|j
dd}| j dddd¡}
d	t |
¡ |	 t |
¡|  }||jdd d¡ }|  |¡}|S )
Nr   r   r   r   é   éÿÿÿÿéþÿÿÿ©r   r   )r6   r#   r9   r   r:   r.   Úexpandr(   r;   r   Úsoftmaxr,   Úviewr+   ZsigmoidÚsumÚ	unsqueezer&   )r/   r<   r=   r>   r?   r#   ÚqÚkZ	pos_scoreZpatch_scoreZgatingr@   r4   r4   r5   r8   P   s    . 
zGPSA.get_attentionc                 C   s^   |   |¡ d¡}| j ¡ d d …d d …df d }t d||f¡| d¡ }|rV||fS |S d S )Nr   rC   ç      à?ú	nm,hnm->h)r8   Úmeanr.   Zsqueezer+   ÚeinsumÚsize)r/   r<   Ú
return_mapÚattn_mapÚ	distancesÚdistr4   r4   r5   Úget_attention_map`   s     zGPSA.get_attention_mapc                 C   sØ   | j jj t | j¡¡ d}t| jd ƒ}|d dkrB|d d n|d }t	|ƒD ]l}t	|ƒD ]^}|||  }d| j
jj|df< d||  | | j
jj|df< d||  | | j
jj|df< q^qR| j
j j| j9  _d S )Nr   rM   r   r   rC   )r$   ÚweightÚdataZcopy_r+   Zeyer   Úintr   Úranger(   r    )r/   Zlocality_distanceZkernel_sizeÚcenterÚh1Úh2Úpositionr4   r4   r5   Ú
local_initi   s     "zGPSA.local_init)Únum_patchesÚreturnc           	      C   sî   t |d ƒ}t d||d¡}t |¡ dd¡t |¡ dd¡ }| ||¡}|j|ddj|dd}|d |d  }| d¡|d d …d d …d d …df< | d¡|d d …d d …d d …df< | d¡|d d …d d …d d …df< | jj	j
}| |¡S )NrM   r   r   rC   r   rE   r   )rY   r+   r-   ÚarangerH   ÚrepeatÚrepeat_interleaverJ   r#   rW   ÚdeviceÚto)	r/   r`   Úimg_sizer.   ÚindÚindxÚindyÚinddre   r4   r4   r5   r7   w   s    $"""
zGPSA.get_rel_indices)r   Fr   r   r   )F)Ú__name__Ú
__module__Ú__qualname__r   rA   r8   rV   r_   rY   r+   ZTensorr7   Ú__classcell__r4   r4   r2   r5   r   )   s        ù
	r   c                       s0   e Zd Zd
‡ fdd„	Zddd„Zdd	„ Z‡  ZS )ÚMHSAr   Fr   c                    sb   t ƒ  ¡  || _|| }|d | _tj||d |d| _t |¡| _t ||¡| _	t |¡| _
d S )Nr   r   r   )r   r   r   r   r!   r"   Úqkvr%   r&   r'   r)   )r/   r   r   r0   r&   r)   r1   r2   r4   r5   r   †   s    

zMHSA.__init__c                 C   s   |j \}}}|  |¡ ||d| j|| j ¡ ddddd¡}|d |d |d   }}}	|| dd¡ | j }
|
jdd d¡}
t	|d	 ƒ}t
 |¡ dd¡t
 |¡ dd¡ }| ||¡}|j|ddj|dd}|d |d  }|d	 }| |j¡}t
 d
||
f¡| }|r||
fS |S d S )Nr   r   r   r   rB   rD   rC   rE   rM   rN   )r6   rq   r9   r   r:   r;   r   rG   rO   rY   r+   rb   rH   rc   rd   rf   re   rP   )r/   r<   rR   r=   r>   r?   rq   rK   rL   r$   rS   rg   rh   ri   rj   rk   rT   rU   r4   r4   r5   rV   ˜   s     .$zMHSA.get_attention_mapc           
      C   s¨   |j \}}}|  |¡ ||d| j|| j ¡ ddddd¡}| d¡\}}}|| dd¡ | j }	|	jdd}	|  	|	¡}	|	|  dd¡ |||¡}|  
|¡}|  |¡}|S )	Nr   r   r   r   rB   rD   rC   rE   )r6   rq   r9   r   r:   Zunbindr;   r   rG   r&   r'   r)   )
r/   r<   r=   r>   r?   rq   rK   rL   r$   r@   r4   r4   r5   rA   ­   s    .


zMHSA.forward)r   Fr   r   )F)rl   rm   rn   r   rV   rA   ro   r4   r4   r2   r5   rp   …   s       ú
rp   c                	       s:   e Zd Zdddddejeddf	‡ fdd„	Zdd	„ Z‡  ZS )
ÚBlockç      @Fr   Tr   c                    s’   t ƒ  ¡  |	|ƒ| _|
| _| jr8t||||||d| _nt|||||d| _|dkr\t|ƒnt 	¡ | _
|	|ƒ| _t|| ƒ}t||||d| _d S )N)r   r0   r&   r)   r    )r   r0   r&   r)   r   )Zin_featuresZhidden_featuresÚ	act_layerZdrop)r   r   Únorm1Úuse_gpsar   r@   rp   r   r!   ÚIdentityÚ	drop_pathÚnorm2rY   r	   Úmlp)r/   r   r   Ú	mlp_ratior0   r)   r&   rx   rt   Ú
norm_layerrv   r    Zmlp_hidden_dimr2   r4   r5   r   ¾   s8    

ú
	û
üzBlock.__init__c                 C   s8   ||   |  |  |¡¡¡ }||   |  |  |¡¡¡ }|S ©N)rx   r@   ru   rz   ry   ©r/   r<   r4   r4   r5   rA   ê   s    zBlock.forward)	rl   rm   rn   r!   ZGELUr
   r   rA   ro   r4   r4   r2   r5   rr   ¼   s   ô,rr   c                       sÀ   e Zd ZdZddddddddd	d
ddddddedddf‡ fdd„	Zdd„ Zejj	dd„ ƒZ
ejj	d$dd„ƒZejj	d%dd„ƒZejj	dd„ ƒZd&dd„Zdd„ Zd'edœd d!„Zd"d#„ Z‡  ZS )(r   zI Vision Transformer with support for patch or hybrid CNN input stage
    éà   é   r   éè  Útokeni   é   rs   Fr   Nr   Tc                    s¨  t ƒ  ¡  |dv sJ ‚ˆˆ9 ‰|| _|| _ˆ| _ˆ | _| _ˆ| _|| _|d urdt	|||ˆd| _
nt|||ˆd| _
| j
j}|| _t t ddˆ¡¡| _tj|d| _| jrÒt t d|ˆ¡¡| _t| jdd dd	„ t d
||¡D ƒ‰t ‡ ‡‡‡‡‡‡‡‡‡	f
dd	„t|ƒD ƒ¡| _ˆˆƒ| _tˆd
ddg| _t |¡| _|d
krXt ˆ|¡nt ¡ | _ t| jdd |  !| j"¡ |  #¡ D ]\}}t$|dƒr„| %¡  q„d S )N)Ú Úavgr‚   )rg   Úin_chansÚ	embed_dim)rg   Ú
patch_sizer†   r‡   r   )Úpç{®Gáz”?©Ústdc                 S   s   g | ]}|  ¡ ‘qS r4   )Úitem)Ú.0r<   r4   r4   r5   Ú
<listcomp>)  ó    z#ConVit.__init__.<locals>.<listcomp>r   c                    s0   g | ](}t ˆˆˆˆ	ˆˆ ˆ| ˆ|ˆk ˆd 
‘qS ))
r   r   r{   r0   r)   r&   rx   r|   rv   r    )rr   )rŽ   Úi©
Úattn_drop_rateZdprr‡   Úlocal_up_to_layerr    r{   r|   r   Úproj_drop_rater0   r4   r5   r   *  s   õöÚhead)Znum_chsZ	reductionÚmoduler_   )&r   r   Únum_classesÚglobal_poolr”   Znum_featuresr‡   r    Úuse_pos_embedr   Úpatch_embedr   r`   r!   r*   r+   r-   Ú	cls_tokenr%   Úpos_dropÚ	pos_embedr   ZlinspaceZ
ModuleListrZ   ÚblocksÚnormÚdictZfeature_infoÚ	head_dropr"   rw   r–   ÚapplyÚ_init_weightsZnamed_modulesÚhasattrr_   )r/   rg   rˆ   r†   r˜   r™   r‡   Údepthr   r{   r0   Z	drop_rateZpos_drop_rater•   r“   Zdrop_path_rateZhybrid_backboner|   r”   r    rš   r`   ÚnÚmr2   r’   r5   r   ô   sN    
ÿ
ü ô

 zConVit.__init__c                 C   sr   t |tjƒrBt|jdd t |tjƒrn|jd urntj |jd¡ n,t |tjƒrntj |jd¡ tj |jd¡ d S )NrŠ   r‹   r   r   )	Ú
isinstancer!   r"   r   rW   r   ÚinitZ	constant_r
   )r/   r¨   r4   r4   r5   r¤   D  s    zConVit._init_weightsc                 C   s   ddhS )Nrž   rœ   r4   ©r/   r4   r4   r5   Úno_weight_decayM  s    zConVit.no_weight_decayc                 C   s   t dddgdS )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)iŸ† )ÚstemrŸ   )r¡   )r/   Zcoarser4   r4   r5   Úgroup_matcherQ  s    þzConVit.group_matcherc                 C   s   |rJ dƒ‚d S )Nz$gradient checkpointing not supportedr4   )r/   Úenabler4   r4   r5   Úset_grad_checkpointingX  s    zConVit.set_grad_checkpointingc                 C   s   | j S r}   )r–   r«   r4   r4   r5   Úget_classifier\  s    zConVit.get_classifierc                 C   sD   || _ |d ur |dv sJ ‚|| _|dkr6t | j|¡nt ¡ | _d S )N)r„   r‚   r…   r   )r˜   r™   r!   r"   r‡   rw   r–   )r/   r˜   r™   r4   r4   r5   Úreset_classifier`  s
    zConVit.reset_classifierc                 C   s€   |   |¡}| jr|| j }|  |¡}| j |jd dd¡}t| jƒD ],\}}|| j	krht
j||fdd}||ƒ}qD|  |¡}|S )Nr   rC   r   rE   )r›   rš   rž   r   rœ   rF   r6   Ú	enumeraterŸ   r”   r+   Úcatr    )r/   r<   Z
cls_tokensÚuZblkr4   r4   r5   Úforward_featuresg  s    





zConVit.forward_features)Ú
pre_logitsc                 C   sX   | j r<| j dkr,|d d …dd …f jddn|d d …df }|  |¡}|rN|S |  |¡S )Nr…   r   rE   r   )r™   rO   r¢   r–   )r/   r<   r·   r4   r4   r5   Úforward_headt  s    6
zConVit.forward_headc                 C   s   |   |¡}|  |¡}|S r}   )r¶   r¸   r~   r4   r4   r5   rA   z  s    

zConVit.forward)F)T)N)F)rl   rm   rn   Ú__doc__r
   r   r¤   r+   ZjitÚignorer¬   r®   r°   r±   r²   r¶   Úboolr¸   rA   ro   r4   r4   r2   r5   r   ð   sF   ëP	


Fc                 K   s(   |  dd ¡rtdƒ‚tt| |fi |¤ŽS )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.)ÚgetÚRuntimeErrorr   r   )ÚvariantÚ
pretrainedÚkwargsr4   r4   r5   Ú_create_convit€  s    rÁ   r„   c              
   K   s   | ddd t tddddœ	|¥S )Nr   )r   r   r   Tzpatch_embed.projr–   )	Úurlr˜   Z
input_sizeZ	pool_sizerO   rŒ   Zfixed_input_sizeZ
first_convÚ
classifierr   )rÂ   rÀ   r4   r4   r5   Ú_cfg‡  s    üûrÄ   ztimm/)Z	hf_hub_id)zconvit_tiny.fb_in1kzconvit_small.fb_in1kzconvit_base.fb_in1k)ra   c                 K   s4   t ddddd}tf d| dœt |fi |¤Ž¤Ž}|S )Né
   r   é0   rB   ©r”   r    r‡   r   Úconvit_tiny©r¾   r¿   ©r¡   rÁ   ©r¿   rÀ   Z
model_argsÚmodelr4   r4   r5   rÈ   ™  s
    ÿ rÈ   c                 K   s4   t ddddd}tf d| dœt |fi |¤Ž¤Ž}|S )NrÅ   r   rÆ   é	   rÇ   Úconvit_smallrÉ   rÊ   rË   r4   r4   r5   rÎ   ¡  s
    ÿ rÎ   c                 K   s4   t ddddd}tf d| dœt |fi |¤Ž¤Ž}|S )NrÅ   r   rÆ   r€   rÇ   Úconvit_baserÉ   rÊ   rË   r4   r4   r5   rÏ   ©  s
    ÿ rÏ   )F)r„   )F)F)F)$r¹   Ú	functoolsr   r+   Ztorch.nnr!   Z	timm.datar   r   Ztimm.layersr   r   r   r	   r
   Z_builderr   Z_features_fxr   Ú	_registryr   r   Zvision_transformer_hybridr   Ú__all__ÚModuler   rp   rr   r   rÁ   rÄ   Zdefault_cfgsrÈ   rÎ   rÏ   r4   r4   r4   r5   Ú<module>   s:   [74 


ü