a
    dS                     @   sb  d Z ddlZddlmZ ddlmZmZmZmZm	Z	 ddl
Z
ddlmZ ddlm  mZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZ ddlm Z  dd	l!m"Z" dd
l#m$Z$ ddl%m&Z&m'Z' dgZ(G dd dej)Z*G dd dej)Z+G dd dej)Z,G dd dej)Z-G dd dej)Z.ee/e/f dddZ0e"ee/e/f ee/e/f dddZ1G dd dej)Z2G dd  d ej)Z3G d!d" d"ej)Z4G d#d dej)Z5d<d%d&Z6d=d(d)Z7e'e7d*d+e7d,d+e7d-d+e7d.d+e7d/d+d0Z8e&d>e5d1d2d3Z9e&d?e5d1d4d5Z:e&d@e5d1d6d7Z;e&dAe5d1d8d9Z<e&dBe5d1d:d;Z=dS )Ca)   Global Context ViT

From scratch implementation of GCViT in the style of timm swin_transformer_v2_cr.py

Global Context Vision Transformers -https://arxiv.org/abs/2206.09959

@article{hatamizadeh2022global,
  title={Global Context Vision Transformers},
  author={Hatamizadeh, Ali and Yin, Hongxu and Kautz, Jan and Molchanov, Pavlo},
  journal={arXiv preprint arXiv:2206.09959},
  year={2022}
}

Free of any code related to NVIDIA GCVit impl at https://github.com/NVlabs/GCVit.
The license for this code release is Apache 2.0 with no commercial restrictions.

However, weight files adapted from NVIDIA GCVit impl ARE under a non-commercial share-alike license
(https://creativecommons.org/licenses/by-nc-sa/4.0/) until I have a chance to train new ones...

Hacked together by / Copyright 2022, Ross Wightman
    N)partial)CallableListOptionalTupleUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPath	to_2tuple	to_ntupleMlpClassifierHeadLayerNorm2dget_attnget_act_layerget_norm_layer
RelPosBias_assert   )build_model_with_cfg)register_notrace_function)named_apply)register_modelgenerate_default_cfgsGlobalContextVitc                       s6   e Zd ZdZddddejf fdd	Zdd	 Z  ZS )
MbConvBlockzR A depthwise separable / fused mbconv style residual block with SE, `no norm.
    Ng      ?seFc           	   	      s   t    t|d}t|tr&|dks.|dkr>d|d< d|d< t|}|pL|}t|| }tj||dd	d	||d
| _	| | _
||fi || _tj||d	d	d|d| _d S )N	act_layerr   Zecag      ?Zrd_ratioFbias   r   )groupsr!   r   r!   )super__init__dict
isinstancestrr   intnnConv2dconv_dwactr   conv_pw)	selfin_chsout_chsZexpand_ratio
attn_layerr!   r    Zattn_kwargsZmid_chs	__class__ Z/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/gcvit.pyr&   ,   s    	

zMbConvBlock.__init__c                 C   s8   |}|  |}| |}| |}| |}|| }|S N)r-   r.   r   r/   )r0   xZshortcutr6   r6   r7   forwardC   s    



zMbConvBlock.forward)	__name__
__module____qualname____doc__r+   GELUr&   r:   __classcell__r6   r6   r4   r7   r   )   s   r   c                       s0   e Zd Zddejef fdd	Zdd Z  ZS )Downsample2dNconvc                    s   t    |p|}|d ur"||nt | _t||d| _|dv sFJ |dkrhtj||ddddd| _nB|d	kr||ks|J tj	dddd
| _n||ksJ tj
dd| _|d ur||nt | _d S )Nr   )rB   maxavgrB   r"      r   Fr$   rC   kernel_sizeZstridepaddingrG   )r%   r&   r+   Identitynorm1r   
conv_blockr,   	reduction	MaxPool2d	AvgPool2dnorm2)r0   dimdim_outrM   r    
norm_layerr4   r6   r7   r&   N   s    
zDownsample2d.__init__c                 C   s,   |  |}| |}| |}| |}|S r8   )rK   rL   rM   rP   r0   r9   r6   r6   r7   r:   f   s
    



zDownsample2d.forward)	r;   r<   r=   r+   r?   r   r&   r:   r@   r6   r6   r4   r7   rA   M   s   rA   c                       s.   e Zd Zddejf fdd	Zdd Z  ZS )FeatureBlockr   rC   c                    s   t    |}td|}|dkr0ttjdd}nttjdddd}t | _t	|D ]J}| j
d|d  t||d |rT| j
d	|d  |  |d8 }qTd S )
Nr   rD   rE   rI   r"   rF   rB   r   pool)r%   r&   rC   r   r+   rO   rN   
SequentialblocksrangeZ
add_moduler   )r0   rQ   levelsrM   r    Z
reductionsZpool_fnir4   r6   r7   r&   o   s    


 zFeatureBlock.__init__c                 C   s
   |  |S r8   )rX   rT   r6   r6   r7   r:      s    zFeatureBlock.forward)r;   r<   r=   r+   r?   r&   r:   r@   r6   r6   r4   r7   rU   n   s
   rU   c                       s<   e Zd Zddejefeeeed fddZdd Z	  Z
S )Stemr"   `   r1   r2   r    rS   c                    s4   t    tj||dddd| _t|||d| _d S )Nr"   rE   r   rF   )r    rS   )r%   r&   r+   r,   conv1rA   down)r0   r1   r2   r    rS   r4   r6   r7   r&      s    
zStem.__init__c                 C   s   |  |}| |}|S r8   )r_   r`   rT   r6   r6   r7   r:      s    

zStem.forward)r;   r<   r=   r+   r?   r   r*   r   r&   r:   r@   r6   r6   r4   r7   r\      s   r\   c                	       sN   e Zd Zd
eeeeef eeeed fddZdee	j
 ddd	Z  ZS )WindowAttentionGlobalT        )rQ   	num_headswindow_size
use_globalqkv_bias	attn_drop	proj_dropc                    s   t    t|}|| _|| _|| | _| jd | _|| _t||d| _	| jrft
j||d |d| _nt
j||d |d| _t
|| _t
||| _t
|| _d S )Ng      )rd   rc   rE   r$   r"   )r%   r&   r   rd   rc   head_dimscalere   r   rel_posr+   LinearqkvZDropoutrg   projrh   )r0   rQ   rc   rd   re   rf   rg   rh   r4   r6   r7   r&      s    


zWindowAttentionGlobal.__init__Nq_globalc                 C   sV  |j \}}}| jr|d urt|j d |j d kd | |}|||d| j| jddddd}|d\}}|	||j d  ddd}	|	||| j| jdddd}	n:| |||d| j| jddddd}
|
d\}	}}|	| j
 }	|	|dd  }| |}|jdd	}| |}|| dd|||}| |}| |}|S )
Nz*x and q_global seq lengths should be equalrE   r   r"   r      )rQ   )shapere   r   rm   Zreshaperc   ri   permuteZunbindrepeatrj   Z	transpose
contiguousrk   Zsoftmaxrg   rn   rh   )r0   r9   rp   BNCkvkvqrm   attnr6   r6   r7   r:      s&    
$"*




zWindowAttentionGlobal.forward)TTrb   rb   )N)r;   r<   r=   r*   r   boolfloatr&   r   torchTensorr:   r@   r6   r6   r4   r7   ra      s       
ra   )rd   c                 C   sj   | j \}}}}| |||d  |d ||d  |d |} | dddddd d|d |d |}|S )Nr   r   r"   rE   rr      rq   rt   viewru   rw   )r9   rd   rx   HWrz   windowsr6   r6   r7   window_partition   s    ,,r   )rd   img_sizec                 C   sf   |\}}| j d }| d||d  ||d  |d |d |}|dddddd d|||}|S )Nrq   r   r   r"   rE   rr   r   r   )r   rd   r   r   r   rz   r9   r6   r6   r7   window_reverse   s
    
,$r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )
LayerScaleh㈵>Fc                    s*   t    || _t|t| | _d S r8   )r%   r&   inplacer+   	Parameterr   Zonesgamma)r0   rQ   Zinit_valuesr   r4   r6   r7   r&      s    
zLayerScale.__init__c                 C   s   | j r|| jS || j S r8   )r   Zmul_r   rT   r6   r6   r7   r:      s    zLayerScale.forward)r   F)r;   r<   r=   r&   r:   r@   r6   r6   r4   r7   r      s   r   c                       s   e Zd Zddddddddeejejfeeeef eee	e
e
ee	 e	e	e	eeed fddZdeej d	d
dZdeej d	ddZ  ZS )GlobalContextVitBlock         @TNrb   )rQ   	feat_sizerc   rd   	mlp_ratiore   rf   layer_scalerh   rg   	drop_pathr3   r    rS   c              	      s   t    t|}t|}|| _t|d |d  |d |d   | _||| _|||||||
|	d| _|d urzt||nt	
 | _|dkrt|nt	
 | _||| _t|t|| ||	d| _|d urt||nt	
 | _|dkrt|nt	
 | _d S )Nr   r   )rc   rd   re   rf   rg   rh   rb   )Zin_featuresZhidden_featuresr    Zdrop)r%   r&   r   rd   r*   Znum_windowsrK   r   r   r+   rJ   ls1r   
drop_path1rP   r   mlpls2
drop_path2)r0   rQ   r   rc   rd   r   re   rf   r   rh   rg   r   r3   r    rS   r4   r6   r7   r&      s*    
&
	
zGlobalContextVitBlock.__init__ro   c           	      C   sZ   |j \}}}}t|| j}|d| jd | jd  |}| ||}t|| j||f}|S )Nrq   r   r   )rt   r   rd   r   r   r   )	r0   r9   rp   rx   r   r   rz   Zx_winZattn_winr6   r6   r7   _window_attn  s    z"GlobalContextVitBlock._window_attnc              
   C   sF   ||  | | | || }|| | | | | }|S r8   )r   r   r   rK   r   r   r   rP   )r0   r9   rp   r6   r6   r7   r:     s    " zGlobalContextVitBlock.forward)N)N)r;   r<   r=   ra   r+   r?   	LayerNormr*   r   r   r   r   r   r&   r   r   r   r:   r@   r6   r6   r4   r7   r      s:   
)r   c                       s   e Zd Zdddddddddejejefeeeeef eeef e	e	e	e
e	ee
 e
e
eee
 e
f eeed fddZd	d
 Z  ZS )GlobalContextVitStageTFr   Nrb   )depthrc   r   rd   
downsampleglobal_norm
stage_normr   rf   r   rh   rg   r   r    rS   norm_layer_clc                    s   t    |rDtd |d| _d d d d d fn
t | _| _ttt	
tt }t|| _|rnt | _t 	
fddt|D | _|r؈nt | _| _| _d| _d S )NrE   )rQ   rR   rS   r   r   c                    sH   g | ]@}t 
|d  dk	ttr8| n dqS )rE   r   )rQ   rc   r   rd   r   rf   re   r   rh   rg   r   r    rS   )r   r(   list).0r[   r    rg   rQ   r   r   r   r   r   rc   rh   rf   rd   r6   r7   
<listcomp>K  s    
z2GlobalContextVitStage.__init__.<locals>.<listcomp>F)r%   r&   rA   r   r+   rJ   r   r   r*   mathlog2minrU   global_blockr   Z
ModuleListrY   rX   normrQ   grad_checkpointing)r0   rQ   r   rc   r   rd   r   r   r   r   rf   r   rh   rg   r   r    rS   r   Zfeat_levelsr4   r   r7   r&   %  s,    

$
zGlobalContextVitStage.__init__c                 C   s   |  |}| |}|dddd}| |dddd}| jD ],}| jrbtj sbt		||}q@|||}q@| 
|}|dddd }|S )Nr   rE   r"   r   )r   r   ru   r   rX   r   r   jitZis_scripting
checkpointr   rw   )r0   r9   Zglobal_queryZblkr6   r6   r7   r:   b  s    



zGlobalContextVitStage.forward)r;   r<   r=   r+   r?   r   r   r*   r   r   r   r   r   r   r   r&   r:   r@   r6   r6   r4   r7   r   $  s>   

=r   c                       s  e Zd Zd,eeeeeef eedf eedf eeedf eedf eeee eeeeeeeed fddZ	d-ddZ
ejjdd Zejjd.ddZejjd/ddZejjd d! Zd0d"d#Zejejd$d%d&Zd1ed'd(d)Zejejd$d*d+Z  ZS )2r   r"     rD          r      r   N@   r"   rr      r   rE   rr      r         @Trb    gelulayernorm2d	layernormr   .)in_chansnum_classesglobal_poolr   window_ratiord   	embed_dimdepthsrc   r   rf   r   	drop_rateproj_drop_rateattn_drop_ratedrop_path_rater    rS   r   norm_epsc                    s  t    t|}tt||d}tt||d}t  tdd  D }|| _|| _|| _	t
|}t|d|d   | _|d urt||}n*|d usJ t fddt||D }t||||d| _d	d td
|t||D }g }t|D ]|}||d k}dt|d d
 }|t|| || |	| |d
 | |d | f|| |d
k||
|||||| |||d qtj| | _t| j|||d| _|rtt| j|d|  d S )N)Zepsc                 s   s   | ]}|d  V  qdS )rr   Nr6   )r   dr6   r6   r7   	<genexpr>      z,GlobalContextVit.__init__.<locals>.<genexpr>rE   r   c                    s$   g | ]} d  |  d | fqS )r   r   r6   )r   rr   r6   r7   r     r   z-GlobalContextVit.__init__.<locals>.<listcomp>r^   c                 S   s   g | ]}|  qS r6   )tolist)r   r9   r6   r6   r7   r     r   r   )rQ   r   rc   r   rd   r   r   r   rf   r   rh   rg   r   r    rS   r   	pool_typer   )scheme)r%   r&   r   r   r   r   tupler   r   r   lenr*   num_featuresr   r\   stemr   ZlinspacesumsplitrY   rC   appendr   r+   rW   stagesr   headr   _init_weights)r0   r   r   r   r   r   rd   r   r   rc   r   rf   r   r   r   r   r   Zweight_initr    rS   r   r   r   Z
num_stagesZdprr   r[   Z
last_stageZstage_scaler4   r   r7   r&   u  s`    
"
zGlobalContextVit.__init__vitc                 C   s   |dkrXt |tjrtj|j |jd urd|v rHtjj|jdd qtj|j n6t |tjrtjj|jdd |jd urtj|j d S )Nr   r   gư>)stdg{Gz?)	r(   r+   rl   initZxavier_uniform_Zweightr!   Znormal_Zzeros_)r0   modulenamer   r6   r6   r7   r     s    

zGlobalContextVit._init_weightsc                 C   s   dd |   D S )Nc                    s*   h | ]"\ }t  fd ddD r qS )c                 3   s   | ]}| v V  qd S r8   r6   )r   nr|   r6   r7   r     r   z=GlobalContextVit.no_weight_decay.<locals>.<setcomp>.<genexpr>)Zrelative_position_bias_tablezrel_pos.mlp)any)r   _r6   r   r7   	<setcomp>  s   z3GlobalContextVit.no_weight_decay.<locals>.<setcomp>)Znamed_parametersr0   r6   r6   r7   no_weight_decay  s    z GlobalContextVit.no_weight_decayFc                 C   s   t ddd}|S )Nz^stemz^stages\.(\d+))r   rX   )r'   )r0   ZcoarseZmatcherr6   r6   r7   group_matcher  s
    zGlobalContextVit.group_matcherc                 C   s   | j D ]
}||_qd S r8   )r   r   )r0   enablesr6   r6   r7   set_grad_checkpointing  s    
z'GlobalContextVit.set_grad_checkpointingc                 C   s   | j jS r8   )r   Zfcr   r6   r6   r7   get_classifier  s    zGlobalContextVit.get_classifierc                 C   s2   || _ |d u r| jjj}t| j||| jd| _d S )Nr   )r   r   r   r   r   r   r   )r0   r   r   r6   r6   r7   reset_classifier  s    
z!GlobalContextVit.reset_classifier)r9   returnc                 C   s   |  |}| |}|S r8   )r   r   rT   r6   r6   r7   forward_features  s    

z!GlobalContextVit.forward_features
pre_logitsc                 C   s   | j ||dS )Nr   )r   )r0   r9   r   r6   r6   r7   forward_head  s    zGlobalContextVit.forward_headc                 C   s   |  |}| |}|S r8   )r   r   rT   r6   r6   r7   r:     s    

zGlobalContextVit.forward)r"   r   rD   r   r   Nr   r   r   r   TNrb   rb   rb   rb   r   r   r   r   r   )r   )F)T)N)F)r;   r<   r=   r*   r)   r   r   r   r   r&   r   r   r   ignorer   r   r   r   r   r   r   r   r:   r@   r6   r6   r4   r7   r   t  sp                        




P



Fc                 K   s,   | dd rtdtt| |fi |}|S )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.)getRuntimeErrorr   r   )variant
pretrainedkwargsmodelr6   r6   r7   _create_gcvit   s    r   r   c                 K   s    | dddddt tdddd	|S )
Nr   )r"   r   r   )r   r   g      ?Zbicubicz
stem.conv1zhead.fcT)urlr   Z
input_sizeZ	pool_sizeZcrop_pctinterpolationmeanr   Z
first_conv
classifierZfixed_input_sizer   )r   r   r6   r6   r7   _cfg  s    r  z}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_xxtiny_224_nvidia-d1d86009.pth)r   z|https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_xtiny_224_nvidia-274b92b7.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_tiny_224_nvidia-ac783954.pthz|https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_small_224_nvidia-4e98afa2.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_base_224_nvidia-f009139b.pth)zgcvit_xxtiny.in1kzgcvit_xtiny.in1kzgcvit_tiny.in1kzgcvit_small.in1kzgcvit_base.in1k)r   c                 K   s&   t f ddd|}tdd| i|S )N)rE   rE      rE   r   r   rc   gcvit_xxtinyr   )r  r'   r   r   r   Zmodel_kwargsr6   r6   r7   r     s    r  c                 K   s&   t f ddd|}tdd| i|S )N)r"   rr   r  r   r   r  gcvit_xtinyr   )r  r  r  r6   r6   r7   r  )  s    r  c                 K   s&   t f ddd|}tdd| i|S )Nr   r   r  
gcvit_tinyr   )r	  r  r  r6   r6   r7   r	  2  s    r	  c                 K   s,   t f dddddd|}td	d| i|S )
Nr   )r"   r        r]   rE   r   r   rc   r   r   r   gcvit_smallr   )r  r  r  r6   r6   r7   r  ;  s    r  c                 K   s,   t f dddddd|}td	d| i|S )
Nr   )rr   r   r   r      rE   r   r  
gcvit_baser   )r  r  r  r6   r6   r7   r  G  s    r  )F)r   )F)F)F)F)F)>r>   r   	functoolsr   typingr   r   r   r   r   r   Ztorch.nnr+   Ztorch.utils.checkpointutilsr   Z	timm.datar	   r
   Ztimm.layersr   r   r   r   r   r   r   r   r   r   r   Z_builderr   Z_features_fxr   Z_manipulater   	_registryr   r   __all__Moduler   rA   rU   r\   ra   r*   r   r   r   r   r   r   r   r  Zdefault_cfgsr  r  r	  r  r  r6   r6   r6   r7   <module>   sn   4$!8"
8P 

