a
    dJ                     @   s  d Z ddlZddlmZ ddlmZmZ ddlmZm	Z	m
Z
mZmZmZmZ ddlmZ ddlmZ ddlmZmZ d	gZG d
d dejZG dd dejZG dd dejZG dd	 d	ejZdddZd ddZeeddedddZed!edddZed"edddZ dS )#z Visformer

Paper: Visformer: The Vision-friendly Transformer - https://arxiv.org/abs/2104.12533

From original at https://github.com/danczs/Visformer

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    NIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)	to_2tupletrunc_normal_DropPath
PatchEmbedLayerNorm2dcreate_classifieruse_fused_attn   )build_model_with_cfg)checkpoint_seq)register_modelgenerate_default_cfgs	Visformerc                       s4   e Zd Zddejdddf fdd	Zdd Z  ZS )	
SpatialMlpN           Fc           	   	      s   t    |p|}|p|}t|}|| _|| _|| _| jrX|dk rP|d d }n|d }|| _|| _tj	||ddddd| _
| | _t|d | _| jrtj	||ddd| jdd	| _| | _nd | _d | _tj	||ddddd| _t|d | _d S )
N         r   r   Fstridepaddingbias   )r   r   groupsr   )super__init__r   in_featuresout_featuresspatial_convhidden_featuresgroupnnConv2dconv1act1Dropoutdrop1conv2act2conv3drop3)	selfr    r#   r!   	act_layerdropr$   r"   Z
drop_probs	__class__ ^/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/visformer.pyr      s2    


zSpatialMlp.__init__c                 C   sT   |  |}| |}| |}| jd ur<| |}| |}| |}| |}|S N)r'   r(   r*   r+   r,   r-   r.   r/   xr4   r4   r5   forward=   s    







zSpatialMlp.forward)__name__
__module____qualname__r%   GELUr   r9   __classcell__r4   r4   r2   r5   r      s   &r   c                       s8   e Zd ZU ejje ed< d	 fdd	Zdd Z	  Z
S )
	Attention
fused_attnr         ?r   c                    s   t    || _|| _t|| | }|| _|d | _tdd| _t	j
||| d ddddd| _t	|| _t	j
| j| j |ddddd| _t	|| _d S )	Ng      T)Zexperimentalr   r   r   Fr   )r   r   dim	num_headsroundhead_dimscaler   r@   r%   r&   qkvr)   	attn_dropproj	proj_drop)r/   rB   rC   head_dim_ratiorH   rJ   rE   r2   r4   r5   r   L   s    

  zAttention.__init__c           
      C   s   |j \}}}}| ||d| j| jdddddd}|d\}}}| jrvtj	j
j| | | | jjd}n4||dd | j }	|	jdd	}	| |	}	|	| }|dddd|d||}| |}| |}|S )
Nr   r   r   r      )Z	dropout_p)rB   )shaperG   ZreshaperC   rE   ZpermuteZunbindr@   torchr%   Z
functionalZscaled_dot_product_attention
contiguousrH   pZ	transposerF   ZsoftmaxrI   rJ   )
r/   r8   BCHWqkvattnr4   r4   r5   r9   Z   s     *


zAttention.forward)r   rA   r   r   )r:   r;   r<   rP   jitFinalbool__annotations__r   r9   r>   r4   r4   r2   r5   r?   I   s   
r?   c                
       s<   e Zd Zdddddejedddf
 fdd	Zdd	 Z  ZS )
BlockrA         @r   r   Fc                    s   t    || _|dkr t|nt | _|r<d | _d | _n|	|| _t	|||||d| _|	|| _
t|t|| |||
|d| _d S )Nr   )rC   rK   rH   rJ   )r    r#   r0   r1   r$   r"   )r   r   r"   r   r%   ZIdentity	drop_pathnorm1rZ   r?   norm2r   intmlp)r/   rB   rC   rK   	mlp_ratiorJ   rH   ra   r0   
norm_layerr$   attn_disabledr"   r2   r4   r5   r   q   s.    



zBlock.__init__c                 C   sB   | j d ur$|| |  | | }|| | | | }|S r6   )rZ   ra   rb   re   rc   r7   r4   r4   r5   r9      s    
zBlock.forward)	r:   r;   r<   r%   r=   r	   r   r9   r>   r4   r4   r2   r5   r_   p   s   )r_   c                       s   e Zd Zddddddddd	d
d
d
d
d
eddddddddf fdd	Zdd Zejjd$ddZ	ejjd%ddZ
ejjdd Zd&ddZdd Zd'edd d!Zd"d# Z  ZS )(r         r              r   r`   r   Z111TFr   avgNc                    s  t    t|}|| _| _|| _|| _|| _|| _t	|t
tfr^|\| _| _| _t|}n"|d  | _| _|| j | j | _|| _d| _dd td||D | jrd | _t|||dd| _fdd|D }n| jd u rd | _t|d |d |dd| _fd	d|D }ntttj|| jd
ddddt| jtjdd| _dd |D }t|d | jd |dd| _fdd|D }| jr| jrttjdg|R  | _n ttjdd g|R  | _tj|d| _ nd | _tj 	
f
ddt!| jD  | _"| jst|d d |dd| _#fdd|D }| jrttjdg|R  | _$nd | _$nd | _#tj 	
f
ddt!| j| j| j D  | _%| js:t|d d |dd| _&fdd|D }| jr2ttjdd g|R  | _'nd | _'nd | _&tj 	
f
ddt!| j| j |D  | _(| jrnd | _)| j)| _*t+| j)| j|d\}}|| _,t|
| _-|| _.| jrt/| jdd | jst/| j$dd t/| j'dd | 0| j1 d S )Nr   Fc                 S   s   g | ]}|  qS r4   )item.0r8   r4   r4   r5   
<listcomp>       z&Visformer.__init__.<locals>.<listcomp>r   )img_size
patch_sizein_chans	embed_dimrg   flattenc                    s   g | ]}|  qS r4   r4   rq   rv   r4   r5   rs      rt   r   c                    s   g | ]}| d   qS r   r4   rq   rz   r4   r5   rs      rt      r   T)Zinplacec                 S   s   g | ]}|d  qS r{   r4   rq   r4   r4   r5   rs      rt   rM   c                    s   g | ]}| d   qS )rM   r4   rq   rz   r4   r5   rs      rt   r   )rR   c                    sB   g | ]:}t d  d | d dk	d dkdqS )r   g      ?r   01rB   rC   rK   rf   rJ   rH   ra   rg   r$   rh   r"   r_   rr   i
attn_drop_rate
attn_stagedprrx   r$   rf   rg   rC   proj_drop_rater"   r4   r5   rs      s   

r   c                    s   g | ]}| d   qS r   r4   rq   rz   r4   r5   rs     rt   c                    s>   g | ]6}t d  | d dk	d dkdqS )rA   r   r}   r~   r   r   r   r   r4   r5   rs   "  s   

c                    s   g | ]}| d   qS r   r4   rq   rz   r4   r5   rs   =  rt   c                    sB   g | ]:}t d  d | d  dk	d  dkdqS )r   rA   r}   r~   r   r   r   r   r4   r5   rs   D  s   

Z	pool_type{Gz?std)2r   r   r   num_classesrx   init_channelsru   vit_stem	conv_init
isinstancelisttupleZ
stage_num1Z
stage_num2Z
stage_num3sumuse_pos_embedgrad_checkpointingrP   Zlinspacestemr   patch_embed1r%   Z
Sequentialr&   BatchNorm2dZReLU	Parameterzeros
pos_embed1r)   pos_droprangestage1patch_embed2
pos_embed2stage2patch_embed3
pos_embed3stage3num_featuresnormr
   global_pool	head_dropheadr   apply_init_weights)r/   ru   rv   rw   r   r   rx   depthrC   rf   Z	drop_rateZpos_drop_rater   r   Zdrop_path_raterg   r   r   r"   r   r$   r   r   
embed_normr   r2   )r   r   r   rx   r$   rf   rg   rC   rv   r   r"   r5   r      s    



  
 
" 
zVisformer.__init__c                 C   s   t |tjr6t|jdd |jd urtj|jd nPt |tjr| j	r^tjj
|jddd nt|jdd |jd urtj|jd d S )Nr   r   r   Zfan_outZrelu)modeZnonlinearityr   )r   r%   ZLinearr   Zweightr   initZ	constant_r&   r   Zkaiming_normal_)r/   mr4   r4   r5   r   f  s    

zVisformer._init_weightsc                 C   s   t d|rdndd fddgdS )Nz^patch_embed1|pos_embed1|stemz^stage(\d+)\.(\d+))z^(?:patch_embed|pos_embed)(\d+))r   )z^norm)i )r   blocks)dict)r/   Zcoarser4   r4   r5   group_matchers  s    zVisformer.group_matcherc                 C   s
   || _ d S r6   )r   )r/   enabler4   r4   r5   set_grad_checkpointing~  s    z Visformer.set_grad_checkpointingc                 C   s   | j S r6   )r   )r/   r4   r4   r5   get_classifier  s    zVisformer.get_classifierc                 C   s$   || _ t| j| j |d\| _| _d S )Nr   )r   r
   r   r   r   )r/   r   r   r4   r4   r5   reset_classifier  s    zVisformer.reset_classifierc                 C   s  | j d ur|  |}| |}| jd ur8| || j }| jrVtj sVt| j	|}n
| 	|}| j
d ur| 
|}| jd ur| || j }| jrtj st| j|}n
| |}| jd ur| |}| jd ur| || j }| jrtj st| j|}n
| |}| |}|S r6   )r   r   r   r   r   rP   r[   Zis_scriptingr   r   r   r   r   r   r   r   r   r7   r4   r4   r5   forward_features  s0    













zVisformer.forward_features)
pre_logitsc                 C   s&   |  |}| |}|r|S | |S r6   )r   r   r   )r/   r8   r   r4   r4   r5   forward_head  s    

zVisformer.forward_headc                 C   s   |  |}| |}|S r6   )r   r   r7   r4   r4   r5   r9     s    

zVisformer.forward)F)T)ro   )F)r:   r;   r<   r	   r   r   rP   r[   ignorer   r   r   r   r   r]   r   r9   r>   r4   r4   r2   r5   r      sH    E


$Fc                 K   s,   | dd rtdtt| |fi |}|S )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.)getRuntimeErrorr   r   )variant
pretrainedZdefault_cfgkwargsmodelr4   r4   r5   _create_visformer  s    r    c                 K   s    | ddddddt tddd	|S )
Nrk   )r   ri   ri   )r|   r|   g?ZbicubicTzstem.0r   )urlr   Z
input_sizeZ	pool_sizeZcrop_pctinterpolationZfixed_input_sizemeanr   Z
first_conv
classifierr   )r   r   r4   r4   r5   _cfg  s    r   ztimm/)Z	hf_hub_id)zvisformer_tiny.in1kzvisformer_small.in1k)returnc                 K   sD   t ddddddddtjd	tjd
}tdd| it |fi |}|S )Nrj      r|   rM   rM   r   r`   r   011100Tr   rx   r   rC   rf   r$   r   r"   rg   r   r   visformer_tinyr   )r   r   r%   r   r   r   r   Z	model_cfgr   r4   r4   r5   r     s    
r   c                 K   sD   t ddddddddtjd	tjd
}tdd| it |fi |}|S )Nrl   rm   r   r   r`   r   r   r   Tr   visformer_smallr   )r   r   r   r4   r4   r5   r     s    
r   )FN)r   )F)F)!__doc__rP   Ztorch.nnr%   Z	timm.datar   r   Ztimm.layersr   r   r   r   r	   r
   r   Z_builderr   Z_manipulater   	_registryr   r   __all__Moduler   r?   r_   r   r   r   Zdefault_cfgsr   r   r4   r4   r4   r5   <module>   s0   	$3'1  

	