a
    d]                     @   s  d Z ddlZddlmZmZmZmZ ddlZddlm	Z	 ddl
m	  mZ ddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZ ddlmZ dd	lmZmZ dd
lm Z  dgZ!ee"e"f ej#dddZ$G dd de	j%Z&G dd de	j%Z'G dd de	j%Z(G dd de	j%Z)dGddZ*ee*ddde*ddddde*d dd!d"e*d#dde*d$dddde*d%dd&dde*d'dd!d"e*d(deed)e*d*deed)e*d+dd!eed,e*d-dd.eed/e*d0dd.eed/e*d1dd!eed,d2Z+d3d4 Z,dHd6d7Z-edIe)d8d9d:Z.edJe)d8d;d<Z/edKe)d8d=d>Z0edLe)d8d?d@Z1edMe)d8dAdBZ2edNe)d8dCdDZ3edOe)d8dEdFZ4dS )Pa   BEiT: BERT Pre-Training of Image Transformers (https://arxiv.org/abs/2106.08254)

Model from official source: https://github.com/microsoft/unilm/tree/master/beit

@inproceedings{beit,
title={{BEiT}: {BERT} Pre-Training of Image Transformers},
author={Hangbo Bao and Li Dong and Songhao Piao and Furu Wei},
booktitle={International Conference on Learning Representations},
year={2022},
url={https://openreview.net/forum?id=p-BhZSz59o4}
}

BEiT-v2 from https://github.com/microsoft/unilm/tree/master/beit2

@article{beitv2,
title={{BEiT v2}: Masked Image Modeling with Vector-Quantized Visual Tokenizers},
author={Zhiliang Peng and Li Dong and Hangbo Bao and Qixiang Ye and Furu Wei},
year={2022},
eprint={2208.06366},
archivePrefix={arXiv},
primaryClass={cs.CV}
}

At this point only the 1k fine-tuned classification weights and model configs have been added,
see original source above for pre-training models and procedure.

Modifications by / Copyright 2021 Ross Wightman, original copyrights below
    N)CallableOptionalTupleUnion)
checkpoint)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)
PatchEmbedMlpSwiGLU	LayerNormDropPathtrunc_normal_use_fused_attn   )build_model_with_cfg)generate_default_cfgsregister_modelcheckpoint_filter_fnBeit)window_sizereturnc              	   C   s  d| d  d d| d  d  d }| d | d  }t t t | d t | d g}t |d}|d d d d d f |d d d d d f  }|ddd }|d d d d df  | d d 7  < |d d d d df  | d d 7  < |d d d d df  d| d  d 9  < t j|d fd |jd}|	d|dd dd f< |d |ddd f< |d |dd df< |d |d< |S )N   r   r      )sizedtype)r   r   )
torchstackZmeshgridZarangeflattenpermute
contiguouszerosr   sum)r   num_relative_distancewindow_areaZcoordsZcoords_flattenZrelative_coordsrelative_position_index r(   Y/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/beit.pygen_relative_position_index;   s&    $,&&*r*   c                
       sp   e Zd ZU ejje ed< deeee	e	e
eeef  e
e d fddZd	d
 Zde
ej dddZ  ZS )	Attention
fused_attn   F        N)dim	num_headsqkv_bias	attn_drop	proj_dropr   attn_head_dimc           
         s>  t    || _|| }|d ur$|}|| j }	|d | _t | _tj||	d dd| _|rt	t
|	| _| jdt
|	dd t	t
|	| _nd | _d | _d | _|r|| _d|d  d	 d|d	  d	  d | _t	t
| j|| _| d
t| nd | _d | _d | _t|| _t|	|| _t|| _d S )Ng      r   F)biask_bias)
persistentr   r   r   r'   )super__init__r0   scaler   r,   nnLinearqkv	Parameterr   r#   q_biasregister_bufferv_biasr6   r   r%   relative_position_bias_tabler*   r'   Dropoutr2   projr3   )
selfr/   r0   r1   r2   r3   r   r4   Zhead_dimZall_head_dim	__class__r(   r)   r9   T   s:    



&zAttention.__init__c                 C   s`   | j | jd | jd | jd  d | jd | jd  d d}|ddd }|dS )Nr   r   r   r   )rB   r'   viewr   r!   r"   Z	unsqueezerE   Zrelative_position_biasr(   r(   r)   _get_rel_pos_bias   s    
zAttention._get_rel_pos_biasshared_rel_pos_biasc                 C   sX  |j \}}}| jd ur,t| j| j| jfnd }tj|| jj	|d}|
||d| jdddddd}|d\}}	}
| jrd }| jd ur|  }|d ur|| }n|d ur|}tj||	|
|| jjd}n`|| j }||	d	d }| jd ur||   }|d ur|| }|jdd
}| |}||
 }|dd
|||}| |}| |}|S )N)inputweightr5   r   r   r   r   r      )Z	attn_maskZ	dropout_pr/   )shaper?   r   catr6   rA   FZlinearr=   rN   Zreshaper0   r!   Zunbindr,   rB   rJ   Zscaled_dot_product_attentionr2   pr:   Z	transposeZsoftmaxrD   r3   )rE   xrL   BNCr1   r=   qkvrel_pos_biasattnr(   r(   r)   forward   s>    $"







zAttention.forward)r-   Fr.   r.   NN)N)__name__
__module____qualname__r   jitFinalbool__annotations__intfloatr   r   r9   rJ   Tensorr_   __classcell__r(   r(   rF   r)   r+   Q   s$   
      ,r+   c                       s   e Zd Zddddddddejeddfeeeeeeeeee	e e
e
e	eeef  e	e d fddZde	ej dd	d
Z  ZS )BlockF      @r.   N)r/   r0   r1   	mlp_ratio	scale_mlp
swiglu_mlpr3   r2   	drop_pathinit_values	act_layer
norm_layerr   r4   c              	      s   t    ||| _t|||||||d| _|	dkr<t|	nt | _||| _	|rxt
|t|| |rj|nd |d| _n$t|t|| ||r|nd |d| _|	dkrt|	nt | _|
rt|
t| | _t|
t| | _nd\| _| _d S )N)r0   r1   r2   r3   r   r4   r.   )in_featureshidden_featuresrs   drop)rt   ru   rr   rs   rv   )NN)r8   r9   norm1r+   r^   r   r;   Identity
drop_path1norm2r   rg   mlpr
   
drop_path2r>   r   Zonesgamma_1gamma_2)rE   r/   r0   r1   rm   rn   ro   r3   r2   rp   rq   rr   rs   r   r4   rF   r(   r)   r9      s@    








zBlock.__init__rK   c              	   C   s   | j d u rD|| | j| ||d }|| | | | }nD|| | j | j| ||d  }|| | j| | |  }|S )NrK   )r}   ry   r^   rw   r|   r{   rz   r~   )rE   rV   rL   r(   r(   r)   r_      s    
$ zBlock.forward)N)r`   ra   rb   r;   ZGELUr   rg   re   rh   r   r   r   r9   r   ri   r_   rj   r(   r(   rF   r)   rk      s:   7rk   c                       s$   e Zd Z fddZdd Z  ZS )RelativePositionBiasc                    sn   t    || _|d |d  | _d|d  d d|d  d  d }tt||| _| 	dt
| d S )Nr   r   r   r   r'   )r8   r9   r   r&   r;   r>   r   r#   rB   r@   r*   )rE   r   r0   r%   rF   r(   r)   r9      s    
$zRelativePositionBias.__init__c                 C   s:   | j | jd | jd | jd d}|ddd S )Nr   r   r   r   )rB   r'   rH   r&   r!   r"   rI   r(   r(   r)   r_      s    zRelativePositionBias.forward)r`   ra   rb   r9   r_   rj   r(   r(   rF   r)   r      s   	r   c                       s$  e Zd ZdZddddddddd	d
dddddddedd	dddfeeeeef f eeeeef f eeeeeee	e
e	e	e
e
e
e
e
eee
 e	e	e	e
d fddZdd Zdd Zejjdd Zejjd'ddZejjd(ddZejjdd Zd)ddZd d! Zd*e	d"d#d$Zd%d& Z  ZS )+r   zI Vision Transformer with support for patch or hybrid CNN input stage
          r     avg      Trl   Fr.   NgMbP?)img_size
patch_sizein_chansnum_classesglobal_pool	embed_dimdepthr0   r1   rm   ro   rn   	drop_ratepos_drop_rateproj_drop_rateattn_drop_ratedrop_path_raters   rq   use_abs_pos_embuse_rel_pos_biasuse_shared_rel_pos_biashead_init_scalec                    s  t    |
_|
_ 
_
_d
_d
_t|||d
_	
j	j
}ttdd
_|rzttd|d nd 
_tj|d
_|rt
j	jd
_nd 
_dd td||D t 	
fd	dt|D 
_
jd
k}|rt n
_|r*nt 
_t|
_|dkrVt|nt 
_
 
j! 
jd urt"
jdd t"
jdd 
#  t$
jtjrt"
jj%dd 
jj%j&'| 
jj(j&'| d S )Nr   F)r   r   r   r   )rU   )r   r0   c                 S   s   g | ]}|  qS r(   )item).0rV   r(   r(   r)   
<listcomp>?      z!Beit.__init__.<locals>.<listcomp>r   c                    s<   g | ]4}t 	 | r0
jjnd dqS )N)r/   r0   r1   rm   rn   ro   r3   r2   rp   rs   rq   r   )rk   patch_embed	grid_size)r   ir   Zdprr   rq   rm   rs   r0   r   r1   rn   rE   ro   r   r(   r)   r   @  s   r   {Gz?std))r8   r9   r   r   Znum_featuresr   num_prefix_tokensgrad_checkpointingr	   r   num_patchesr;   r>   r   r#   	cls_token	pos_embedrC   pos_dropr   r   r]   ZlinspaceZ
ModuleListrangeblocksrx   normfc_norm	head_dropr<   headapply_init_weightsr   fix_init_weight
isinstancerN   dataZmul_r5   )rE   r   r   r   r   r   r   r   r0   r1   rm   ro   rn   r   r   r   r   r   rs   rq   r   r   r   r   r   Zuse_fc_normrF   r   r)   r9   	  sP    
"
&

 zBeit.__init__c                 C   sL   dd }t | jD ]4\}}||jjjj|d  ||jjjj|d  qd S )Nc                 S   s   |  td|  d S )Ng       @)Zdiv_mathsqrt)paramlayer_idr(   r(   r)   rescalec  s    z%Beit.fix_init_weight.<locals>.rescaler   )	enumerater   r^   rD   rN   r   r{   Zfc2)rE   r   r   Zlayerr(   r(   r)   r   b  s    zBeit.fix_init_weightc                 C   sr   t |tjrBt|jdd t |tjrn|jd urntj|jd n,t |tjrntj|jd tj|jd d S )Nr   r   r         ?)	r   r;   r<   r   rN   r5   initZ	constant_r   )rE   mr(   r(   r)   r   j  s    zBeit._init_weightsc                 C   s0   ddh}|   D ]\}}d|v r|| q|S )Nr   r   rB   )Znamed_parametersadd)rE   Znwdn_r(   r(   r)   no_weight_decays  s
    zBeit.no_weight_decayc                 C   s
   || _ d S N)r   )rE   enabler(   r(   r)   set_grad_checkpointing{  s    zBeit.set_grad_checkpointingc                 C   s   t dddgd}|S )Nz-^cls_token|pos_embed|patch_embed|rel_pos_bias)z^blocks\.(\d+)N)z^norm)i )stemr   )dict)rE   ZcoarseZmatcherr(   r(   r)   group_matcher  s
    zBeit.group_matcherc                 C   s   | j S r   )r   )rE   r(   r(   r)   get_classifier  s    zBeit.get_classifierc                 C   s8   || _ |d ur|| _|dkr*t| j|nt | _d S )Nr   )r   r   r;   r<   r   rx   r   )rE   r   r   r(   r(   r)   reset_classifier  s    zBeit.reset_classifierc                 C   s   |  |}tj| j|jd dd|fdd}| jd urB|| j }| |}| jd ur^|  nd }| j	D ]0}| j
rtj st|||d}qh|||d}qh| |}|S )Nr   r   r   rQ   rK   )r   r   rS   r   expandrR   r   r   r]   r   r   rc   Zis_scriptingr   r   )rE   rV   r]   Zblkr(   r(   r)   forward_features  s    
$




zBeit.forward_features)
pre_logitsc                 C   sd   | j r>| j dkr.|d d | jd f jddn|d d df }| |}| |}|rZ|S | |S )Nr   r   rQ   r   )r   r   meanr   r   r   )rE   rV   r   r(   r(   r)   forward_head  s
    8

zBeit.forward_headc                 C   s   |  |}| |}|S r   )r   r   )rE   rV   r(   r(   r)   r_     s    

zBeit.forward)T)F)N)F)r`   ra   rb   __doc__r   r   rg   r   strre   rh   r   r   r9   r   r   r   rc   ignorer   r   r   r   r   r   r   r_   rj   r(   r(   rF   r)   r     s~   Y	


 c                 K   s    | ddd dddddddd	|S )
Nr   )r   r   r   g?ZbicubicT)      ?r   r   zpatch_embed.projr   )urlr   
input_sizeZ	pool_sizecrop_pctinterpolationZfixed_input_sizer   r   Z
first_conv
classifierr(   )r   kwargsr(   r(   r)   _cfg  s    r   znhttps://conversationhub.blob.core.windows.net/beit-share-public/beit/beit_base_patch16_224_pt22k_ft22kto1k.pthztimm/)r   	hf_hub_idznhttps://conversationhub.blob.core.windows.net/beit-share-public/beit/beit_base_patch16_384_pt22k_ft22kto1k.pth)r     r   r   )r   r   r   r   zjhttps://conversationhub.blob.core.windows.net/beit-share-public/beit/beit_base_patch16_224_pt22k_ft22k.pthiQU  )r   r   r   zohttps://conversationhub.blob.core.windows.net/beit-share-public/beit/beit_large_patch16_224_pt22k_ft22kto1k.pthzohttps://conversationhub.blob.core.windows.net/beit-share-public/beit/beit_large_patch16_384_pt22k_ft22kto1k.pthzohttps://conversationhub.blob.core.windows.net/beit-share-public/beit/beit_large_patch16_512_pt22k_ft22kto1k.pth)r      r   zkhttps://conversationhub.blob.core.windows.net/beit-share-public/beit/beit_large_patch16_224_pt22k_ft22k.pthzqhttps://conversationhub.blob.core.windows.net/beit-share-public/beitv2/beitv2_base_patch16_224_pt1k_ft21kto1k.pth)r   r   r   r   zlhttps://conversationhub.blob.core.windows.net/beit-share-public/beitv2/beitv2_base_patch16_224_pt1k_ft1k.pthzmhttps://conversationhub.blob.core.windows.net/beit-share-public/beitv2/beitv2_base_patch16_224_pt1k_ft21k.pth)r   r   r   r   r   zrhttps://conversationhub.blob.core.windows.net/beit-share-public/beitv2/beitv2_large_patch16_224_pt1k_ft21kto1k.pthgffffff?)r   r   r   r   r   zmhttps://conversationhub.blob.core.windows.net/beit-share-public/beitv2/beitv2_large_patch16_224_pt1k_ft1k.pthznhttps://conversationhub.blob.core.windows.net/beit-share-public/beitv2/beitv2_large_patch16_224_pt1k_ft21k.pth)z)beit_base_patch16_224.in22k_ft_in22k_in1kz)beit_base_patch16_384.in22k_ft_in22k_in1kz$beit_base_patch16_224.in22k_ft_in22kz*beit_large_patch16_224.in22k_ft_in22k_in1kz*beit_large_patch16_384.in22k_ft_in22k_in1kz*beit_large_patch16_512.in22k_ft_in22k_in1kz%beit_large_patch16_224.in22k_ft_in22kz*beitv2_base_patch16_224.in1k_ft_in22k_in1kz$beitv2_base_patch16_224.in1k_ft_in1kz%beitv2_base_patch16_224.in1k_ft_in22kz+beitv2_large_patch16_224.in1k_ft_in22k_in1kz%beitv2_large_patch16_224.in1k_ft_in1kz&beitv2_large_patch16_224.in1k_ft_in22kc                 C   s   d| v r| d } t | |S )Nmoduler   )Z
state_dictmodelr(   r(   r)   _beit_checkpoint_filter_fn  s    r   Fc                 K   s0   | dd rtdtt| |fdti|}|S )NZfeatures_onlyz.features_only not implemented for BEiT models.Zpretrained_filter_fn)getRuntimeErrorr   r   r   )variant
pretrainedr   r   r(   r(   r)   _create_beit  s    r   )r   c              
   K   s:   t ddddddddd}tdd
| it |fi |}|S )Nr   r   r   rO   FT皙?r   r   r   r0   rm   r   r   rq   beit_base_patch16_224r   )r   r   r   r   r   Z
model_argsr   r(   r(   r)   r     s    
r   c              
   K   s:   t ddddddddd}tdd
| it |fi |}|S )Nr   r   r   r   FTr   r   r   r   r   r0   r   r   rq   beit_base_patch16_384r   )r   r   r   r(   r(   r)   r     s    
r   c              	   K   s8   t dddddddd}td
d	| it |fi |}|S )Nr         FTh㈵>r   r   r   r0   r   r   rq   beit_large_patch16_224r   )r   r   r   r(   r(   r)   r      s    r   c              
   K   s:   t ddddddddd}tdd
| it |fi |}|S )Nr   r   r   r   FTr   r   beit_large_patch16_384r   )r   r   r   r(   r(   r)   r   )  s    
r   c              
   K   s:   t ddddddddd}tdd
| it |fi |}|S )Nr   r   r   r   FTr   r   beit_large_patch16_512r   )r   r   r   r(   r(   r)   r   2  s    
r   c              
   K   s:   t ddddddddd}tdd
| it |fi |}|S )Nr   r   r   rO   FTr   r   beitv2_base_patch16_224r   )r   r   r   r(   r(   r)   r   ;  s    
r   c              	   K   s8   t dddddddd}td
d	| it |fi |}|S )Nr   r   r   FTr   r   beitv2_large_patch16_224r   )r   r   r   r(   r(   r)   r   D  s    r   )r   )F)F)F)F)F)F)F)F)5r   r   typingr   r   r   r   r   Ztorch.nnr;   Ztorch.nn.functionalZ
functionalrT   Ztorch.utils.checkpointr   Z	timm.datar   r   Ztimm.layersr	   r
   r   r   r   r   r   Z_builderr   	_registryr   r   Zvision_transformerr   __all__rg   ri   r*   Moduler+   rk   r   r   r   Zdefault_cfgsr   r   r   r   r   r   r   r   r   r(   r(   r(   r)   <module>   s   ($`C *
B
