a
    d^                     @   s  d Z ddlmZ ddlmZ ddlmZ ddlZddlZddlm	Z	 ddl
mZmZ ddlmZmZmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ dgZG dd de	jZG dd de	jZG dd de	jZG dd de	jZ dd Z!edAee"e"f e#dddZ$G dd de	jZ%dBddZ&dCd d!Z'ee'd"d#e'd"d$d%e'd"d&d$d'd(e'd"d#e'd"d$d%e'd"d&d$d'd(e'd"d#e'd"d$d%e'd"d#e'd"d#e'd"d#d)Z(edDe%d*d+d,Z)edEe%d*d-d.Z*edFe%d*d/d0Z+edGe%d*d1d2Z,edHe%d*d3d4Z-edIe%d*d5d6Z.edJe%d*d7d8Z/edKe%d*d9d:Z0edLe%d*d;d<Z1edMe%d*d=d>Z2edNe%d*d?d@Z3dS )Oa   CrossViT Model

@inproceedings{
    chen2021crossvit,
    title={{CrossViT: Cross-Attention Multi-Scale Vision Transformer for Image Classification}},
    author={Chun-Fu (Richard) Chen and Quanfu Fan and Rameswar Panda},
    booktitle={International Conference on Computer Vision (ICCV)},
    year={2021}
}

Paper link: https://arxiv.org/abs/2103.14899
Original code: https://github.com/IBM/CrossViT/blob/main/models/crossvit.py

NOTE: model names have been renamed from originals to represent actual input res all *_224 -> *_240 and *_384 -> *_408

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    )partial)List)TupleNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPath	to_2tupletrunc_normal__assert   )build_model_with_cfg)register_notrace_function)register_modelgenerate_default_cfgs)BlockCrossVitc                       s*   e Zd ZdZd fdd	Zd	d
 Z  ZS )
PatchEmbedz Image to Patch Embedding
                Fc                    sP  t    t|}t|}|d |d  |d |d   }|| _|| _|| _|r8|d dkrttj||d ddddtj	dd	tj|d |d
 ddddtj	dd	tj|d
 |dddd| _
nr|d dkrLttj||d ddddtj	dd	tj|d |d
 dd
ddtj	dd	tj|d
 |dd
dd| _
ntj||||d| _
d S )Nr   r            r   )kernel_sizestridepaddingT)Zinplace   r   )r   r   )super__init__r	   img_size
patch_sizenum_patchesnn
SequentialZConv2dZReLUproj)selfr!   r"   in_chans	embed_dim
multi_convr#   	__class__ ]/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/crossvit.pyr    1   s2    
 



zPatchEmbed.__init__c                 C   s   |j \}}}}t|| jd kd| d| d| jd  d| jd  d	 t|| jd kd| d| d| jd  d| jd  d	 | |ddd}|S )Nr   zInput image size (*z) doesn't match model (r   z).r   )shaper   r!   r&   flatten	transpose)r'   xBCHWr-   r-   r.   forwardM   s    ((zPatchEmbed.forward)r   r   r   r   F)__name__
__module____qualname____doc__r    r8   __classcell__r-   r-   r+   r.   r   -   s   r   c                       s&   e Zd Zd fdd	Zdd Z  ZS )	CrossAttention   F        c                    s   t    || _|| }|d | _tj|||d| _tj|||d| _tj|||d| _t	|| _
t||| _t	|| _d S )Ng      )bias)r   r    	num_headsscaler$   LinearwqwkwvDropout	attn_dropr&   	proj_drop)r'   dimrB   qkv_biasrI   rJ   Zhead_dimr+   r-   r.   r    Y   s    

zCrossAttention.__init__c           	      C   s   |j \}}}| |d d dddf |d| j|| j dddd}| |||| j|| j dddd}| |||| j|| j dddd}||dd | j }|j	dd}| 
|}|| dd|d|}| |}| |}|S )	Nr   r   .r   r   rK   )r0   rE   ZreshaperB   ZpermuterF   rG   r2   rC   ZsoftmaxrI   r&   rJ   )	r'   r3   r4   Nr5   qkvattnr-   r-   r.   r8   n   s    <**


zCrossAttention.forward)r?   Fr@   r@   )r9   r:   r;   r    r8   r=   r-   r-   r+   r.   r>   X   s       r>   c                       s8   e Zd Zdddddejejf fdd	Zdd Z  ZS )CrossAttentionBlock      @Fr@   c
           
         sF   t    |	|| _t|||||d| _|dkr8t|nt | _d S )N)rB   rL   rI   rJ   r@   )	r   r    norm1r>   rT   r   r$   Identity	drop_path)
r'   rK   rB   	mlp_ratiorL   rJ   rI   rY   	act_layer
norm_layerr+   r-   r.   r       s    

zCrossAttentionBlock.__init__c                 C   s0   |d d dddf |  | | | }|S )Nr   r   .)rY   rT   rW   )r'   r3   r-   r-   r.   r8      s    ,zCrossAttentionBlock.forward)	r9   r:   r;   r$   GELU	LayerNormr    r8   r=   r-   r-   r+   r.   rU      s   rU   c                       sJ   e Zd Zddddejejf fdd	Zeej	 eej	 dddZ
  ZS )MultiScaleBlockFr@   c                    sr  t    t|}|| _t | _t|D ]f}g }t|| D ]2}|t	|| || || ||||	| |d q>t|dkr*| jtj
|  q*t| jdkrd | _t | _t|D ]j}|| ||d |  krdrt g}n,||| |
 t|| ||d |  g}| jtj
|  qt | _t|D ]}|d | }|| }|d dkr| jt|| ||| ||||	d |d nTg }t|d D ]0}|t|| ||| ||||	d |d q| jtj
|  q6t | _t|D ]x}||d |  || kr$dr$t g}n4|||d |  |
 t||d |  || g}| jtj
|  qd S )N)rK   rB   rZ   rL   rJ   rI   rY   r\   r   r   FrN   )r   r    lennum_branchesr$   
ModuleListblocksrangeappendr   r%   projsrX   rD   fusionrU   revert_projs)r'   rK   patchesdepthrB   rZ   rL   rJ   rI   rY   r[   r\   ra   dtmpiZd_Znh_r+   r-   r.   r       s    




,


 zMultiScaleBlock.__init__)r3   returnc                 C   s(  g }t | jD ]\}}||||  qtjttj g }t | jD ],\}}|||| d d dddf  qHg }t t	| j
| jD ]\}\}}	tj|| ||d | j  d d dd df fdd}
||
}
|	|
d d dddf }tj||| d d dd df fdd}
||
 q|S )Nr   r   .rO   )	enumeraterc   re   torchjitZannotater   Tensorrf   ziprg   rh   catra   )r'   r3   Zouts_brm   blockZproj_cls_tokenr&   Zoutsrg   Zrevert_projrl   Zreverted_proj_cls_tokenr-   r-   r.   r8      s    &6(zMultiScaleBlock.forward)r9   r:   r;   r$   r]   r^   r    r   rq   rs   r8   r=   r-   r-   r+   r.   r_      s   	Wr_   c                 C   s   dd t | |D S )Nc                 S   s(   g | ] \}}|d  | |d  | qS )r   r   r-   ).0rm   pr-   r-   r.   
<listcomp>      z(_compute_num_patches.<locals>.<listcomp>)rt   )r!   ri   r-   r-   r.   _compute_num_patches  s    r{   F)ss
crop_scalec                 C   s   | j dd \}}||d ks*||d kr|r|d |kr|d |krtt||d  d tt||d  d  }}| dddd|||d  |||d  f } ntjjj| |ddd} | S )	a~  
    Pulled out of CrossViT.forward_features to bury conditional logic in a leaf node for FX tracing.
    Args:
        x (Tensor): input image
        ss (tuple[int, int]): height and width to scale to
        crop_scale (bool): whether to crop instead of interpolate to achieve the desired scale. Defaults to False
    Returns:
        Tensor: the "scaled" image batch tensor
    rM   Nr   r          @ZbicubicF)sizemodeZalign_corners)r0   introundrq   r$   Z
functionalZinterpolate)r3   r|   r}   r6   r7   ZcuZclr-   r-   r.   scale_image  s    22r   c                       s   e Zd ZdZdddddddd	d
ddddddddeejdddf fdd	Zdd Ze	j
jdd Ze	j
jd(ddZe	j
jd)ddZe	j
jdd Zd*ddZee	j d d!d"Zd+ee	j ee	jd#d$d%Zd&d' Z  ZS ),r   zI Vision Transformer with support for patch or hybrid CNN input stage
    r   )      ?r   )r?   r   r     )     )r   r   r   r   r   )   r   )r~   r~   rV   FTr@   gư>)Zepstokenc           !         sj  t    |dv sJ _|_t|_t|}fdd|D _|_tj|}t	|_
 _t _t _tj
D ]X}td| ttdd||   |  td| ttdd |  qtj| D ]$\}}}jt|||||
d qtj|d_td	d |D }d
d td||D }d}t _t|D ]b\}}t|d d |d  }||||  }t ||||	||||d
} ||7 }j|  qdt fddtj
D _ t|_!t fddtj
D _"tj
D ]6}t#t$d| dd t#t$d| dd q"%j& d S )Nr   avgc                    s$   g | ] t  fd djD qS )c                    s   g | ]}t |  qS r-   )r   )rw   Zsjsir-   r.   ry   I  rz   z0CrossVit.__init__.<locals>.<listcomp>.<listcomp>)tupler!   )rw   r'   r   r.   ry   I  rz   z%CrossVit.__init__.<locals>.<listcomp>
pos_embed_r   
cls_token_)r!   r"   r(   r)   r*   )rx   c                 S   s   g | ]}t |d d qS )rM   N)sumrw   r3   r-   r-   r.   ry   b  rz   c                 S   s   g | ]}|  qS r-   )itemr   r-   r-   r.   ry   c  rz   r   rN   )rB   rZ   rL   rJ   rI   rY   r\   c                    s   g | ]} | qS r-   r-   rw   rm   )r)   r\   r-   r.   ry   x  rz   c                    s,   g | ]$}d kr t  | nt  qS r   )r$   rD   rX   r   )r)   num_classesr-   r.   ry   z  s   {Gz?std)'r   r    r   global_poolr	   r!   img_size_scaledr}   r{   r`   ra   r)   r   Znum_featuresr$   rb   patch_embedrd   setattr	Parameterrq   zerosrt   re   r   rH   pos_dropZlinspacerc   rp   maxr_   norm	head_dropheadr
   getattrapply_init_weights)!r'   r!   	img_scaler"   r(   r   r)   rj   rB   rZ   r*   r}   rL   Z	drop_rateZpos_drop_rateZproj_drop_rateZattn_drop_rateZdrop_path_rater\   r   r#   rm   Zim_srx   rk   Ztotal_depthZdprZdpr_ptridxZ	block_cfgZ
curr_depthZdpr_blkr+   )r)   r\   r   r'   r.   r    ,  sr    




.(	
"
zCrossVit.__init__c                 C   sr   t |tjrBt|jdd t |tjrn|jd urntj|jd n,t |tjrntj|jd tj|jd d S )Nr   r   r   r   )	
isinstancer$   rD   r
   ZweightrA   initZ	constant_r^   )r'   mr-   r-   r.   r     s    zCrossVit._init_weightsc                 C   sZ   t  }t| jD ]D}|d|  t| d| d }|d ur|jr|d|  q|S )Nr   r   )setrd   ra   addr   Zrequires_grad)r'   outrm   per-   r-   r.   no_weight_decay  s    zCrossVit.no_weight_decayc                 C   s   t dddgdS )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)i )stemrc   )dict)r'   Zcoarser-   r-   r.   group_matcher  s    zCrossVit.group_matcherc                 C   s   |rJ dd S )Nz$gradient checkpointing not supportedr-   )r'   enabler-   r-   r.   set_grad_checkpointing  s    zCrossVit.set_grad_checkpointingc                 C   s   | j S N)r   r   r-   r-   r.   get_classifier  s    zCrossVit.get_classifierNc                    sF    _ |d ur |dv sJ |_t fddtjD _d S )Nr   c                    s.   g | ]&} d kr"t j|  nt  qS r   )r$   rD   r)   rX   r   r   r'   r-   r.   ry     rz   z-CrossVit.reset_classifier.<locals>.<listcomp>)r   r   r$   rb   rd   ra   r   )r'   r   r   r-   r   r.   reset_classifier  s    zCrossVit.reset_classifierro   c           
         s   |j d }g  t| jD ]\}}|}| j| }t||| j}||}|dkrR| jn| j}||dd}t	j
||fdd}|dkr| jn| j}|| }| |} | qt| jD ]\}}	|	  q fddt| jD   S )Nr   rN   r   rO   c                    s   g | ]\}}| | qS r-   r-   )rw   rm   r   xsr-   r.   ry     rz   z-CrossVit.forward_features.<locals>.<listcomp>)r0   rp   r   r   r   r}   Zcls_token_0Zcls_token_1expandrq   ru   Zpos_embed_0Zpos_embed_1r   re   rc   r   )
r'   r3   r4   rm   r   Zx_r|   Z
cls_tokens	pos_embedr   r-   r   r.   forward_features  s$    



zCrossVit.forward_features)r   
pre_logitsro   c                    s    j dkrdd D ndd D  fddD |sNt jd tjrftjdd D dd	S tjtjfd
dt	 jD dd	dd	S )Nr   c                 S   s(   g | ] }|d d dd f j ddqS )Nr   rO   )meanr   r-   r-   r.   ry     rz   z)CrossVit.forward_head.<locals>.<listcomp>c                 S   s   g | ]}|d d df qS )Nr   r-   r   r-   r-   r.   ry     rz   c                    s   g | ]}  |qS r-   )r   r   r   r-   r.   ry     rz   r   c                 S   s   g | ]}|qS r-   r-   r   r-   r-   r.   ry     rz   r   rO   c                    s   g | ]\}}| | qS r-   r-   )rw   rm   r   r   r-   r.   ry     rz   )
r   r   r   r$   rX   rq   ru   r   stackrp   )r'   r   r   r-   )r'   r   r.   forward_head  s
    &zCrossVit.forward_headc                 C   s   |  |}| |}|S r   )r   r   )r'   r3   r   r-   r-   r.   r8     s    

zCrossVit.forward)F)T)N)F)r9   r:   r;   r<   r   r$   r^   r    r   rq   rr   ignorer   r   r   r   r   r   rs   r   boolr   r8   r=   r-   r-   r+   r.   r   (  sD   X	
	

	c                 K   s4   | dd rtddd }tt| |fd|i|S )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.c                 S   sD   i }|   D ]2}d|v s d|v r.|dd}n|}| | ||< q|S )Nr   Z	cls_token.rn   )keysreplace)Z
state_dictZnew_state_dictkeyZnew_keyr-   r-   r.   pretrained_filter_fn  s    z._create_crossvit.<locals>.pretrained_filter_fnr   )getRuntimeErrorr   r   )variant
pretrainedkwargsr   r-   r-   r.   _create_crossvit  s    
r    c                 K   s   | ddd dt tdddd
|S )Nr   )r      r   g      ?T)zpatch_embed.0.projzpatch_embed.1.proj)zhead.0zhead.1)
urlr   
input_sizeZ	pool_sizecrop_pctr   r   Zfixed_input_size
first_conv
classifierr   )r   r   r-   r-   r.   _cfg  s    r   ztimm/)	hf_hub_id)zpatch_embed.0.proj.0zpatch_embed.1.proj.0)r   r   )r     r   r   )r   r   r   r   )zcrossvit_15_240.in1kzcrossvit_15_dagger_240.in1kzcrossvit_15_dagger_408.in1kzcrossvit_18_240.in1kzcrossvit_18_dagger_240.in1kzcrossvit_18_dagger_408.in1kzcrossvit_9_240.in1kzcrossvit_9_dagger_240.in1kzcrossvit_base_240.in1kzcrossvit_small_240.in1kzcrossvit_tiny_240.in1kr   c                 K   sZ   t dddgddgg dg dg dgddgg dd	}tf d
| dt |fi |}|S )Nr   g?r   r   `   r   r   r   r   r   r   r   r   r   r"   r)   rj   rB   rZ   crossvit_tiny_240r   r   r   r   r   r   Z
model_argsmodelr-   r-   r.   r     s    " r   c                 K   sZ   t dddgddgg dg dg dgddgg dd	}tf d
| dt |fi |}|S )Nr   r   r   r   r   r   r   r   r   crossvit_small_240r   r   r   r-   r-   r.   r     s    " r   c                 K   sZ   t dddgddgg dg dg dgddgg dd}tf d	| d
t |fi |}|S )Nr   r   r   r   r   r   r   r   crossvit_base_240r   r   r   r-   r-   r.   r   %  s    " r   c                 K   sZ   t dddgddgg dg dg dgddgg dd	}tf d
| dt |fi |}|S )Nr   r   r         r   r   r   r   r   r   r   r   crossvit_9_240r   r   r   r-   r-   r.   r   .  s    " r   c                 K   sZ   t dddgddgg dg dg dgddgg dd	}tf d
| dt |fi |}|S )Nr   r   r   r   r   r      r   r   r   r   crossvit_15_240r   r   r   r-   r-   r.   r   7  s    " r   c              	   K   sb   t f dddgddgg dg dg dgddgg dd	|}tf d
| dt |fi |}|S )Nr   r   r   r     r   r   r   r   r   r   crossvit_18_240r   r   r   r-   r-   r.   r   @  s    " r   c              	   K   s\   t dddgddgg dg dg dgddgg dd	d
}tf d| dt |fi |}|S )Nr   r   r   r   r   r   r   r   Tr   r"   r)   rj   rB   rZ   r*   crossvit_9_dagger_240r   r   r   r-   r-   r.   r   I  s    " r   c              	   K   s\   t dddgddgg dg dg dgddgg dd	d
}tf d| dt |fi |}|S )Nr   r   r   r   r   r   r   r   Tr   crossvit_15_dagger_240r   r   r   r-   r-   r.   r   R  s    " r   c              	   K   s\   t dddgddgg dg dg dgddgg dd	d
}tf d| dt |fi |}|S )Nr   g?r   r   r   r   r   r   r   Tr   crossvit_15_dagger_408r   r   r   r-   r-   r.   r   [  s    " r   c              	   K   s\   t dddgddgg dg dg dgddgg dd	d
}tf d| dt |fi |}|S )Nr   r   r   r   r   r   r   r   Tr   crossvit_18_dagger_240r   r   r   r-   r-   r.   r   d  s    " r   c              	   K   s\   t dddgddgg dg dg dgddgg dd	d
}tf d| dt |fi |}|S )Nr   r   r   r   r   r   r   r   Tr   crossvit_18_dagger_408r   r   r   r-   r-   r.   r   m  s    " r   )F)F)r   )F)F)F)F)F)F)F)F)F)F)F)4r<   	functoolsr   typingr   r   rq   Z	torch.hubZtorch.nnr$   Z	timm.datar   r   Ztimm.layersr   r	   r
   r   Z_builderr   Z_features_fxr   	_registryr   r   Zvision_transformerr   __all__Moduler   r>   rU   r_   r{   r   r   r   r   r   r   Zdefault_cfgsr   r   r   r   r   r   r   r   r   r   r   r-   r-   r-   r.   <module>   s   +)o ,

