a
    dv                     @   s  d Z ddlZddlZddlZddlmZ ddlm  mZ	 ddl
mZ ddlmZmZ ddlmZmZmZmZmZ ddlmZ ddlmZmZ d	gZG d
d dejZG dd dejZG dd dejZG dd dejZG dd dejZ G dd dejZ!dd Z"dYddZ#G dd dejZ$G dd dejZ%dZd#d$Z&d[d%d&Z'G d'd	 d	ejZ(d\d(d)Z)d]d+d,Z*ee*d-d.d/d0e*d-d1d2d3d4e*d-d5d/d0e*d-d6d2d3d4e*d-d7d/d0e*d-d8d2d9d4e*d-d:d/d0e*d-d;d<d9d4e*d-d=d/d0e*d-d>d<d9d4e*d-d?d<d@d4dAZ+ed^e(dBdCdDZ,ed_e(dBdEdFZ-ed`e(dBdGdHZ.edae(dBdIdJZ/edbe(dBdKdLZ0edce(dBdMdNZ1edde(dBdOdPZ2edee(dBdQdRZ3edfe(dBdSdTZ4edge(dBdUdVZ5edhe(dBdWdXZ6dS )ia5   Vision OutLOoker (VOLO) implementation

Paper: `VOLO: Vision Outlooker for Visual Recognition` - https://arxiv.org/abs/2106.13112

Code adapted from official impl at https://github.com/sail-sg/volo, original copyright in comment below

Modifications and additions for timm by / Copyright 2022, Ross Wightman
    N)
checkpointIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPathMlp	to_2tuple	to_ntupletrunc_normal_   )build_model_with_cfg)register_modelgenerate_default_cfgsVOLOc                       s&   e Zd Zd	 fdd	Zdd Z  ZS )
OutlookAttention   r   F        c	           
         s   t    || }	|| _|| _|| _|| _|	d | _tj|||d| _	t||d | | _
t|| _t||| _t|| _tj|||d| _tj||dd| _d S )N      ࿩bias   )kernel_sizepaddingstrideT)r   r   Z	ceil_mode)super__init__	num_headsr   r   r   scalennLinearvattnDropout	attn_dropproj	proj_dropZUnfoldunfoldZ	AvgPool2dpool)
selfdimr   r   r   r   qkv_biasr#   r%   head_dim	__class__ Y/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/volo.pyr   (   s    

zOutlookAttention.__init__c           
      C   sp  |j \}}}}| |dddd}t|| j t|| j  }}| ||| j|| j | j	| j	 || ddddd}| 
|dddddddd}	| |	||| | j| j	| j	 | j	| j	 ddddd}	|	| j }	|	jdd}	| |	}	|	| ddddd||| j	 | j	 || }tj|||f| j	| j| jd}| |dddd}| |}|S )	Nr   r   r      r   r)   )Zoutput_sizer   r   r   )shaper    permutemathceilr   r&   reshaper   r   r'   r!   r   softmaxr#   Ffoldr   r$   r%   )
r(   xBHWCr    hwr!   r.   r.   r/   forwardE   s0    "

"




0
zOutlookAttention.forward)r   r   r   Fr   r   __name__
__module____qualname__r   rB   __classcell__r.   r.   r,   r/   r   &   s         r   c                       s:   e Zd Zdddddejejdf fdd	Zdd Z  ZS )		Outlookerr         @r   Fc              	      sp   t    |
|| _t|||||||d| _|dkr<t|nt | _|
|| _	t
|| }t|||	d| _d S )N)r   r   r   r*   r#   r   in_featureshidden_features	act_layer)r   r   norm1r   r!   r   r   Identity	drop_pathnorm2intr   mlp)r(   r)   r   r   r   r   	mlp_ratior#   rP   rM   
norm_layerr*   mlp_hidden_dimr,   r.   r/   r   a   s&    



zOutlooker.__init__c                 C   s8   ||  | | | }||  | | | }|S NrP   r!   rN   rS   rQ   r(   r;   r.   r.   r/   rB      s    zOutlooker.forward	rD   rE   rF   r   ZGELU	LayerNormr   rB   rG   r.   r.   r,   r/   rH   `   s   $rH   c                       s&   e Zd Zd fdd	Zdd Z  ZS )		Attention   Fr   c                    sb   t    || _|| }|d | _tj||d |d| _t|| _t||| _	t|| _
d S )Nr   r   r   )r   r   r   r   r   r   qkvr"   r#   r$   r%   )r(   r)   r   r*   r#   r%   r+   r,   r.   r/   r      s    

zAttention.__init__c                 C   s   |j \}}}}| |||| d| j|| j ddddd}|d\}}}	||dd | j }
|
jdd}
| 	|
}
|
|	 dd||||}| 
|}| |}|S )	Nr   r0   r   r   r   r1   r2   )r3   r^   r7   r   r4   unbind	transposer   r8   r#   r$   r%   )r(   r;   r<   r=   r>   r?   r^   qkr    r!   r.   r.   r/   rB      s    2


zAttention.forward)r]   Fr   r   rC   r.   r.   r,   r/   r\      s       r\   c                       s6   e Zd Zddddejejf fdd	Zdd Z  ZS )Transformer      @Fr   c	           
         sj   t    ||| _t||||d| _|dkr6t|nt | _||| _	t
|| }	t||	|d| _d S )N)r   r*   r#   r   rJ   )r   r   rN   r\   r!   r   r   rO   rP   rQ   rR   r   rS   )
r(   r)   r   rT   r*   r#   rP   rM   rU   rV   r,   r.   r/   r      s    


zTransformer.__init__c                 C   s8   ||  | | | }||  | | | }|S rW   rX   rY   r.   r.   r/   rB      s    zTransformer.forwardrZ   r.   r.   r,   r/   rd      s   rd   c                       s&   e Zd Zd	 fdd	Zdd Z  ZS )
ClassAttentionr]   NFr   c                    s   t    || _|d ur || _n|| }|| _|d | _tj|| j| j d |d| _tj|| j| j |d| _t	|| _
t| j| j || _t	|| _d S )Nr   r0   r   )r   r   r   r+   r   r   r   kvrb   r"   r#   r$   r%   )r(   r)   r   r+   r*   r#   r%   r,   r.   r/   r      s    	

zClassAttention.__init__c                 C   s   |j \}}}| |||d| j| jddddd}|d\}}| |d d d dd d f || jd| j}|| j |	dd }	|	j
dd}	| |	}	|	| 	dd|d| j| j }
| |
}
| |
}
|
S )	Nr0   r   r   r   r   r_   r1   r2   )r3   rg   r7   r   r+   r4   r`   rb   r   ra   r8   r#   r$   r%   )r(   r;   r<   Nr?   rg   rc   r    rb   r!   	cls_embedr.   r.   r/   rB      s    *0
"

zClassAttention.forward)r]   NFr   r   rC   r.   r.   r,   r/   rf      s        rf   c                       s:   e Zd Zddddddejejf fdd	Zdd Z  ZS )	
ClassBlockNre   Fr   c                    sp   t    |
|| _t||||||d| _|dkr:t|nt | _|
|| _	t
|| }t|||	|d| _d S )N)r   r+   r*   r#   r%   r   )rK   rL   rM   drop)r   r   rN   rf   r!   r   r   rO   rP   rQ   rR   r   rS   )r(   r)   r   r+   rT   r*   rk   r#   rP   rM   rU   rV   r,   r.   r/   r      s&    

	
zClassBlock.__init__c                 C   sj   |d d d df }||  | | | }||  | | | }tj||d d dd f gddS )Nr   r2   )rP   r!   rN   rS   rQ   torchcat)r(   r;   ri   r.   r.   r/   rB     s    zClassBlock.forwardrZ   r.   r.   r,   r/   rj      s   "rj   c                 K   s   | dkrt f i |S d S )Nca)rj   )Z
block_typeZkargsr.   r.   r/   	get_block#  s    ro   c                 C   s   | d | }| d | }t d| }t || }t || }t j|}t j|}	t ||d  d|}
t |	|d  d|}t ||d  d|}t |	|d  d|}|
|||fS )zt
    get bounding box as token labeling (https://github.com/zihangJiang/TokenLabeling)
    return: bounding box
    r   r0         ?r   )npsqrtrR   randomrandintZclip)sizelamr   r>   r=   Zcut_ratZcut_wZcut_hZcxcybbx1bby1bbx2bby2r.   r.   r/   	rand_bbox(  s    r|   c                       s*   e Zd ZdZd fd	d
	Zdd Z  ZS )
PatchEmbedzs Image to Patch Embedding.
    Different with ViT use 1 conv layer, we use 4 conv layers to do patch embedding
       Fr   r]   r   @     c                    s   t    |dv sJ |rttj||d|dddt|tjddtj||dddddt|tjddtj||dddddt|tjdd	| _nd | _tj|||| || d	| _|| ||  | _	d S )
N)r   r]         r   F)r   r   r   r   T)Zinplacer   r   r   )
r   r   r   
SequentialConv2dZBatchNorm2dZReLUconvr$   Znum_patches)r(   img_size	stem_convstem_stride
patch_sizein_chans
hidden_dim	embed_dimr,   r.   r/   r   D  s&    




zPatchEmbed.__init__c                 C   s"   | j d ur|  |}| |}|S rW   )r   r$   rY   r.   r.   r/   rB   c  s    


zPatchEmbed.forward)r~   Fr   r]   r   r   r   rD   rE   rF   __doc__r   rB   rG   r.   r.   r,   r/   r}   ?  s          r}   c                       s*   e Zd ZdZd fdd	Zdd Z  ZS )
DownsamplezF Image to Patch Embedding, downsampling between stage1 and stage2
    r0   c                    s"   t    tj||||d| _d S )Nr   )r   r   r   r   r$   )r(   Zin_embed_dimZout_embed_dimr   r,   r.   r/   r   n  s    
zDownsample.__init__c                 C   s.   | dddd}| |}| dddd}|S )Nr   r   r   r0   )r4   r$   rY   r.   r.   r/   rB   r  s    
zDownsample.forward)r0   r   r.   r.   r,   r/   r   j  s   r   r   r0   rI   Fr   c                 K   sh   g }t || D ]H}||t|d|   t|d  }|| |||||||	|
|d	 qtj| }|S )zI
    generate outlooker layer in stage1
    return: outlooker layers
    Nr   )r   r   r   r   rT   r*   r#   rP   rangesumappendr   r   )block_fnindexr)   layersr   r   r   r   rT   r*   r#   drop_path_ratekwargsblocks	block_idx	block_dprr.   r.   r/   outlooker_blocksy  s     $

r   c	                 K   sb   g }
t || D ]B}||t|d|   t|d  }|
| ||||||d qtj|
 }
|
S )zN
    generate transformer layers in stage2
    return: transformer layers
    Nr   )rT   r*   r#   rP   r   )r   r   r)   r   r   rT   r*   r#   r   r   r   r   r   r.   r.   r/   transformer_blocks  s    $

r   c                       s   e Zd ZdZddddddddd	d	d
dddddejddddf fdd	Zdd Zej	j
dd Zej	j
d+ddZej	j
d,ddZej	j
dd Zd-ddZdd Zd d! Zd"d# Zd$d% Zd.ed&d'd(Zd)d* Z  ZS )/r   z7
    Vision Outlooker, the main class of our model
    r~   r     tokenr]   r   N)TFFFrI   Fr   )rn   rn   Tr0   c                    sx  t    t|}t|t|}|| _|| _|| _|| _d | _	|rbd| _
|dksbJ dd| _tdd|||d d	| _|d | | |d
 | | f}ttd
|d |d
 d | _tj|d| _g }tt|D ]}|| r$tt|| || |  d	}|| n2tt|| || | | d
}|| |
| r|t| |d
  d qt|| _d | _d urt fddttD | _ttd
d
d | _t | jdd |r|dkrt!| j	|nt" | _#nd | _#| j	| _$t|| _%|dkrPt!| j	|nt" | _&t | jdd | '| j( d S )Nr1   rp   r   z)return all tokens if mix_token is enabledFTr0   r   )r   r   r   r   r   r   r   )p)rT   r*   r#   rU   )rT   r*   r   r#   rU   c                    s4   g | ],}t | d  d  d   ddqS )r1   r   )r)   r   rT   r*   r#   rP   rU   )ro   ).0iattn_drop_rate
embed_dimsrT   rU   r   post_layersr*   r.   r/   
<listcomp>"  s   
z!VOLO.__init__.<locals>.<listcomp>{Gz?std))r   r   lenr	   r   num_classesglobal_pool	mix_tokenpooling_scalenum_featuresbetagrad_checkpointingr}   patch_embedr   	Parameterrl   zeros	pos_embedr"   pos_dropr   r   rH   r   r   rd   r   Z
ModuleListnetworkpost_network	cls_tokenr
   r   rO   aux_headnorm	head_dropheadapply_init_weights)r(   r   r   r   r   r   r   stem_hidden_dimr   r   ZdownsamplesZoutlook_attentionrT   r*   Z	drop_rateZpos_drop_rater   r   rU   r   Zuse_aux_headZuse_mix_tokenr   Z
num_layersZ
patch_gridr   r   Zstager,   r   r/   r     s    


 $

 



$"zVOLO.__init__c                 C   sD   t |tjr@t|jdd t |tjr@|jd ur@tj|jd d S )Nr   r   r   )
isinstancer   r   r
   Zweightr   initZ	constant_)r(   mr.   r.   r/   r   ?  s    zVOLO._init_weightsc                 C   s   ddhS )Nr   r   r.   r(   r.   r.   r/   no_weight_decayE  s    zVOLO.no_weight_decayc                 C   s   t dddgg ddS )Nz ^cls_token|pos_embed|patch_embed)z^network\.(\d+)\.(\d+)N)z^network\.(\d+)r   ))z
^cls_tokenr   )z^post_network\.(\d+)N)z^norm)i )stemr   Zblocks2)dict)r(   Zcoarser.   r.   r/   group_matcherI  s    zVOLO.group_matcherc                 C   s
   || _ d S rW   )r   )r(   enabler.   r.   r/   set_grad_checkpointingX  s    zVOLO.set_grad_checkpointingc                 C   s   | j S rW   )r   r   r.   r.   r/   get_classifier\  s    zVOLO.get_classifierc                 C   sb   || _ |d ur|| _|dkr*t| j|nt | _| jd ur^|dkrTt| j|nt | _d S )Nr   )r   r   r   r   r   rO   r   r   )r(   r   r   r.   r.   r/   reset_classifier`  s     
zVOLO.reset_classifierc                 C   st   t | jD ]H\}}|dkr.|| j }| |}| jrJtj sJt||}q
||}q
|j	\}}}}|
|d|}|S )Nr0   r1   )	enumerater   r   r   r   rl   jitis_scriptingr   r3   r7   )r(   r;   idxblockr<   r=   r>   r?   r.   r.   r/   forward_tokensh  s    


zVOLO.forward_tokensc                 C   sb   |j \}}}| j|dd}tj||gdd}| jD ](}| jrTtj sTt	||}q4||}q4|S )Nr1   r   r2   )
r3   r   expandrl   rm   r   r   r   r   r   )r(   r;   r<   rh   r?   Z
cls_tokensr   r.   r.   r/   forward_clsw  s    

zVOLO.forward_clsc                 C   sV  |  |}|dddd}| jr| jrtj| j| j}|jd | j |jd | j  }}t	|
 || jd\}}}}| }	| j| | j|  }
}| j| | j|  }}|ddd|
|||ddf |	dd|
|||ddf< |	}nd\}}}}| |}| jdur| |}| |}| jdkrB|jdd	}n"| jd
kr`|dddf }n|}| jdu rt|S | |ddddf }| js|d|dd   S | jrD| jrD||jd |||jd }| }	|ddd||||ddf |	dd||||ddf< |	}||jd || |jd }||||||ffS )z A separate forward fn for training with mix_token (if a train script supports).
        Combining multiple modes in as single forward with different return types is torchscript hell.
        r   r0   r   r   )r   N)r   r   r   r   avgr2   r         ?r1   )r   r4   r   Ztrainingrq   rs   r   r3   r   r|   ru   cloneZflipr   r   r   r   r   meanr   maxr7   )r(   r;   rv   Zpatch_hZpatch_wrx   ry   rz   r{   Ztemp_xZsbbx1Zsbby1Zsbbx2Zsbby2Zx_clsZx_auxr.   r.   r/   forward_train  sB    
"B


BzVOLO.forward_trainc                 C   sB   |  |dddd}| |}| jd ur4| |}| |}|S )Nr   r0   r   r   )r   r4   r   r   r   r   rY   r.   r.   r/   forward_features  s    



zVOLO.forward_features)
pre_logitsc                 C   s   | j dkr|jdd}n | j dkr4|d d df }n|}| |}|rJ|S | |}| jd ur| |d d dd f }|d|dd   }|S )Nr   r   r2   r   r   r   )r   r   r   r   r   r   )r(   r;   r   outZauxr.   r.   r/   forward_head  s    




zVOLO.forward_headc                 C   s   |  |}| |}|S )z1 simplified forward (without mix token training) )r   r   rY   r.   r.   r/   rB     s    

zVOLO.forward)F)T)N)F)rD   rE   rF   r   r   r[   r   r   rl   r   ignorer   r   r   r   r   r   r   r   r   boolr   rB   rG   r.   r.   r,   r/   r     sN   }


4c                 K   s(   | dd rtdtt| |fi |S )NZfeatures_onlyz<features_only not implemented for Vision Transformer models.)getRuntimeErrorr   r   )variant
pretrainedr   r.   r.   r/   _create_volo  s    r    c                 K   s    | ddd dddt tddd|S )	Nr   )r   r~   r~   Q?ZbicubicTzpatch_embed.conv.0)r   r   )urlr   
input_sizeZ	pool_sizecrop_pctinterpolationZfixed_input_sizer   r   Z
first_conv
classifierr   )r   r   r.   r.   r/   _cfg  s    r   ztimm/zLhttps://github.com/sail-sg/volo/releases/download/volo_1/d1_224_84.2.pth.tarr   )	hf_hub_idr   r   zLhttps://github.com/sail-sg/volo/releases/download/volo_1/d1_384_85.2.pth.tarrp   )r   r   r   )r   r   r   r   zLhttps://github.com/sail-sg/volo/releases/download/volo_1/d2_224_85.2.pth.tarzLhttps://github.com/sail-sg/volo/releases/download/volo_1/d2_384_86.0.pth.tarzLhttps://github.com/sail-sg/volo/releases/download/volo_1/d3_224_85.4.pth.tarzLhttps://github.com/sail-sg/volo/releases/download/volo_1/d3_448_86.3.pth.tar)r     r   zLhttps://github.com/sail-sg/volo/releases/download/volo_1/d4_224_85.7.pth.tarzMhttps://github.com/sail-sg/volo/releases/download/volo_1/d4_448_86.79.pth.targffffff?zMhttps://github.com/sail-sg/volo/releases/download/volo_1/d5_224_86.10.pth.tarzLhttps://github.com/sail-sg/volo/releases/download/volo_1/d5_448_87.0.pth.tarzMhttps://github.com/sail-sg/volo/releases/download/volo_1/d5_512_87.07.pth.tar)r      r   )zvolo_d1_224.sail_in1kzvolo_d1_384.sail_in1kzvolo_d2_224.sail_in1kzvolo_d2_384.sail_in1kzvolo_d3_224.sail_in1kzvolo_d3_448.sail_in1kzvolo_d4_224.sail_in1kzvolo_d4_448.sail_in1kzvolo_d5_224.sail_in1kzvolo_d5_448.sail_in1kzvolo_d5_512.sail_in1k)returnc                 K   s,   t f dddd|}tdd| i|}|S ) VOLO-D1 model, Params: 27M r   r   r]   r0      r   r   r         r   r   r   r   r   volo_d1_224r   )r   r   r   r   r   Z
model_argsmodelr.   r.   r/   r      s    r   c                 K   s,   t f dddd|}tdd| i|}|S )r   r   r   r   r   volo_d1_384r   )r  r   r   r.   r.   r/   r  (  s    r  c                 K   s,   t f dddd|}tdd| i|}|S ) VOLO-D2 model, Params: 59M r   r   
   r      r   r   r   r]   r   r   r   r   volo_d2_224r   )r  r   r   r.   r.   r/   r  0  s    r  c                 K   s,   t f dddd|}tdd| i|}|S )r  r  r  r  r   volo_d2_384r   )r	  r   r   r.   r.   r/   r	  8  s    r	  c                 K   s,   t f dddd|}tdd| i|}|S ) VOLO-D3 model, Params: 86M r]   r]   r   r   r  r  r   volo_d3_224r   )r  r   r   r.   r.   r/   r  @  s    r  c                 K   s,   t f dddd|}tdd| i|}|S )r
  r  r  r  r   volo_d3_448r   )r  r   r   r.   r.   r/   r  H  s    r  c                 K   s,   t f dddd|}tdd| i|}|S ) VOLO-D4 model, Params: 193M r  r      r  r  r   r   r   r   r   volo_d4_224r   )r  r   r   r.   r.   r/   r  P  s    r  c                 K   s,   t f dddd|}tdd| i|}|S )r  r  r  r  r   volo_d4_448r   )r  r   r   r.   r.   r/   r  X  s    r  c                 K   s0   t f dddddd|}td	d| i|}|S )
h VOLO-D5 model, Params: 296M
    stem_hidden_dim=128, the dim in patch embedding is 128 for VOLO-D5
    r   r      r   r  r  r      r   r   r   rT   r   volo_d5_224r   )r  r   r   r.   r.   r/   r  `  s    r  c                 K   s0   t f dddddd|}td	d| i|}|S )
r  r  r  r  r   r  r  volo_d5_448r   )r  r   r   r.   r.   r/   r  l  s    r  c                 K   s0   t f dddddd|}td	d| i|}|S )
r  r  r  r  r   r  r  volo_d5_512r   )r  r   r   r.   r.   r/   r  x  s    r  )r   )r   r   r   r0   rI   Fr   r   )rI   Fr   r   )F)r   )F)F)F)F)F)F)F)F)F)F)F)7r   r5   numpyrq   rl   Ztorch.nnr   Ztorch.nn.functionalZ
functionalr9   Ztorch.utils.checkpointr   Z	timm.datar   r   Ztimm.layersr   r   r   r	   r
   Z_builderr   	_registryr   r   __all__Moduler   rH   r\   rd   rf   rj   ro   r|   r}   r   r   r   r   r   r   Zdefault_cfgsr   r  r  r	  r  r  r  r  r  r  r  r.   r.   r.   r/   <module>   s   :+%*+
+        
+    
  

0