a
    db:                     @   s  d Z ddlZddlZddlmZ ddlmZmZ ddlZddlm	Z	 ddl
mZmZ ddlmZmZmZ dd	lmZ dd
lmZmZ ddlmZ dgZG dd de	jZG dd de	jZG dd de	jZG dd de	jZG dd de	jZdd Z d4ddZ!d5ddZ"ee"dde"dde"dde"dde"dd d!e"dd d!e"dd d!e"dd d!d"Z#ed6ed#d$d%Z$ed7ed#d&d'Z%ed8ed#d(d)Z&ed9ed#d*d+Z'ed:ed#d,d-Z(ed;ed#d.d/Z)ed<ed#d0d1Z*ed=ed#d2d3Z+dS )>a   Pooling-based Vision Transformer (PiT) in PyTorch

A PyTorch implement of Pooling-based Vision Transformers as described in
'Rethinking Spatial Dimensions of Vision Transformers' - https://arxiv.org/abs/2103.16302

This code was adapted from the original version at https://github.com/naver-ai/pit, original copyright below.

Modifications for timm by / Copyright 2020 Ross Wightman
    N)partial)SequenceTuple)nnIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)trunc_normal_	to_2tuple	LayerNorm   )build_model_with_cfg)register_modelgenerate_default_cfgs)BlockPoolingVisionTransformerc                       sH   e Zd ZdZ fddZeejejf eejejf dddZ  Z	S )SequentialTuplezI This module exists to work around torchscript typing issues list -> listc                    s   t t| j|  d S N)superr   __init__)selfargs	__class__ X/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/pit.pyr   "   s    zSequentialTuple.__init__xreturnc                 C   s   | D ]}||}q|S r   r   )r   r   moduler   r   r   forward%   s    
zSequentialTuple.forward)
__name__
__module____qualname____doc__r   r   torchTensorr    __classcell__r   r   r   r   r       s   r   c                       sF   e Zd Zd fdd	Zeejejf eejejf dddZ  ZS )	TransformerN        c
           
         s^   t t|   | || _|	r(|	nt | _tj fddt|D  | _	d S )Nc                    s2   g | ]*}t d  | ttjdddqS )Tư>Zeps)dimZ	num_heads	mlp_ratioZqkv_bias	proj_drop	attn_dropZ	drop_path
norm_layer)r   r   r   r   ).0ir/   drop_path_prob	embed_dimheadsr-   r.   r   r   
<listcomp>=   s   z(Transformer.__init__.<locals>.<listcomp>)
r   r(   r   poolr   Identitynorm
Sequentialrangeblocks)
r   Zbase_dimdepthr6   r-   r8   r.   r/   r4   r0   r   r3   r   r   ,   s    zTransformer.__init__r   c                 C   s   |\}}|j d }| jd ur,| ||\}}|j \}}}}|ddd}tj||fdd}| |}| |}|d d d |f }|d d |d f }|dd||||}||fS )Nr      )r,   )	shaper8   flattenZ	transposer%   catr:   r=   Zreshape)r   r   
cls_tokensZtoken_lengthBCHWr   r   r   r    J   s    



zTransformer.forward)Nr)   r)   NN	r!   r"   r#   r   r   r%   r&   r    r'   r   r   r   r   r(   +   s        r(   c                       s8   e Zd Zd fdd	Zeejejf dddZ  ZS )Poolingzerosc              	      sB   t t|   tj|||d |d |||d| _t||| _d S )Nr   r?   )kernel_sizepaddingstridepadding_modegroups)r   rI   r   r   Conv2dconvLinearfc)r   Z
in_featureZout_featurerM   rN   r   r   r   r   _   s    	zPooling.__init__r   c                 C   s   |  |}| |}||fS r   )rQ   rS   )r   r   	cls_tokenr   r   r   r    m   s    

zPooling.forward)rJ   rH   r   r   r   r   rI   ^   s   rI   c                       s2   e Zd Zd
eeeed fddZdd	 Z  ZS )ConvEmbedding         r   )img_size
patch_sizerM   rL   c                    s   t t|   |}t|| _t|| _t| jd d|  | jd  | d | _t| jd d|  | jd  | d | _	| j| j	f| _
tj|||||dd| _d S )Nr   r?   r   T)rK   rM   rL   bias)r   rV   r   r
   rZ   r[   mathfloorheightwidthZ	grid_sizer   rP   rQ   )r   Zin_channelsZout_channelsrZ   r[   rM   rL   r   r   r   r   t   s    	

,,zConvEmbedding.__init__c                 C   s   |  |}|S r   )rQ   r   r   r   r   r   r       s    
zConvEmbedding.forward)rW   rX   rY   r   )r!   r"   r#   intr   r    r'   r   r   r   r   rV   s   s       rV   c                       s   e Zd ZdZd'eeeeee ee ee ed fddZdd Z	e
jjdd Ze
jjd(ddZe
jjd)ddZdd Zd*ddZd d! Zd+ee
jd"d#d$Zd%d& Z  ZS ),r   z Pooling-based Vision Transformer

    A PyTorch implement of 'Rethinking Spatial Dimensions of Vision Transformers'
        - https://arxiv.org/abs/2103.16302
    rW   rX   rY   overlap0   re   re   r?         r?   rh   rY   rh        tokenFr)   )rZ   r[   rM   	stem_type	base_dimsr>   r6   r-   c                    s6  t t|   |dv sJ || _|| _|d |d  }|	| _|| _|rJdnd| _g | _t	|
||||| _
ttd|| j
j| j
j| _ttd| j|| _tj|d| _g }dd td|t||D }|}tt|D ]}d }|| ||  }|dkrt||dd}|t|| || || |||||| d	g7 }|}|  jt||d d|  d
| dg7  _qt| | _tj|d |d  dd| _ | | _!| _"t|| _#|	dkrt$| j"|	nt% | _&d | _'|r|	dkrt$| j"| jnt% | _'d| _(t)| jdd t)| jdd | *| j+ d S )N)rl   r   r?   r   )pc                 S   s   g | ]}|  qS r   )tolist)r1   r   r   r   r   r7          z5PoolingVisionTransformer.__init__.<locals>.<listcomp>)rM   )r8   r.   r/   r4   transformers.)Znum_chsZ	reductionr   r*   r+   Fg{Gz?)std),r   r   r   rn   r6   num_classesglobal_poolZ
num_tokensZfeature_inforV   patch_embedr   	Parameterr%   Zrandnr_   r`   	pos_embedrU   ZDropoutpos_dropZlinspacesumsplitr<   lenrI   r(   dictr   transformersr   r:   Znum_featuresr5   	head_droprR   r9   head	head_distdistilled_trainingr	   apply_init_weights)r   rZ   r[   rM   rm   rn   r>   r6   r-   ru   Zin_chansrv   	distilledZ	drop_rateZpos_drop_drateZproj_drop_rateZattn_drop_rateZdrop_path_rater5   r   ZdprZprev_dimr2   r8   r   r   r   r      sb     "


.
"$z!PoolingVisionTransformer.__init__c                 C   s0   t |tjr,tj|jd tj|jd d S )Nr   g      ?)
isinstancer   r   initZ	constant_r\   Zweight)r   mr   r   r   r      s    z&PoolingVisionTransformer._init_weightsc                 C   s   ddhS )Nry   rU   r   r   r   r   r   no_weight_decay   s    z(PoolingVisionTransformer.no_weight_decayTc                 C   s
   || _ d S r   )r   r   enabler   r   r   set_distilled_training   s    z/PoolingVisionTransformer.set_distilled_trainingc                 C   s   |rJ dd S )Nz$gradient checkpointing not supportedr   r   r   r   r   set_grad_checkpointing   s    z/PoolingVisionTransformer.set_grad_checkpointingc                 C   s    | j d ur| j| j fS | jS d S r   )r   r   r   r   r   r   get_classifier   s    
z'PoolingVisionTransformer.get_classifierNc                 C   sV   || _ |dkrt| j|nt | _| jd urR|dkrHt| j| j nt | _d S )Nr   )ru   r   rR   r5   r9   r   r   )r   ru   rv   r   r   r   reset_classifier   s     
z)PoolingVisionTransformer.reset_classifierc                 C   sP   |  |}| || j }| j|jd dd}| ||f\}}| |}|S )Nr   rs   )rw   rz   ry   rU   expandr@   r   r:   )r   r   rC   r   r   r   forward_features   s    

z)PoolingVisionTransformer.forward_features)
pre_logitsr   c                 C   s   | j d ur| jdksJ |d d df |d d df  }}| |}| |}|sf| |}|  |}| jr| jrtj s||fS || d S n6| jdkr|d d df }| |}|s| |}|S d S )Nrl   r   r   r?   )	r   rv   r   r   r   Ztrainingr%   jitZis_scripting)r   r   r   Zx_distr   r   r   forward_head  s"    
"






z%PoolingVisionTransformer.forward_headc                 C   s   |  |}| |}|S r   )r   r   ra   r   r   r   r      s    

z PoolingVisionTransformer.forward)rW   rX   rY   rc   rd   rf   ri   rh   rj   rk   rl   Fr)   r)   r)   r)   r)   )T)T)N)F)r!   r"   r#   r$   rb   strr   floatr   r   r%   r   ignorer   r   r   r   r   r   boolr&   r   r    r'   r   r   r   r   r      sP                    N

c                 C   s<   i }t d}|  D ] \}}|dd |}|||< q|S )z preprocess checkpoints zpools\.(\d)\.c                 S   s   dt | dd  dS )Nrr   r   z.pool.)rb   group)expr   r   r   <lambda>-  rq   z&checkpoint_filter_fn.<locals>.<lambda>)recompileitemssub)Z
state_dictmodelZout_dictZp_blockskvr   r   r   checkpoint_filter_fn$  s    

r   Fc                 K   s@   t td}|d|}tt| |fttdd|dd|}|S )Nrk   out_indiceshookT)Zfeature_clsZ
no_rewriter   )Zpretrained_filter_fnZfeature_cfg)tupler<   popr   r   r   r~   )variant
pretrainedkwargsZdefault_out_indicesr   r   r   r   r   _create_pit2  s    r    c                 K   s    | ddd dddt tddd|S )	Nrj   )rk   rW   rW   g?ZbicubicTzpatch_embed.convr   )urlru   Z
input_sizeZ	pool_sizeZcrop_pctinterpolationZfixed_input_sizemeanrt   Z
first_conv
classifierr   )r   r   r   r   r   _cfgA  s    r   ztimm/)	hf_hub_id)r   r   )r   r   )zpit_ti_224.in1kzpit_xs_224.in1kzpit_s_224.in1kzpit_b_224.in1kzpit_ti_distilled_224.in1kzpit_xs_distilled_224.in1kzpit_s_distilled_224.in1kzpit_b_distilled_224.in1krT   c                 K   s>   t ddg dg dg ddd}td| fi t |fi |S )	N      @   r   r   rk   rg   rh   rh   rY   rX   rh   r[   rM   rn   r>   r6   r-   	pit_b_224r~   r   r   r   Z
model_argsr   r   r   r   a  s    r   c                 K   s>   t ddg dg dg ddd}td| fi t |fi |S )	NrX   rY   rd   rf   rk   rg      rh   r   	pit_s_224r   r   r   r   r   r   n  s    r   c                 K   s>   t ddg dg dg ddd}td| fi t |fi |S )	NrX   rY   rd   rf   ri   rh   r   
pit_xs_224r   r   r   r   r   r   {  s    r   c                 K   s>   t ddg dg dg ddd}td| fi t |fi |S )	NrX   rY       r   r   rf   ri   rh   r   
pit_ti_224r   r   r   r   r   r     s    r   c              	   K   s@   t ddg dg dg dddd}td	| fi t |fi |S )
Nr   r   r   r   r   rh   Tr[   rM   rn   r>   r6   r-   r   pit_b_distilled_224r   r   r   r   r   r     s    	r   c              	   K   s@   t ddg dg dg dddd}td	| fi t |fi |S )
NrX   rY   rd   rf   r   rh   Tr   pit_s_distilled_224r   r   r   r   r   r     s    	r   c              	   K   s@   t ddg dg dg dddd}td	| fi t |fi |S )
NrX   rY   rd   rf   ri   rh   Tr   pit_xs_distilled_224r   r   r   r   r   r     s    	r   c              	   K   s@   t ddg dg dg dddd}td	| fi t |fi |S )
NrX   rY   r   rf   ri   rh   Tr   pit_ti_distilled_224r   r   r   r   r   r     s    	r   )F)r   )F)F)F)F)F)F)F)F),r$   r]   r   	functoolsr   typingr   r   r%   r   Z	timm.datar   r   Ztimm.layersr	   r
   r   Z_builderr   	_registryr   r   Zvision_transformerr   __all__r;   r   Moduler(   rI   rV   r   r   r   r   Zdefault_cfgsr   r   r   r   r   r   r   r   r   r   r   r   <module>   sv   3 

