a  d@s,dZddlZddlmZmZmZmZddlZddlm Z ddl m m Z ddl mmZddlmZmZddlmZmZmZmZmZmZmZddlmZddlmZdd lm Z m!Z!m"Z"d gZ#ee$ee$e$ffZ%ee$e$fd d d Z&eee$e$fee$e$fdddZ'Gddde j(Z)Gddde j(Z*Gddde j(Z+Gddde j(Z,Gdd d e j(Z-ddZ.dYddZ/dZd d!Z0e e0d"d#d$e0d"d%d&d'd(d)e0d"d*d$e0d"d+d&d'd(d)e0d"d,d$e0d"d-d$e0d"d.d$e0d"d/d$e0d"d0d$e0d"d1d$e0d"d2d3d4d5d6e0d"d7d3d4d5d6d8 Z1e!d[e-d9d:d;Z2e!d\e-d9dd?Z4e!d^e-d9d@dAZ5e!d_e-d9dBdCZ6e!d`e-d9dDdEZ7e!dae-d9dFdGZ8e!dbe-d9dHdIZ9e!dce-d9dJdKZ:e!dde-d9dLdMZ;e!dee-d9dNdOZdRdSdTdUdVdWdXdS)gaK Swin Transformer V2 A PyTorch impl of : `Swin Transformer V2: Scaling Up Capacity and Resolution` - https://arxiv.org/abs/2111.09883 Code/weights from https://github.com/microsoft/Swin-Transformer, original copyright/license info below Modifications and additions for timm hacked together by / Copyright 2022, Ross Wightman N)CallableOptionalTupleUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD) PatchEmbedMlpDropPath to_2tuple trunc_normal__assertClassifierHead)build_model_with_cfg)register_notrace_function)generate_default_cfgsregister_modelregister_model_deprecationsSwinTransformerV2) window_sizecCsj|j\}}}}||||d|d||d|d|}|ddddddd|d|d|}|S)z Args: x: (B, H, W, C) window_size (int): window size Returns: windows: (num_windows*B, window_size, window_size, C) rrshapeviewpermute contiguous)xrBHWCwindowsr(h/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/swin_transformer_v2.pywindow_partition"s ,,r*)rimg_sizecCsf|\}}|jd}|d||d||d|d|d|}|ddddddd|||}|S)z Args: windows: (num_windows * B, window_size[0], window_size[1], C) window_size (Tuple[int, int]): Window size img_size (Tuple[int, int]): Image size Returns: x: (B, H, W, C) rrrrrrrr)r'rr+r$r%r&r"r(r(r)window_reverse1s  ,$r,csDeZdZdZdddddgffdd Zd eejdd d ZZ S) WindowAttentiona Window based multi-head self attention (W-MSA) module with relative position bias. It supports both of shifted and non-shifted window. Args: dim (int): Number of input channels. window_size (tuple[int]): The height and width of the window. num_heads (int): Number of attention heads. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 proj_drop (float, optional): Dropout ratio of output. Default: 0.0 pretrained_window_size (tuple[int]): The height and width of the window in pre-training. Trc st||_||_||_||_tt dt |ddf|_ t tj ddddtjddtj d|dd|_tj|jd d |jd tjd }tj|jdd |jdtjd } tt|| gddd d } |d d krL| ddddddd f|d d<| dddddddf|dd<n\| ddddddd f|jd d<| dddddddf|jdd<| d 9} t| tt| d td } |jd | ddt|jd } t|jd} tt| | g} t| d}|dddddf|dddddf}|ddd }|ddddd f|jd d7<|dddddf|jdd7<|ddddd fd|jdd9<|d}|jd|ddtj ||ddd|_|rTtt||_ |jdt|ddtt||_!nd|_ d|_"d|_!t#||_$t |||_%t#||_&tj'dd|_(dS)N rriTbias)ZinplaceFr)Zdtype?relative_coords_table persistentrrelative_position_indexrk_biasdim))super__init__r:rpretrained_window_size num_headsnn ParametertorchlogZones logit_scale SequentialLinearZReLUcpb_mlpZarangefloat32stackZmeshgridr r! unsqueezesignlog2absmathregister_bufferflattensumqkvzerosq_biasv_biasr8ZDropout attn_dropproj proj_dropZSoftmaxsoftmax)selfr:rr>qkv_biasrUrWr=Zrelative_coords_hZrelative_coords_wr4Zcoords_hZcoords_wZcoordsZcoords_flattenZrelative_coordsr7 __class__r(r)r<Qsp " $$,...   ,((,   zWindowAttention.__init__NmaskcCs|j\}}}d}|jdur0t|j|j|jf}tj||jj |d}| ||d|j d ddddd}| d\}} } tj|dd tj| dd d d} tj|jtd d } | | } ||jd|j } | |jd|jd|jd|jd|jdd}| ddd}d t|}| |d} |dur|jd}| d||j |||dd} | d|j ||} || } n || } || } | | dd |||}||}| |}|S)z Args: x: input features with shape of (num_windows*B, N, C) mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None N)inputweightr1rrrrrrr9gY@)max)!rrSrAcatr8rTFZlinearrQr`reshaper>r Zunbind normalizeZ transposeclamprCrMrBexprFr4rr7rr!ZsigmoidrIrXrUrVrW)rYr"r^ZB_Nr&rZrQqkvattnrCrelative_position_bias_tableZrelative_position_biasZnum_winr(r(r)forwards8  "$&  $     zWindowAttention.forward)N) __name__ __module__ __qualname____doc__r<rrAZTensorrp __classcell__r(r(r[r)r-CsFr-c speZdZdZdddddddejejdf fdd Zeee e fee e ffd d d Z d d Z ddZ Z S)SwinTransformerV2Blockz Swin Transformer Block. r@Tr.c st||_t||_||_|||\}}||_||_|jd|jd|_ ||_ t |t|j||| |t| d|_ | ||_ | dkrt| nt|_t|t||| |d|_| ||_| dkrt| nt|_t|jr|j\}}td||df}d}td|jd t|jd |jd t|jd dfD]j}td|jd t|jd |jd t|jd dfD]&}||dd||ddf<|d7}qqDt||j}|d|j }|d|d}||dktd |dktd}nd}|j d |d d dS) a Args: dim: Number of input channels. input_resolution: Input resolution. num_heads: Number of attention heads. window_size: Window size. shift_size: Shift size for SW-MSA. mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: If True, add a learnable bias to query, key, value. proj_drop: Dropout rate. attn_drop: Attention dropout rate. drop_path: Stochastic depth rate. act_layer: Activation layer. norm_layer: Normalization layer. pretrained_window_size: Window size in pretraining. rr)rr>rZrUrWr=r.)Z in_featuresZhidden_features act_layerZdropNrrgY attn_maskFr5)!r;r<r:r input_resolutionr>_calc_window_shiftr shift_size window_area mlp_ratior-rnnorm1r r?Identity drop_path1r intmlpnorm2 drop_path2anyrArRslicer*rrIZ masked_fillfloatrN)rYr:r{r>rr}rrZrWrU drop_pathry norm_layerr=wsssr$r%Zimg_maskZcnthwZ mask_windowsrzr[r(r)r<s`        &zSwinTransformerV2Block.__init__returncCsNt|}t|}ddt|j|D}ddt|j||D}t|t|fS)NcSs g|]\}}||kr|n|qSr(r().0rrr(r(r) z=SwinTransformerV2Block._calc_window_shift..cSs"g|]\}}}||krdn|qSrr()rrrsr(r(r)r r)r zipr{tuple)rYZtarget_window_sizeZtarget_shift_sizerr}r(r(r)r|s z)SwinTransformerV2Block._calc_window_shiftc Cs|j\}}}}t|j}|rBtj||jd |jd fdd}n|}t||j}|d|j|}|j ||j d} | d|jd|jd|} t | |j|j }|rtj||jdd}n|}|S)Nrr)rr)Zshiftsdimsrr]) rrr}rAZrollr*rrr~rnrzr,r{) rYr"r#r$r%r&Z has_shiftZ shifted_xZ x_windowsZ attn_windowsr(r(r)_attn#s & zSwinTransformerV2Block._attncCsd|j\}}}}|||||}||d|}|||||}|||||}|S)Nr)rrrrrfrrrrYr"r#r$r%r&r(r(r)rp?s zSwinTransformerV2Block.forward)rqrrrsrtr?ZGELU LayerNormr<rrr|rrprur(r(r[r)rvsW&rvcs0eZdZdZdejffdd ZddZZS) PatchMergingz Patch Merging Layer. NcsFt||_|pd||_tjd||jdd|_||j|_dS)z Args: dim (int): Number of input channels. out_dim (int): Number of output channels (or 2 * dim if None) norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm rrFr0N)r;r<r:out_dimr?rE reductionnorm)rYr:rrr[r(r)r<Ls  zPatchMerging.__init__cCs|j\}}}}t|ddkd|dt|ddkd|d|||dd|dd|dddddd d}||}||}|S) Nrrz x height (z) is not even.z x width (rrrr)rrrfr rOrrrr(r(r)rpYs2  zPatchMerging.forward) rqrrrsrtr?rr<rprur(r(r[r)rHs rc sFeZdZdZddddddejddf fdd Zd d Zd d ZZ S) SwinTransformerV2Stagez" A Swin Transformer V2 Stage. FrxTr.rc st| _| _|r,tdd|Dn| _| _| _d _|rZt |d _ n|ksfJt _ t  f ddt|D _dS)a  Args: dim: Number of input channels. input_resolution: Input resolution. depth: Number of blocks. num_heads: Number of attention heads. window_size: Local window size. downsample: Use downsample layer at start of the block. mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: If True, add a learnable bias to query, key, value. proj_drop: Projection dropout rate attn_drop: Attention dropout rate. drop_path: Stochastic depth rate. norm_layer: Normalization layer. pretrained_window_size: Local window size in pretraining. output_nchw: Output tensors on NCHW format instead of NHWC. css|]}|dVqdS)rNr(rir(r(r) rz2SwinTransformerV2Stage.__init__..F)r:rrcsTg|]L}t j |ddkr$dn dttrD|nd qS)rr) r:r{r>rr}rrZrWrUrrr=)rvoutput_resolution isinstancelistr rUrrrr>rr=rWrZrYrr(r)rsz3SwinTransformerV2Stage.__init__..N)r;r<r:r{rrdepth output_nchwgrad_checkpointingr downsampler?rZ ModuleListrangeblocks)rYr:rr{rr>rrrrZrWrUrrr=rr[rr)r<gs#   "zSwinTransformerV2Stage.__init__cCs@||}|jD]*}|jr2tjs2t||}q||}q|SN)rrrrAjitZ is_scripting checkpoint)rYr"blkr(r(r)rps    zSwinTransformerV2Stage.forwardcCsX|jD]L}tj|jjdtj|jjdtj|jjdtj|jjdqdS)Nr)rr?init constant_rr1r`r)rYrr(r(r)_init_respostnorms  z(SwinTransformerV2Stage._init_respostnorm) rqrrrsrtr?rr<rprrur(r(r[r)rcs D rcseZdZdZdddddddd d d d d d d dejdfeeeeeee edfe edfee e e e e e e e edfdfdd Z ddZejjddZejjd)ddZejjd*ddZejjddZd+d d!Zd"d#Zd,e d$d%d&Zd'd(ZZS)-rz Swin Transformer V2 A PyTorch impl of : `Swin Transformer V2: Scaling Up Capacity and Resolution` - https://arxiv.org/abs/2111.09883 rravg`rrrrr rwrxTr.g?)rrrr.)r+ patch_sizein_chans num_classes global_pool embed_dimdepthsr>rrrZ drop_rateproj_drop_rateattn_drop_ratedrop_path_raterpretrained_window_sizesc st||_|dvsJ||_d|_t||_|_td|jd|_ g|_ t t t fs|fddt|jDt|||d|dd|_d dtd|t||D}g}d}d}t|jD]}|}|t|||jjd||jjd|f|||dk||| | | | ||||||d g7}|}|dkrN|d9}|j t|d |d |d g7_ qtj||_||j |_t|j ||| |jd|_||j|jD]}| qdS)a/ Args: img_size: Input image size. patch_size: Patch size. in_chans: Number of input image channels. num_classes: Number of classes for classification head. embed_dim: Patch embedding dimension. depths: Depth of each Swin Transformer stage (layer). num_heads: Number of attention heads in different layers. window_size: Window size. mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: If True, add a learnable bias to query, key, value. drop_rate: Head dropout rate. proj_drop_rate: Projection dropout rate. attn_drop_rate: Attention dropout rate. drop_path_rate: Stochastic depth rate. norm_layer: Normalization layer. patch_norm: If True, add normalization after patch embedding. pretrained_window_sizes: Pretrained window sizes of each layer. output_fmt: Output tensor format if not None, otherwise output 'NHWC' by default. )rZNHWCrrcsg|]}td|qS)r)rrrr(r)rrz.SwinTransformerV2.__init__..r)r+rrrr output_fmtcSsg|] }|qSr()tolist)rr"r(r(r)rr)r:rr{rrr>rrrZrWrUrrr=rlayers.)Znum_chsrmodule)Z pool_typerZ input_fmtN)!r;r<rrrlenZ num_layersrrZ num_featuresZ feature_inforrrrr patch_embedrAZlinspacerPsplitrZ grid_sizedictr?rDlayersrrheadapply _init_weightsr)rYr+rrrrrrr>rrrZrrrrrrkwargsZdprrZin_dimZscalerrZblyr[rr)r<sv*    "  &    zSwinTransformerV2.__init__cCsDt|tjr@t|jddt|tjr@|jdur@tj|jddS)Ng{Gz?)stdr)rr?rEr r`r1rr)rYmr(r(r)r1s zSwinTransformerV2._init_weightscs<t}|D](\}tfdddDr|q|S)Ncsg|] }|vqSr(r()rkwnr(r)r;rz5SwinTransformerV2.no_weight_decay..)rFrCro)setZ named_modulesradd)rYZnodrr(rr)no_weight_decay7s  z!SwinTransformerV2.no_weight_decayFcCstd|r dngddS)Nz^absolute_pos_embed|patch_embedz^layers\.(\d+)))z^layers\.(\d+).downsampler)z^layers\.(\d+)\.\w+\.(\d+)N)z^norm)i)stemr)r)rYZcoarser(r(r) group_matcher?szSwinTransformerV2.group_matchercCs|jD] }||_qdSr)rr)rYenablelr(r(r)set_grad_checkpointingJs z(SwinTransformerV2.set_grad_checkpointingcCs|jjSr)rZfc)rYr(r(r)get_classifierOsz SwinTransformerV2.get_classifierNcCs||_|j||dSr)rrreset)rYrrr(r(r)reset_classifierSsz"SwinTransformerV2.reset_classifiercCs"||}||}||}|Sr)rrrrYr"r(r(r)forward_featuresWs   z"SwinTransformerV2.forward_features pre_logitscCs|r|j|ddS||S)NTr)r)rYr"rr(r(r) forward_head]szSwinTransformerV2.forward_headcCs||}||}|Sr)rrrr(r(r)rp`s  zSwinTransformerV2.forward)F)T)N)F)rqrrrsrtr?r_int_or_tuple_2_trstrrrboolrr<rrArignorerrrrrrrrprur(r(r[r)rsd   m     cs|d|}|d|}d|v}i}ddl}|D]J\}tfdddDrTq4|sv|dd d d d ||<q4|S) Nmodel state_dictzhead.fc.weightrcsg|] }|vqSr(r()rrrlr(r)rmrz(checkpoint_filter_fn..)r7r4rzzlayers.(\d+).downsamplecSsdt|dddS)Nrrz .downsample)rgroup)r"r(r(r)qrz&checkpoint_filter_fn..zhead.zhead.fc.)getreitemsrsubreplace)rrZnative_checkpointZout_dictrrmr(rr)checkpoint_filter_fnfs    rFcKsPtddt|ddD}|d|}tt||fttd|dd|}|S) Ncss|]\}}|VqdSrr()rr_r(r(r)ryrz._create_swin_transformer_v2..r)rrrr out_indicesT)Zflatten_sequentialr)Zpretrained_filter_fnZ feature_cfg)r enumeraterpoprrrr)variant pretrainedrZdefault_out_indicesrrr(r(r)_create_swin_transformer_v2xs  rrc Ks"|ddddddttddd d |S) Nr)rr)r2r2g?ZbicubicTzpatch_embed.projzhead.fcZmit) urlr input_size pool_sizecrop_pct interpolationZfixed_input_sizemeanrZ first_conv classifierlicenser)rrr(r(r)_cfgsrztimm/z{https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window12to16_192to256_22kto1k_ft.pth) hf_hub_idrz{https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window12to24_192to384_22kto1k_ft.pth)rr)rrr3)rrrrr z|https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_large_patch4_window12to16_192to256_22kto1k_ft.pthz|https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_large_patch4_window12to24_192to384_22kto1k_ft.pthzfhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_tiny_patch4_window8_256.pthzghttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_tiny_patch4_window16_256.pthzghttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_small_patch4_window8_256.pthzhhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_small_patch4_window16_256.pthzfhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window8_256.pthzghttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window16_256.pthzkhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window12_192_22k.pthiQU)rr)rr)rrrrrzlhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_large_patch4_window12_192_22k.pth) 2swinv2_base_window12to16_192to256.ms_in22k_ft_in1k2swinv2_base_window12to24_192to384.ms_in22k_ft_in1k3swinv2_large_window12to16_192to256.ms_in22k_ft_in1k3swinv2_large_window12to24_192to384.ms_in22k_ft_in1kzswinv2_tiny_window8_256.ms_in1kz swinv2_tiny_window16_256.ms_in1kz swinv2_small_window8_256.ms_in1kz!swinv2_small_window16_256.ms_in1kzswinv2_base_window8_256.ms_in1kz swinv2_base_window16_256.ms_in1k!swinv2_base_window12_192.ms_in22k"swinv2_large_window12_192.ms_in22krcKs.tddddd}tdd|it|fi|S)  rcrrrrrrr>swinv2_tiny_window16_256r)rrrrrZ model_argsr(r(r)rsrcKs.tddddd}tdd|it|fi|S) rr2rrrrswinv2_tiny_window8_256r)rrrr(r(r)rsrcKs.tddddd}tdd|it|fi|S) rrcrrrrrrswinv2_small_window16_256r)r rrr(r(r)r sr cKs.tddddd}tdd|it|fi|S) rr2rrrrswinv2_small_window8_256r)r!rrr(r(r)r!sr!cKs.tddddd}tdd|it|fi|S) rrcrrr2rc rswinv2_base_window16_256r)r%rrr(r(r)r%sr%cKs.tddddd}tdd|it|fi|S) rr2r"rr#rswinv2_base_window8_256r)r&rrr(r(r)r&sr&cKs.tddddd}tdd|it|fi|S) rrr"rr#rswinv2_base_window12_192r)r'rrr(r(r)r'sr'cKs0tdddddd}td d|it|fi|S) rrcr"rr#rrrrrrrr>r!swinv2_base_window12to16_192to256r)r*rrr(r(r)r*sr*cKs0tdddddd}td d|it|fi|S) rrr"rr#r(r)!swinv2_base_window12to24_192to384r)r+rrr(r(r)r+sr+cKs.tddddd}tdd|it|fi|S) rrrrrrr0rswinv2_large_window12_192r)r.rrr(r(r)r.sr.cKs0tdddddd}td d|it|fi|S) rrcrrr,r(r)"swinv2_large_window12to16_192to256r)r/rrr(r(r)r/'sr/cKs0tdddddd}td d|it|fi|S) rrrrr,r(r)"swinv2_large_window12to24_192to384r)r0rrr(r(r)r02sr0rrrrrr)Zswinv2_base_window12_192_22kZ)swinv2_base_window12to16_192to256_22kft1kZ)swinv2_base_window12to24_192to384_22kft1kZswinv2_large_window12_192_22kZ*swinv2_large_window12to16_192to256_22kft1kZ*swinv2_large_window12to24_192to384_22kft1k)F)r)F)F)F)F)F)F)F)F)F)F)F)F)?rtrMtypingrrrrrAZtorch.nnr?Ztorch.nn.functionalZ functionalreZtorch.utils.checkpointutilsrZ timm.datarrZ timm.layersr r r r r rrZ_builderrZ _features_fxr _registryrrr__all__rrr*r,Moduler-rvrrrrrrZ default_cfgsrrr r!r%r&r'r*r+r.r/r0rqr(r(r(r)s $  "~Z* :