a Adʏ @sddlZddlZddlmZddlmmZddlmm Z ddl m Z m Z m Z GdddejZddZddZGd d d ejZGd d d ejZGd ddejZGdddejZGdddejZGdddejZGdddejZGdddejZGdddejZGdddejZedkrdZdZd eed!eZ d"eed!eZ!ed#e e!fed$gd%d&gd%d#d'd( Z"e#e"e#e e!e"$d)e%d!d*e e!fZ&e"e&Z&e#e&j'dS)+N)DropPath to_2tuple trunc_normal_cs0eZdZddejdffdd ZddZZS)MlpNcsNt|p|}|p|}t|||_||_t|||_t||_dSN) super__init__nnLinearfc1actfc2Dropoutdrop)self in_featureshidden_features out_features act_layerr __class__S/var/www/html/stable-diffusion-webui/extensions-builtin/SwinIR/swinir_model_arch.pyr s z Mlp.__init__cCs6||}||}||}||}||}|Sr)r r rrrxrrrforwards      z Mlp.forward)__name__ __module__ __qualname__r GELUr r __classcell__rrrrrs rcCsR|j\}}}}|||||||||}|ddddddd|||}|S)z Args: x: (B, H, W, C) window_size (int): window size Returns: windows: (num_windows*B, window_size, window_size, C) r)shapeviewpermute contiguous)r window_sizeBHWCwindowsrrrwindow_partition!s $r2cCsbt|jd||||}||||||||d}|dddddd|||d}|S)z Args: windows: (num_windows*B, window_size, window_size, C) window_size (int): Window size H (int): Height of image W (int): Width of image Returns: x: (B, H, W, C) rr'r"r#r$r%r&)intr(r)r*r+)r1r,r.r/r-rrrrwindow_reverse0s $r4csBeZdZdZdfdd ZdddZed d d Zd d ZZ S)WindowAttentiona Window based multi-head self attention (W-MSA) module with relative position bias. It supports both of shifted and non-shifted window. Args: dim (int): Number of input channels. window_size (tuple[int]): The height and width of the window. num_heads (int): Number of attention heads. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 proj_drop (float, optional): Dropout ratio of output. Default: 0.0 TNrcst||_||_||_||}|p.|d|_tt d|ddd|dd||_ t |jd} t |jd} t t | | g} t| d} | dddddf| dddddf} | ddd} | dddddf|jdd7<| dddddf|jdd7<| dddddfd|jdd9<| d}|d|tj||d|d|_t||_t|||_t||_t|j d d tjdd |_dS) Ngr$rr"r'relative_position_indexr#bias{Gz?std)dim)rr r<r, num_headsscaler Parametertorchzerosrelative_position_bias_tablearangestackmeshgridflattenr*r+sumregister_bufferr qkvr attn_dropproj proj_droprSoftmaxsoftmax)rr<r,r=qkv_biasqk_scalerJrLhead_dimcoords_hcoords_wcoordscoords_flattenrelative_coordsr6rrrr Os4 & ,((,    zWindowAttention.__init__c Csl|j\}}}||||d|j||jddddd}|d|d|d}}} ||j}||dd} |j|j d |j d|j d|j d|j dd} | ddd } | | d} |dur&|jd} | || | |j||| d d} | d|j||} | | } n | | } || } | | dd|||}||}||}|S) z Args: x: input features with shape of (num_windows*B, N, C) mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None r#r$rr"r%r'N)r(rIreshaper=r*r> transposerBr6r)r,r+ unsqueezerNrJrKrL) rrmaskB_Nr0rIqkvattnrelative_position_biasnWrrrrrs* . &  (     zWindowAttention.forwardreturncCsd|jd|jd|jS)Ndim=, window_size= , num_heads=)r<r,r=rrrr extra_reprszWindowAttention.extra_reprcCspd}|||jd|j7}||j||j|j|7}||j|||j|j7}|||j|j7}|S)Nrr#)r<r=)rr]flopsrrrrks zWindowAttention.flops)TNrr)N) rrr__doc__r rstrrjrkr!rrrrr5As  # !r5c s`eZdZdZddddddddejejf fdd Zd d Zd d Z e dddZ ddZ Z S)SwinTransformerBlocka Swin Transformer Block. Args: dim (int): Number of input channels. input_resolution (tuple[int]): Input resolution. num_heads (int): Number of attention heads. window_size (int): Window size. shift_size (int): Shift size for SW-MSA. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. drop (float, optional): Dropout rate. Default: 0.0 attn_drop (float, optional): Attention dropout rate. Default: 0.0 drop_path (float, optional): Stochastic depth rate. Default: 0.0 act_layer (nn.Module, optional): Activation layer. Default: nn.GELU norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm r@TNrc st||_||_||_||_||_||_t|j|jkrPd|_t|j|_d|jkrh|jksrnJd| ||_ t |t |j|||| | d|_ | dkrt | nt|_| ||_t||}t||| | d|_|jdkr||j}nd}|d|dS)Nrz shift_size must in 0-window_size)r,r=rOrPrJrLr)rrrr attn_mask)rr r<input_resolutionr=r, shift_size mlp_ratiominnorm1r5rrarr Identity drop_pathnorm2r3rmlpcalculate_maskrH)rr<rrr=r,rsrtrOrPrrJrxr norm_layermlp_hidden_dimrqrrrr s0  "     zSwinTransformerBlock.__init__c Cs|\}}td||df}td|j t|j |j t|j df}td|j t|j |j t|j df}d}|D].}|D]$} ||dd|| ddf<|d7}qqzt||j} | d|j|j} | d| d} | | dkt d| dkt d} | S)Nr"rr'r$gYr) r@rAslicer,rsr2r)rZ masked_fillfloat) rx_sizer.r/img_maskh_slicesw_slicescnthw mask_windowsrqrrrr{s(      $z#SwinTransformerBlock.calculate_maskc CsD|\}}|j\}}}|}||}|||||}|jdkrZtj||j |j fdd} n|} t| |j} | d|j|j|} |j|kr|j | |j d} n|j | | | |j d} | d|j|j|} t| |j||} |jdkrtj| |j|jfdd}n| }|||||}|||}|||||}|S)Nr)r"r$)shiftsdimsr')r[)r(rvr)rsr@rollr2r,rrrarqr{todevicer4rxrzry) rrrr.r/r-Lr0shortcut shifted_x x_windows attn_windowsrrrrs,      zSwinTransformerBlock.forwardrdc Cs4d|jd|jd|jd|jd|jd|j S)Nrf, input_resolution=rhrgz , shift_size=z , mlp_ratio=)r<rrr=r,rsrtrirrrrjszSwinTransformerBlock.extra_reprcCsd}|j\}}||j||7}|||j|j}|||j|j|j7}|d|||j|j|j7}||j||7}|S)Nrr$)rrr<r,rarkrt)rrkr.r/rcrrrrks "zSwinTransformerBlock.flops)rrrrlr r LayerNormr r{rrmrjrkr!rrrrrns !*rncsDeZdZdZejffdd ZddZedddZ d d Z Z S) PatchMergingz Patch Merging Layer. Args: input_resolution (tuple[int]): Resolution of input feature. dim (int): Number of input channels. norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm csBt||_||_tjd|d|dd|_|d||_dS)Nr%r$Fr7)rr rrr<r r reductionnorm)rrrr<r|rrrr 5s  zPatchMerging.__init__c Cs6|j\}}|j\}}}|||ks*Jd|ddkrB|ddksXJd|d|d|||||}|ddddddddddf}|ddddddddddf}|ddddddddddf} |ddddddddddf} t||| | gd }||d d |}||}||}|S) z x: B, H*W, C zinput feature has wrong sizer$rzx size (*z) are not even.Nr"r'r%)rrr(r)r@catrr) rrr.r/r-rr0x0x1x2x3rrrr<s  .$$$$  zPatchMerging.forwardrdcCsd|jd|jS)Nzinput_resolution=z, dim=rrr<rirrrrjSszPatchMerging.extra_reprcCsD|j\}}|||j}||d|dd|jd|j7}|S)Nr$r%rrr.r/rkrrrrkVs (zPatchMerging.flops rrrrlr rr rrmrjrkr!rrrrr,s rc sTeZdZdZddddddejddf fdd Zd d Zed d d Z ddZ Z S) BasicLayera/ A basic Swin Transformer layer for one stage. Args: dim (int): Number of input channels. input_resolution (tuple[int]): Input resolution. depth (int): Number of blocks. num_heads (int): Number of attention heads. window_size (int): Local window size. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. drop (float, optional): Dropout rate. Default: 0.0 attn_drop (float, optional): Attention dropout rate. Default: 0.0 drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. rpTNrFc sxt|_|_||_||_t f ddt|D|_ | durn| d|_ nd|_ dS)NcsRg|]J}t |ddkr"dn d ttrD|nd qS)r$r) r<rrr=r,rsrtrOrPrrJrxr|)rn isinstancelist).0i rJr<rrxrrrtr|r=rPrOr,rr |s z'BasicLayer.__init__..)r<r|) rr r<rrdepthuse_checkpointr ModuleListrangeblocks downsample)rr<rrrr=r,rtrOrPrrJrxr|rrrrrr qs "  zBasicLayer.__init__cCsD|jD]$}|jr t|||}q|||}q|jdur@||}|Sr)rr checkpointr)rrrblkrrrrs    zBasicLayer.forwardrdcCsd|jd|jd|jS)Nrfrz, depth=)r<rrrrirrrrjszBasicLayer.extra_reprcCs8d}|jD]}||7}q |jdur4||j7}|SNr)rrkr)rrkrrrrrks   zBasicLayer.flopsrrrrrr]s   rc sLeZdZdZddddddejddddd f fd d Zd d ZddZZ S)RSTBaResidual Swin Transformer Block (RSTB). Args: dim (int): Number of input channels. input_resolution (tuple[int]): Input resolution. depth (int): Number of blocks. num_heads (int): Number of attention heads. window_size (int): Local window size. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. drop (float, optional): Dropout rate. Default: 0.0 attn_drop (float, optional): Attention dropout rate. Default: 0.0 drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. img_size: Input image size. patch_size: Patch size. resi_connection: The convolutional block before residual connection. rpTNrFr%1convcstt|||_||_t||||||||| | | | | |d|_|dkr^t||ddd|_ nj|dkrt t||ddddtj ddd t|d|dddd tj ddd t|d|ddd|_ t ||d |dd |_ t||d |dd |_dS) N)r<rrrr=r,rtrOrPrrJrxr|rrrr#r"3convr%皙?Tnegative_slopeinplacerimg_size patch_sizein_chans embed_dimr|)rrr r<rrrresidual_groupr Conv2dconv Sequential LeakyReLU PatchEmbed patch_embed PatchUnEmbed patch_unembed)rr<rrrr=r,rtrOrPrrJrxr|rrrrresi_connectionrrrr s@ $ z RSTB.__init__c Cs$||||||||Sr)rrrr)rrrrrrrsz RSTB.forwardcCsXd}||j7}|j\}}||||j|jd7}||j7}||j7}|S)Nr )rrkrrr<rrrrkr.r/rrrrks z RSTB.flops) rrrrlr rr rrkr!rrrrrs  'rcs2eZdZdZd fdd Zd d Zd d ZZS)rax Image to Patch Embedding Args: img_size (int): Image size. Default: 224. patch_size (int): Patch token size. Default: 4. in_chans (int): Number of input image channels. Default: 3. embed_dim (int): Number of linear projection output channels. Default: 96. norm_layer (nn.Module, optional): Normalization layer. Default: None rr%r#`Ncstt|}t|}|d|d|d|dg}||_||_||_|d|d|_||_||_|dur~|||_ nd|_ dSNrr") rr rrrpatches_resolution num_patchesrrrrrrrrr|rrrrr s   zPatchEmbed.__init__cCs*|ddd}|jdur&||}|S)Nr$r")rFrYrrrrrr s  zPatchEmbed.forwardcCs.d}|j\}}|jdur*||||j7}|Sr)rrrrrrrrks   zPatchEmbed.flops)rr%r#rNrrrrlr rrkr!rrrrrs rcs2eZdZdZd fdd Zd d Zd d ZZS)raz Image to Patch Unembedding Args: img_size (int): Image size. Default: 224. patch_size (int): Patch token size. Default: 4. in_chans (int): Number of input image channels. Default: 3. embed_dim (int): Number of linear projection output channels. Default: 96. norm_layer (nn.Module, optional): Normalization layer. Default: None rr%r#rNcsntt|}t|}|d|d|d|dg}||_||_||_|d|d|_||_||_dSr) rr rrrrrrrrrrrr %s  zPatchUnEmbed.__init__cCs2|j\}}}|dd||j|d|d}|S)Nr"r$r)r(rYr)r)rrrr-HWr0rrrr2s "zPatchUnEmbed.forwardcCsd}|Srr)rrkrrrrk7szPatchUnEmbed.flops)rr%r#rNrrrrrrs  rcs eZdZdZfddZZS)UpsamplezUpsample module. Args: scale (int): Scale factor. Supported scales: 2^n and 3. num_feat (int): Channel number of intermediate features. c sg}||d@dkr\ttt|dD]0}|t|d|ddd|tdq(nF|dkr|t|d|ddd|tdntd|dt t |j |dS) Nr"rr$r%r#rzscale z/ is not supported. Supported scales: 2^n and 3.) rr3mathlogappendr r PixelShuffle ValueErrorrrr )rr>num_featm_rrrr DszUpsample.__init__)rrrrlr r!rrrrr<srcs*eZdZdZdfdd ZddZZS)UpsampleOneStepa*UpsampleOneStep module (the difference with Upsample is that it always only has 1conv + 1pixelshuffle) Used in lightweight SR to save parameters. Args: scale (int): Scale factor. Supported scales: 2^n and 3. num_feat (int): Channel number of intermediate features. Nc sT||_||_g}|t||d|ddd|t|tt|j|dS)Nr$r#r") rrrrr rrrrr )rr>r num_out_chrrrrrrr \s  zUpsampleOneStep.__init__cCs$|j\}}|||jdd}|S)Nr#r)rrrrrrrrkds zUpsampleOneStep.flops)N)rrrrlr rkr!rrrrrRs rcseZdZdZddddddddd d d d d ejd d d ddddffdd ZddZej j ddZ ej j ddZ ddZ ddZddZd d!ZZS)"SwinIRa SwinIR A PyTorch impl of : `SwinIR: Image Restoration Using Swin Transformer`, based on Swin Transformer. Args: img_size (int | tuple(int)): Input image size. Default 64 patch_size (int | tuple(int)): Patch size. Default: 1 in_chans (int): Number of input image channels. Default: 3 embed_dim (int): Patch embedding dimension. Default: 96 depths (tuple(int)): Depth of each Swin Transformer layer. num_heads (tuple(int)): Number of attention heads in different layers. window_size (int): Window size. Default: 7 mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4 qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. Default: None drop_rate (float): Dropout rate. Default: 0 attn_drop_rate (float): Attention dropout rate. Default: 0 drop_path_rate (float): Stochastic depth rate. Default: 0.1 norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. ape (bool): If True, add absolute position embedding to the patch embedding. Default: False patch_norm (bool): If True, add normalization after patch embedding. Default: True use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False upscale: Upscale factor. 2/3/4/8 for image SR, 1 for denoising and compress artifact reduction img_range: Image range. 1. or 255. upsampler: The reconstruction reconstruction module. 'pixelshuffle'/'pixelshuffledirect'/'nearest+conv'/None resi_connection: The convolutional block before residual connection. '1conv'/'3conv' @r"r#rrrrrorpTNrg?Fr$?rc  stt||}|}d}||_|dkrFd}t|dddd|_ntdddd|_||_ ||_ ||_ t ||ddd|_t||_||_||_||_||_||_t|||||jr|ndd|_|jj}|jj}||_t|||||jr|ndd|_|jr$t td|||_t|jddt j| d|_ d d t!d | t"|D}t #|_$t%|jD]v}t&||d |df||||||j| | | | |t"|d|t"|d|d|d||||d }|j$'|qb||j|_(|d krt ||ddd|_)nl|dkrrt *t ||ddddt j+dddt |d|dddd t j+dddt |d|ddd|_)|j dkrt *t ||dddt j+dd|_,t-|||_.t ||ddd|_/n|j dkrt0||||d |df|_.n|j dkrt *t ||dddt j+dd|_,t ||ddd|_1|j dkrRt ||ddd|_2t ||ddd|_3t ||ddd|_/t j+ddd|_4nt ||ddd|_/|5|j6dS)Nrr#)gw#?g8EGr?gB`"?r"rr9r:)pcSsg|] }|qSr)item)rrrrrrz#SwinIR.__init__..r)r<rrrr=r,rtrOrPrrJrxr|rrrrrrrr%rTr pixelshuffle)rpixelshuffledirect nearest+conv)7rrr img_ranger@Tensorr)meanrAupscale upsamplerr,r r conv_firstlen num_layersrape patch_norm num_featuresrtrrrrrrr?absolute_pos_embedrrpos_droplinspacerGrlayersrrrrconv_after_bodyrrconv_before_upsamplerupsample conv_lastrconv_up1conv_up2conv_hrlreluapply _init_weights) rrrrrdepthsr=r,rtrOrP drop_rateattn_drop_ratedrop_path_rater|rrrrrrrkwargs num_in_chrrrgb_meanrrdpri_layerlayerrrrr s    &            zSwinIR.__init__cCsrt|tjrBt|jddt|tjrn|jdurntj|jdn,t|tjrntj|jdtj|jddS)Nr9r:rr) rr r rweightr8init constant_r)rrrrrrs  zSwinIR._init_weightscCsdhS)Nrrrirrrno_weight_decayszSwinIR.no_weight_decaycCsdhS)NrBrrirrrno_weight_decay_keywords szSwinIR.no_weight_decay_keywordscCsV|\}}}}|j||j|j}|j||j|j}t|d|d|fd}|S)Nrreflect)sizer,Fpad)rrrrr mod_pad_h mod_pad_wrrrcheck_image_sizes zSwinIR.check_image_sizecCsh|jd|jdf}||}|jr.||j}||}|jD]}|||}q>||}|||}|S)Nr$r#)r(rrrrrrr)rrrr rrrforward_featuress       zSwinIR.forward_featuresc Cs|jdd\}}||}|j||_||j|j}|jdkr~||}||||}| |}| | |}n|jdkr||}||||}| |}n|jdkrJ||}||||}| |}| | tjjj|ddd}|jdkr2| |tjjj|ddd}| | ||}n,||}||||}|| |}||j|j}|ddddd||jd||jfS)Nr$rrrnearest) scale_factormoder%)r(rrtype_asrrrrrrrrrrr@r functional interpolaterrr)rrr.r/x_firstresrrrr%s4              zSwinIR.forwardcCs|d}|j\}}|||d|jd7}||j7}|jD]}||7}q<|||d|j|j7}||j7}|S)Nrr#r)rrrrkrr)rrkr.r/r rrrrkJs  z SwinIR.flops)rrrrlr rr rr@jitignorerrrrrrkr!rrrrrjs" x   %r__main__r%ir"ir$rr<r) rrr,rrrr=rtrgeAr#)(rr@torch.nnr Ztorch.nn.functionalrrtorch.utils.checkpointutilsrtimm.models.layersrrrModulerr2r4r5rnrrrrrrrrrrrr,heightwidthmodelprintrkrandnrr(rrrrsD c 1FL+"m