a  d]@sdZddlmZddlmZmZmZddlZddlm Z ddl m m Z ddl mZmZddlmZmZmZmZmZmZddlmZdd lmZdd lmZmZd gZGd d d e jZ Gddde jZ!Gddde jZ"Gddde jZ#Gddde jZ$Gdd d e jZ%dDddZ&dEddZ'ee'dde'dde'dde'dde'dde'dde'ddd dd!d"e'ddd dd!d"e'ddd dd!d"e'ddd dd!d"e'dd!d#e'ddd#d$ Z(e%d%d&d'Z)dFd)d*Z*edGe%d+d,d-Z+edHe%d+d.d/Z,edIe%d+d0d1Z-edJe%d+d2d3Z.edKe%d+d4d5Z/edLe%d+d6d7Z0edMe%d+d8d9Z1edNe%d+d:d;Z2edOe%d+dd?Z4edQe%d+d@dAZ5edRe%d+dBdCZ6dS)Sa[ FocalNet As described in `Focal Modulation Networks` - https://arxiv.org/abs/2203.11926 Significant modifications and refactoring from the original impl at https://github.com/microsoft/FocalNet This impl is/has: * fully convolutional, NCHW tensor layout throughout, seemed to have minimal performance impact but more flexible * re-ordered downsample / layer so that striding always at beginning of layer (stage) * no input size constraints or input resolution/H/W tracking through the model * torchscript fixed and a number of quirks cleaned up * feature extraction support via `features_only=True` )partial)CallableOptionalTupleNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)MlpDropPath LayerNorm2d trunc_normal_ClassifierHeadNormMlpClassifierHead)build_model_with_cfg) named_apply)generate_default_cfgsregister_modelFocalNetc sFeZdZdddddefeeeeeeeedfdd Zdd Z Z S) FocalModulationTF)dim focal_level focal_factorbias use_post_normnormalize_modulator proj_drop norm_layerc s&t||_||_||_||_||_||_|||jdg|_t j |d||jdd|d|_ t j ||d|d|_ t |_t j ||dd|_t ||_t |_g|_t|jD]N} |j| |j} |jt t j ||| || dddt |j| q|jr| |nt |_dS)Nrr) kernel_sizer)r F)r groupspaddingr)super__init__r focal_windowrrrr input_splitnnConv2dfhGELUactprojZDropoutr ModuleList focal_layersZ kernel_sizesrangeappend SequentialIdentitynorm) selfrr%rrrrrrrkr  __class__]/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/focalnet.pyr$%s. "    zFocalModulation.__init__c Cs||}t||jd\}}}d}t|jD]0\}}||}|||dd||df}q.||jddd}|||dd|jdf}|j r||jd}|| |} | | } | | } | | } | S)Nrr)rT)Zkeepdim)r)torchsplitr& enumerater/r,meanrrr*r4r-r) r5xqctxZgatesZctx_alllZ focal_layerZ ctx_globalZx_outr9r9r:forwardMs "   zFocalModulation.forward) __name__ __module__ __qualname__r intboolfloatrr$rD __classcell__r9r9r7r:r$s"(rcs&eZdZdfdd ZddZZS) LayerScale2dh㈵>Fcs*t||_t|t||_dSN)r#r$inplacer' Parameterr<Zonesgamma)r5rZ init_valuesrOr7r9r:r$is zLayerScale2d.__init__cCs*|jdddd}|jr"||S||S)Nr)rQviewrOmul_)r5r@rQr9r9r:rDnszLayerScale2d.forward)rMF)rErFrGr$rDrKr9r9r7r:rLhsrLcs^eZdZdZdddddddddejef eeeee e e eeee e d fd d Z d d Z Z S) FocalNetBlockz% Focal Modulation Network Block. @rr;F-C6?r) r mlp_ratiorr%ruse_post_norm_in_modulationrlayerscale_valuer drop_path act_layerrc s(t||_||_||_||_||_|s4| |nt|_ t |||j||| | d|_ |rd| |nt|_ |durt ||nt|_| dkrt| nt|_|s| |nt|_t|t||| | dd|_|r| |nt|_|durt ||nt|_| dkrt| nt|_dS)ap Args: dim: Number of input channels. mlp_ratio: Ratio of mlp hidden dim to embedding dim. focal_level: Number of focal levels. focal_window: Focal window size at first focal level. use_post_norm: Whether to use layer norm after modulation. use_post_norm_in_modulation: Whether to use layer norm in modulation. layerscale_value: Initial layerscale value. proj_drop: Dropout rate. drop_path: Stochastic depth rate. act_layer: Activation layer. norm_layer: Normalization layer. )r%rrrrrNrT)Z in_featuresZhidden_featuresr\ZdropZuse_conv)r#r$rrXr%rrr'r3norm1r modulation norm1_postrLls1r drop_path1norm2r rHmlp norm2_postls2 drop_path2) r5rrXrr%rrYrrZrr[r\rr7r9r:r$ws<   zFocalNetBlock.__init__c Cs`|}||}||}||}||||}|||||| |}|SrN) r]r^r_rar`rfrerdrcrb)r5r@Zshortcutr9r9r:rDs   &zFocalNetBlock.forward)rErFrG__doc__r'r+r rHrJrIrr$rDrKr9r9r7r:rUss6?rUcsveZdZdZdddddddddddef eeeeeeeeeeeeeeedfd d Z e j j dd d Z d dZZS) FocalNetStagez4 A basic Focal Transformer layer for one stage. rVTrFrWrrout_dimdepthrX downsamplerr%use_overlap_downrrYrrZrr[rc svt||_||_d|_|r6t|d|d|_n t|_t  f ddt |D|_ dS)a8 Args: dim: Number of input channels. out_dim: Number of output channels. depth: Number of blocks. mlp_ratio: Ratio of mlp hidden dim to embedding dim. downsample: Downsample layer at start of the layer. focal_level: Number of focal levels focal_window: Focal window size at first focal level use_overlap_down: User overlapped convolution in downsample layer. use_post_norm: Whether to use layer norm after modulation. use_post_norm_in_modulation: Whether to use layer norm in modulation. layerscale_value: Initial layerscale value proj_drop: Dropout rate for projections. drop_path: Stochastic depth rate. norm_layer: Normalization layer. Frin_chsout_chsstrideoverlaprcs<g|]4}t ttr.|nd qS)) rrXrr%rrYrrZrr[r)rU isinstancelist.0i r[rr%rZrXrrrjrrrYr9r: sz*FocalNetStage.__init__..N) r#r$rrkgrad_checkpointing Downsamplerlr'r3r.r0blocks)r5rrjrkrXrlrr%rmrrYrrZrr[rr7rxr:r$s #   "zFocalNetStage.__init__cCs ||_dSrN)rz)r5enabler9r9r:set_grad_checkpointing sz$FocalNetStage.set_grad_checkpointingcCs@||}|jD]*}|jr2tjs2t||}q||}q|SrN)rlr|rzr<jitZ is_scripting checkpoint)r5r@Zblkr9r9r:rDs    zFocalNetStage.forward)T)rErFrGrgr rHrJrIrr$r<rignorer~rDrKr9r9r7r:rhsBD rhcs8eZdZd eeeeeedfdd ZddZZ S) r{FNrncs~t||_d}|}|rJ|dvs(J|dkr:d\}}n|dkrJd\}}tj|||||d|_|durp||nt|_dS) a Args: in_chs: Number of input image channels. out_chs: Number of linear projection output channels. stride: Downsample stride. overlap: Use overlapping convolutions if True. norm_layer: Normalization layer. r)rrr)rr)r;r)r rqr"N)r#r$rqr'r(r-r3r4)r5rorprqrrrr"r r7r9r:r$s   zDownsample.__init__cCs||}||}|SrN)r-r4r5r@r9r9r:rD;s  zDownsample.forward)rFN) rErFrGrHrIrrr$rDrKr9r9r7r:r{sr{cseZdZdZdddddddd d d d d d d d d d deeddfeeeeeedfe eedfeedfe e e e e ee e e e e e e dfdd Z ejjddZejjd'ddZejjd(ddZejjddZd)ddZd d!Zd*e d"d#d$Zd%d&ZZS)+rz," Focal Modulation Networks (FocalNets) r;avg`rrrrV)rrrrr;r;r;r;FN?rg?rM)Zeps.)in_chans num_classes global_pool embed_dimdepthsrX focal_levels focal_windowsrmrrYrhead_hidden_sizehead_init_scalerZ drop_rateproj_drop_ratedrop_path_ratercstt||_fddt|jD||_|_d|_g|_t |d| |d|_ d}ddt d|t |D}g}t|jD]}|}t||||||dk||||| | | | |||t |d|t |d|d|d }|}||g7}|jt|d d |d |d g7_qtj||_| rdt|_t|j|| |||d|_n ||j|_t|j|||d|_ttt|d|dS)a` Args: in_chans: Number of input image channels. num_classes: Number of classes for classification head. embed_dim: Patch embedding dimension. depths: Depth of each Focal Transformer layer. mlp_ratio: Ratio of mlp hidden dim to embedding dim. focal_levels: How many focal levels at all stages. Note that this excludes the finest-grain level. focal_windows: The focal window size at all stages. use_overlap_down: Whether to use convolutional embedding. use_post_norm: Whether to use layernorm after modulation (it helps stablize training of large models) layerscale_value: Value for layer scale. drop_rate: Dropout rate. drop_path_rate: Stochastic depth rate. norm_layer: Normalization layer. csg|]}d|qS)rr9rurr9r:rynz%FocalNet.__init__..rRr)rorprrrcSsg|] }|qSr9)item)rvr@r9r9r:ry}rNrrirrlayers.)Znum_chsZ reductionmodule)Z hidden_size pool_typerr)rr)r)r#r$lenZ num_layersr0rrZ num_featuresZ feature_infor{stemr<Zlinspacesumrhdictr'r2layersr3r4rheadr rr _init_weights)r5rrrrrrXrrrmrrYrrrrZrrrrZin_dimZdprrZi_layerrjZlayerr7rr:r$Esr&   & *   zFocalNet.__init__cCsdhS)Nr9r5r9r9r:no_weight_decayszFocalNet.no_weight_decaycCstd|rddgngddS)Nz^stem)z^layers\.(\d+)Nz^norm)i))z^layers\.(\d+).downsample)r)z^layers\.(\d+)\.\w+\.(\d+)Nr)rr|)r)r5Zcoarser9r9r: group_matcherszFocalNet.group_matcherTcCs"||_|jD]}|j|dq dS)N)r})rzrr~)r5r}rCr9r9r:r~s zFocalNet.set_grad_checkpointingcCs|jjSrN)rZfcrr9r9r:get_classifierszFocalNet.get_classifiercCs|jj||ddS)N)r)rreset)r5rrr9r9r:reset_classifierszFocalNet.reset_classifiercCs"||}||}||}|SrN)rrr4rr9r9r:forward_featuress   zFocalNet.forward_features pre_logitscCs|j||dS)Nr)r)r5r@rr9r9r: forward_headszFocalNet.forward_headcCs||}||}|SrN)rrrr9r9r:rDs  zFocalNet.forward)F)T)N)F)rErFrGrgrr rHstrrrJrIrrr$r<rrrrr~rrrrrDrKr9r9r7r:rAsj    h     rcCst|tjr4t|jdd|jdurtj|jnZt|tjrt|jdd|jdurftj|j|rd|vr|jj ||jj |dS)Ng{Gz?)stdhead.fc) rsr'r(r ZweightrinitZzeros_ZLineardatarT)rnamerr9r9r:rs     rrc Ks |dddddttdddd |S) Nr)r;r)rrg?Zbicubicz stem.projrZmit) urlr input_size pool_sizecrop_pct interpolationr?rZ first_conv classifierlicenser)rkwargsr9r9r:_cfgsrztimm/) hf_hub_id)r;r) riRU)rrrrr)rr) zfocalnet_tiny_srf.ms_in1kzfocalnet_small_srf.ms_in1kzfocalnet_base_srf.ms_in1kzfocalnet_tiny_lrf.ms_in1kzfocalnet_small_lrf.ms_in1kzfocalnet_base_lrf.ms_in1kzfocalnet_large_fl3.ms_in22kzfocalnet_large_fl4.ms_in22kzfocalnet_xlarge_fl3.ms_in22kzfocalnet_xlarge_fl4.ms_in22kzfocalnet_huge_fl3.ms_in22kzfocalnet_huge_fl4.ms_in22k)modelcCs|d|}d|vr|Sddl}i}|}|D]\}}|dd|}|dd}|dd d |}d |vr||vr|d d |}|dd}|dd}||vr|||kr||j|jkr|||j}|||<q4|S)Nrzstem.proj.weightrz gamma_([0-9])z ls\1.gammaZ patch_embedrzlayers.(\d+).downsamplecSsdt|dddS)Nrrz .downsample)rHgroup)r@r9r9r:!rz&checkpoint_filter_fn..r4z norm([0-9])z norm\1_postzln.znorm.rr) getre state_dictitemssubreplaceZnumelshapeZreshape)rrrZout_dictZ dest_dictr6vr9r9r:checkpoint_filter_fns$    , rFcKsPtddt|ddD}|d|}tt||fttd|dd|}|S) Ncss|]\}}|VqdSrNr9)rvrw_r9r9r: -rz#_create_focalnet..r)rrr;r out_indicesT)Zflatten_sequentialr)Zpretrained_filter_fnZ feature_cfg)tupler>rpoprrrr)variant pretrainedrZdefault_out_indicesrrr9r9r:_create_focalnet,s  r)returncKs*tfgddd|}tdd|i|S)Nrrrrfocalnet_tiny_srfr)rrrrrZ model_kwargsr9r9r:r8srcKs*tfgddd|}tdd|i|S)Nrrrrrfocalnet_small_srfr)rrrr9r9r:r>srcKs*tfgddd|}tdd|i|S)Nrrfocalnet_base_srfr)rrrr9r9r:rDsrcKs0tfgddgdd|}tdd|i|S)Nrrrrrrfocalnet_tiny_lrfr)rrrr9r9r:rJsrcKs0tfgddgdd|}tdd|i|S)Nrrrrfocalnet_small_lrfr)rrrr9r9r:rPsrcKs0tfgddgdd|}tdd|i|S)Nrrrrfocalnet_base_lrfr)rrrr9r9r:rVsrc Ks>tfgddgddgddddd|}td d |i|S) NrrrTrWrrrrrrmrZfocalnet_large_fl3r)rrrr9r9r:r]src Ks6tfgddgddddd|}td d|i|S) NrrrrrrTrWrrrrrmrZfocalnet_large_fl4r)rrrr9r9r:resrc Ks>tfgddgddgddddd|}td d |i|S) NrrrrTrWrfocalnet_xlarge_fl3r)rrrr9r9r:rmsrc Ks6tfgddgddddd|}td d|i|S) NrrrTrWrfocalnet_xlarge_fl4r)rrrr9r9r:rusrc Ks@tfgddgddgdddddd|}td d |i|S) Nr`rr;rTrW)rrrrrrYrmrZfocalnet_huge_fl3r)rrrr9r9r:r}src Ks8tfgddgdddddd|}td d|i|S) NrrrTrW)rrrrrYrmrZfocalnet_huge_fl4r)rrrr9r9r:rsr)Nr)r)F)F)F)F)F)F)F)F)F)F)F)F)F)7rg functoolsrtypingrrrr<Ztorch.nnr'Ztorch.utils.checkpointutilsrZ timm.datarrZ timm.layersr r r r r rZ_builderrZ _manipulater _registryrr__all__ModulerrLrUrhr{rrrZ default_cfgsrrrrrrrrrrrrrrr9r9r9r:s     D RV&  #