a  dt/@sdZddlZddlmZddlmZddlmZmZm Z m Z m Z gdZ ddZ Gdd d ejZGd d d ejZGd d d ejZGdddejZGdddeZGdddejZdS)z[ EfficientNet, MobileNetV3, etc Blocks Hacked together by / Copyright 2019, Ross Wightman N) functional) create_conv2dDropPathmake_divisiblecreate_act_layerget_norm_act_layer) SqueezeExcite ConvBnActDepthwiseSeparableConvInvertedResidualCondConvResidual EdgeResidualcCs$|sdS||dksJ||SdS)Nr) group_sizeZchannelsrri/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/_efficientnet_blocks.py num_groupssrcs:eZdZdZddejejddffdd ZddZZ S)ra Squeeze-and-Excitation w/ specific features for EfficientNet/MobileNet family Args: in_chs (int): input channels to layer rd_ratio (float): ratio of squeeze reduction act_layer (nn.Module): activation layer of containing block gate_layer (Callable): attention gate function force_act_layer (nn.Module): override block's activation fn if this is set/bound rd_round_fn (Callable): specify a fn to calculate rounding of reduced chs g?Ncsvtt||dur*|pt}|||}|p0|}tj||ddd|_t|dd|_tj||ddd|_ t||_ dS)NrT)Zbiasinplace) superr__init__roundnnZConv2d conv_reduceract1 conv_expandgate)selfin_chsZrd_ratioZ rd_channels act_layerZ gate_layerZforce_act_layerZ rd_round_fn __class__rrr%s zSqueezeExcite.__init__cCs:|jddd}||}||}||}|||S)N)T)Zkeepdim)meanrrrr)rxZx_serrrforward2s    zSqueezeExcite.forward) __name__ __module__ __qualname____doc__rReLUZSigmoidrr& __classcell__rrr rrs   rcsFeZdZdZdddddejejdffdd Zd d Zd d Z Z S) r z@ Conv + Norm Layer + Activation w/ optional skip connection rrFc  sxtt|t| | } t||} |o4|dko4||k|_t|||||| |d|_| |dd|_| rjt | nt |_ dS)NrstridedilationgroupspaddingTr) rr rrrhas_skiprconvbn1rrIdentity drop_path)rrout_chsZ kernel_sizer0r1rpad_typeskipr norm_layerdrop_path_ratenorm_act_layerr2r rrr=s  zConvBnAct.__init__cCs0|dkrtdd|jjdStdd|jjdSdS)N expansionr6r&moduleZ hook_typeZnum_chsr-)dictr5 out_channelsrlocationrrr feature_infoJszConvBnAct.feature_infocCs0|}||}||}|jr,|||}|SN)r5r6r4r8rr%shortcutrrrr&Ps   zConvBnAct.forward r'r(r)r*rr+ BatchNorm2drrFr&r,rrr rr :s   r c sNeZdZdZddddddddejejddf fdd Zd d Zd d Z Z S)r z DepthwiseSeparable block Used for DS convs in MobileNet-V1 and in the place of IR blocks that have no expansion (factor of 1.0). This is an alternative to having a IR with an optional first pw conv. r#rr-FNr.c stt|t| | }t||}|dko6||ko6| |_| |_t|||||||d|_||dd|_ | rv| || dnt |_ t||| |d|_ ||d|jd|_|rt|nt |_dS)Nr)r0r1r3r2Trrr3)r apply_act)rr rrrr4Z has_pw_actrconv_dwr6rr7seconv_pwbn2rr8)rrr9dw_kernel_sizer0r1rr:noskippw_kernel_sizeZpw_actrr<se_layerr=r>r2r rrr^s  zDepthwiseSeparableConv.__init__cCs0|dkrtdd|jjdStdd|jjdSdS)Nr?rQ forward_prer@r-)rBrQ in_channelsrCrDrrrrFssz#DepthwiseSeparableConv.feature_infocCsN|}||}||}||}||}||}|jrJ|||}|SrG)rOr6rPrQrRr4r8rHrrrr&ys     zDepthwiseSeparableConv.forwardrJrrr rr Ys r csReZdZdZdddddddddejejdddffd d Zd d Zd dZ Z S)r a Inverted residual block w/ optional SE Originally used in MobileNet-V2 - https://arxiv.org/abs/1801.04381v4, this layer is often referred to as 'MBConv' for (Mobile inverted bottleneck conv) and is also used in * MNasNet - https://arxiv.org/abs/1807.11626 * EfficientNet - https://arxiv.org/abs/1905.11946 * MobileNet-V3 - https://arxiv.org/abs/1905.02244 r#rr-F?Nr.cstt|t| | }|pi}t|| }t||}||koJ|dkoJ| |_t||| fd|i||_||dd|_ t|||f||||d||_ ||dd|_ |r||| dnt |_t||| fd|i||_||dd|_|rt|nt |_dS) Nrr3Trr/rLFrN)rr rrrrr4rrQr6rOrRrr7rPconv_pwlbn3rr8)rrr9rSr0r1rr:rT exp_ratioexp_kernel_sizerUrr<rV conv_kwargsr=r>mid_chsr2r rrrs*   zInvertedResidual.__init__cCs0|dkrtdd|jjdStdd|jjdSdSNr?r[rWr@r-rBr[rXrCrDrrrrFszInvertedResidual.feature_infocCsb|}||}||}||}||}||}||}||}|jr^|||}|SrG) rQr6rOrRrPr[r\r4r8rHrrrr&s       zInvertedResidual.forwardrJrrr rr s   r csJeZdZdZdddddddddejejddd ffd d Zd d ZZ S)r z, Inverted residual block w/ CondConv routingr#rr-FrYNrr.csV||_t|jd}tt|j|||||||| || | | || ||dt||j|_dS)N) num_experts)rSr0r1rr:rrTr]r^rUrVr<r_r=)rcrBrr rrZLinear routing_fn)rrr9rSr0r1rr:rTr]r^rUrr<rVrcr=r_r rrrs    zCondConvResidual.__init__cCs|}t|dd}t||}|||}||}|||}| |}| |}| ||}| |}|j r|||}|S)Nr)FZadaptive_avg_pool2dflattentorchZsigmoidrdrQr6rOrRrPr[r\r4r8)rr%rIZ pooled_inputsZrouting_weightsrrrr&s       zCondConvResidual.forward) r'r(r)r*rr+rKrr&r,rrr rr s    r c sPeZdZdZdddddddddejejdd f fd d Zd d ZddZ Z S)r a( Residual block with expansion convolution followed by pointwise-linear w/ stride Originally introduced in `EfficientNet-EdgeTPU: Creating Accelerator-Optimized Neural Networks with AutoML` - https://ai.googleblog.com/2019/08/efficientnet-edgetpu-creating.html This layer is also called FusedMBConv in the MobileDet, EfficientNet-X, and EfficientNet-V2 papers * MobileDet - https://arxiv.org/abs/2004.14525 * EfficientNet-X - https://arxiv.org/abs/2102.05610 * EfficientNet-V2 - https://arxiv.org/abs/2104.00298 r#rrr-FrYNr.c stt|t| | }|dkr.t|| }n t|| }t||}||koX|dkoX| |_t|||||||d|_||dd|_ |r||| dnt |_ t||| |d|_ ||dd |_|rt|nt |_dS) Nrrr/TrrLrMFrZ)rr rrrrr4rconv_expr6rr7rPr[rRrr8)rrr9r^r0r1rr:Z force_in_chsrTr]rUrr<rVr=r>r`r2r rrrs   zEdgeResidual.__init__cCs0|dkrtdd|jjdStdd|jjdSdSrarbrDrrrrF szEdgeResidual.feature_infocCsN|}||}||}||}||}||}|jrJ|||}|SrG)rhr6rPr[rRr4r8rHrrrr&s     zEdgeResidual.forwardrJrrr rr s  r )r*rgZtorch.nnrrreZ timm.layersrrrrr__all__rModulerr r r r r rrrrs   !,;#