a  dM@s dZddlZddlZddlZddlmZddlmZddlm Z ddl Tddl m Z m Z mZmZmZgdZeeZd Zd Zd Zeeed Zd dZddZd-ddZd.ddZddZddZddZ d/d d!Z!d0d"d#Z"Gd$d%d%Z#d1d(d)Z$d2e j%d*d+d,Z&dS)3z EfficientNet, MobileNetV3, etc Builder Assembles EfficieNet and related network feature blocks from string definitions. Handles stride, dilation calculations, and selects feature extraction points. Hacked together by / Copyright 2019, Ross Wightman Ndeepcopy)partial)*) CondConv2dget_condconv_initializer get_act_layerget_attnmake_divisible)EfficientNetBuilderdecode_arch_defefficientnet_init_weightsresolve_bn_argsresolve_act_layerround_channelsBN_MOMENTUM_TF_DEFAULTBN_EPS_TF_DEFAULTFgGz?gMbP?)momentumepscCstSN) _BN_ARGS_TFcopyrrj/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/_efficientnet_builder.pyget_bn_args_tf&srcCs@i}|dd}|dur ||d<|dd}|dur<||d<|S)N bn_momentumrbn_epsr)pop)kwargsZbn_argsrrrrrr*s  rrelucCst|d|S)N act_layer)r r)rdefaultrrrr5sr??cCs|s|St|||||dS)z2Round number of filters based on depth multiplier.) round_limit)r )Zchannels multiplierZdivisorZ channel_minr&rrrr9srcCs|rt|dSr)_loggerinfo)msg conditionrrr _log_info_if@sr,cCs(|rt|Sdd|dDSdS)NcSsg|] }t|qSr)int).0krrr Iz _parse_ksize...)isdigitr-split)ssrrr _parse_ksizeEsr6c Cst|tsJ|d}|d}|dd}i}d}|D]}|dkrJd}q8|dkrXd}q8|d r|d}|dd}|d krtd }nJ|d krtd }n8|dkrtd}n&|dkrtd}n|dkr8td}nq8|||<q8td|} t| dkr8| dd\}}|||<q8d |vr |d nd} d|vr:t|dnd} d|vrTt|dnd} d|vrnt|dnd} t|d}t |t|dt|d| d}|dkr| t t|d| | t |dd |vrt |d nd!|dud"d#|vrt|d#|d$<n|d%ks|d&krf| t t|d| d |vrDt |d nd!|d&k|d&kp\|dud'n|d(kr| t t|d| t |d| d |vrt |d nd!|dud)n<|d*kr| t t|d|dud+ndsJd,|d-|vr|d-|d.<||fS)/a@ Decode block definition string Gets a list of block arg (dicts) through a string notation of arguments. E.g. ir_r2_k3_s2_e1_i32_o16_se0.25_noskip All args can exist in any order with the exception of the leading string which is assumed to indicate the block type. leading string - block type ( ir = InvertedResidual, ds = DepthwiseSep, dsa = DeptwhiseSep with pw act, cn = ConvBnAct) r - number of repeat blocks, k - kernel size, s - strides (1-9), e - expansion ratio, c - output channels, se - squeeze/excitation ratio n - activation fn ('re', 'r6', 'hs', or 'sw') Args: block_str: a string representation of block arguments. Returns: A list of block args (dicts) Raises: ValueError: if the string def not properly specified (TODO) _rrNnoskipFskipTnrer Zr6Zrelu6ZhsZ hard_swishswZswishmiZmishz(\d.*)apZfcrcs) block_typeout_chsstrider!irr/ese)dw_kernel_sizeexp_kernel_sizepw_kernel_size exp_ratiose_ratior8cc num_expertsdsdsa)rKrMrOZpw_actr8er)rLrMrN force_in_chsrOr8cn) kernel_sizer9zUnknown block type (%s)Zgs group_size) isinstancestrr4 startswithr r;lenr6r-dictupdatefloat) block_stropsrDoptionsr9opkeyvvaluesplitsr!rLrMrU num_repeat block_argsrrr_decode_block_strLs                               rjceilc st|}|dkr$tdt||}ntt||}g}|dddD]4}tdt|||}||||8}||8}qH|ddd}g} t||D]$\} | fddt | Dq| S)a Per-stage depth scaling Scales the block repeats in each stage. This depth scaling impl maintains compatibility with the EfficientNet scaling method, while allowing sensible scaling for other models that may have multiple block arg definitions in each stage. roundrNcsg|] }tqSrr)r.r7barrr0r1z&_scale_stage_depth..) summaxrlr-mathrkappendzipextendrange) stack_argsrepeatsdepth_multiplier depth_truncrhZnum_repeat_scaledZrepeats_scaledrArsZ sa_scaledreprrnr_scale_stage_depths   r}cCs&g}t|tr$t|t|ks2Jn|ft|}tt||D]\}\}} t|tsZJg} g} |D]n} t| tsxJt| \} }| dddkr|dkr| d|9<|dur| d|| | | |qf|r |dks|t|dkr | t | | d|q@| t | | | |q@|S)a6 Decode block architecture definition strings -> block kwargs Args: arch_def: architecture definition strings, list of list of strings depth_multiplier: network depth multiplier depth_trunc: networ depth truncation mode when applying multiplier experts_multiplier: CondConv experts multiplier fix_first_last: fix first and last block depths when multiplier is applied group_size: group size override for all blocks that weren't explicitly set in arch string Returns: list of list of block kwargs rQrrNrXr#) rYtupler\ enumeratertlistrZrjget setdefaultrsr})Zarch_defryrzZexperts_multiplierZfix_first_lastrXZ arch_args stack_idxZ block_stringsr'rwrxr`ror|rrrr s*      r c @s<eZdZdZddeddddddf ddZd d Zd d ZdS) r a/ Build Trunk Blocks This ended up being somewhat of a cross between https://github.com/tensorflow/tpu/blob/master/models/official/mnasnet/mnasnet_models.py and https://github.com/facebookresearch/maskrcnn-benchmark/blob/master/maskrcnn_benchmark/modeling/backbone/fbnet_builder.py FNrJc Cs||_||_||_||_||_||_t||_z|jdddd|_Wnt y^d|_Yn0||_ | dkr|t dd} | |_ | d vsJt|_d|_g|_dS) Nr$r#Zrd_ratioTFZ depthwisez>feature_location=='depthwise' is deprecated, using 'expansion' expansion) bottleneckrr) output_stridepad_type round_chs_fn se_from_expr! norm_layerr se_layer se_has_ratio TypeErrordrop_path_rater(warningfeature_location_DEBUG_BUILDERverbosein_chsfeatures) selfrrrrr!rrrrrrr__init__s*      zEfficientNetBuilder.__init__cCs|j||}|d}|j|d<||d|d<d|vrV|drV||d|d<|j|d<|ddurt|dn|j|d<|ddusJ|j|d<||d<|d kr|d }|r|jdur|js|| d d }|j rt |j|d |d<n |j|d<|dkrHt d |t||j| ddr8tfi|n tfi|}n|dks\|dkrt d |t||jtfi|}nv|dkrt d |t||jtfi|}nD|d krt d |t||jtfi|}ndsJd||d|_|S)NrDrrErUrr!rrrVrOrNr#rrrGz InvertedResidual {}, Args: {}rQrrRrSz! DepthwiseSeparable {}, Args: {}rTz EdgeResidual {}, Args: {}z ConvBnAct {}, Args: {}Fz-Uknkown block type (%s) while building model.)rrrrrr!rrrrrrr,formatrZrZCondConvResidualZInvertedResidualZDepthwiseSeparableConvZ EdgeResidualZ ConvBnAct)rro block_idxZ block_countrZbtrOblockrrr _make_block8sD       ,   zEfficientNetBuilder._make_blockcCs^tdt||j||_tdd|D}d}d}d}g}|ddddkr|td|d||jd krhd nd d }|j|t |D]\} } | dt|k} td | |jt | t sJg} t | D]z\} }| dt| k}td | |j|ddvsJ| dkrd|d<d}|rN| d}|t|kpL||dddk}|}|ddkr||d}||j kr||d}d|d<td |j |jn|}||d<||kr|}||||}| ||r>tf| d|d||j}d| d| }|dd }|r*d||gn||d<|j||d7}q|tj| q|S)aY Build the blocks Args: in_chs: Number of input-channels passed to first block model_block_args: A list of lists, outer list defines stages, inner list contains strings defining block configuration(s) Return: List of block stacks (each stack wrapped in nn.Sequential) z&Building model trunk with %d stages...cSsg|] }t|qSr)r\)r.xrrrr0nr1z0EfficientNetBuilder.__call__..rr>rrFZact1rZforwardr)moduleZnum_chsstage reductionZ hook_typez Stack: {}z Block: {})rr>Fz= Converting stride to dilation to maintain output_stride=={}Zdilation)rrzblocks.r2r)r,r\rrrpr]rrrsrrrYrrr feature_inforjoinnnZ Sequential)rrZmodel_block_argsZtotal_block_countZtotal_block_idxZcurrent_strideZcurrent_dilationZstagesrrrwZ last_stackblocksrriZ last_blockZextract_featuresZnext_stack_idxZ next_dilationZnext_output_strider module_nameZ leaf_namerrr__call__csx            zEfficientNetBuilder.__call__)__name__ __module__ __qualname____doc__rrrrrrrrr s  +r rTcspt|trn|jd|jd|j|r2|jtfdd|j|j}||j|j durlt j |j nt|t j r|jd|jd|j|r|jt j |jdtd|j durt j |j nt|t jrt j |jt j |j nht|t jrl|jdd}d|vr8|jd}dt|}t j |j| |t j |j dS) a) Weight initialization as per Tensorflow official implementations. Args: m (nn.Module): module to init n (str): module name fix_group_fanout (bool): enable correct (matching Tensorflow TPU impl) fanout calculation w/ group convs Handles layers in EfficientNet, EfficientNet-CondConv, MixNet, MnasNet, MobileNetV3, etc: * https://github.com/tensorflow/tpu/blob/master/models/official/mnasnet/mnasnet_model.py * https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/efficientnet_model.py rrcstj|dtdS)Nr@)rinitnormal_rrsqrt)wZfan_outrrr1z#_init_weight_goog..NrZ routing_fnr#)rYrrWZ out_channelsgroupsrrQZ weight_shapeZweightZbiasrrZzeros_ZConv2drrrrZ BatchNorm2dZones_ZLinearsizeZuniform_)mr:Zfix_group_fanoutZinit_weight_fnZfan_inZ init_rangerrr_init_weight_googs8          r)modelcCs(|pt}|D]\}}|||qdSr)rZ named_modules)rZinit_fnr:rrrrrsr)r )r#r$Nr%)r#rk)r#rkrFN)rT)N)'rloggingrrr;rr functoolsrZtorch.nnrZ_efficientnet_blocksZ timm.layersrrr r r __all__ getLoggerrr(rrrr]rrrrrr,r6rjr}r r rModulerrrrrs@       t ( . )