a  d^@sdZddlZddlmZddlZddlmZddlmZm Z ddl m Z m Z m Z mZmZmZmZddlmZddlmZmZdd lmZmZmZd d gZGd d d ejZGd ddejZGdddejZGdddejZ GdddejZ!Gdd d ejZ"dleje#e$dddZ%ddZ&dmddZ'dnd d!Z(ee(e(e(e(d"d#d$e(d"d%d&d'e(e(d"d(d$e(d"d)d&d'e(d"d*d+d,d-d.d/d0e(d"d1d+d,d-d.d2e(ee d3e(d"d4ee d5e(d"d6ee d5e(d"d7ee d5e(d"d8ee d5e(d"d9ee d5e(d"d:ee d5e(d"d;ee d5e(d"dee d5e(d"d?ee d5e(d"d@ee d5e(e(d"dAd$e(dBZ)edoe"dCdDdEZ*edpe"dCdFdGZ+edqe"dCdHdIZ,edre"dCdJdKZ-edse"dCdLdMZ.edte"dCdNdOZ/edue"dCdPdQZ0edve"dCdRdSZ1edwe"dCdTdUZ2edxe"dCdVdWZ3edye"dCdXdYZ4edze"dCdZd[Z5ed{e"dCd\d]Z6ed|e"dCd^d_Z7ed}e"dCd`daZ8ee9dbdcdddedfdgdhdidjdUdWdk dS)~uj MLP-Mixer, ResMLP, and gMLP in PyTorch This impl originally based on MLP-Mixer paper. Official JAX impl: https://github.com/google-research/vision_transformer/blob/linen/vit_jax/models_mixer.py Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 @article{tolstikhin2021, title={MLP-Mixer: An all-MLP Architecture for Vision}, author={Tolstikhin, Ilya and Houlsby, Neil and Kolesnikov, Alexander and Beyer, Lucas and Zhai, Xiaohua and Unterthiner, Thomas and Yung, Jessica and Keysers, Daniel and Uszkoreit, Jakob and Lucic, Mario and Dosovitskiy, Alexey}, journal={arXiv preprint arXiv:2105.01601}, year={2021} } Also supporting ResMlp, and a preliminary (not verified) implementations of gMLP Code: https://github.com/facebookresearch/deit Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404 @misc{touvron2021resmlp, title={ResMLP: Feedforward networks for image classification with data-efficient training}, author={Hugo Touvron and Piotr Bojanowski and Mathilde Caron and Matthieu Cord and Alaaeldin El-Nouby and Edouard Grave and Armand Joulin and Gabriel Synnaeve and Jakob Verbeek and Hervé Jégou}, year={2021}, eprint={2105.03404}, } Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050 @misc{liu2021pay, title={Pay Attention to MLPs}, author={Hanxiao Liu and Zihang Dai and David R. So and Quoc V. Le}, year={2021}, eprint={2105.08050}, } A thank you to paper authors for releasing code and weights. Hacked together by / Copyright 2021 Ross Wightman N)partial)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD) PatchEmbedMlpGluMlpGatedMlpDropPath lecun_normal_ to_2tuple)build_model_with_cfg) named_applycheckpoint_seq)generate_default_cfgsregister_modelregister_model_deprecations MixerBlockMlpMixercsBeZdZdZdeeejddejddffdd Z dd Z Z S) rz Residual Block w/ token mixing and channel MLPs Based on: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 ?@ư>Zepsc sztfddt|D\} } ||_||| ||d|_|dkrPt|nt|_||_ || ||d|_ dS)Ncsg|]}t|qS)int).0xdimr^/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/timm/models/mlp_mixer.py Hz'MixerBlock.__init__.. act_layerdropr) super__init__r norm1 mlp_tokensr nnIdentity drop_pathnorm2 mlp_channels) selfr seq_len mlp_ratio mlp_layer norm_layerr%r&r-Z tokens_dimZ channels_dim __class__rr!r(<s   zMixerBlock.__init__c CsH|||||dddd}|||||}|SNr )r-r*r) transposer/r.r0rrrr!forwardOs*zMixerBlock.forward) __name__ __module__ __qualname____doc__rrr+ LayerNormGELUr(r; __classcell__rrr5r!r8s cs$eZdZfddZddZZS)Affinecs>tttdd|f|_ttdd|f|_dS)Nr ) r'r(r+ Parametertorchonesalphazerosbeta)r0r r5rr!r(Vs zAffine.__init__cCst|j|j|SN)rEZaddcmulrIrGr:rrr!r;[szAffine.forward)r<r=r>r(r;rBrrr5r!rCUs rCcs:eZdZdZdeeejdddffdd ZddZ Z S) ResBlockz Residual MLP block w/ LayerScale and Affine 'norm' Based on: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404 g-C6?rc stt||} |||_t|||_| dkr>t| nt|_ |||_ ||| ||d|_ t |t ||_t |t ||_dS)Nrr$)r'r(rr)r+Linear linear_tokensr r,r-r.r/rDrErFls1ls2) r0r r1r2r3r4r% init_valuesr&r- channel_dimr5rr!r(ds    zResBlock.__init__c CsT|||j|||dddd}|||j|||}|Sr7)r-rOrNr)r9rPr/r.r:rrr!r;zs0 zResBlock.forward) r<r=r>r?rrCr+rAr(r;rBrrr5r!rK_srKcs6eZdZdZejffdd ZddZddZZ S)SpatialGatingUnitzc Spatial Gating Unit Based on: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050 cs.t|d}|||_t|||_dS)Nr8)r'r(normr+rMproj)r0r r1r4Zgate_dimr5rr!r(s  zSpatialGatingUnit.__init__cCs(tjj|jjddtj|jjdS)Nrstd)r+initnormal_rUweightones_biasr0rrr! init_weightsszSpatialGatingUnit.init_weightscCs>|jddd\}}||}||dd}||ddS)Nr8r)chunkrTrUr9)r0ruvrrr!r;s zSpatialGatingUnit.forward) r<r=r>r?r+r@r(r^r;rBrrr5r!rSsrScsBeZdZdZdeeejddejddffdd Z dd Z Z S) SpatialGatingBlockzp Residual Block w/ Spatial Gating Based on: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050 rLrrrc s^tt||} |||_tt|d} ||| || |d|_|dkrPt|nt |_ dS)N)r1)r%Z gate_layerr&r) r'r(rrTrrSr/r r+r,r-) r0r r1r2r3r4r%r&r-rRZsgur5rr!r(s    zSpatialGatingBlock.__init__cCs|||||}|SrJ)r-r/rTr:rrr!r;szSpatialGatingBlock.forward) r<r=r>r?rrr+r@rAr(r;rBrrr5r!rds rdcseZdZdddddddeeeejdd ejd d d d d d ffd d Z e j j d"ddZ e j j d#ddZe j j d$ddZe j j ddZd%ddZddZd&edddZd d!ZZS)'rrrrrFavgc st|_|___d_t||||r:ndd_t j f ddt |D_ _ t | _|dkrt jnt _j|ddS)NF)img_size patch_sizein_chans embed_dimr4c s(g|] }jjdqS))r3r4r%r&r-)stemZ num_patches)r_ r% block_layerdrop_path_rateror3r2r4proj_drop_rater0rr!r"s z%MlpMixer.__init__..r)nlhb)r'r( num_classes global_poolZ num_featuresrograd_checkpointingrrpr+Z SequentialrangeblocksrTZDropout head_droprMr,headr^)r0rwrlrnrm num_blocksror2rsr3r4r%Z drop_raterurtrvZ stem_normrxr5rrr!r(s&       zMlpMixer.__init__cCs.|rt|j nd}ttt|d|ddS)Nr) head_bias)module)mathlogrwrr _init_weights)r0rvrrrr!r^szMlpMixer.init_weightscCstdddgdS)Nz^stem)z^blocks\.(\d+)N)z^norm)i)rpr{)dict)r0Zcoarserrr! group_matcherszMlpMixer.group_matcherTcCs ||_dSrJ)ry)r0enablerrr!set_grad_checkpointingszMlpMixer.set_grad_checkpointingcCs|jSrJ)r}r]rrr!get_classifierszMlpMixer.get_classifierNcCsD||_|dur |dvsJ||_|dkr6t|j|nt|_dS)N)rkr)rwrxr+rMror,r})r0rwrxrrr!reset_classifiers  zMlpMixer.reset_classifiercCs@||}|jr(tjs(t|j|}n ||}||}|SrJ)rpryrEjitZ is_scriptingrr{rTr:rrr!forward_featuress    zMlpMixer.forward_features) pre_logitscCs2|jdkr|jdd}||}|r(|S||S)Nrkr r)rxmeanr|r})r0rrrrr! forward_head s   zMlpMixer.forward_headcCs||}||}|SrJ)rrr:rrr!r;s  zMlpMixer.forward)F)F)T)N)F)r<r=r>rrrr+r@rAr(rErignorer^rrrrrboolrr;rBrrr5r!rs< 4      rF)rnamercCs t|tjr|dr6tj|jtj|j|nj|r^t |j|jdurtj|jnBtj |j|jdurd|vrtjj |jddntj|jnzt|tj rt |j|jdurtj|jnJt|tj tjtjfrtj|jtj|jnt|dr|dS)zA Mixer weight initialization (trying to match Flax defaults) r}NZmlprrVr^) isinstancer+rM startswithrXZzeros_rZZ constant_r\r Zxavier_uniform_rYZConv2dr@Z BatchNorm2dZ GroupNormr[hasattrr^)rrrZflaxrrr!rs,         rcCsd|vr|i}|D]b\}}|dd}|dd}|dd}|dd }|d s`|d rn|d d d }|||<q|S|S)z Remap checkpoints if needed zpatch_embed.proj.weightz patch_embed.zstem.zattn.zlinear_tokens.zmlp.z mlp_channels.Zgamma_Zlsz.alphaz.betar r_)itemsreplaceendswithZreshape)Z state_dictmodelZout_dictkrcrrr!checkpoint_filter_fn;s     rcKs0|ddrtdtt||fdti|}|S)NZ features_onlyz3features_only not implemented for MLP-Mixer models.Zpretrained_filter_fn)get RuntimeErrorr rr)variant pretrainedkwargsrrrr! _create_mixerLs rrc Ks |ddddddddddd |S) Nre)rgrfrf?ZbicubicT)rrrz stem.projr}) urlrwZ input_sizeZ pool_sizecrop_pct interpolationZfixed_input_sizerrWZ first_conv classifierr)rrrrr!_cfgZsrztimm/zlhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_b16_224-76587d61.pth) hf_hub_idrzrhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_b16_224_in21k-617b3de2.pthiSU)rrrwzlhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_l16_224-92f9adc4.pthzrhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_mixer_l16_224_in21k-846aa33c.pthzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/mixer_b16_224_miil_in21k-2a558a71.pth)rrr)?rrrZbilineari+)rrrrWrrrwzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/mixer_b16_224_miil-9229a591.pth)rrrrWrr)rrWzohttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/gmixer_24_224_raa-7daf7ae6.pth)rrrrWz9https://dl.fbaipublicfiles.com/deit/resmlp_12_no_dist.pthz9https://dl.fbaipublicfiles.com/deit/resmlp_24_no_dist.pthz9https://dl.fbaipublicfiles.com/deit/resmlp_36_no_dist.pthz:https://dl.fbaipublicfiles.com/deit/resmlpB_24_no_dist.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_12_dist.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_24_dist.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_36_dist.pthz7https://dl.fbaipublicfiles.com/deit/resmlpB_24_dist.pthz6https://dl.fbaipublicfiles.com/deit/resmlpB_24_22k.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_12_dino.pthz6https://dl.fbaipublicfiles.com/deit/resmlp_24_dino.pthznhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/gmlp_s16_224_raa-10536d42.pth)zmixer_s32_224.untrainedzmixer_s16_224.untrainedzmixer_b32_224.untrained mixer_b16_224.goog_in21k_ft_in1kzmixer_b16_224.goog_in21kzmixer_l32_224.untrained mixer_l16_224.goog_in21k_ft_in1kzmixer_l16_224.goog_in21kmixer_b16_224.miil_in21k mixer_b16_224.miil_in21k_ft_in1kzgmixer_12_224.untrainedzgmixer_24_224.ra3_in1kzresmlp_12_224.fb_in1kzresmlp_24_224.fb_in1kzresmlp_36_224.fb_in1kzresmlp_big_24_224.fb_in1kresmlp_12_224.fb_distilled_in1kresmlp_24_224.fb_distilled_in1kresmlp_36_224.fb_distilled_in1k#resmlp_big_24_224.fb_distilled_in1k"resmlp_big_24_224.fb_in22k_ft_in1kzresmlp_12_224.fb_dinozresmlp_24_224.fb_dinozgmlp_ti16_224.untrainedzgmlp_s16_224.ra3_in1kzgmlp_b16_224.untrained)returncKs,tfdddd|}tdd|i|}|S)zv Mixer-S/32 224x224 Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 rirjrmr~ro mixer_s32_224r)rrrrrZ model_argsrrrr!rsrcKs,tfdddd|}tdd|i|}|S)zw Mixer-S/16 224x224 Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 rhrirjr mixer_s16_224r)rrrrrr!rsrcKs,tfdddd|}tdd|i|}|S)zw Mixer-B/32 224x224 Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 r r mixer_b32_224r)rrrrrr!rsrcKs,tfdddd|}tdd|i|}|S)z Mixer-B/16 224x224. ImageNet-1k pretrained weights. Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 rhrrr mixer_b16_224r)rrrrrr!rsrcKs,tfdddd|}tdd|i|}|S)zx Mixer-L/32 224x224. Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 rr mixer_l32_224r)rrrrrr!rsrcKs,tfdddd|}tdd|i|}|S)z Mixer-L/16 224x224. ImageNet-1k pretrained weights. Paper: 'MLP-Mixer: An all-MLP Architecture for Vision' - https://arxiv.org/abs/2105.01601 rhrrr mixer_l16_224r)rrrrrr!rsrc Ks4tfddddttjd|}tdd|i|}|S) zV Glu-Mixer-12 224x224 Experiment by Ross Wightman, adding SwiGLU to MLP-Mixer rhrrrrmr~ror2r3r% gmixer_12_224r)rrrr+ZSiLUrrrrr!rsrc Ks4tfddddttjd|}tdd|i|}|S) zV Glu-Mixer-24 224x224 Experiment by Ross Wightman, adding SwiGLU to MLP-Mixer rhrrrr gmixer_24_224r)rrrrrr!rsrc Ks2tfddddttd|}tdd|i|}|S) zx ResMLP-12 Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404 rhrrrLrmr~ror2rsr4 resmlp_12_224r)r)rrKrCrrrrr!rs rc Ks:tfddddttddtd|}td d |i|}|S) zx ResMLP-24 Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404 rhrrrLgh㈵>rQr resmlp_24_224r)rrrrKrCrrrrr!r(s rc Ks:tfddddttddtd|}td d |i|}|S) zx ResMLP-36 Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404 rh$rrLrrr resmlp_36_224r)rrrrrr!r4s rc Ks:tfddddttddtd|}td d |i|}|S) zz ResMLP-B-24 Paper: `ResMLP: Feedforward networks for image classification...` - https://arxiv.org/abs/2105.03404 rirrrLrrrresmlp_big_24_224r)rrrrrr!r@s rc Ks2tfddddttd|}tdd|i|}|S) zU gMLP-Tiny Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050 rhrmr~ror2rsr3 gmlp_ti16_224r)rrrdrrrrrr!rLs rc Ks2tfddddttd|}tdd|i|}|S) zV gMLP-Small Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050 rhrrr gmlp_s16_224r)rrrrrr!rXs rc Ks2tfddddttd|}tdd|i|}|S) zU gMLP-Base Paper: `Pay Attention to MLPs` - https://arxiv.org/abs/2105.08050 rhrrjrr gmlp_b16_224r)rrrrrr!rds rrrrrrrrrr) Zmixer_b16_224_in21kZmixer_l16_224_in21kZmixer_b16_224_miilZmixer_b16_224_miil_in21kZresmlp_12_distilled_224Zresmlp_24_distilled_224Zresmlp_36_distilled_224Zresmlp_big_24_distilled_224Zresmlp_big_24_224_in22ft1kZresmlp_12_224_dinoZresmlp_24_224_dino)rF)F)r)F)F)F)F)F)F)F)F)F)F)F)F)F)F)F):r?r functoolsrrEZtorch.nnr+Z timm.datarrZ timm.layersrrrrr r r Z_builderr Z _manipulaterr _registryrrr__all__ModulerrCrKrSrdrstrfloatrrrrZ default_cfgsrrrrrrrrrrrrrrrr<rrrr!sH(  $  !f"    d