a dw} @sUddlZddlmZddlmZddlmZmZmZm Z m Z m Z ddl Z ddl mZddlmZmZddlmZmZddlmZd d lmZmZmZd d lmZd d lmZmZgd Z Gddde Z!GdddeZ"Gdddej#Z$Gdddej#Z%Gdddej#Z&e'e'e'e'e'e ee(ee&d ddZ)deiZ*ee+efe,d<ie*dddZ-Gd d!d!eZ.Gd"d#d#eZ/Gd$d%d%eZ0Gd&d'd'eZ1Gd(d)d)eZ2eed*e.j3fd+dd,d-e e.e(ee&d.d/d0Z4eed*e/j3fd+dd,d-e e/e(ee&d.d1d2Z5eed*e0j3fd+dd,d-e e0e(ee&d.d3d4Z6eed*e1j3fd+dd,d-e e1e(ee&d.d5d6Z7eed7d+dd,d-e e2e(ee&d.d8d9Z8d@e'e'dd?Z9dS)AN) OrderedDict)partial)AnyCallableDictList NamedTupleOptional)Conv2dNormActivationMLP)ImageClassificationInterpolationMode)_log_api_usage_once)register_modelWeights WeightsEnum)_IMAGENET_CATEGORIES)_ovewrite_named_paramhandle_legacy_interface) VisionTransformerViT_B_16_WeightsViT_B_32_WeightsViT_L_16_WeightsViT_L_32_WeightsViT_H_14_Weightsvit_b_16vit_b_32vit_l_16vit_l_32vit_h_14c@sVeZdZUeed<eed<eed<ejZedej fed<ej Z edej fed<dS)ConvStemConfig out_channels kernel_sizestride. norm_layeractivation_layerN) __name__ __module__ __qualname__int__annotations__nnZ BatchNorm2dr&rModuleZReLUr'r/r/n/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/torchvision/models/vision_transformer.pyr" s r"cs:eZdZdZdZeeedfdd ZfddZZ S)MLPBlockzTransformer MLP block.r )in_dimmlp_dimdropoutcsdtj|||gtjd|d|D]:}t|tjr$tj|j |j dur$tjj |j ddq$dS)N)r'Zinplacer4ư>std) super__init__r-ZGELUmodules isinstanceLinearinitZxavier_uniform_weightbiasnormal_)selfr2r3r4m __class__r/r0r9-s    zMLPBlock.__init__c s|dd}|dus|dkrxtdD]R} dD]H} |d| dd| } |d| d| } | |vr,|| || <q,q$t|||||||dS)Nversionr )r>r?Zlinear_r.)getrangepopr8_load_from_state_dict) rAZ state_dictprefixZlocal_metadatastrictZ missing_keysZunexpected_keysZ error_msgsrEitypeZold_keyZnew_keyrCr/r0rK6s"  zMLPBlock._load_from_state_dict) r(r)r*__doc___versionr+floatr9rK __classcell__r/r/rCr0r1(s r1c sZeZdZdZeejddfeeeeee de jj fdfdd Z e j dd d ZZS) EncoderBlockzTransformer encoder block.r5Zeps.) num_heads hidden_dimr3r4attention_dropoutr&csVt||_|||_tj|||dd|_t||_|||_ t ||||_ dS)NT)r4Z batch_first) r8r9rVln_1r-ZMultiheadAttentionself_attentionDropoutr4ln_2r1mlp)rArVrWr3r4rXr&rCr/r0r9Ys    zEncoderBlock.__init__inputcCsjt|dkd|j||}|j|||dd\}}||}||}||}||}||S)NrG2Expected (batch_size, seq_length, hidden_dim) got F)Z need_weights) torch_assertdimshaperYrZr4r\r])rAr_x_yr/r/r0forwardns    zEncoderBlock.forwardr(r)r*rPrr- LayerNormr+rRrrar.r9TensorrhrSr/r/rCr0rTVs rTc s^eZdZdZeejddfeeeeeeee de jj fdfdd Z e j dd d ZZS) Encoderz?Transformer Model Encoder for sequence to sequence translation.r5rU.) seq_length num_layersrVrWr3r4rXr&c s~tttd||jdd|_t||_ t } t |D] } t ||||||| d| <qBt | |_|||_dS)Nrg{Gz?r6Zencoder_layer_)r8r9r- Parameterraemptyr@ pos_embeddingr[r4rrIrT Sequentiallayersln) rArmrnrVrWr3r4rXr&rsrNrCr/r0r9}s    zEncoder.__init__r^cCs<t|dkd|j||j}||||S)NrGr`)rarbrcrdrqrtrsr4)rAr_r/r/r0rhs zEncoder.forwardrir/r/rCr0rlzs rlcseZdZdZddddeejdddfeeeeeeeeee ee de jj fe e ed fd d Ze je jd d d Ze jdddZZS)rz;Vision Transformer as per https://arxiv.org/abs/2010.11929.iNr5rU.) image_size patch_sizernrVrWr3r4rX num_classesrepresentation_sizer&conv_stem_configsc  stt|t||dkd||_||_||_||_||_ ||_ | |_ | |_ | |_ | durt} d}t| D]:\}}| d|t||j|j|j|j |jd|j}qx| dtj||dd| |_ntjd|||d |_||d }ttdd||_|d7}t|||||||| |_||_t}| durRt || |d <n,t || |d <t!|d <t | | |d <t||_"t#|jtjr|jj$|jjd|jjd}tj%j&|jj't()d|d|jj*durbtj%+|jj*nj|jj,durbt#|jj,tjrbtj%j-|jj,j'dt()d|jj,jd|jj,j*durbtj%+|jj,j*t.|j"d rt#|j"j/tj r|j"j/j0}tj%j&|j"j/j't()d|dtj%+|j"j/j*t#|j"j1tj rtj%+|j"j1j'tj%+|j"j1j*dS)Nrz&Input shape indivisible by patch size!rGZ conv_bn_relu_) in_channelsr#r$r%r&r' conv_lastr)r{r#r$)r{r#r$r%r head pre_logitsZactr6rug@)meanr7)2r8r9rrarbrvrwrWr3rXr4rxryr&r-rr enumerateZ add_moduler r#r$r%r'ZConv2d conv_projrozeros class_tokenrlencoderrmrr<ZTanhheadsr;r{r=Z trunc_normal_r>mathsqrtr?Zzeros_r|r@hasattrr~Z in_featuresr})rArvrwrnrVrWr3r4rXrxryr&rzZseq_projZ prev_channelsrNZconv_stem_layer_configrmZ heads_layersZfan_inrCr/r0r9s            zVisionTransformer.__init__)rereturnc Cs|j\}}}}|j}t||jkd|jd|dt||jkd|jd|d||}||}||}|||j||}|ddd}|S)NzWrong image height! Expected z but got !zWrong image width! Expected rr r) rdrwrarbrvrreshaperWpermute) rArenchwpZn_hZn_wr/r/r0_process_input s"" z VisionTransformer._process_input)recCs^||}|jd}|j|dd}tj||gdd}||}|dddf}||}|S)Nrrrc)rrdrexpandracatrr)rArerZbatch_class_tokenr/r/r0rh!s    zVisionTransformer.forward)r(r)r*rPrr-rjr+rRr rrar.rr"r9rkrrhrSr/r/rCr0rs.   ir) rwrnrVrWr3weightsprogresskwargsrc Ks|durTt|dt|jd|jdd|jddks>Jt|d|jdd|dd}tf||||||d|} |r| |j|d | S) Nrx categoriesmin_sizerrrv)rvrwrnrVrWr3)r)rlenmetarJrZload_state_dictZget_state_dict) rwrnrVrWr3rrrrvmodelr/r/r0_vision_transformer4s$    rr _COMMON_METAz(https://github.com/facebookresearch/SWAGz:https://github.com/facebookresearch/SWAG/blob/main/LICENSE)recipelicensec @seZdZedeeddiedddddd d id d d ddZedeeddej die dddddd idddddZ edeeddej die ddddddd id d dd dZ eZ d!S)"rz9https://download.pytorch.org/models/vit_b_16-c867db91.pthr crop_sizei(rrzNhttps://github.com/pytorch/vision/tree/main/references/classification#vit_b_16 ImageNet-1KgS㥛DT@g1ZW@zacc@1zacc@5gMb1@g(\t@ These weights were trained from scratch by using a modified version of `DeIT `_'s training recipe.  num_paramsrr_metrics_ops _file_size_docsurlZ transformsrz>https://download.pytorch.org/models/vit_b_16_swag-9ac1b537.pthr resize_size interpolationi^-)rrg~jtSU@giX@gˡEK@g|?5^t@ These weights are learnt via transfer learning by end-to-end fine-tuning the original `SWAG `_ weights on ImageNet-1K data. rrrrrrzAhttps://download.pytorch.org/models/vit_b_16_lc_swag-4e70ced5.pth+https://github.com/pytorch/vision/pull/5793gbX9xT@gQ X@ These weights are composed of the original frozen `SWAG `_ trunk weights and a linear classifier learnt on top of them trained on ImageNet-1K data. rrrrrrrNr(r)r*rrr r IMAGENET1K_V1rBICUBIC_COMMON_SWAG_METAIMAGENET1K_SWAG_E2E_V1IMAGENET1K_SWAG_LINEAR_V1DEFAULTr/r/r/r0r_s rc @sHeZdZedeeddiedddddd d id d d ddZeZdS)rz9https://download.pytorch.org/models/vit_b_32-d86f8d99.pthrri1BrzNhttps://github.com/pytorch/vision/tree/main/references/classification#vit_b_32rg|?5^R@gW@rgA`Т@gl u@rrrN r(r)r*rrr rrrr/r/r/r0rs( rc @seZdZedeedddieddddd d d id d dddZedeeddej die dddddd idddddZ edeeddej die ddddddd id d d d!dZ eZ d"S)#rz9https://download.pytorch.org/models/vit_l_16-852ce7e3.pthr)rri#rzNhttps://github.com/pytorch/vision/tree/main/references/classification#vit_l_16rg|?5^S@gFԨW@rgףp= N@g;O$@a These weights were trained from scratch by using a modified version of TorchVision's `new training recipe `_. rrz>https://download.pytorch.org/models/vit_l_16_swag-4f3808c9.pthri0)rrgjtV@gT㥛ĠX@gƟv@gy&11@rrzAhttps://download.pytorch.org/models/vit_l_16_lc_swag-4d563306.pthrgMbXIU@g^I [X@rrNrr/r/r/r0rs rc @sHeZdZedeeddiedddddd d id d d ddZeZdS)rz9https://download.pytorch.org/models/vit_l_32-c7638314.pthrri[ErzNhttps://github.com/pytorch/vision/tree/main/references/classification#vit_l_32rg|?5>S@gGzDW@rgK7.@gE@rrrNrr/r/r/r0rs( rc @seZdZedeeddejdiedddddd id d d d dZ edeeddejdieddddddd idddddZ e Z dS)rz>https://download.pytorch.org/models/vit_h_14_swag-80465313.pthri%)rrrgS#V@g#~jX@rg~jŏ@gK7I@rrrzAhttps://download.pytorch.org/models/vit_h_14_lc_swag-c1eb923e.pthrri@%rgZd;OmU@gQnX@g= ףpd@gI k֢@rrN) r(r)r*rrr rrrrrrr/r/r/r0r2s`r pretrained)rT)rr)rrrrc Ks(t|}tfddddd||d|S)a Constructs a vit_b_16 architecture from `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale `_. Args: weights (:class:`~torchvision.models.ViT_B_16_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.ViT_B_16_Weights` below for more details and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.vision_transformer.VisionTransformer`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.ViT_B_16_Weights :members:   rwrnrVrWr3rr)rverifyrrrrr/r/r0rks rc Ks(t|}tfddddd||d|S)a Constructs a vit_b_32 architecture from `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale `_. Args: weights (:class:`~torchvision.models.ViT_B_32_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.ViT_B_32_Weights` below for more details and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.vision_transformer.VisionTransformer`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.ViT_B_32_Weights :members: rrrr)rrrrr/r/r0rs rc Ks(t|}tfddddd||d|S)a Constructs a vit_l_16 architecture from `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale `_. Args: weights (:class:`~torchvision.models.ViT_L_16_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.ViT_L_16_Weights` below for more details and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.vision_transformer.VisionTransformer`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.ViT_L_16_Weights :members: rr)rrrrr/r/r0rs rc Ks(t|}tfddddd||d|S)a Constructs a vit_l_32 architecture from `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale `_. Args: weights (:class:`~torchvision.models.ViT_L_32_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.ViT_L_32_Weights` below for more details and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.vision_transformer.VisionTransformer`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.ViT_L_32_Weights :members: rrrrrr)rrrrr/r/r0r s r )rNc Ks(t|}tfddddd||d|S)a Constructs a vit_h_14 architecture from `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale `_. Args: weights (:class:`~torchvision.models.ViT_H_14_Weights`, optional): The pretrained weights to use. See :class:`~torchvision.models.ViT_H_14_Weights` below for more details and possible values. By default, no pre-trained weights are used. progress (bool, optional): If True, displays a progress bar of the download to stderr. Default is True. **kwargs: parameters passed to the ``torchvision.models.vision_transformer.VisionTransformer`` base class. Please refer to the `source code `_ for more details about this class. .. autoclass:: torchvision.models.ViT_H_14_Weights :members: rriir)rrrrr/r/r0r!s r!bicubicFzOrderedDict[str, torch.Tensor])rvrw model_stateinterpolation_mode reset_headsrcCsh|d}|j\}}}|dkr,td|j||dd} | |krd|d8}| d8} |ddddddf} |ddddddf} | ddd} tt|} | | |krtd| | d|| d|| | } ||} tjj | | |d d }|d|| }|ddd}t j | |gdd }||d<|rdt }| D]\}}|d s@|||<q@|}|S) aThis function helps interpolate positional embeddings during checkpoint loading, especially when you want to apply a pre-trained model on images with different resolution. Args: image_size (int): Image size of the new model. patch_size (int): Patch size of the new model. model_state (OrderedDict[str, torch.Tensor]): State dict of the pre-trained model. interpolation_mode (str): The algorithm used for upsampling. Default: bicubic. reset_heads (bool): If true, not copying the state of heads. Default: False. Returns: OrderedDict[str, torch.Tensor]: A state dict which can be loaded into the new model. zencoder.pos_embeddingrz%Unexpected position embedding shape: r NrzPseq_length is not a perfect square! Instead got seq_length_1d * seq_length_1d = z and seq_length = T)sizemodeZ align_cornersrr)rd ValueErrorrr+rrrr-Z functionalZ interpolaterarritems startswith)rvrwrrrrqrrmrWZnew_seq_lengthZpos_embedding_tokenZpos_embedding_imgZ seq_length_1dZnew_seq_length_1dZnew_pos_embedding_imgZnew_pos_embeddingZmodel_state_copykvr/r/r0interpolate_embeddingssF     r)rF):r collectionsr functoolsrtypingrrrrrr raZtorch.nnr-Zops.miscr r Ztransforms._presetsr rutilsrZ_apirrrZ_metar_utilsrr__all__r"r1r.rTrlrr+boolrrstrr,rrrrrrrrrrr r!rr/r/r/r0s       .$& !OP9$ $ $ $ $$