a  dB0@s0dZddlZddlZddlmZmZmZddlZddl Z ddl m m Z ddlZ ddl m Z ddlmZddlmZddlmZdd lmZmZmZmZmZmZdd lmZdd lmZm Z m!Z!m"Z"d d l#m$Z$e"%e&Z'dZ(dZ)dZ*gdZ+dZ,dZ-dZ.ddgZ/dZ0dZ1gdZ2dTee3e3fe4e3ee j5e3ej6dddZ7Gddde j8Z9Gddde j8Z:Gdd d e j8Z;Gd!d"d"e j8ZGd'd(d(e>Z?Gd)d*d*e j8Z@Gd+d,d,e j8ZAGd-d.d.e j8ZBGd/d0d0e j8ZCGd1d2d2e j8ZDGd3d4d4e j8ZEGd5d6d6e j8ZFGd7d8d8e j8ZGGd9d:d:e j8ZHGd;d<dd>eZJd?ZKd@ZLe dAeKGdBdCdCeJZMe dDeKGdEdFdFeJZNe dGeKGdHdIdIeJZOe dJeKGdKdLdLeJZPGdMdNdNe j8ZQGdOdPdPe j8ZRe dQeKGdRdSdSeJZSdS)Uz PyTorch WavLM model.N)OptionalTupleUnion)nn)CrossEntropyLoss)ACT2FN)is_deepspeed_zero3_enabled)BaseModelOutputCausalLMOutputSequenceClassifierOutputTokenClassifierOutputWav2Vec2BaseModelOutput XVectorOutput)PreTrainedModel)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardlogging) WavLMConfigrz1patrickvonplaten/wavlm-libri-clean-100h-base-plus)ri$izZ'mister quilter is the aposle of the middle classes and we are glad to welcome his gospel'gQ)@zmicrosoft/wavlm-base-plus-sdzmicrosoft/wavlm-base-plus-svg ףp= ?)zmicrosoft/wavlm-basezmicrosoft/wavlm-base-pluszmicrosoft/wavlm-large)shape mask_prob mask_lengthattention_mask min_masksreturncs|\}dkrtdkr6tdddtjdfdd}|durt|d nfd d t|D}tj |ft d }g} |} | d kr|S|D]v} || } tjj t | d| dd} t | d krd}n| d }t| tj| | tjd |g} | | qt| } t| dddddf|| f} | || } t ddddf}t||| f|| }| |} | dkr҈d| | dk<t|| dd |S)af Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on CPU as part of the preprocessing during training. Args: shape: The shape for which to compute masks. This should be of a tuple of size 2 where the first element is the batch size and the second element is the length of the axis to span. mask_prob: The percentage of the whole axis (between 0 and 1) which will be masked. The number of independently generated mask spans of length `mask_length` is computed by `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the actual percentage will be smaller. mask_length: size of the mask min_masks: minimum number of masked spans attention_mask: A (right-padded) attention mask which independently shortens the feature axis of each batch dimension. rz&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: `csXt|}t|}|kr2}|d|krTt|dd}|S)z;Given input length, compute how many spans should be maskedrr)intmax) input_lengthnum_masked_spanepsilonrrrsequence_lengthq/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/transformers/models/wavlm/modeling_wavlm.pycompute_num_masked_spanrs  z6_compute_mask_indices..compute_num_masked_spanNcsg|]}qSr&r&.0_)r%r&r' z)_compute_mask_indices..dtyperF)replace) ValueErrornprandomZranditemsumdetachtolistrangezerosboolchoicearangelenZ concatenateonesint32appendarray broadcast_toZreshaper Zput_along_axis)rrrrr batch_sizer( input_lengthsZ spec_aug_maskZspec_aug_mask_idxsZmax_num_masked_spanr!r"Zspec_aug_mask_idxZdummy_mask_idxoffsetsr&r#r'_compute_mask_indicesLs\    rGcs&eZdZdfdd ZddZZS)WavLMNoLayerNormConvLayerrcsjt|dkr |j|dnd|_|j||_tj|j|j|j||j||j d|_ t |j |_ dS)Nrr kernel_sizestridebias)super__init__conv_dim in_conv_dim out_conv_dimrConv1d conv_kernel conv_stride conv_biasconvrfeat_extract_activation activationselfconfiglayer_id __class__r&r'rNs  z"WavLMNoLayerNormConvLayer.__init__cCs||}||}|SN)rVrXrZ hidden_statesr&r&r'forwards  z!WavLMNoLayerNormConvLayer.forward)r__name__ __module__ __qualname__rNrb __classcell__r&r&r]r'rHsrHcs&eZdZdfdd ZddZZS)WavLMLayerNormConvLayerrcs|t|dkr |j|dnd|_|j||_tj|j|j|j||j||j d|_ tj |jdd|_ t |j|_dS)NrrrIT)Zelementwise_affine)rMrNrOrPrQrrRrSrTrUrV LayerNorm layer_normrrWrXrYr]r&r'rNs  z WavLMLayerNormConvLayer.__init__cCs:||}|dd}||}|dd}||}|S)Nr))rV transposerjrXr`r&r&r'rbs      zWavLMLayerNormConvLayer.forward)rrcr&r&r]r'rhsrhcs&eZdZdfdd ZddZZS)WavLMGroupNormConvLayerrcst|dkr |j|dnd|_|j||_tj|j|j|j||j||j d|_ t |j |_ tj|j|jdd|_dS)NrrrIT) num_groupsZ num_channelsZaffine)rMrNrOrPrQrrRrSrTrUrVrrWrX GroupNormrjrYr]r&r'rNs   z WavLMGroupNormConvLayer.__init__cCs"||}||}||}|Sr_)rVrjrXr`r&r&r'rbs   zWavLMGroupNormConvLayer.forward)rrcr&r&r]r'rmsrmcs$eZdZfddZddZZS)WavLMPositionalConvEmbeddingcsttj|j|j|j|jd|jd|_tjj }t tjj drNtjj j }t rddl }|jj|jjdd"||jddd|_Wdn1s0Y|j||jj|j||jjn||jddd|_t|j|_t|j|_dS)Nr)rJpaddinggroups weight_normr)Z modifier_rankweight)namedim)rMrNrrR hidden_sizenum_conv_pos_embeddingsZnum_conv_pos_embedding_groupsrVutilsrshasattrZparametrizationsr deepspeedZzeroZGatheredParametersrtZregister_external_parameterZweight_vZweight_gWavLMSamePadLayerrqrrWrX)rZr[rsr{r]r&r'rNs(  0 z%WavLMPositionalConvEmbedding.__init__cCs:|dd}||}||}||}|dd}|SNrr)rlrVrqrXr`r&r&r'rb+s      z$WavLMPositionalConvEmbedding.forwardrcr&r&r]r'rps rpcs$eZdZfddZddZZS)r|cs$t|ddkrdnd|_dSNrrr)rMrNnum_pad_remove)rZrxr]r&r'rN8s zWavLMSamePadLayer.__init__cCs,|jdkr(|ddddd|j f}|S)Nr)rr`r&r&r'rb<s zWavLMSamePadLayer.forwardrcr&r&r]r'r|7s r|cs0eZdZdZfddZddZddZZS)WavLMFeatureEncoderz.Construct the features from raw audio waveformcstjdkr@tddgfddtjdD}n6jdkrdfddtjD}ntd jd t||_ d |_ d |_ dS) Ngrouprr\csg|]}t|ddqS)rr)rHr+ir[r&r'r-Jsz0WavLMFeatureEncoder.__init__..rlayercsg|]}t|dqS)r)rhrrr&r'r-Nr.z`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT) rMrNZfeat_extract_normrmr9Znum_feat_extract_layersr2r ModuleList conv_layersgradient_checkpointing_requires_grad)rZr[rr]rr'rNFs      zWavLMFeatureEncoder.__init__cCs|D] }d|_qd|_dS)NF) parameters requires_gradrrZparamr&r&r'_freeze_parametersWs z&WavLMFeatureEncoder._freeze_parameterscCsj|dddf}|jr"|jr"d|_|jD]<}|jr\|jr\|jr\dd}tjj|||}q(||}q(|S)NTcsfdd}|S)Ncs|Sr_r&inputsmoduler&r'custom_forwardgszRWavLMFeatureEncoder.forward..create_custom_forward..custom_forwardr&rrr&rr'create_custom_forwardfs z:WavLMFeatureEncoder.forward..create_custom_forward)rtrainingrrrtorchry checkpoint)rZ input_valuesraZ conv_layerrr&r&r'rb\s   zWavLMFeatureEncoder.forward)rdrerf__doc__rNrrbrgr&r&r]r'rCs rcseZdZfddZZS)WavLMFeatureExtractorcs8t|td|jjd|jjdjdtdS)Nz The class `zD` has been depreciated and will be removed in Transformers v5. Use `rz ` instead.)rMrNwarningswarnr^rd __bases__ FutureWarningrZr[r]r&r'rNws   zWavLMFeatureExtractor.__init__)rdrerfrNrgr&r&r]r'rvsrcs$eZdZfddZddZZS)WavLMFeatureProjectioncsJttj|jd|jd|_t|jd|j|_ t |j |_ dS)Nr)Zeps) rMrNrrirOlayer_norm_epsrjLinearrw projectionDropoutZfeat_proj_dropoutdropoutrr]r&r'rNs zWavLMFeatureProjection.__init__cCs&||}||}||}||fSr_)rjrr)rZraZnorm_hidden_statesr&r&r'rbs   zWavLMFeatureProjection.forwardrcr&r&r]r'rs rc seZdZdZdeeeeeedfdd Zdej e ej e ej ee ej e ej e e ej fd d dZ ej eejejfej eej ej fdddZeeej dddZej ej dddZZS)WavLMAttentionz=Multi-headed attention from 'Attention Is All You Need' paper@ T embed_dim num_headsr num_buckets max_distancehas_relative_position_biascst||_||_||_|||_|j||jkrNtd|jd|d|jd|_t |||_ t |||_ t |||_ t |||_ ||_||_ttd|jdd|_t |jd|_|rt|j|j|_dS)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).gr)rMrNrrrZhead_dimr2Zscalingrrk_projv_projq_projout_projrr Parameterrr?gru_rel_pos_constgru_rel_pos_linearZ Embeddingrel_attn_embed)rZrrrrrrr]r&r'rNs,    zWavLMAttention.__init__NFr)rar position_biasoutput_attentionsrcCs|\}}}|durH|||}|d|ddd||j||}||jdd|jdf} | dddd} || } | | jddd d} t | j ddd\} } | | |j d d } | ||jdd|}|d||f}|||||\}}|||fS) z'Attention layer with relative attentionNrrr)rr)rrv?g@)size compute_bias unsqueezerepeatviewrrpermuterr6rZsigmoidchunkrtorch_multi_head_self_attention)rZrarrrindexZbszZtgt_lenr,Zgated_hidden_statesZrelative_position_projZgate_aZgate_bZ gate_outputgated_position_bias attn_output attn_weightsr&r&r'rbs"  $ zWavLMAttention.forward)rarrrrcCs|dd}}}|dur&|dnd}d} } d} tj||||j|jtdgt|j j |j j |j j f| | | |j |jj|jj |j|||d|j j|j j|j jd\} } | dd} | dur| dddf| jdd|jf| jdd} | | fS)zCsimple wrapper around torch's multi_head_attention_forward functionrrNFT)Zuse_separate_proj_weightZ q_proj_weightZ k_proj_weightZ v_proj_weight)rlneFZmulti_head_attention_forwardrrremptycatrrLrrrrrtrrCr)rZrarrrquerykeyvalueZkey_padding_maskZbias_kZbias_vZ add_zero_attnrrr&r&r'rsB    "z.WavLMAttention.torch_multi_head_self_attention) query_length key_lengthrcCsvtj|tjddddf}tj|tjddddf}||}||}||jjj}||}|gd}|S)Nr/)rrr) rr=long_relative_positions_buckettorrtdevicer)rZrrZcontext_positionZmemory_positionZrelative_positionZrelative_position_bucketvaluesr&r&r'rs  zWavLMAttention.compute_bias)relative_positionsrcCs|jd}|dktj|}t|}|d}||k}t||}|t|j|}|||}||tj}t |t ||d}|t |||7}|Sr~) rrrrabslogfloatmathrminZ full_likewhere)rZrrZrelative_bucketsZ max_exactZis_smallZrelative_positions_if_largeZrelative_position_if_larger&r&r'r"s   z)WavLMAttention._relative_positions_bucket)rrrT)NNFr)rdrerfrrrr;rNrTensorrrrb FloatTensorr LongTensorZ BoolTensorrrrrgr&r&r]r'rs@' + 7 rcs$eZdZfddZddZZS)WavLMFeedForwardcsptt|j|_t|j|j|_ t |j t rDt |j |_n|j |_t|j|j|_t|j|_dSr_)rMrNrrZactivation_dropoutintermediate_dropoutrrwZintermediate_sizeintermediate_dense isinstanceZ hidden_actstrrintermediate_act_fn output_densehidden_dropoutoutput_dropoutrr]r&r'rN9s  zWavLMFeedForward.__init__cCs6||}||}||}||}||}|Sr_)rrrrrr`r&r&r'rbFs      zWavLMFeedForward.forwardrcr&r&r]r'r8s rcs0eZdZd eedfdd Zd dd ZZS) WavLMEncoderLayerTr[rcsntt|j|j|j|j|j|d|_t |j |_ t j |j|jd|_t||_t j |j|jd|_dSNrrrMrNrrwZnum_attention_headsZattention_dropoutrZmax_bucket_distance attentionrrrrrirrjr feed_forwardfinal_layer_normrZr[rr]r&r'rNQs  zWavLMEncoderLayer.__init__NFrc Csl|}|j|||||d\}}}||}||}||}|||}||}||f}|rh||f7}|S)Nrrrr)rrrjrr) rZrarrrr attn_residualroutputsr&r&r'rb`s"     zWavLMEncoderLayer.forward)T)NNFrrdrerfrr;rNrbrgr&r&r]r'rPsrcs0eZdZd eedfdd Zd ddZZS) WavLMEncoderLayerStableLayerNormTrcsntt|j|j|j|j|j|d|_t |j |_ t j |j|jd|_t||_t j |j|jd|_dSrrrr]r&r'rNzs  z)WavLMEncoderLayerStableLayerNorm.__init__NFcCsf|}||}|j||||d\}}}||}||}||||}||f}|rb||f7}|S)N)rrr)rjrrrr)rZrarrrrrrr&r&r'rbs    z(WavLMEncoderLayerStableLayerNorm.forward)T)NNFrr&r&r]r'rysrcs&eZdZfddZdddZZS) WavLMEncodercsft|_t|_tjjjd|_ t j |_ t fddtjD|_d|_dS)Nrcsg|]}t|dkdqSr)r)rrrr&r'r-r.z)WavLMEncoder.__init__..FrMrNr[rppos_conv_embedrrirwrrjrrrrr9num_hidden_layerslayersrrr]rr'rNs  zWavLMEncoder.__init__NFTcsX|rdnd}rdnd}|dur*d||<||}||}||}||}t} d} t|jD]\} } |rz||f}tjdd} |j o| dko| |j j k}|r| r|j r|j r܇fdd}t jj|| ||| }n| ||| | d}|dd\}} |rd }rd||df}qd|r,||f}|sJtd d |||fDSt|||d S) Nr&rrrcsfdd}|S)Ncsg|RSr_r&rrrr&r'rszKWavLMEncoder.forward..create_custom_forward..custom_forwardr&rrrr'rsz3WavLMEncoder.forward..create_custom_forwardrrNNcss|]}|dur|VqdSr_r&r+vr&r&r' r.z'WavLMEncoder.forward..last_hidden_statera attentions)rrjrr enumeraterr3r4uniformrr[ layerdroprrryrtupler rZrarroutput_hidden_states return_dictZall_hidden_statesZall_self_attentionsZposition_embeddingsZdeepspeed_zero3_is_enabledrrrZdropout_probabilityZskip_the_layerrZ layer_outputsr&rr'rbsZ          zWavLMEncoder.forward)NFFTrcr&r&r]r'rs  rcs&eZdZfddZdddZZS) WavLMEncoderStableLayerNormcsft|_t|_tjjjd|_ t j |_ t fddtjD|_d|_dS)Nrcsg|]}t|dkdqSr)rrrr&r'r-sz8WavLMEncoderStableLayerNorm.__init__..Frrr]rr'rNs   z$WavLMEncoderStableLayerNorm.__init__NFTcsT|rdnd}rdnd}|dur*d||<||}||}||}t} d} t|jD]\} } |rp||f}tjdd} |jo| dko| |j j k}|r| r|j r|jr҇fdd}t j j|| ||| }n| ||| d}|dd\}} |rd}rZ||df}qZ||}|r(||f}|sFtd d |||fDSt|||d S) Nr&rrcsfdd}|S)Ncsg|RSr_r&rrr&r'r)szZWavLMEncoderStableLayerNorm.forward..create_custom_forward..custom_forwardr&rrrr'r(szBWavLMEncoderStableLayerNorm.forward..create_custom_forward)rrrrrcss|]}|dur|VqdSr_r&rr&r&r'rIr.z6WavLMEncoderStableLayerNorm.forward..r)rrr r rr3r4r rr[r rrryrrjr r rr&rr'rbsT          z#WavLMEncoderStableLayerNorm.forward)NFFTrcr&r&r]r'rs  rcs4eZdZdZfddZeddZddZZS)WavLMGumbelVectorQuantizerz Vector quantization using gumbel softmax. See [CATEGORICAL REPARAMETERIZATION WITH GUMBEL-SOFTMAX](https://arxiv.org/pdf/1611.01144.pdf) for more information. cst|j|_|j|_|j|jdkrDtd|jd|jdt t d|j|j|j|j|_ t |jd|j|j|_d|_dS)Nrz`config.codevector_dim z5 must be divisible by `config.num_codevector_groups` z for concatenation.rr)r)rMrNZnum_codevector_groupsrnZnum_codevectors_per_groupnum_varsZcodevector_dimr2rrrr codevectorsrrO weight_proj temperaturerr]r&r'rNUs  z#WavLMGumbelVectorQuantizer.__init__cCs8|jdd}ttj|t|ddd }|S)NrrgHz>r))meanrexpr6r)ZprobsZmarginal_probs perplexityr&r&r'_compute_perplexityjs (z.WavLMGumbelVectorQuantizer._compute_perplexityc Cs|j\}}}||}||||jd}|jrtjj||j dd}| |}t j ||||jddd}| |}nH|jdd}|j|jd|ddd}||||jd}| |}|||d}|d|j} | |||j|jd} | d||d} | |fS)Nr)T)tauhardrrrrk)rrrrnrr functionalZgumbel_softmaxrrZtype_asrsoftmaxrargmaxZ new_zerosZscatter_rrrr6) rZrarDr%rwZcodevector_probsZcodevector_soft_distrZcodevector_idxZcodevectors_per_grouprr&r&r'rbps*       z"WavLMGumbelVectorQuantizer.forward) rdrerfrrN staticmethodrrbrgr&r&r]r'rOs   rcs$eZdZfddZddZZS) WavLMAdaptercsptjjkr8tjj|_tj|_n d|_|_t fddt j D|_ j |_ dS)Nc3s|]}tVqdSr_)WavLMAdapterLayerr*rr&r'rr.z(WavLMAdapter.__init__..)rMrNoutput_hidden_sizerwrrprojriproj_layer_normrr9num_adapter_layersrr rr]rr'rNs    zWavLMAdapter.__init__cCsr|jdur(|jdur(||}||}|dd}|jD]&}tj}|jrX||jkr:||}q:|dd}|Sr})r$r%rlrr3r4rr )rZrarZlayerdrop_probr&r&r'rbs       zWavLMAdapter.forwardrcr&r&r]r'r!s r!cs$eZdZfddZddZZS)r"cs0ttj|jd|j|j|jdd|_dS)Nrr)rKrq)rMrNrrRr#Zadapter_kernel_sizeadapter_striderVrr]r&r'rNs zWavLMAdapterLayer.__init__cCs||}tjj|dd}|S)Nrr)rVrrZglur`r&r&r'rbs zWavLMAdapterLayer.forwardrcr&r&r]r'r"s r"c@sleZdZdZeZdZdZdgZdZ ddZ de e j efeed d d Zdee j d d dZdddZdS)WavLMPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. wavlmrZ position_idsTc Cst|tr>|jjjjddd|jjjtj |j njt|t rtj j|j jddtd|j jd|j jdtj |j jdnt|trtd|jj}tj j |jj| |dtj j |jj| |dnt|tjr|jjjd|jjd|jdur|jjnt|tjtjfrN|jj|jjdnZt|tjrtj |j|jdurt|j|j|jd}tj j |j| |ddS) zInitialize the weightsrr)rstdrr)abNr)rrrrtdataZnormal_rLZzero_rinituniform_rrprVrsqrtrJZ in_channelsZ constant_rrZ in_featuresrr[Zinitializer_rangeriroZfill_rRZkaiming_normal_rr)rZrkr&r&r' _init_weightss6       z"WavLMPreTrainedModel._init_weightsN)rE add_adaptercCsn|dur|jjn|}dd}t|jj|jjD]\}}||||}q.|rjt|jjD]}||d|jj}qT|S)zH Computes the output length of the convolutional layers NcSstj|||dddS)Nfloor)Z rounding_moder)rdivr!rJrKr&r&r'_conv_out_lengthszOWavLMPreTrainedModel._get_feat_extract_output_lengths.._conv_out_lengthr)r[r3ziprSrTr9r&r')rZrEr3r7rJrKr,r&r&r' _get_feat_extract_output_lengthssz5WavLMPreTrainedModel._get_feat_extract_output_lengths)feature_vector_lengthrcCs|jdddddf}|j||d}|tj}|jd}tj||f|j|jd}d|tj |jd|jd|df<| dgd dg }|S)Nr)rr3r)r0rr)r) Zcumsumr9rrrrr:r0rr=Zflipr;)rZr:rr3Znon_padded_lengthsZoutput_lengthsrDr&r&r'"_get_feature_vector_attention_mask s  "z7WavLMPreTrainedModel._get_feature_vector_attention_maskFcCst|tttfr||_dSr_)rrrrr)rZrrr&r&r'_set_gradient_checkpointing sz0WavLMPreTrainedModel._set_gradient_checkpointing)N)N)F)rdrerfrr config_classZbase_model_prefixZmain_input_nameZ_keys_to_ignore_on_load_missingZsupports_gradient_checkpointingr2rrrrrr;r9r<r=r&r&r&r'r(s "  r(a WavLM was proposed in [WavLM: Unified Speech Representation Learning with Labeled and Unlabeled Data](https://arxiv.org/abs/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei, Michael Zeng, Xuedong Huang. This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving etc.). This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`WavLMConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. aI Args: input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details. attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask == True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should **not** be passed to avoid degraded performance when doing batched inference. For such models `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these models also yield slightly different results depending on whether `input_values` is padded or not. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. z_The bare WavLM Model transformer outputting raw hidden-states without any specific head on top.c seZdZedfdd ZddZddZdeje eje ej d d d Z e e eeeed ed de eje eje eje ee ee eeeefdddZZS) WavLMModelrcst|||_t||_t||_|jdks:|jdkrRt t |j |_|jrdt||_n t||_|jr|t|nd|_|dS)Nr)rMrNr[rfeature_extractorrfeature_projectionmask_time_probmask_feature_probrrrrrwr/masked_spec_embedZdo_stable_layer_normrencoderrr3r!adapter post_initrr]r&r'rNcs     zWavLMModel.__init__cCstdt|dSz Calling this function will disable the gradient computation for the feature encoder so that its parameters will not be updated during training. The method `freeze_feature_extractor` is deprecated and will be removed in Transformers v5.Please use the equivalent `freeze_feature_encoder` method instead.Nrrrfreeze_feature_encoderrZr&r&r'freeze_feature_extractorws z#WavLMModel.freeze_feature_extractorcCs|jdS Calling this function will disable the gradient computation for the feature encoder so that its parameter will not be updated during training. N)r@rrLr&r&r'rKsz!WavLMModel.freeze_feature_encoderN)ramask_time_indicesrcCst|jdds|S|\}}}|dur<|j|j||<nZ|jjdkr|jrt||f|jj|jj ||jj d}t j ||j t jd}|j|j||<|jjdkr|jrt||f|jj|jj|jjd}t j ||j t jd}|dddfd|d}d||<|S) z Masks extracted features along time axis and/or along feature axis according to [SpecAugment](https://arxiv.org/abs/1904.08779). Zapply_spec_augmentTNr)rrrr)rr0)rrrr))getattrr[rrDrr0rBrrGZmask_time_lengthZmask_time_min_masksrZtensorrr;rCZmask_feature_lengthZmask_feature_min_masksexpand)rZrarPrrDr%rwZmask_feature_indicesr&r&r'_mask_hidden_statess4 zWavLMModel._mask_hidden_statesaudior output_typer>modalityexpected_output)rrrPrrrrc Cs|dur |n|jj}|dur |n|jj}|dur4|n|jj}||}|dd}|durp|j|jd|dd}||\}}|j |||d}|j |||||d} | d}|j dur| |}|s||f| ddSt ||| j | jdS) NrrFr;)rPrrrrrr)rextract_featuresrar )r[rruse_return_dictr@rlr<rrArSrErFrrar ) rZrrrPrrrrZraZencoder_outputsr&r&r'rbs@     zWavLMModel.forward)NN)NNNNN)rdrerfrrNrMrKrrrrrSrWAVLM_INPUTS_DOCSTRINGr_CHECKPOINT_FOR_DOCr_CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPErr;rrrbrgr&r&r]r'r?]s@   .  r?zcWavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).c seZdZd fdd ZddZddZeeee e e e e dd eejeejeeeeeeeejeee fd d d ZZS) WavLMForCTCNcst|t||_t|j|_|jdur@t d|j dt |drV|j rV|j n|j}t||j|_|durt|jdddurt d|dn8|durt|jdddurtdn|dur|||dS)NzYou are trying to instantiate z with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `WavLMForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.r3Zadapter_attn_dimzCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.)rMrNr?r)rrZ final_dropoutr vocab_sizer2r^rzr3r#rwrlm_headrQr[loggerinfoZ load_adapterrG)rZr[Z target_langr#r]r&r'rNs"      zWavLMForCTC.__init__cCstdt|dSrOrINrJrLr&r&r'rMs z$WavLMForCTC.freeze_feature_extractorcCs|jjdSrNr)r@rrLr&r&r'rK#sz"WavLMForCTC.freeze_feature_encoder)rrVr>rXZ expected_lossrrrrrlabelsrc Cs||dur |n|jj}|j|||||d}|d}||}||} d} |dur8||jjkrttd|jj|dur|ntj |tj d}| | d tj } |dk} | d} || }tjj| dtjddd}tjjjd d 6tjj||| | |jj|jj|jjd } Wdn1s.0Y|sh| f|td}| durd| f|S|St| | |j|jd S) a labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size - 1]`. NrYrz$Label values must be <= vocab_size: r/r))rvr0rF)Zenabled)ZblankZ reductionZ zero_infinitylosslogitsrar )r[r[r)rrbr rar2rZ ones_likerr9r6rZ masked_selectrrZ log_softmaxfloat32rlbackendsZcudnnflagsZctc_lossZ pad_token_idZctc_loss_reductionZctc_zero_infinity_HIDDEN_STATES_START_POSITIONr rar )rZrrrrrrhrrarkrjrEZ labels_maskZtarget_lengthsZflattened_targetsZ log_probsoutputr&r&r'rb*sL     &  zWavLMForCTC.forward)N)NNNNN)rdrerfrNrMrKrr\rr]r r^_CTC_EXPECTED_OUTPUT_CTC_EXPECTED_LOSSrrrr;rrrbrgr&r&r]r'r`s2   r`z WavLM Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. c seZdZfddZddZddZddZeee e e e d d de eje eje ee ee ee ejeee fd d dZZS)WavLMForSequenceClassificationcst|t|dr$|jr$tdt||_|jd}|jrTt t |||_ t |j|j|_t |j|j|_|dS)Nr3z\Sequence classification does not support the use of WavLM adapters (config.add_adapter=True)r)rMrNrzr3r2r?r)ruse_weighted_layer_sumrrrr? layer_weightsrrwZclassifier_proj_size projector num_labels classifierrGrZr[ num_layersr]r&r'rNs   z'WavLMForSequenceClassification.__init__cCstdt|dSrHrJrLr&r&r'rMs z7WavLMForSequenceClassification.freeze_feature_extractorcCs|jjdSrNrfrLr&r&r'rKsz5WavLMForSequenceClassification.freeze_feature_encodercCs|jD] }d|_q dSz Calling this function will disable the gradient computation for the base model so that its parameters will not be updated during training. Only the classification head will be updated. FNr)rrrr&r&r'freeze_base_modelsz0WavLMForSequenceClassification.freeze_base_modelrT)rrVr>rWNrgcCsf|dur |n|jj}|jjr dn|}|j|||||d}|jjr|t}tj|dd}tjj |j dd} || dddj dd}n|d}| |}|dur|jdd} n<||jd|} d|| <|j dd| j dd dd} || } d} |dur"t}|| d|jj| d} |sR| f|td}| durN| f|S|St| | |j|jd S)  labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). NTrYrrr)rrri)r[r[rtr)rorstackrrrrurr6rvrr<rrxrrwr rar )rZrrrrrrhrra norm_weightsZ pooled_outputZ padding_maskrkrjloss_fctrpr&r&r'rbsF     z&WavLMForSequenceClassification.forward)NNNNN)rdrerfrNrMrKr}rr\rr]r r^rrrr;rrrbrgr&r&r]r'rszs2    rsza WavLM Model with a frame classification head on top for tasks like Speaker Diarization. c seZdZfddZddZddZddZeee e e e d e d deejeejeejeeeeeeeee fd d dZZS) WavLMForAudioFrameClassificationcszt|t|dr$|jr$tdt||_|jd}|jrTt t |||_ t |j|j|_|j|_|dS)Nr3z_Audio frame classification does not support the use of WavLM adapters (config.add_adapter=True)r)rMrNrzr3r2r?r)rrtrrrr?rurrwrwrx init_weightsryr]r&r'rNs   z)WavLMForAudioFrameClassification.__init__cCstdt|dSrerJrLr&r&r'rMs z9WavLMForAudioFrameClassification.freeze_feature_extractorcCs|jjdSrNrfrLr&r&r'rKsz7WavLMForAudioFrameClassification.freeze_feature_encodercCs|jD] }d|_q dSr{r|rr&r&r'r}"sz2WavLMForAudioFrameClassification.freeze_base_modelrTrUN)rrrhrrrrcCs|dur |n|jj}|jjr dn|}|j|||||d}|jjr|t}tj|dd}tjj |j dd} || dddj dd}n|d}| |} d} |durt} | | d|jtj| d|jdd} |s| f|td} | St| | |j|jd S) r~NTrYrrr)r)Zaxisri)r[r[rtr)rorrrrrrurr6rxrrwrr rar )rZrrrhrrrrrarrkrjrrpr&r&r'rb*s: (z(WavLMForAudioFrameClassification.forward)NNNNN)rdrerfrNrMrKr}rr\r_FRAME_CLASS_CHECKPOINTr r^_FRAME_EXPECTED_OUTPUTrrrr;rrrbrgr&r&r]r'rs4    rcs&eZdZdfdd ZddZZS) AMSoftmaxLoss>@皙?csFtt|||_||_||_tjt ||dd|_ t |_ dS)NT)r) rMrrNscalemarginrwrrrZrandnrtrrj)rZZ input_dimrwrrr]r&r'rNjs zAMSoftmaxLoss.__init__c Csx|}tjj|jdd}tjj|dd}t||}||j}tj||j }|j t | ||}| ||}|S)Nrrr)flattenrr normalizertrmmrZone_hotrwrrr;rj) rZrarhrtZ cos_thetapsiZonehotrkrjr&r&r'rbrs   zAMSoftmaxLoss.forward)rrrcr&r&r]r'risrcs&eZdZdfdd ZddZZS) TDNNLayerrcsvt|dkr |j|dn|j||_|j||_|j||_|j||_t |j|j|j|_ t |_ dS)Nrr)rMrNtdnn_dimrPrQ tdnn_kernelrJZ tdnn_dilationdilationrrkernelZReLUrXrYr]r&r'rNs "   zTDNNLayer.__init__cCsV|d}tjj||j|jfd|jf|jdfd}|dd}||}| |}|S)Nr)rKrr) rrrZunfoldrJrPrrlrrXr`r&r&r'rbs     zTDNNLayer.forward)rrcr&r&r]r'rs rzi WavLM Model with an XVector feature extraction head on top for tasks like Speaker Verification. c seZdZfddZddZddZddZeej e fd d d Z e e eeeed ed deejeejeeeeeeeejeeefdddZZS)WavLMForXVectorcstt|_jd}jr.r)r)rMrNr?r)rrtrrrr?rurrwrrvr9r>rtdnnZxvector_output_dimr@rxrrw objectiver)rZr[rzZ tdnn_layersr]rr'rNs    zWavLMForXVector.__init__cCstdt|dSrerJrLr&r&r'rMs z(WavLMForXVector.freeze_feature_extractorcCs|jjdSrNrfrLr&r&r'rKsz&WavLMForXVector.freeze_feature_encodercCs|jD] }d|_q dSr{r|rr&r&r'r}sz!WavLMForXVector.freeze_base_model)rEcCs&dd}|jjD]}|||d}q|S)z? Computes the output length of the TDNN layers cSs|||dS)Nrr&r6r&r&r'r7szBWavLMForXVector._get_tdnn_output_lengths.._conv_out_lengthr)r[r)rZrEr7rJr&r&r'_get_tdnn_output_lengthss z(WavLMForXVector._get_tdnn_output_lengthsrTrUNrgcCs|dur |n|jj}|jjr dn|}|j|||||d}|jjr|t}tj|dd}tjj |j dd} || dddj dd}n|d}| |}|jD] } | |}q|dur|jdd} |jdd} n||j dd} || }g} g} t|D]D\}}| ||d|fjdd| ||d|fjddqt| } t| } tj| | gdd}||}||}d}|dur|||}|s||f|td}|dur|f|S|St||||j|jdS) r~NTrYrrr)r)rjrkZ embeddingsrar )r[r[rtr)rorrrrrrurr6rvrrr*r9rr rArr@rxrrrar )rZrrrrrrhrrarZ tdnn_layerZ mean_featuresZ std_featuresZfeat_extract_output_lengthsZtdnn_output_lengthsrlengthZstatistic_poolingZoutput_embeddingsrkrjrpr&r&r'rbs\            zWavLMForXVector.forward)NNNNN)rdrerfrNrMrKr}rrrrrrr\r_XVECTOR_CHECKPOINTrr^_XVECTOR_EXPECTED_OUTPUTrrr;rrbrgr&r&r]r'rs6    r)Nr)Trrrtypingrrrnumpyr3rZtorch.nn.functionalrrrZtorch.utils.checkpointZtorch.nnrZ activationsrr{r Zmodeling_outputsr r r r rrZmodeling_utilsrryrrrrZconfiguration_wavlmrZ get_loggerrdrcror^r]r_rqrrrrrrZ#WAVLM_PRETRAINED_MODEL_ARCHIVE_LISTrrrZndarrayrGModulerHrhrmrpr|rrrrrrrrrrr!r"r(ZWAVLM_START_DOCSTRINGr\r?r`rsrrrrr&r&r&r's           x( 3 ()%XYG ]%vk