a
    dv                    @   s  d Z ddlZddlZddlmZ ddlmZmZmZ ddl	Z
ddlZddlZddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZmZmZmZm Z m!Z! ddl"m#Z# e $e%Z&dZ'dZ(dZ)g dZ*dZ+dZ,ddgZ-eG dd deZ.dLee/e/f e0e/eej1 e/e
j2dddZ3G dd dej4Z5G dd dej4Z6G d d! d!ej4Z7G d"d# d#ej4Z8G d$d% d%ej4Z9G d&d' d'ej4Z:G d(d) d)e:Z;G d*d+ d+ej4Z<G d,d- d-ej4Z=G d.d/ d/ej4Z>G d0d1 d1ej4Z?G d2d3 d3ej4Z@G d4d5 d5ej4ZAG d6d7 d7ej4ZBG d8d9 d9ej4ZCG d:d; d;ej4ZDG d<d= d=eZEd>ZFd?ZGed@eFG dAdB dBeEZHedCeFG dDdE dEeEZIedFeFG dGdH dHeEZJedIeFG dJdK dKeEZKdS )Mz PyTorch UniSpeech model.    N)	dataclass)OptionalTupleUnion)nn)CrossEntropyLoss   )ACT2FN)is_deepspeed_zero3_enabled)BaseModelOutputCausalLMOutputSequenceClassifierOutputWav2Vec2BaseModelOutput)PreTrainedModel)ModelOutputadd_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings   )UniSpeechConfig   r   z/patrickvonplaten/unispeech-large-1500h-cv-timit)r   i$  i   zW'mister quilter is the apposl of the midle classes and weare glad to welcom his gosepl'gQ+1@z"microsoft/unispeech-large-1500h-cvz0microsoft/unispeech-large-multi-lingual-1500h-cvc                   @   sz   e Zd ZU dZdZeej ed< dZ	ejed< dZ
ejed< dZejed< dZeeej  ed< dZeeej  ed< dS )	UniSpeechForPreTrainingOutputaL  
    Output type of [`UniSpeechForPreTrainingOutput`], with potential hidden states and attentions.

    Args:
        loss (*optional*, returned when model is in train mode, `torch.FloatTensor` of shape `(1,)`):
            Total loss as the sum of the contrastive loss (L_m) and the diversity loss (L_d) as stated in the [official
            paper](https://arxiv.org/pdf/2006.11477.pdf) . (classification) loss.
        projected_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.proj_codevector_dim)`):
            Hidden-states of the model projected to *config.proj_codevector_dim* that can be used to predict the masked
            projected quantized states.
        projected_quantized_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.proj_codevector_dim)`):
            Quantized extracted feature vectors projected to *config.proj_codevector_dim* representing the positive
            target vectors for contrastive loss.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    Nlossprojected_statesprojected_quantized_statescodevector_perplexityhidden_states
attentions)__name__
__module____qualname____doc__r   r   torchFloatTensor__annotations__r   r   r   r   r   r    r'   r'   y/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/transformers/models/unispeech/modeling_unispeech.pyr   B   s   
r   )shape	mask_probmask_lengthattention_mask	min_masksreturnc                    s  | \}dk rt dkr6t d d dtjd   fdd}|durt|d	  nfd
dt|D }tj	|ft
d}g }	|}
|
dkr|S |D ]v}||}tjjt|d  |dd}t|dkrd }n|d }t|tj|
| tjd| g}|	| qt|	}	t|	dddddf ||
f}	|	||
 }	tddddf }t|||
f||
 }|	| }	|	 d kr҈d |	|	d k< t||	dd	 |S )af  
    Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
    ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on
    CPU as part of the preprocessing during training.

    Args:
        shape: The shape for which to compute masks. This should be of a tuple of size 2 where
               the first element is the batch size and the second element is the length of the axis to span.
        mask_prob:  The percentage of the whole axis (between 0 and 1) which will be masked. The number of
                    independently generated mask spans of length `mask_length` is computed by
                    `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the
                    actual percentage will be smaller.
        mask_length: size of the mask
        min_masks: minimum number of masked spans
        attention_mask: A (right-padded) attention mask which independently shortens the feature axis of
                        each batch dimension.
    r   z&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: `c                    sX   t |     }t|}| kr2 }| d  |k rTt| d  d}|S )z;Given input length, compute how many spans should be maskedr   r   )intmax)input_lengthnum_masked_spanepsilonr+   r*   r-   sequence_lengthr'   r(   compute_num_masked_span   s    
z6_compute_mask_indices.<locals>.compute_num_masked_spanNc                    s   g | ]} qS r'   r'   .0_)r6   r'   r(   
<listcomp>       z)_compute_mask_indices.<locals>.<listcomp>dtyper   F)replace)
ValueErrornprandomZranditemsumdetachtolistrangezerosboolchoicearangelenZconcatenateonesint32appendarrayZbroadcast_toreshaper1   Zput_along_axis)r)   r*   r+   r,   r-   
batch_sizer7   input_lengthsZspec_aug_maskZspec_aug_mask_idxsZmax_num_masked_spanr2   r3   Zspec_aug_mask_idxZdummy_mask_idxoffsetsr'   r4   r(   _compute_mask_indicesg   s\    

rV   c                       s&   e Zd Zd fdd	Zdd Z  ZS )UniSpeechNoLayerNormConvLayerr   c                    sj   t    |dkr |j|d  nd| _|j| | _tj| j| j|j| |j| |j	d| _
t|j | _d S )Nr   r   kernel_sizestridebias)super__init__conv_dimin_conv_dimout_conv_dimr   Conv1dconv_kernelconv_stride	conv_biasconvr	   feat_extract_activation
activationselfconfiglayer_id	__class__r'   r(   r]      s    
z&UniSpeechNoLayerNormConvLayer.__init__c                 C   s   |  |}| |}|S N)re   rg   ri   r   r'   r'   r(   forward   s    

z%UniSpeechNoLayerNormConvLayer.forward)r   r    r!   r"   r]   rp   __classcell__r'   r'   rl   r(   rW      s   rW   c                       s&   e Zd Zd fdd	Zdd Z  ZS )UniSpeechLayerNormConvLayerr   c                    s|   t    |dkr |j|d  nd| _|j| | _tj| j| j|j| |j| |j	d| _
tj| jdd| _t|j | _d S )Nr   r   rX   T)Zelementwise_affine)r\   r]   r^   r_   r`   r   ra   rb   rc   rd   re   	LayerNorm
layer_normr	   rf   rg   rh   rl   r'   r(   r]      s    
z$UniSpeechLayerNormConvLayer.__init__c                 C   s:   |  |}|dd}| |}|dd}| |}|S )Nr8   )re   	transposeru   rg   ro   r'   r'   r(   rp     s    


z#UniSpeechLayerNormConvLayer.forward)r   rq   r'   r'   rl   r(   rs      s   rs   c                       s&   e Zd Zd fdd	Zdd Z  ZS )UniSpeechGroupNormConvLayerr   c                    s   t    |dkr |j|d  nd| _|j| | _tj| j| j|j| |j| |j	d| _
t|j | _tj| j| jdd| _d S )Nr   r   rX   T)
num_groupsZnum_channelsZaffine)r\   r]   r^   r_   r`   r   ra   rb   rc   rd   re   r	   rf   rg   	GroupNormru   rh   rl   r'   r(   r]     s    
z$UniSpeechGroupNormConvLayer.__init__c                 C   s"   |  |}| |}| |}|S rn   )re   ru   rg   ro   r'   r'   r(   rp   "  s    


z#UniSpeechGroupNormConvLayer.forward)r   rq   r'   r'   rl   r(   rx     s   rx   c                       s$   e Zd Z fddZdd Z  ZS ) UniSpeechPositionalConvEmbeddingc                    s   t    tj|j|j|j|jd |jd| _tjj	}t
tjjdrNtjjj	}t rdd l}|jj| jjdd" || jddd| _W d    n1 s0    Y  |j| | jj |j| | jj n|| jddd| _t|j| _t|j | _d S )Nr   )rY   paddinggroupsweight_normr   )Zmodifier_rankweight)namedim)r\   r]   r   ra   hidden_sizenum_conv_pos_embeddingsZnum_conv_pos_embedding_groupsre   utilsr~   hasattrZparametrizationsr
   	deepspeedZzeroZGatheredParametersr   Zregister_external_parameterZweight_vZweight_gUniSpeechSamePadLayerr|   r	   rf   rg   )ri   rj   r~   r   rl   r'   r(   r]   +  s(    

0z)UniSpeechPositionalConvEmbedding.__init__c                 C   s:   | dd}| |}| |}| |}| dd}|S Nr   r   )rw   re   r|   rg   ro   r'   r'   r(   rp   F  s    


z(UniSpeechPositionalConvEmbedding.forwardrq   r'   r'   rl   r(   r{   *  s   r{   c                       s$   e Zd Z fddZdd Z  ZS )r   c                    s$   t    |d dkrdnd| _d S )Nr   r   r   )r\   r]   num_pad_remove)ri   r   rl   r'   r(   r]   S  s    
zUniSpeechSamePadLayer.__init__c                 C   s,   | j dkr(|d d d d d | j  f }|S )Nr   )r   ro   r'   r'   r(   rp   W  s    
zUniSpeechSamePadLayer.forwardrq   r'   r'   rl   r(   r   R  s   r   c                       s0   e Zd ZdZ fddZdd Zdd Z  ZS )UniSpeechFeatureEncoderz.Construct the features from raw audio waveformc                    s   t     jdkr@t ddg fddt jd D  }n6 jdkrd fddt jD }ntd	 j d
t|| _	d| _
d| _d S )Ngroupr   rk   c                    s   g | ]}t  |d  dqS )r   r   )rW   r:   irj   r'   r(   r<   e  s   z4UniSpeechFeatureEncoder.__init__.<locals>.<listcomp>r   layerc                    s   g | ]}t  |d qS )r   )rs   r   r   r'   r(   r<   j  s   z`config.feat_extract_norm` is z), but has to be one of ['group', 'layer']FT)r\   r]   Zfeat_extract_normrx   rH   Znum_feat_extract_layersrA   r   
ModuleListconv_layersgradient_checkpointing_requires_grad)ri   rj   r   rl   r   r(   r]   a  s    




z UniSpeechFeatureEncoder.__init__c                 C   s   |   D ]
}d|_qd| _d S )NF)
parametersrequires_gradr   ri   paramr'   r'   r(   _freeze_parametersu  s    z*UniSpeechFeatureEncoder._freeze_parametersc                 C   sj   |d d d f }| j r"| jr"d|_| jD ]<}| j r\| jr\| jr\dd }tjj|||}q(||}q(|S )NTc                    s    fdd}|S )Nc                     s    |  S rn   r'   inputsmoduler'   r(   custom_forward  s    zVUniSpeechFeatureEncoder.forward.<locals>.create_custom_forward.<locals>.custom_forwardr'   r   r   r'   r   r(   create_custom_forward  s    z>UniSpeechFeatureEncoder.forward.<locals>.create_custom_forward)r   trainingr   r   r   r$   r   
checkpoint)ri   input_valuesr   Z
conv_layerr   r'   r'   r(   rp   z  s    

zUniSpeechFeatureEncoder.forward)r    r!   r"   r#   r]   r   rp   rr   r'   r'   rl   r(   r   ^  s   r   c                       s   e Zd Z fddZ  ZS )UniSpeechFeatureExtractorc                    s8   t  | td| jj d| jjd j dt d S )NzThe class `zD` has been depreciated and will be removed in Transformers v5. Use `r   z
` instead.)r\   r]   warningswarnrm   r    	__bases__FutureWarningri   rj   rl   r'   r(   r]     s    z"UniSpeechFeatureExtractor.__init__)r    r!   r"   r]   rr   r'   r'   rl   r(   r     s   r   c                       s$   e Zd Z fddZdd Z  ZS )UniSpeechFeatureProjectionc                    sJ   t    tj|jd |jd| _t|jd |j| _	t
|j| _d S )Nr8   Zeps)r\   r]   r   rt   r^   layer_norm_epsru   Linearr   
projectionDropoutZfeat_proj_dropoutdropoutr   rl   r'   r(   r]     s    
z#UniSpeechFeatureProjection.__init__c                 C   s&   |  |}| |}| |}||fS rn   )ru   r   r   )ri   r   Znorm_hidden_statesr'   r'   r(   rp     s    


z"UniSpeechFeatureProjection.forwardrq   r'   r'   rl   r(   r     s   r   c                       s   e Zd ZdZdeeeeed fddZej	eedd	d
Z
dej	eej	 eeej	  eej	 eej	 eeej	eej	 eeej	  f dddZ  ZS )UniSpeechAttentionz=Multi-headed attention from 'Attention Is All You Need' paper        FT)	embed_dim	num_headsr   
is_decoderr[   c                    s   t    || _|| _|| _|| | _| j| | jkrNtd| j d| d| jd | _|| _t	j
|||d| _t	j
|||d| _t	j
|||d| _t	j
|||d| _d S )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      )r[   )r\   r]   r   r   r   head_dimrA   scalingr   r   r   k_projv_projq_projout_proj)ri   r   r   r   r   r[   rl   r'   r(   r]     s"    


zUniSpeechAttention.__init__)tensorseq_lenbszc                 C   s    | ||| j| jdd S r   )viewr   r   rw   
contiguous)ri   r   r   r   r'   r'   r(   _shape  s    zUniSpeechAttention._shapeN)r   key_value_statespast_key_valuer,   layer_head_maskoutput_attentionsr.   c                 C   sx  |du}|  \}}	}
| || j }|r\|dur\|d jd |jd kr\|d }|d }n|r| | |d|}| | |d|}n|dur| | |d|}| | |d|}tj|d |gdd}tj|d |gdd}n(| | |d|}| | |d|}| j	r ||f}|| j
 d| jf}| ||	|j| }|j| }|j| }| d}t||dd}|  || j
 |	|fkrtd|| j
 |	|f d|   |dur |  |d|	|fkrtd	|d|	|f d|   ||| j
|	|| }||| j
 |	|}tjj|dd}|dur|  | j
fkrhtd
| j
f d|   |dddd||| j
|	| }||| j
 |	|}|r||| j
|	|}||| j
 |	|}nd}tjj|| j| jd}t||}|  || j
 |	| jfkr4td|| j
 |	| jf d|   ||| j
|	| j}|dd}|||	| j}| |}|||fS )z#Input shape: Batch x Time x ChannelNr   r   r   r8   r   z$Attention weights should be of size z	, but is z!Attention mask should be of size z/Head mask for a single layer should be of size )pr   z `attn_output` should be of size )sizer   r   r)   r   r   r   r$   catr   r   r   r   rR   Zbmmrw   rA   r   
functionalsoftmaxr   r   r   r   )ri   r   r   r   r,   r   r   Zis_cross_attentionr   Ztgt_lenr;   Zquery_statesZ
key_statesZvalue_statesZ
proj_shapeZsrc_lenattn_weightsZattn_weights_reshapedZ
attn_probsZattn_outputr'   r'   r(   rp     s    





"
zUniSpeechAttention.forward)r   FT)NNNNF)r    r!   r"   r#   r0   floatrJ   r]   r$   Tensorr   r   r   rp   rr   r'   r'   rl   r(   r     s4           r   c                       s$   e Zd Z fddZdd Z  ZS )UniSpeechFeedForwardc                    sp   t    t|j| _t|j|j| _	t
|jtrDt|j | _n|j| _t|j|j| _t|j| _d S rn   )r\   r]   r   r   Zactivation_dropoutintermediate_dropoutr   r   Zintermediate_sizeintermediate_dense
isinstanceZ
hidden_actstrr	   intermediate_act_fnoutput_densehidden_dropoutoutput_dropoutr   rl   r'   r(   r]   L  s    
zUniSpeechFeedForward.__init__c                 C   s6   |  |}| |}| |}| |}| |}|S rn   )r   r   r   r   r   ro   r'   r'   r(   rp   Y  s    




zUniSpeechFeedForward.forwardrq   r'   r'   rl   r(   r   K  s   r   c                       s&   e Zd Z fddZdddZ  ZS )UniSpeechEncoderLayerc                    sf   t    t|j|j|jdd| _t|j	| _
tj|j|jd| _t|| _tj|j|jd| _d S )NFr   r   r   r   r   )r\   r]   r   r   num_attention_headsattention_dropout	attentionr   r   r   r   rt   r   ru   r   feed_forwardfinal_layer_normr   rl   r'   r(   r]   e  s    

zUniSpeechEncoderLayer.__init__NFc                 C   sf   |}| j |||d\}}}| |}|| }| |}|| | }| |}|f}|rb||f7 }|S Nr,   r   )r   r   ru   r   r   ri   r   r,   r   Zattn_residualr   r;   outputsr'   r'   r(   rp   r  s    



zUniSpeechEncoderLayer.forward)NFrq   r'   r'   rl   r(   r   d  s   r   c                       s,   e Zd Z fddZejdddZ  ZS )UniSpeechAttnAdapterLayerc                    sZ   t    |j| _|j| _t| j| _t	| j| j| _
t | _t	| j| j| _dS )z
        Implements adapter modules directly with 3D tensor weight as parameters and without using ModuleList to speed
        up training throughput.
        N)r\   r]   adapter_attn_dimZ	input_dimr   Z
hidden_dimr   rt   normr   linear_1ZReLUact_fnlinear_2r   rl   r'   r(   r]     s    

z"UniSpeechAttnAdapterLayer.__init__)r   c                 C   s,   |  |}| |}| |}| |}|S rn   )r   r   r   r   ro   r'   r'   r(   rp     s
    



z!UniSpeechAttnAdapterLayer.forward)r    r!   r"   r]   r$   r%   rp   rr   r'   r'   rl   r(   r     s   r   c                       s8   e Zd Z fddZdejeej edddZ  Z	S )	$UniSpeechEncoderLayerStableLayerNormc                    s   t    t|j|j|jdd| _t|j	| _
tj|j|jd| _t|| _tj|j|jd| _t|dd d ur~t|| _nd | _d S )NFr   r   r   )r\   r]   r   r   r   r   r   r   r   r   r   rt   r   ru   r   r   r   getattrr   adapter_layerr   rl   r'   r(   r]     s    

z-UniSpeechEncoderLayerStableLayerNorm.__init__NF)r   r,   r   c                 C   sz   |}|  |}| j|||d\}}}| |}|| }|| | | }| jd urb|| | }|f}|rv||f7 }|S r   )ru   r   r   r   r   r   r   r'   r'   r(   rp     s    



z,UniSpeechEncoderLayerStableLayerNorm.forward)NF)
r    r!   r"   r]   r$   r   r   rJ   rp   rr   r'   r'   rl   r(   r     s     r   c                       s<   e Zd Z fddZd	ejeej eeedddZ	  Z
S )
UniSpeechEncoderc                    sf   t     | _t | _tj j jd| _	t
 j| _t fddt jD | _d| _d S )Nr   c                    s   g | ]}t  qS r'   )r   r9   r   r'   r(   r<     r=   z-UniSpeechEncoder.__init__.<locals>.<listcomp>Fr\   r]   rj   r{   pos_conv_embedr   rt   r   r   ru   r   r   r   r   rH   num_hidden_layerslayersr   r   rl   r   r(   r]     s    

 zUniSpeechEncoder.__init__NFT)r   r,   r   output_hidden_statesreturn_dictc                    s  |rdnd } rdnd }|d ur| ddd|jd }d|| < d|d d d d d d f j|jd }|t|jj }||jd d|jd |jd }| 	|}	||	 }| 
|}| |}t }
| jD ]}|r||f }tjdd}| jr
|| jjk r
dnd	}|r|
rd| jrN| jrN fd
d}tjj||||}n||| d}|d }|rnd} r||d f }q|r||f }|stdd |||fD S t|||dS )Nr'   r8   r   r   r         ?r>   TFc                    s    fdd}|S )Nc                     s    g | R  S rn   r'   r   r   r   r'   r(   r     s    zOUniSpeechEncoder.forward.<locals>.create_custom_forward.<locals>.custom_forwardr'   r   r   r   r(   r     s    z7UniSpeechEncoder.forward.<locals>.create_custom_forwardr   NNc                 s   s   | ]}|d ur|V  qd S rn   r'   r:   vr'   r'   r(   	<genexpr>  r=   z+UniSpeechEncoder.forward.<locals>.<genexpr>last_hidden_stater   r   )	unsqueezerepeatr)   tor?   r$   finfominexpandr   ru   r   r
   r   rB   rC   uniformr   rj   	layerdropr   r   r   tupler   ri   r   r,   r   r   r   Zall_hidden_statesZall_self_attentionsZexpand_attention_maskZposition_embeddingsZdeepspeed_zero3_is_enabledr   Zdropout_probabilityZskip_the_layerr   Zlayer_outputsr'   r   r(   rp     sZ    
&





zUniSpeechEncoder.forward)NFFT)r    r!   r"   r]   r$   r   r   r   rJ   rp   rr   r'   r'   rl   r(   r     s       r   c                       s&   e Zd Z fddZdddZ  ZS )	UniSpeechEncoderStableLayerNormc                    sf   t     | _t | _tj j jd| _	t
 j| _t fddt jD | _d| _d S )Nr   c                    s   g | ]}t  qS r'   )r   r9   r   r'   r(   r<   /  r=   z<UniSpeechEncoderStableLayerNorm.__init__.<locals>.<listcomp>Fr   r   rl   r   r(   r]   (  s    

z(UniSpeechEncoderStableLayerNorm.__init__NFTc                    s  |rdnd } rdnd }|d ur| ddd|jd }d|| < d|d d d d d d f j|jd }|t|jj }||jd d|jd |jd }| 	|}	||	 }| 
|}t }
| jD ]}|r||f }tjdd}| jr|| jjk rdnd	}|r|
rV| jr@| jr@ fd
d}tjj||||}n||| d}|d }|r`d} r||d f }q| |}|r||f }|stdd |||fD S t|||dS )Nr'   r8   r   r   r   r   r>   TFc                    s    fdd}|S )Nc                     s    g | R  S rn   r'   r   r   r'   r(   r   ^  s    z^UniSpeechEncoderStableLayerNorm.forward.<locals>.create_custom_forward.<locals>.custom_forwardr'   r   r   r   r(   r   ]  s    zFUniSpeechEncoderStableLayerNorm.forward.<locals>.create_custom_forwardr   r   c                 s   s   | ]}|d ur|V  qd S rn   r'   r   r'   r'   r(   r   z  r=   z:UniSpeechEncoderStableLayerNorm.forward.<locals>.<genexpr>r   )r   r   r)   r   r?   r$   r   r   r  r   r   r
   r   rB   rC   r  r   rj   r  r   r   r   ru   r  r   r  r'   r   r(   rp   3  sZ    
&





z'UniSpeechEncoderStableLayerNorm.forward)NFFTrq   r'   r'   rl   r(   r  '  s       r  c                       s4   e Zd ZdZ fddZedd Zdd Z  ZS )UniSpeechGumbelVectorQuantizerz
    Vector quantization using gumbel softmax. See [CATEGORICAL REPARAMETERIZATION WITH
    GUMBEL-SOFTMAX](https://arxiv.org/pdf/1611.01144.pdf) for more information.
    c                    s   t    |j| _|j| _|j| j dkrDtd|j d| j dt	t
d| j| j |j| j | _t|jd | j| j | _d| _d S )Nr   z`config.codevector_dim z5 must be divisible by `config.num_codevector_groups` z for concatenationr   r8   r   )r\   r]   Znum_codevector_groupsry   Znum_codevectors_per_groupnum_varscodevector_dimrA   r   	Parameterr$   r%   codevectorsr   r^   weight_projtemperaturer   rl   r'   r(   r]     s    

z'UniSpeechGumbelVectorQuantizer.__init__c                 C   s8   | j dd}ttj|t|d  dd  }|S )Nr   r   gHz>r8   )meanr$   exprE   log)ZprobsZmarginal_probs
perplexityr'   r'   r(   _compute_perplexity  s    (z2UniSpeechGumbelVectorQuantizer._compute_perplexityc                 C   s  |j \}}}| |}||| | j d}| jr~tjj| | j	dd
|}tj||| | jd dd}| |}nH|jdd}|j|j  d|ddd}||| | jd}| |}||| d}|d| j }	|	|| | j| jd}
|
d||d}
|
|fS )Nr8   T)tauhardr   r   r   rv   )r)   r  r   ry   r   r   r   Zgumbel_softmaxr   r  type_asr$   r   r  ZargmaxZ	new_zerosZscatter_r   r  r  rE   )ri   r   rS   r6   r   Zcodevector_probsZcodevector_soft_distr  Zcodevector_idxZcodevectors_per_groupr  r'   r'   r(   rp     s0    

z&UniSpeechGumbelVectorQuantizer.forward)	r    r!   r"   r#   r]   staticmethodr  rp   rr   r'   r'   rl   r(   r    s
   
r  c                   @   sb   e Zd ZdZeZdZdZdgZdZ	dd Z
eejef dd	d
ZeejdddZdddZdS )UniSpeechPreTrainedModelz
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    	unispeechr   Zposition_idsTc              	   C   s  t |tr>|jjjjddd |jjj  tj	
|j njt |trtj	j|jjddtd|jjd |jj   d tj	|jjd nt |trtd|jj }tj	j
|jj| |d tj	j
|jj| |d nt |tjr|jjjd| jjd |jdur|jj  nt |tjtjfrN|jj  |jjd nZt |tjrtj	|j |jdurt|j|j|jd   }tj	j
|j| |d dS )	zInitialize the weightsr   r   )r  Zstdr   r   )abNr   )r   r  r  r   dataZnormal_r[   Zzero_r   inituniform_r  r{   re   mathsqrtrY   Zin_channelsZ	constant_r   r   Zin_featuresr   rj   Zinitializer_rangert   rz   fill_ra   Zkaiming_normal_r}   )ri   r   kr'   r'   r(   _init_weights  s6    

 
z&UniSpeechPreTrainedModel._init_weights)rT   c                 C   s4   dd }t | jj| jjD ]\}}||||}q|S )zH
        Computes the output length of the convolutional layers
        c                 S   s   t j| | |ddd S )Nfloor)Zrounding_moder   )r$   div)r2   rY   rZ   r'   r'   r(   _conv_out_length  s    zSUniSpeechPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_length)ziprj   rb   rc   )ri   rT   r%  rY   rZ   r'   r'   r(    _get_feat_extract_output_lengths  s    z9UniSpeechPreTrainedModel._get_feat_extract_output_lengths)feature_vector_lengthr,   c                 C   s   |j ddd d df }| |tj}|jd }tj||f|j|jd}d|tj	|jd |jd|d f< |
dg d
dg }|S )Nr8   r   r   )r?   devicer   )r)  )Zcumsumr'  r   r$   longr)   rI   r?   r)  rL   ZfliprJ   )ri   r(  r,   Znon_padded_lengthsZoutput_lengthsrS   r'   r'   r(   "_get_feature_vector_attention_mask  s    
"z;UniSpeechPreTrainedModel._get_feature_vector_attention_maskFc                 C   s   t |tttfr||_d S rn   )r   r   r  r   r   )ri   r   valuer'   r'   r(   _set_gradient_checkpointing  s    z4UniSpeechPreTrainedModel._set_gradient_checkpointingN)F)r    r!   r"   r#   r   config_classZbase_model_prefixZmain_input_nameZ_keys_to_ignore_on_load_missingZsupports_gradient_checkpointingr"  r   r$   
LongTensorr0   r'  r+  r-  r'   r'   r'   r(   r    s   !r  a  
    UniSpeech was proposed in [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled
    Data](https://arxiv.org/abs/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei,
    Michael Zeng, Xuedong Huang.

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving etc.).

    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`UniSpeechConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aI  
    Args:
        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
            into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install
            soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and
            conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details.
        attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0,
            1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)

            <Tip warning={true}>

            `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask ==
            True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should
            **not** be passed to avoid degraded performance when doing batched inference. For such models
            `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these
            models also yield slightly different results depending on whether `input_values` is padded or not.

            </Tip>

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
zcThe bare UniSpeech Model transformer outputting raw hidden-states without any specific head on top.c                       s   e Zd Zed fddZdejeej eej dddZ	e
eeeeeded	deej eej eej ee ee ee eeef d
ddZ  ZS )UniSpeechModelr   c                    sz   t  | || _t|| _t|| _|jdks:|jdkrRt	
t|j | _|jrdt|| _n
t|| _|   d S )Nr   )r\   r]   rj   r   feature_extractorr   feature_projectionmask_time_probmask_feature_probr   r
  r$   r%   r   r  masked_spec_embedZdo_stable_layer_normr  encoderr   	post_initr   rl   r'   r(   r]   U  s    


zUniSpeechModel.__init__N)r   mask_time_indicesr,   c                 C   s  t | jdds|S | \}}}|dur<| j|j||< nZ| jjdkr| jrt||f| jj| jj	|| jj
d}tj||jtjd}| j|j||< | jjdkr| jrt||f| jj| jj| jjd}tj||jtjd}|dddf d|d}d||< |S )	z
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://arxiv.org/abs/1904.08779).
        Zapply_spec_augmentTNr   )r*   r+   r,   r-   )r)  r?   )r*   r+   r-   r8   )r   rj   r   r5  r   r?   r3  r   rV   Zmask_time_lengthZmask_time_min_masksr$   r   r)  rJ   r4  Zmask_feature_lengthZmask_feature_min_masksr  )ri   r   r8  r,   rS   r6   r   Zmask_feature_indicesr'   r'   r(   _mask_hidden_statesg  s4    z"UniSpeechModel._mask_hidden_statesaudio)r   output_typer.  modalityexpected_output)r   r,   r8  r   r   r   r.   c           
      C   s   |d ur|n| j j}|d ur |n| j j}|d ur4|n| j j}| |}|dd}|d url| |jd |}| |\}}| j	|||d}| j
|||||d}	|	d }|s||f|	dd   S t|||	j|	jdS )Nr   r   )r8  r,   r,   r   r   r   r   )r   extract_featuresr   r   )rj   r   r   use_return_dictr1  rw   r+  r)   r2  r9  r6  r   r   r   )
ri   r   r,   r8  r   r   r   r?  r   Zencoder_outputsr'   r'   r(   rp     s8    
zUniSpeechModel.forward)NN)NNNNN)r    r!   r"   r   r]   r$   r%   r   r/  r9  r   UNISPEECH_INPUTS_DOCSTRINGr   _CHECKPOINT_FOR_DOCr   _CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPEr   rJ   r   r   rp   rr   r'   r'   rl   r(   r0  P  s<     .
     
r0  zPUniSpeech Model with a vector-quantization module and ctc loss for pre-training.c                       s   e Zd Zed fddZedddZdd Zd	d
 Ze	de
je
je
jedddZeeeeeddee
j ee
j ee ee ee eeef dddZ  ZS )UniSpeechForPreTrainingr   c                    s~   t  | t|| _t|j| _t|| _	t
|j|j| _t
|j|j| _t
|j|j| _t|j| _|   d S rn   )r\   r]   r0  r  r   r   Zfeat_quantizer_dropoutdropout_featuresr  	quantizerr   r	  Zproj_codevector_dim	project_qr   project_hidZnum_ctc_classesctc_projfinal_dropoutr   r7  r   rl   r'   r(   r]     s    

z UniSpeechForPreTraining.__init__)r  c                 C   s   || j _dS )zb
        Set the Gumbel softmax temperature to a given value. Only necessary for training
        N)rG  r  )ri   r  r'   r'   r(   set_gumbel_temperature  s    z.UniSpeechForPreTraining.set_gumbel_temperaturec                 C   s   t dt |   dS z
        Calling this function will disable the gradient computation for the feature encoder so that its parameters will
        not be updated during training.
        The method `freeze_feature_extractor` is deprecated and will be removed in Transformers v5.Please use the equivalent `freeze_feature_encoder` method instead.Nr   r   r   freeze_feature_encoderri   r'   r'   r(   freeze_feature_extractor  s
    z0UniSpeechForPreTraining.freeze_feature_extractorc                 C   s   | j j  dS 
        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
        not be updated during training.
        Nr  r1  r   rQ  r'   r'   r(   rP    s    z.UniSpeechForPreTraining.freeze_feature_encoderr   )target_featuresnegative_featurespredicted_featuresr  c                 C   s@   t j| |gdd} t j| |  dd}|| }|| }|S )z
        Compute logits for contrastive loss based using cosine similarity as the distance measure between
        `[positive_feature, negative_features]` and `[predicted_features]`. Additionally, temperature can be applied.
        r   r   r8   )r$   r   Zcosine_similarityr   r  )rV  rW  rX  r  logitsr'   r'   r(   compute_contrastive_logits  s
    
z2UniSpeechForPreTraining.compute_contrastive_logits)r;  r.  N)r   r,   r   r   r   r.   c                 C   sB  |dur|n| j j}| j|||||d}|d }| |d }| |\}	}
| |	}	| |	}	t|	d|	d
| j j}|dd}t| |j}|dd}|d}||d|	| d }| |}| |}d}|s*|dur|||	|
f|dd  S ||	|
f|dd  S t|||	|
|j|jdS )	a  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.
        sampled_negative_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_negatives)`, *optional*):
            Indices indicating which quantized target vectors are used as negative sampled vectors in contrastive loss.
            Required input for pre-training.

        Returns:

        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoFeatureExtractor, UniSpeechForPreTraining

        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("microsoft/unispeech-large-1500h-cv")
        >>> model = UniSpeechForPreTraining.from_pretrained("microsoft/unispeech-large-1500h-cv")
        >>> # TODO: Add full pretraining example
        ```Nr>  r   r   r8   r   r   )r   r   r   r   r   r   )rj   r@  r  rF  rG  rH  rI  r$   emptyr   r   Zreplace_probrw   Z	bernoullirJ   r   r)  r   Zmasked_fillr   rJ  r   r   r   )ri   r   r,   r   r   r   r   Ztransformer_featuresr?  Zquantized_featuresr   Zprob_replace_matrixZsampled_replace_matrixrY  r   r'   r'   r(   rp     sL    






zUniSpeechForPreTraining.forward)r   )NNNN)r    r!   r"   r   r]   r0   rL  rR  rP  r  r$   r%   rZ  r   rA  r   r   rC  r   r   rJ   r   r   rp   rr   r'   r'   rl   r(   rE    s4    
    
rE  zgUniSpeech Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).c                       s   e Zd Zd fdd	Zdd Zdd Zeeee	e
eeeddeej eej ee ee ee eej eee
f d	d
dZ  ZS )UniSpeechForCTCNc                    s   t  | t|| _t|j| _|jd u r@t	d| j
 dt|drV|jrV|jn|j}t||j| _|d urt| jdd d u rt	d| dn8|d u rt| jdd d urtd n|d ur| | |   d S )NzYou are trying to instantiate z with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `UniSpeechForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.add_adapterr   zCannot pass `target_lang`: z- if `config.adapter_attn_dim` is not defined.z)By default `target_lang` is set to 'eng'.)r\   r]   r0  r  r   r   rK  r   
vocab_sizerA   rm   r   r]  output_hidden_sizer   r   lm_headr   rj   loggerinfoZload_adapterr7  )ri   rj   Ztarget_langr_  rl   r'   r(   r]   e  s"    


zUniSpeechForCTC.__init__c                 C   s   t dt |   dS )rT  rN  NrO  rQ  r'   r'   r(   rR    s
    z(UniSpeechForCTC.freeze_feature_extractorc                 C   s   | j j  dS rS  rU  rQ  r'   r'   r(   rP    s    z&UniSpeechForCTC.freeze_feature_encoder)r   r;  r.  r=  Zexpected_lossr   r,   r   r   r   labelsr.   c              
   C   s|  |dur|n| j j}| j|||||d}|d }| |}| |}	d}
|dur8| | j jkrttd| j j |dur|ntj	|tj
d}| |dtj
}|dk}|d}||}tjj|	dtjddd}tjjjd	d
6 tjj||||| j j| j j| j jd}
W d   n1 s.0    Y  |sh|	f|td  }|
durd|
f| S |S t|
|	|j|jdS )a  
        labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
            Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
            the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
            All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
            config.vocab_size - 1]`.
        Nr>  r   z$Label values must be <= vocab_size: r>   r8   )r   r?   r   F)Zenabled)ZblankZ	reductionZzero_infinityr   rY  r   r   )rj   r@  r  r   r`  r1   r^  rA   r$   Z	ones_liker*  r'  rE   r   Zmasked_selectr   r   Zlog_softmaxfloat32rw   backendsZcudnnflagsZctc_lossZpad_token_idZctc_loss_reductionZctc_zero_infinity_HIDDEN_STATES_START_POSITIONr   r   r   )ri   r   r,   r   r   r   rd  r   r   rY  r   rT   Zlabels_maskZtarget_lengthsZflattened_targetsZ	log_probsoutputr'   r'   r(   rp     sL    




&
zUniSpeechForCTC.forward)N)NNNNN)r    r!   r"   r]   rR  rP  r   rA  r   rB  r   rC  _CTC_EXPECTED_OUTPUT_CTC_EXPECTED_LOSSr   r$   r   rJ   r   r   rp   rr   r'   r'   rl   r(   r\  _  s2   
     
r\  z
    UniSpeech Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like
    SUPERB Keyword Spotting.
    c                       s   e Zd Z fddZdd Zdd Zdd Zeee	e
eed	d
deej eej ee ee ee eej eeef dddZ  ZS )"UniSpeechForSequenceClassificationc                    s   t  | t|dr$|jr$tdt|| _|jd }|jrTt	
t|| | _t	|j|j| _t	|j|j| _|   d S )Nr]  z`Sequence classification does not support the use of UniSpeech adapters (config.add_adapter=True)r   )r\   r]   r   r]  rA   r0  r  r   use_weighted_layer_sumr   r
  r$   rN   layer_weightsr   r   Zclassifier_proj_size	projector
num_labels
classifierr7  )ri   rj   Z
num_layersrl   r'   r(   r]     s    

z+UniSpeechForSequenceClassification.__init__c                 C   s   t dt |   dS rM  rO  rQ  r'   r'   r(   rR    s
    z;UniSpeechForSequenceClassification.freeze_feature_extractorc                 C   s   | j j  dS rS  rU  rQ  r'   r'   r(   rP    s    z9UniSpeechForSequenceClassification.freeze_feature_encoderc                 C   s   | j  D ]
}d|_q
dS )z
        Calling this function will disable the gradient computation for the base model so that its parameters will not
        be updated during training. Only the classification head will be updated.
        FN)r  r   r   r   r'   r'   r(   freeze_base_model  s    z4UniSpeechForSequenceClassification.freeze_base_modelr:  )r   r;  r.  r<  Nrc  c                 C   sf  |dur|n| j j}| j jr dn|}| j|||||d}| j jr|t }tj|dd}tjj	| j
dd}	||	ddd jdd}n|d }| |}|du r|jdd}
n<| |jd |}d|| < |jdd|jdddd }
| |
}d}|dur"t }||d| j j|d}|sR|f|td  }|durN|f| S |S t|||j|jd	S )
a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NTr>  r   r   r8   r   r   re  )rj   r@  rn  r  ri  r$   stackr   r   r   ro  r   rE   rp  r  r+  r)   rr  r   rq  r   r   r   )ri   r   r,   r   r   r   rd  r   r   Znorm_weightsZpooled_outputZpadding_maskrY  r   Zloss_fctrj  r'   r'   r(   rp     sF    

 

z*UniSpeechForSequenceClassification.forward)NNNNN)r    r!   r"   r]   rR  rP  rs  r   rA  r   rB  r   rC  r   r$   r   rJ   r   r   rp   rr   r'   r'   rl   r(   rm    s2   
     
rm  )Nr   )Lr#   r  r   Zdataclassesr   typingr   r   r   numpyrB   r$   Ztorch.utils.checkpointr   Ztorch.nnr   Zactivationsr	   r   r
   Zmodeling_outputsr   r   r   r   Zmodeling_utilsr   r   r   r   r   r   r   r   Zconfiguration_unispeechr   Z
get_loggerr    ra  ri  rC  rB  rD  rk  rl  Z'UNISPEECH_PRETRAINED_MODEL_ARCHIVE_LISTr   r0   r   r/  ZndarrayrV   ModulerW   rs   rx   r{   r   r   r   r   r   r   r   r   r   r   r  r  r  ZUNISPEECH_START_DOCSTRINGrA  r0  rE  r\  rm  r'   r'   r'   r(   <module>   s    
(  
x(6 #.X[FP%y  