a  d@sdZddlmZddlmZmZmZddlZddlZddlm Z ddl m Z m Z m Z ddlmZdd lmZmZmZmZmZdd lmZdd lmZdd lmZmZmZmZd dlm Z e!e"Z#dZ$dZ%dZ&dZ'dZ(dZ)dZ*dZ+dZ,dZ-gdZ.Gddde j/Z0Gdddej1j2Z3Gddde4Z5d d!Z6Gd"d#d#ej1j2Z7Gd$d%d%e j/Z8Gd&d'd'e j/Z9Gd(d)d)e j/Z:Gd*d+d+e j/Z;Gd,d-d-e j/ZGd2d3d3e j/Z?d4d5Z@ejAjBd6d7ZCejAjBd8d9ZDejAjBd:d;ZEGdd?d?e j/ZGGd@dAdAeZHdBZIdCZJedDeIGdEdFdFeHZKedGeIGdHdIdIeHZLGdJdKdKe j/ZMGdLdMdMe j/ZNGdNdOdOe j/ZOedPeIGdQdRdReHZPedSeIGdTdUdUeHZQedVeIGdWdXdXeHZRdS)Yz PyTorch DeBERTa model.)Sequence)OptionalTupleUnionN)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)BaseModelOutputMaskedLMOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)softmax_backward_data)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardlogging) DebertaConfigrmicrosoft/deberta-basez!lsanochkin/deberta-large-feedbackz' Paris'z0.54z#Palak/microsoft_deberta-large_squadz' a nice puppet'gQ? )rzmicrosoft/deberta-largezmicrosoft/deberta-xlargezmicrosoft/deberta-base-mnlizmicrosoft/deberta-large-mnlizmicrosoft/deberta-xlarge-mnlics0eZdZfddZddZeddZZS) ContextPoolercs2tt|j|j|_t|j|_||_ dSN) super__init__rLinearZpooler_hidden_sizedense StableDropoutZpooler_dropoutdropoutconfigselfr$ __class__u/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/transformers/models/deberta/modeling_deberta.pyrCs  zContextPooler.__init__cCs8|dddf}||}||}t|jj|}|SNr)r#r!r r$Zpooler_hidden_act)r& hidden_statesZ context_token pooled_outputr)r)r*forwardIs   zContextPooler.forwardcCs|jjSr)r$ hidden_sizer&r)r)r* output_dimSszContextPooler.output_dim)__name__ __module__ __qualname__rr.propertyr1 __classcell__r)r)r'r*rBs  rc@s4eZdZdZeddZeddZeddZdS) XSoftmaxa Masked Softmax which is optimized for saving memory Args: input (`torch.tensor`): The input tensor that will apply softmax. mask (`torch.IntTensor`): The mask matrix where 0 indicate that element will be ignored in the softmax calculation. dim (int): The dimension that will apply softmax Example: ```python >>> import torch >>> from transformers.models.deberta.modeling_deberta import XSoftmax >>> # Make a tensor >>> x = torch.randn([4, 20, 100]) >>> # Create a mask >>> mask = (x > 0).int() >>> # Specify the dimension to apply softmax >>> dim = -1 >>> y = XSoftmax.apply(x, mask, dim) ```cCsX||_|tj}||tt|jj}t ||j}| |d| ||Sr+) dimtotorchbool masked_filltensorfinfodtypeminsoftmaxZ masked_fill_save_for_backward)r&inputmaskr8Zrmaskoutputr)r)r*r.ts  zXSoftmax.forwardcCs$|j\}t||||j|}|ddfSr) saved_tensorsrr8)r& grad_outputrEZ inputGradr)r)r*backwardszXSoftmax.backwardc Csddlmm}ddlm}m}|jd||jdd}|jd|d|jdtj dtj d d ||jd d}|||||jdt t | jd } ||| |} ||| ||jdtj dtjd d S) Nr)r<rAZCastZLong)Zto_iSubConstantrr?)Zvalue_tZBool)Ztorch.onnx.symbolic_helperZonnxZsymbolic_helperZtorch.onnx.symbolic_opset9r<rAopZcast_pytorch_to_onnxr:r=int64r>typer?r@r;) gr&rDr8Zsym_helpr<rAZmask_cast_valueZr_maskrEr)r)r*symbolics"( zXSoftmax.symbolicN)r2r3r4__doc__ staticmethodr.rHrPr)r)r)r*r7Xs  r7c@seZdZddZdS)DropoutContextcCsd|_d|_d|_d|_dS)NrrT)r#rDscale reuse_maskr0r)r)r*rszDropoutContext.__init__N)r2r3r4rr)r)r)r*rSsrScCst|ts|}d}n |j}||j9}|jr0|jnd}|dkrd|durddt|d| tj }t|tr~|jdur~||_||fS)Nrr) isinstancerSr#rTrUrDr:Z empty_likeZ bernoulli_r9r;)rCZ local_contextr#rDr)r)r*get_masks     rWc@sTeZdZdZeddZeddZeejj ejj e e e fejj dddZd S) XDropoutzlOptimized dropout function to save computation and memory by using mask operation instead of multiplication.cCsHt||\}}dd||_|dkr@||||d|jS|SdS)Ng?rr)rWrTrBr<)ctxrC local_ctxrDr#r)r)r*r.s  zXDropout.forwardcCs4|jdkr(|j\}||d|jdfS|dfSdS)Nrr)rTrFr<)rYrGrDr)r)r*rHs zXDropout.backward)rOrCrZreturncCs4ddlm}|}t|tr |j}d}|||||S)Nr)symbolic_opset12T)Z torch.onnxr\rVrSr#)rOrCrZr\Z dropout_pZtrainr)r)r*rPs   zXDropout.symbolicN)r2r3r4rQrRr.rHr:Z_CZGraphValuerfloatrSrPr)r)r)r*rXs  rXcsBeZdZdZfddZddZddZdd d Zd d ZZ S)r"z Optimized dropout module for stabilizing the training Args: drop_prob (float): the dropout probabilities cs t||_d|_d|_dSr+)rr drop_probcount context_stack)r&r_r'r)r*rs zStableDropout.__init__cCs$|jr |jdkr t||S|S)zr Call the module Args: x (`torch.tensor`): The input tensor to apply dropout r)trainingr_rXapply get_context)r&xr)r)r*r.szStableDropout.forwardcCsd|_d|_dSr+)r`rar0r)r)r* clear_contextszStableDropout.clear_contextTrcCs2|jdurg|_d|_|jD]}||_||_qdSr+)rar`rUrT)r&rUrTcr)r)r* init_contexts   zStableDropout.init_contextcCsX|jdurN|jt|jkr(|jt|j|j}|j|_|jd7_|S|jSdS)Nr)rar`lenappendrSr_r#)r&rYr)r)r*rds  zStableDropout.get_context)Tr) r2r3r4rQrr.rfrhrdr6r)r)r'r*r"s    r"cs*eZdZdZdfdd ZddZZS)DebertaLayerNormzBLayerNorm module in the TF style (epsilon inside the square root).-q=cs8ttt||_tt||_||_ dSr) rrr Parameterr:onesweightzerosbiasvariance_epsilon)r&sizeepsr'r)r*r s zDebertaLayerNorm.__init__cCsj|j}|}|jddd}||djddd}||t||j}||}|j||j }|S)NT)Zkeepdim) r?r^meanpowr:sqrtrrr9rorq)r&r,Z input_typerwZvarianceyr)r)r*r.s zDebertaLayerNorm.forward)rlr2r3r4rQrr.r6r)r)r'r*rk srkcs$eZdZfddZddZZS)DebertaSelfOutputcs<tt|j|j|_t|j|j|_t |j |_ dSr) rrrr r/r!rklayer_norm_eps LayerNormr"hidden_dropout_probr#r%r'r)r*rs zDebertaSelfOutput.__init__cCs&||}||}|||}|Srr!r#r~r&r,Z input_tensorr)r)r*r.%s  zDebertaSelfOutput.forwardr2r3r4rr.r6r)r)r'r*r|s r|cs&eZdZfddZdddZZS)DebertaAttentioncs(tt||_t||_||_dSr)rrDisentangledSelfAttentionr&r|rEr$r%r'r)r*r-s   zDebertaAttention.__init__FNc CsN|j||||||d}|r"|\}}|dur.|}|||} |rF| |fS| SdS)N) query_states relative_posrel_embeddings)r&rE) r&r,attention_maskoutput_attentionsrrrZ self_output att_matrixattention_outputr)r)r*r.3s  zDebertaAttention.forward)FNNNrr)r)r'r*r,s  rcs0eZdZfddZejejdddZZS)DebertaIntermediatecsBtt|j|j|_t|jt r6t |j|_ n|j|_ dSr) rrrr r/intermediate_sizer!rV hidden_actstrr intermediate_act_fnr%r'r)r*rRs   zDebertaIntermediate.__init__)r,r[cCs||}||}|Sr)r!rr&r,r)r)r*r.Zs  zDebertaIntermediate.forward)r2r3r4rr:Tensorr.r6r)r)r'r*rQs rcs$eZdZfddZddZZS) DebertaOutputcsBtt|j|j|_t|j|j|_ t |j |_ ||_ dSr)rrrr rr/r!rkr}r~r"rr#r$r%r'r)r*ras   zDebertaOutput.__init__cCs&||}||}|||}|Srrrr)r)r*r.hs  zDebertaOutput.forwardrr)r)r'r*r`s rcs&eZdZfddZdddZZS) DebertaLayercs,tt||_t||_t||_dSr)rrr attentionr intermediaterrEr%r'r)r*rps   zDebertaLayer.__init__NFc CsL|j||||||d}|r"|\}}||} || |} |rD| |fS| SdS)Nrrrr)rrrE) r&r,rrrrrrrZintermediate_outputZ layer_outputr)r)r*r.vs   zDebertaLayer.forward)NNNFrr)r)r'r*ros  rcsDeZdZdZfddZddZddZdd d Zdd dZZ S)DebertaEncoderz8Modified BertEncoder with relative position bias supportcs~ttfddtjD|_tdd|_|jrttdd|_ |j dkr^j |_ t |j dj |_ d|_dS) Ncsg|] }tqSr))r.0_r$r)r* z+DebertaEncoder.__init__..relative_attentionFmax_relative_positionsrurrv)rrrZ ModuleListrangeZnum_hidden_layerslayergetattrrrmax_position_embeddings Embeddingr/rgradient_checkpointingr%r'rr*rs   zDebertaEncoder.__init__cCs|jr|jjnd}|Sr)rrro)r&rr)r)r*get_rel_embeddingsz DebertaEncoder.get_rel_embeddingcCsL|dkr2|dd}||dd}n|dkrH|d}|S)Nrvrrur )r8 unsqueezesqueeze)r&rZextended_attention_maskr)r)r*get_attention_masks    z!DebertaEncoder.get_attention_maskNcCsB|jr>|dur>|dur |dn|d}t||d|j}|S)Nr)rrsbuild_relative_positiondevice)r&r,rrqr)r)r* get_rel_posszDebertaEncoder.get_rel_posTFc sT||}||||}|r dnd}r,dnd} t|trD|d} n|} |} t|jD]\} } |rp||f}|jr|jrfdd}t j j || | |||| }n| | |||| d}r|\}}|dur|}t|tr| dt |jkr|| dnd} n|} rZ| |f} qZ|r(||f}|sFt dd||| fDSt||| d S) Nr)rcsfdd}|S)Ncsg|RSrr))inputs)modulerr)r*custom_forwardszMDebertaEncoder.forward..create_custom_forward..custom_forwardr))rrr)rr*create_custom_forwardsz5DebertaEncoder.forward..create_custom_forward)rrrrrcss|]}|dur|VqdSrr))rvr)r)r* rz)DebertaEncoder.forward..Zlast_hidden_stater, attentions)rrrVrr enumeraterrrbr:utils checkpointritupler )r&r,routput_hidden_statesrrr return_dictZall_hidden_statesZall_attentionsZnext_kvriZ layer_modulerZatt_mr)rr*r.sZ            $  zDebertaEncoder.forward)NN)TFNNT) r2r3r4rQrrrrr.r6r)r)r'r*rs  rcCsjtj|tj|d}tj|tj|d}|dddf|dd|d}|d|ddf}|d}|S)a Build relative position according to the query and key We assume the absolute position of query \(P_q\) is range from (0, query_size) and the absolute position of key \(P_k\) is range from (0, key_size), The relative positions from query to key is \(R_{q \rightarrow k} = P_q - P_k\) Args: query_size (int): the length of query key_size (int): the length of key Return: `torch.LongTensor`: A tensor with shape [1, query_size, key_size] r?rNrrur)r:arangelongviewrepeatr)Z query_sizeZkey_sizerZq_idsZk_idsZ rel_pos_idsr)r)r*rs $ rcCs*||d|d|d|dgS)Nrrrvruexpandrs)c2p_pos query_layerrr)r)r*c2p_dynamic_expandsrcCs*||d|d|d|dgS)Nrrrr)rr key_layerr)r)r*p2c_dynamic_expandsrcCs*||dd|d|dfS)Nrvrr) pos_indexp2c_attrr)r)r*pos_dynamic_expandsrcs:eZdZdZfddZddZd dd Zd d ZZS) ra Disentangled self-attention module Parameters: config (`str`): A model config class instance with the configuration to build a new model. The schema is similar to *BertConfig*, for more details, please refer [`DebertaConfig`] cst|j|jdkr4td|jd|jd|j|_t|j|j|_|j|j|_tj |j|jddd|_ t t j |jt jd|_t t j |jt jd|_|jdur|jng|_t|d d|_t|d d|_|jrtj |j|jdd|_tj |j|jdd|_|jrt|d d |_|jd kr:|j|_t|j|_d|jvrhtj |j|jdd|_d|jvrt |j|j|_t|j|_dS)NrzThe hidden size (z6) is not a multiple of the number of attention heads ()r FrqrKr talking_headrrurc2pp2c) rrr/num_attention_heads ValueErrorintZattention_head_sizeZ all_head_sizerr in_projrmr:rpr^q_biasv_bias pos_att_typerrrhead_logits_projhead_weights_projrrr"r pos_dropoutpos_proj pos_q_projZattention_probs_dropout_probr#r%r'r)r*r/s:      z"DisentangledSelfAttention.__init__cCs4|dd|jdf}||}|ddddS)Nrurrvrr )rsrrpermute)r&reZ new_x_shaper)r)r*transpose_for_scoresRs z.DisentangledSelfAttention.transpose_for_scoresFNcs\|dur.}|jddd\}} } nddjjjjdddfdd tdDdgddd|jdjd } fd d td dD\} } fd d | | | fD\}} } |jddddf}| j ddddf} d}d t j }t t j|dt jd |}||j|jd }t || dd}jr|}|| |||}|dur||}jrƈ|dddd ddd d}t||d}|}jr|dddd ddd d}t || }|ddd d}|ddd}||}|rT||fS|SdS)a Call the module Args: hidden_states (`torch.FloatTensor`): Input states to the module usually the output from previous layer, it will be the Q,K and V in *Attention(Q,K,V)* attention_mask (`torch.BoolTensor`): An attention mask matrix of shape [*B*, *N*, *N*] where *B* is the batch size, *N* is the maximum sequence length in which element [i,j] = *1* means the *i* th token in the input can attend to the *j* th token. output_attentions (`bool`, optional): Whether return the attention matrix. query_states (`torch.FloatTensor`, optional): The *Q* state in *Attention(Q,K,V)*. relative_pos (`torch.LongTensor`): The relative position encoding between the tokens in the sequence. It's of shape [*B*, *N*, *N*] with values ranging in [*-max_relative_positions*, *max_relative_positions*]. rel_embeddings (`torch.FloatTensor`): The embedding of relative distances. It's a tensor of shape [\(2 \times \text{max_relative_positions}\), *hidden_size*]. Nr rur8cSs4|dur t|||St||SdSr)r:matmult)wbrer)r)r*linearsz1DisentangledSelfAttention.forward..linearrcs0g|](tjfddtjDddqS)csg|]}|dqS)r r)rr)kwsr)r*rrz@DisentangledSelfAttention.forward...rr)r:catrr)r)r&r)rr*rrz5DisentangledSelfAttention.forward..rKcs.g|]&}||j|jdqS)rK)r9r?r)r,rqkvbqkvwr)r*rrrcsg|]}|qSr))r)rrer0r)r*rrrrvru)rrchunkrorrr9r?rrrirr:ryr=rsr^r transposerrdisentangled_att_biasrrrr7rcr#r contiguousr)r&r,rrrrrZqprrZ value_layerrrrZrel_att scale_factorrTZattention_scoresZattention_probsZ context_layerZnew_context_layer_shaper))r,rrrr&rr*r.WsF&  ""   " "  z!DisentangledSelfAttention.forwardcCsp|dur&|d}t||d|j}|dkrD|dd}n6|dkr\|d}n|dkrztd|tt|d|d|j}| |j}||j||j|ddfd}d}d|j vr@| |} | | } t|| d d} t||d|dd} tj| d t| ||d } || 7}d |j vrl||} | | } | ttj| d tjd |} |d|dkrt|d|d|j} n|} t| |d|dd}t|| d dj |jd }tj|d t|||d d d}|d|dkrd|dddddddfd }tj|dt|||d }||7}|S) Nrrvrr rz2Relative position ids must be of dim 2 or 3 or 4. rru)r8indexrrK)rsrrr8rrr@maxrrr9rrrr:rrclampgatherrrryr=r^r?rr)r&rrrrrrZatt_spanZscoreZ pos_key_layerZc2p_attrZpos_query_layerZr_posZp2c_posrrr)r)r*rsV           $"z/DisentangledSelfAttention.disentangled_att_bias)FNNN) r2r3r4rQrrr.rr6r)r)r'r*r$s #  [rcs*eZdZdZfddZdddZZS)DebertaEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.cstt|dd}t|d|j|_tj|j|j|d|_t|dd|_ |j sXd|_ nt|j |j|_ |j dkrt|j |j|_ |j|jkrtj|j|jdd|_t|j|j|_t|j|_||_|d t|j d dS) N pad_token_idrembedding_size) padding_idxposition_biased_inputTFr position_ids)rru)rrrr/rrr vocab_sizeword_embeddingsrposition_embeddingsrtype_vocab_sizetoken_type_embeddingsr embed_projrkr}r~r"rr#r$Zregister_bufferr:rr)r&r$rr'r)r*rs      zDebertaEmbeddings.__init__Nc CsT|dur|}n|dd}|d}|durH|jddd|f}|durftj|tj|jjd}|durx||}|jdur||}n t|}|} |j r| |7} |j j dkr| |} | | 7} |j |j jkr|| } || } |durF|| kr2|dkr(|dd}|d}|| j}| |} || } | S)Nrurrrrrv)rsrr:rprrrrZ zeros_likerr$rrrr/rr~r8rrr9r?r#) r& input_idstoken_type_idsrrD inputs_embeds input_shapeZ seq_lengthr embeddingsrr)r)r*r.s>            zDebertaEmbeddings.forward)NNNNNr{r)r)r'r*rs rc@s:eZdZdZeZdZdgZdgZdZ ddZ d d d Z d S) DebertaPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. debertarrTcCs|t|tjr:|jjjd|jjd|jdurx|jj n>t|tj rx|jjjd|jjd|j durx|jj|j  dS)zInitialize the weights.g)rwZstdN) rVrr rodataZnormal_r$Zinitializer_rangerqZzero_rr)r&rr)r)r* _init_weights<s    z$DebertaPreTrainedModel._init_weightsFcCst|tr||_dSr)rVrr)r&rvaluer)r)r*_set_gradient_checkpointingIs z2DebertaPreTrainedModel._set_gradient_checkpointingN)F) r2r3r4rQr config_classZbase_model_prefix_keys_to_ignore_on_load_missing"_keys_to_ignore_on_load_unexpectedZsupports_gradient_checkpointingr rr)r)r)r*r 0s r a The DeBERTa model was proposed in [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. It's build on top of BERT/RoBERTa with two improvements, i.e. disentangled attention and enhanced mask decoder. With those two improvements, it out perform BERT/RoBERTa on a majority of tasks with 80GB pretraining data. This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`DebertaConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. zaThe bare DeBERTa Model transformer outputting raw hidden-states without any specific head on top.cseZdZfddZddZddZddZee d e e e e d deejeejeejeejeejeeeeeeeee fd d dZZS) DebertaModelcs8t|t||_t||_d|_||_|dSr+) rrrr rencoderz_stepsr$ post_initr%r'r)r*rs    zDebertaModel.__init__cCs|jjSrr rr0r)r)r*get_input_embeddingssz!DebertaModel.get_input_embeddingscCs ||j_dSrrr&Znew_embeddingsr)r)r*set_input_embeddingssz!DebertaModel.set_input_embeddingscCs tddS)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel z7The prune function is not implemented in DeBERTa model.N)NotImplementedError)r&Zheads_to_pruner)r)r* _prune_headsszDebertaModel._prune_headsbatch_size, sequence_lengthr output_typerN) rrrrrrrrr[c  s|dur |njj}|dur |njj}|dur4|njj}|durV|durVtdn4|durh|} n"|dur|dd} ntd|dur|jn|j} |durtj| | d}|durtj | tj | d}j |||||d} j | |d||d} | d } j d kr| d }fd d tj D}| d}j }j |}j | }|d dD]$}|||d |||d}| |q`| d}|s|f| |rd nddSt||r| jnd| jdS)NzDYou cannot specify both input_ids and inputs_embeds at the same timeruz5You have to specify either input_ids or inputs_embeds)rr)rrrrDrT)rrrrrcsg|]}jjdqSr)rrrr0r)r*rrz(DebertaModel.forward..Frrvr)r$rruse_return_dictrrsrr:rnrprr rrrrrrrjr r,r)r&rrrrrrrrrrZembedding_outputZencoder_outputsZencoded_layersr,ZlayersrrZrel_posrsequence_outputr)r0r*r.sp      zDebertaModel.forward)NNNNNNNN)r2r3r4rrrrrDEBERTA_INPUTS_DOCSTRINGformatr_CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOCrr:rr;rrr.r6r)r)r'r*rs:   rz5DeBERTa Model with a `language modeling` head on top.cseZdZdgZgdZfddZddZddZee d e e e ed eed deejeejeejeejeejeejeeeeeeeee fd ddZZS)DebertaForMaskedLMpooler)rzpredictions.decoder.biaszcls.predictions.decoder.weightcs,t|t||_t||_|dSr)rrrr DebertaOnlyMLMHeadclsrr%r'r)r*rs   zDebertaForMaskedLM.__init__cCs |jjjSrr) predictionsdecoderr0r)r)r*get_output_embeddings sz(DebertaForMaskedLM.get_output_embeddingscCs||jj_dSrr*rr)r)r*set_output_embeddingssz(DebertaForMaskedLM.set_output_embeddingsrz[MASK])rrrrDexpected_output expected_lossN rrrrrlabelsrrrr[c  Cs| dur | n|jj} |j|||||||| d} | d} || } d} |durnt}|| d|jj|d} | s| f| dd}| dur| f|S|St| | | j| j dS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` Nrrrrrrrrrurlosslogitsr,r) r$r r r)rrrr r,r)r&rrrrrr2rrroutputsr!prediction_scoresZmasked_lm_lossloss_fctrEr)r)r*r.s4  zDebertaForMaskedLM.forward) NNNNNNNNN)r2r3r4rrrr-r.rr"r#r_CHECKPOINT_FOR_MASKED_LMr r%_MASKED_LM_EXPECTED_OUTPUT_MASKED_LM_EXPECTED_LOSSrr:rr;rrr.r6r)r)r'r*r&sF    r&cs$eZdZfddZddZZS)DebertaPredictionHeadTransformcsVtt|j|j|_t|jtr6t |j|_ n|j|_ tj |j|j d|_ dS)N)rt) rrrr r/r!rVrrr transform_act_fnr~r}r%r'r)r*rPs   z'DebertaPredictionHeadTransform.__init__cCs"||}||}||}|Sr)r!r>r~rr)r)r*r.Ys   z&DebertaPredictionHeadTransform.forwardrr)r)r'r*r=Os r=cs$eZdZfddZddZZS)DebertaLMPredictionHeadcsLtt||_tj|j|jdd|_t t |j|_ |j |j_ dS)NFr) rrr= transformrr r/rr,rmr:rprqr%r'r)r*rbs   z DebertaLMPredictionHead.__init__cCs||}||}|Sr)r@r,rr)r)r*r.os  zDebertaLMPredictionHead.forwardrr)r)r'r*r?as r?cs$eZdZfddZddZZS)r(cstt||_dSr)rrr?r+r%r'r)r*rws zDebertaOnlyMLMHead.__init__cCs||}|Sr)r+)r&r!r8r)r)r*r.{s zDebertaOnlyMLMHead.forwardrr)r)r'r*r(vs r(z DeBERTa Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. cseZdZfddZddZddZeede e e e dd e eje eje eje eje eje eje ee ee eeee fd d d ZZS) DebertaForSequenceClassificationcs~t|t|dd}||_t||_t||_|jj}t |||_ t|dd}|durd|j j n|}t||_|dS)N num_labelsrvZ cls_dropout)rrrrBrr rr'r1rr classifierr$rr"r#r)r&r$rBr1Zdrop_outr'r)r*rs      z)DebertaForSequenceClassification.__init__cCs |jSr)r rr0r)r)r*rsz5DebertaForSequenceClassification.get_input_embeddingscCs|j|dSr)r rrr)r)r*rsz5DebertaForSequenceClassification.set_input_embeddingsrrNr1c  CsJ| dur | n|jj} |j|||||||| d} | d} || } || } || } d}|dur|jjdurx|jdkrt }| d |j } || | d}n| dks|ddkrT|dk}|}|ddkrBt| d||d| d}t|d| d}t}|| d|j| d}ntd | }n"td}|| |d }n|jjdkrt }|jdkr|| |}n || |}nN|jjdkrt}|| d|j| d}n|jjdkrt}|| |}| s6| f| dd}|dur2|f|S|St|| | j| jd S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). N)rrrrrrrrrruZ regressionZsingle_label_classificationZmulti_label_classificationr4)r$r r r'r#rCZ problem_typerBrr rr9r?r8rsZnonzerorr:rrrr^r=Z LogSoftmaxsumrwrrrr,r)r&rrrrrr2rrrr7Z encoder_layerr-r6r5Zloss_fnZ label_indexZlabeled_logitsr9Z log_softmaxrEr)r)r*r.sh             z(DebertaForSequenceClassification.forward) NNNNNNNNN)r2r3r4rrrrr"r#rr$rr%rr:rr;rrr.r6r)r)r'r*rAs<   rAz DeBERTa Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. cseZdZdgZfddZeedee e e dd e e je e je e je e je e je e je ee ee eeee fd dd ZZS) DebertaForTokenClassificationr'csJt||j|_t||_t|j|_t |j |j|_ | dSr) rrrBrr rZDropoutrr#r r/rCrr%r'r)r*rs   z&DebertaForTokenClassification.__init__rrNr1c  Cs| dur | n|jj} |j|||||||| d} | d} || } || } d} |durvt}|| d|j|d} | s| f| dd}| dur| f|S|St| | | j | j dS)z labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. Nr3rrurr4) r$r r r#rCrrrBrr,r)r&rrrrrr2rrrr7r!r6r5r9rEr)r)r*r. s0    z%DebertaForTokenClassification.forward) NNNNNNNNN)r2r3r4rrrr"r#rr$rr%rr:rr;rrr.r6r)r)r'r*rEs:  rEz DeBERTa Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). cseZdZdgZfddZeedee e e e e eedd eejeejeejeejeejeejeejeeeeeeeee fd dd ZZS) DebertaForQuestionAnsweringr'cs<t||j|_t||_t|j|j|_| dSr) rrrBrr rr r/ qa_outputsrr%r'r)r*rKs   z$DebertaForQuestionAnswering.__init__r)rrrr/r0Zqa_target_start_indexZqa_target_end_indexN) rrrrrstart_positions end_positionsrrrr[c  CsN| dur | n|jj} |j||||||| | d} | d} || } | jddd\}}|d}|d}d}|dur|durt|dkr|d}t|dkr|d}|d}| d|}| d|}t |d}|||}|||}||d}| s8||f| dd}|dur4|f|S|St |||| j | j d S) a start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. Nr3rrrur)Z ignore_indexrv)r5 start_logits end_logitsr,r)r$r r rGsplitrrrirsrrrr,r)r&rrrrrrHrIrrrr7r!r6rJrKZ total_lossZ ignored_indexr9Z start_lossZend_lossrEr)r)r*r.UsN!           z#DebertaForQuestionAnswering.forward) NNNNNNNNNN)r2r3r4rrrr"r#r_CHECKPOINT_FOR_QArr%_QA_EXPECTED_OUTPUT_QA_EXPECTED_LOSS_QA_TARGET_START_INDEX_QA_TARGET_END_INDEXrr:rr;rrr.r6r)r)r'r*rFAsF   rF)SrQcollections.abcrtypingrrrr:Ztorch.utils.checkpointrZtorch.nnrrr Z activationsr Zmodeling_outputsr r rrrZmodeling_utilsrZ pytorch_utilsrrrrrrZconfiguration_debertarZ get_loggerr2loggerr%r$r:r;r<rMrNrOrPrQZ%DEBERTA_PRETRAINED_MODEL_ARCHIVE_LISTModulerZautogradZFunctionr7objectrSrWrXr"rkr|rrrrrrZjitscriptrrrrrr ZDEBERTA_START_DOCSTRINGr"rr&r=r?r(rArErFr)r)r)r*s        ?'1%"k   CJ,nQ oD