a  þdsJã@sädZddlZddlZddlmZddlmZmZmZddl Z ddl m Z ddl m Z ddl mZdd lmZmZmZmZdd lmZdd lmZmZmZdd lmZmZmZmZd dlm Z e !e"¡Z#dZ$dZ%dZ&dZ'gd¢Z(dd„Z)Gdd„de j*ƒZ+Gdd„de j*ƒZ,Gdd„de j*ƒZ-Gdd„de j*ƒZ.Gdd„de j*ƒZ/Gd d!„d!e j*ƒZ0Gd"d#„d#e j*ƒZ1Gd$d%„d%e j*ƒZ2Gd&d'„d'e j*ƒZ3eGd(d)„d)eƒƒZ4eGd*d+„d+eƒƒZ5eGd,d-„d-eƒƒZ6eGd.d/„d/eƒƒZ7Gd0d1„d1e j*ƒZ8Gd2d3„d3e j*ƒZ9Gd4d5„d5e j*ƒZ:Gd6d7„d7e j*ƒZ;Gd8d9„d9e j*ƒZGdd?„d?e?ƒZ@ed@e=ƒGdAdB„dBe?ƒƒZAedCe=ƒGdDdE„dEe?ƒƒZBedFe=ƒGdGdH„dHe?ƒƒZCedIe=ƒGdJdK„dKe?ƒƒZDdLZEedMe=ƒGdNdO„dOe?ƒƒZFdS)Pz PyTorch REALM model.éN)Ú dataclass)ÚOptionalÚTupleÚUnion)Únn)ÚCrossEntropyLossé)ÚACT2FN)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚMaskedLMOutputÚ ModelOutput)ÚPreTrainedModel)Úapply_chunking_to_forwardÚ find_pruneable_heads_and_indicesÚprune_linear_layer)Úadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstringsé)Ú RealmConfigú(google/realm-cc-news-pretrained-embedderú'google/realm-cc-news-pretrained-encoderú&google/realm-cc-news-pretrained-scorerr)rrrz&google/realm-cc-news-pretrained-openqazgoogle/realm-orqa-nq-openqazgoogle/realm-orqa-nq-readerzgoogle/realm-orqa-wq-openqazgoogle/realm-orqa-wq-readerc CsŒzddl}ddl}ddl}Wnty:t d¡‚Yn0tj |¡}t  d|›¡|j   |¡}g}g} |D]@\} } t  d| ›d| ›¡|j   || ¡} |  | ¡|   | ¡qpt|| ƒD]È\} } t|tƒröd| vröt  d| ›d |jj›d ¡q¼|  d ¡s|  d ¡r2t|tƒr2|  d d¡} |  dd¡} |  d ¡sJ|  d ¡rbt|tƒrb|  d d¡} |  d¡rÜt|tƒr~dnd} |  d| ›d¡} |  d| ›d¡} |  d| ›d¡} |  d| ›d¡} |  d| ›d¡} |  d¡rjt|tƒrødnd}|  d|›d¡} |  d|›d ¡} |  d!|›d"¡} |  d#|›d$¡} |  d%|›d¡} |  d&|›d$¡} nD|  d'¡r®t|tƒr†dnd}|  d(|›d ¡} |  d)|›d"¡} |  d*¡} td+d,„| Dƒƒrät  dd* | ¡›¡q¼|}| D]Þ}| d-|¡r | d.|¡}n|g}|dd/ks.|dd0kr:t|d1ƒ}nl|dd2ksV|dd3krbt|d4ƒ}nDzt||dƒ}Wn0ty¤t  dd* | ¡›¡YqìYn0t|ƒd5krìt|d6ƒ}||}qì|d7d…d8krêt|d1ƒ}n|d/krþ|  | ¡} z,|j!| j!ks(Jd9|j!›d:| j!›d;ƒ‚Wn>t"yh}z$|j#|j!| j!f7_#‚WYd}~n d}~00t  d<| ›¡t$ %| ¡|_&q¼|S)=z'Load tf checkpoints in a pytorch model.rNz™Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.z&Converting TensorFlow checkpoint from zLoading TF weight z with shape Úreaderz Skipping z as it is not z 's parameterZbertÚclszbert/z reader/realm/zcls/z reader/cls/zrealm/Úzreader/zreader/module/bert/zreader/module/cls/z reader/dense/zqa_outputs/dense_intermediate/zreader/dense_1/zqa_outputs/dense_output/zreader/layer_normalizationzqa_outputs/layer_normalizationzmodule/module/module/z embedder/z!module/module/module/module/bert/zmodule/module/module/LayerNorm/zcls/LayerNorm/zmodule/module/module/dense/z cls/dense/z,module/module/module/module/cls/predictions/zcls/predictions/zmodule/module/module/bert/z%module/module/module/cls/predictions/zmodule/module/zmodule/module/LayerNorm/zmodule/module/dense/ú/css|]}|dvVqdS))Zadam_vZadam_mZAdamWeightDecayOptimizerZAdamWeightDecayOptimizer_1Z global_stepN©)Ú.0Únrrúq/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/transformers/models/realm/modeling_realm.pyÚ |sÿz+load_tf_weights_in_realm..z [A-Za-z]+_\d+z_(\d+)ÚkernelÚgammaÚweightZ output_biasÚbetaÚbiasériõÿÿÿZ _embeddingszPointer shape z and array shape z mismatchedzInitialize PyTorch weight )'ÚreÚnumpyZ tensorflowÚ ImportErrorÚloggerÚerrorÚosÚpathÚabspathÚinfoZtrainZlist_variablesZ load_variableÚappendÚzipÚ isinstanceÚ RealmReaderÚ __class__Ú__name__Ú startswithÚRealmForOpenQAÚreplaceÚRealmKnowledgeAugEncoderÚ RealmEmbedderÚsplitÚanyÚjoinÚ fullmatchÚgetattrÚAttributeErrorÚlenÚintÚ transposeÚshapeÚAssertionErrorÚargsÚtorchZ from_numpyÚdata)ÚmodelÚconfigZtf_checkpoint_pathr*ÚnpÚtfZtf_pathZ init_varsÚnamesZarraysÚnamerGÚarrayZ reader_prefixZembedder_prefixÚpointerZm_nameZ scope_namesÚnumÚerrr"Úload_tf_weights_in_realm:sª  ÿ     $  $     þ          ÿþrVcsTeZdZdZ‡fdd„Zd eejeejeejeeje ej dœdd„Z ‡Z S) ÚRealmEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.cs¶tƒ ¡tj|j|j|jd|_t |j|j¡|_ t |j |j¡|_ tj |j|j d|_ t |j¡|_t|ddƒ|_| dt |j¡ d¡¡|jdtj|j ¡tjdd d dS) N)Ú padding_idx©ZepsÚposition_embedding_typeÚabsoluteÚ position_ids)réÿÿÿÿÚtoken_type_ids©ÚdtypeF)Ú persistent)ÚsuperÚ__init__rÚ EmbeddingÚ vocab_sizeÚ hidden_sizeZ pad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsZtype_vocab_sizeÚtoken_type_embeddingsÚ LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutrBrZÚregister_bufferrJÚarangeÚexpandÚzerosr\ÚsizeÚlong©ÚselfrM©r7rr"rc©s ÿzRealmEmbeddings.__init__Nr)Ú input_idsr^r\Ú inputs_embedsÚpast_key_values_lengthÚreturnc Csø|dur| ¡}n| ¡dd…}|d}|durL|jdd…|||…f}|dur t|dƒrŠ|jdd…d|…f}| |d|¡} | }ntj|tj|jjd}|dur²|  |¡}|  |¡} || } |j dkrà|  |¡} | | 7} |  | ¡} | | ¡} | S)Nr]rr^r©r`Údevicer[)rtr\Úhasattrr^rrrJrsrur~rgrjrZrirkro) rwryr^r\rzr{Ú input_shapeÚ seq_lengthÚbuffered_token_type_idsÚ buffered_token_type_ids_expandedrjÚ embeddingsrirrr"Úforwardºs,        zRealmEmbeddings.forward)NNNNr) r8Ú __module__Ú __qualname__Ú__doc__rcrrJÚ LongTensorÚ FloatTensorrEÚTensorr…Ú __classcell__rrrxr"rW¦s úùrWc s‚eZdZd ‡fdd„ Zejejdœdd„Zd ejeejeejeejeejee e ejee e ejdœd d „Z ‡Z S) ÚRealmSelfAttentionNcsþtƒ ¡|j|jdkr>t|dƒs>td|j›d|j›dƒ‚|j|_t|j|jƒ|_|j|j|_t   |j|j¡|_ t   |j|j¡|_ t   |j|j¡|_ t  |j¡|_|p¸t|ddƒ|_|jdksÐ|jd krò|j|_t  d |jd |j¡|_|j|_dS) NrZembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)rZr[Ú relative_keyÚrelative_key_queryr)r)rbrcrfÚnum_attention_headsrÚ ValueErrorrEÚattention_head_sizeÚ all_head_sizerÚLinearÚqueryÚkeyÚvaluermZattention_probs_dropout_probrorBrZrhrdÚdistance_embeddingÚ is_decoder©rwrMrZrxrr"rcæs*  ÿÿÿzRealmSelfAttention.__init__)Úxr|cCs6| ¡dd…|j|jf}| |¡}| dddd¡S)Nr]rr)rr)rtr‘r“ÚviewÚpermute)rwrœZ new_x_shaperrr"Útranspose_for_scoress z'RealmSelfAttention.transpose_for_scoresF©Ú hidden_statesÚattention_maskÚ head_maskÚencoder_hidden_statesÚencoder_attention_maskÚpast_key_valueÚoutput_attentionsr|cCsÒ| |¡}|du} | r4|dur4|d} |d} |}n | r^| | |¡¡} | | |¡¡} |}nv|dur´| | |¡¡} | | |¡¡} tj|d| gdd} tj|d| gdd} n | | |¡¡} | | |¡¡} | |¡} |du} |jrô| | f}t | |  dd¡¡}|j dks |j dkr| j d| j d}}| r^tj |dtj |j d  dd¡}ntj|tj |j d  dd¡}tj|tj |j d  dd¡}||}| ||jd¡}|j| jd }|j dkrät d | |¡}||}n4|j dkrt d | |¡}t d | |¡}|||}|t |j¡}|dur:||}tjj|dd}| |¡}|durf||}t || ¡}| dddd ¡ ¡}| ¡dd…|jf}| |¡}|r¶||fn|f}|jrÎ||f}|S)Nrrr)©Údimr]éþÿÿÿrrr}r_zbhld,lrd->bhlrzbhrd,lrd->bhlrr) r–rŸr—r˜rJÚcatršÚmatmulrFrZrGÚtensorrur~rrqr™rhÚtor`ÚeinsumÚmathÚsqrtr“rZ functionalZsoftmaxroržÚ contiguousrtr”)rwr¡r¢r£r¤r¥r¦r§Zmixed_query_layerZis_cross_attentionZ key_layerZ value_layerZ query_layerÚ use_cacheZattention_scoresZ query_lengthZ key_lengthZposition_ids_lZposition_ids_rZdistanceZpositional_embeddingZrelative_position_scoresZrelative_position_scores_queryZrelative_position_scores_keyZattention_probsZ context_layerZnew_context_layer_shapeÚoutputsrrr"r…sn   ÿ          zRealmSelfAttention.forward)N)NNNNNF) r8r†r‡rcrJr‹rŸrrŠrÚboolr…rŒrrrxr"rås$ø÷rcs4eZdZ‡fdd„Zejejejdœdd„Z‡ZS)ÚRealmSelfOutputcsBtƒ ¡t |j|j¡|_tj|j|jd|_t |j ¡|_ dS©NrY) rbrcrr•rfÚdenserkrlrmrnrorvrxrr"rcms zRealmSelfOutput.__init__©r¡Ú input_tensorr|cCs&| |¡}| |¡}| ||¡}|S©N©r¸rork©rwr¡rºrrr"r…ss  zRealmSelfOutput.forward©r8r†r‡rcrJr‹r…rŒrrrxr"r¶ls r¶c sveZdZd ‡fdd„ Zdd„Zd ejeejeejeejeejee e ejee e ejdœdd „Z ‡Z S) ÚRealmAttentionNcs.tƒ ¡t||d|_t|ƒ|_tƒ|_dS)N©rZ)rbrcrrwr¶ÚoutputÚsetÚ pruned_headsr›rxrr"rc|s  zRealmAttention.__init__cCs²t|ƒdkrdSt||jj|jj|jƒ\}}t|jj|ƒ|j_t|jj|ƒ|j_t|jj |ƒ|j_ t|j j |dd|j _ |jjt|ƒ|j_|jj|jj|j_ |j  |¡|_dS)Nrrr¨)rDrrwr‘r“rÃrr–r—r˜rÁr¸r”Úunion)rwÚheadsÚindexrrr"Ú prune_heads‚s ÿzRealmAttention.prune_headsFr c Cs<| |||||||¡}| |d|¡} | f|dd…} | S)Nrr)rwrÁ) rwr¡r¢r£r¤r¥r¦r§Z self_outputsÚattention_outputr´rrr"r…”s ù zRealmAttention.forward)N)NNNNNF) r8r†r‡rcrÇrJr‹rrŠrrµr…rŒrrrxr"r¿{s$ø÷r¿cs0eZdZ‡fdd„Zejejdœdd„Z‡ZS)ÚRealmIntermediatecsBtƒ ¡t |j|j¡|_t|jt ƒr6t |j|_ n|j|_ dSr») rbrcrr•rfÚintermediate_sizer¸r5Ú hidden_actÚstrr Úintermediate_act_fnrvrxrr"rc®s   zRealmIntermediate.__init__©r¡r|cCs| |¡}| |¡}|Sr»)r¸rÍ©rwr¡rrr"r…¶s  zRealmIntermediate.forwardr¾rrrxr"rÉ­s rÉcs4eZdZ‡fdd„Zejejejdœdd„Z‡ZS)Ú RealmOutputcsBtƒ ¡t |j|j¡|_tj|j|jd|_t  |j ¡|_ dSr·) rbrcrr•rÊrfr¸rkrlrmrnrorvrxrr"rc¾s zRealmOutput.__init__r¹cCs&| |¡}| |¡}| ||¡}|Sr»r¼r½rrr"r…Äs  zRealmOutput.forwardr¾rrrxr"rнs rÐc steZdZ‡fdd„Zd ejeejeejeejeejeeeejee eejdœdd„Z dd „Z ‡Z S) Ú RealmLayercsrtƒ ¡|j|_d|_t|ƒ|_|j|_|j|_|jrZ|jsLt|›dƒ‚t|dd|_ t |ƒ|_ t |ƒ|_ dS)Nrz> should be used as a decoder model if cross attention is addedr[rÀ)rbrcÚchunk_size_feed_forwardÚ seq_len_dimr¿Ú attentionršÚadd_cross_attentionr’ÚcrossattentionrÉÚ intermediaterÐrÁrvrxrr"rcÍs   zRealmLayer.__init__NFr c Cs|dur|dd…nd}|j|||||d} | d} |jrP| dd…} | d} n | dd…} d} |jrÞ|durÞt|dƒsˆtd|›dƒ‚|durœ|d d…nd}| | ||||||¡}|d} | |dd…} |d} | | } t|j|j|j| ƒ}|f| } |jr| | f} | S) Nr)©r§r¦rrr]rÖz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`rª) rÔršrr’rÖrÚfeed_forward_chunkrÒrÓ)rwr¡r¢r£r¤r¥r¦r§Zself_attn_past_key_valueZself_attention_outputsrÈr´Zpresent_key_valueZcross_attn_present_key_valueZcross_attn_past_key_valueZcross_attention_outputsÚ layer_outputrrr"r…ÛsP û     ÿù ÿ  zRealmLayer.forwardcCs| |¡}| ||¡}|Sr»)r×rÁ)rwrÈZintermediate_outputrÚrrr"rÙs  zRealmLayer.feed_forward_chunk)NNNNNF) r8r†r‡rcrJr‹rrŠrrµr…rÙrŒrrrxr"rÑÌs$ ø÷ ArÑcs†eZdZ‡fdd„Zd ejeejeejeejeejeeeejee ee ee ee e eeje fdœ dd„Z ‡Z S) Ú RealmEncodercs:tƒ ¡ˆ|_t ‡fdd„tˆjƒDƒ¡|_d|_dS)Ncsg|] }tˆƒ‘qSr)rÑ)r Ú_©rMrr"Ú 'óz)RealmEncoder.__init__..F) rbrcrMrZ ModuleListÚrangeÚnum_hidden_layersÚlayerÚgradient_checkpointingrvrxrÝr"rc$s  zRealmEncoder.__init__NFT) r¡r¢r£r¤r¥Úpast_key_valuesr³r§Úoutput_hidden_statesÚ return_dictr|c  st| rdnd} ˆrdnd} ˆr(|jjr(dnd} |jrJ|jrJ|rJt d¡d}|rRdnd}t|jƒD]Î\}}| rv| |f} |dur†||nd}|durš||nd‰|jrÖ|jrÖ‡‡fdd„}tj j   ||ƒ|||||¡}n||||||ˆˆƒ}|d}|r||df7}ˆr`| |df} |jjr`| |d f} q`| r@| |f} | sbt d d „||| | | fDƒƒSt ||| | | d S) NrzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fcs‡‡‡fdd„}|S)Ncsˆg|¢ˆ‘ˆ‘RŽSr»r)Úinputs)Úmoduler§r¦rr"Úcustom_forwardMszKRealmEncoder.forward..create_custom_forward..custom_forwardr)rèrérØ)rèr"Úcreate_custom_forwardLsz3RealmEncoder.forward..create_custom_forwardrr]rr)css|]}|dur|VqdSr»r)r Úvrrr"r#qsøz'RealmEncoder.forward..)Úlast_hidden_staterär¡Ú attentionsÚcross_attentions) rMrÕrãÚtrainingr-Z warning_onceÚ enumeraterârJÚutilsÚ checkpointÚtupler )rwr¡r¢r£r¤r¥rär³r§råræZall_hidden_statesZall_self_attentionsZall_cross_attentionsZnext_decoder_cacheÚiZ layer_moduleZlayer_head_maskrêZ layer_outputsrrØr"r…*sv   ÿ   ú ù  ûþ ûzRealmEncoder.forward) NNNNNNFFT)r8r†r‡rcrJr‹rrŠrrµrr r…rŒrrrxr"rÛ#s. õôrÛcs0eZdZ‡fdd„Zejejdœdd„Z‡ZS)Ú RealmPoolercs*tƒ ¡t |j|j¡|_t ¡|_dSr»)rbrcrr•rfr¸ZTanhÚ activationrvrxrr"rc‡s zRealmPooler.__init__rÎcCs(|dd…df}| |¡}| |¡}|S)Nr)r¸rö)rwr¡Zfirst_token_tensorÚ pooled_outputrrr"r…Œs  zRealmPooler.forwardr¾rrrxr"rõ†s rõc@sLeZdZUdZdZejed<dZe e ejed<dZ e e ejed<dS)ÚRealmEmbedderOutputa* Outputs of [`RealmEmbedder`] models. Args: projected_score (`torch.FloatTensor` of shape `(batch_size, config.retriever_proj_size)`): Projected score. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. NÚprojected_scorer¡rí) r8r†r‡rˆrùrJrŠÚ__annotations__r¡rrrírrrr"rø•s røc@s<eZdZUdZdZejed<dZejed<dZ ejed<dS)ÚRealmScorerOutputa' Outputs of [`RealmScorer`] models. Args: relevance_score (`torch.FloatTensor` of shape `(batch_size, config.num_candidates)`): The relevance score of document candidates (before softmax). query_score (`torch.FloatTensor` of shape `(batch_size, config.retriever_proj_size)`): Query score derived from the query embedder. candidate_score (`torch.FloatTensor` of shape `(batch_size, config.num_candidates, config.retriever_proj_size)`): Candidate score derived from the embedder. NÚrelevance_scoreÚ query_scoreÚcandidate_score) r8r†r‡rˆrürJrŠrúrýrþrrrr"rû°s  rûc@s¼eZdZUdZdZejed<dZejed<dZ ejed<dZ ej ed<dZ ej ed<dZ ejed<dZejed <dZejed <dZejed <dZeeejed <dZeeejed <dS)ÚRealmReaderOutputa+ Outputs of [`RealmReader`] models. Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `start_positions`, `end_positions`, `has_answers` are provided): Total loss. retriever_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `start_positions`, `end_positions`, `has_answers` are provided): Retriever loss. reader_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `start_positions`, `end_positions`, `has_answers` are provided): Reader loss. retriever_correct (`torch.BoolTensor` of shape `(config.searcher_beam_size,)`, *optional*): Whether or not an evidence block contains answer. reader_correct (`torch.BoolTensor` of shape `(config.reader_beam_size, num_candidates)`, *optional*): Whether or not a span candidate contains answer. block_idx (`torch.LongTensor` of shape `()`): The index of the retrieved evidence block in which the predicted answer is most likely. candidate (`torch.LongTensor` of shape `()`): The index of the retrieved span candidates in which the predicted answer is most likely. start_pos (`torch.IntTensor` of shape `()`): Predicted answer starting position in *RealmReader*'s inputs. end_pos (`torch.IntTensor` of shape `()`): Predicted answer ending position in *RealmReader*'s inputs. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. NÚlossÚretriever_lossÚ reader_lossÚretriever_correctÚreader_correctÚ block_idxÚ candidateÚ start_posÚend_posr¡rí)r8r†r‡rˆrrJrŠrúrrrÚ BoolTensorrrr‰rrÚint32rr¡rrrírrrr"rÿÃs #rÿc@s,eZdZUdZdZeed<dZej ed<dS)ÚRealmForOpenQAOutputzï Outputs of [`RealmForOpenQA`] models. Args: reader_output (`dict`): Reader output. predicted_answer_ids (`torch.LongTensor` of shape `(answer_sequence_length)`): Predicted answer ids. NÚ reader_outputÚpredicted_answer_ids) r8r†r‡rˆr Údictrúr rJr‰rrrr"r õs  r cs$eZdZ‡fdd„Zdd„Z‡ZS)ÚRealmPredictionHeadTransformcsVtƒ ¡t |j|j¡|_t|jtƒr6t |j|_ n|j|_ tj |j|j d|_ dSr·) rbrcrr•rfr¸r5rËrÌr Útransform_act_fnrkrlrvrxrr"rcs   z%RealmPredictionHeadTransform.__init__cCs"| |¡}| |¡}| |¡}|Sr»)r¸rrkrÏrrr"r…s   z$RealmPredictionHeadTransform.forward©r8r†r‡rcr…rŒrrrxr"rs rcs$eZdZ‡fdd„Zdd„Z‡ZS)ÚRealmLMPredictionHeadcsLtƒ ¡t|ƒ|_tj|j|jdd|_t  t   |j¡¡|_ |j |j_ dS)NF)r() rbrcrÚ transformrr•rfreÚdecoderÚ ParameterrJrsr(rvrxrr"rcs   zRealmLMPredictionHead.__init__cCs| |¡}| |¡}|Sr»)rrrÏrrr"r…%s  zRealmLMPredictionHead.forwardrrrrxr"rs rcs$eZdZ‡fdd„Zdd„Z‡ZS)ÚRealmOnlyMLMHeadcstƒ ¡t|ƒ|_dSr»)rbrcrÚ predictionsrvrxrr"rc,s zRealmOnlyMLMHead.__init__cCs| |¡}|Sr»)r)rwÚsequence_outputÚprediction_scoresrrr"r…0s zRealmOnlyMLMHead.forwardrrrrxr"r+s rcs$eZdZ‡fdd„Zdd„Z‡ZS)ÚRealmScorerProjectioncs>tƒ ¡t|ƒ|_t |j|j¡|_tj |j|j d|_ dSr·) rbrcrrrr•rfÚretriever_proj_sizer¸rkrlrvrxrr"rc6s  zRealmScorerProjection.__init__cCs| |¡}| |¡}|Sr»)r¸rkrÏrrr"r…<s  zRealmScorerProjection.forwardrrrrxr"r5s rcs$eZdZ‡fdd„Zdd„Z‡ZS)ÚRealmReaderProjectioncsXtƒ ¡||_t |j|jd¡|_t |jd¡|_tj |j|j d|_ t  ¡|_ dS)Nr)rrY)rbrcrMrr•rfZspan_hidden_sizeÚdense_intermediateÚ dense_outputrkZreader_layer_norm_epsÚlayer_normalizationZReLUÚrelurvrxrr"rcCs  zRealmReaderProjection.__init__cs¬‡fdd„}tjfdd„}ˆ |¡}|jddd\}}||ƒ\}}} tj|d|d } tj|d|d } | | } ˆ | ¡} ˆ | ¡} ˆ | ¡ d¡} | || | j d 7} | ||fS) Ncs„ˆj\}‰‡‡fdd„‰t‡fdd„tˆjjƒDƒŽ\}}t |d¡}t |d¡}tjˆd|d}tjˆd|d}||}|||fS)aK Generate span candidates. Args: masks: [num_retrievals, max_sequence_len] Returns: starts: [num_spans] ends: [num_spans] span_masks: [num_retrievals, num_spans] whether spans locate in evidence block. cs6tjˆ|dˆjd}tj|dˆˆjd}||fS)Nr©r~)rJrqr~)ÚwidthZcurrent_startsZ current_ends)ÚmasksÚmax_sequence_lenrr"Ú_spans_given_widthYszRRealmReaderProjection.forward..span_candidates.._spans_given_widthc3s|]}ˆ|dƒVqdS)rNr)r Úw)r%rr"r#^rßzIRealmReaderProjection.forward..span_candidates..rr]©r©rÆ)rGr4ràrMÚmax_span_widthrJr«Ú index_select)r#rÜZstartsZendsZ start_masksZ end_masksZ span_masks©rw)r%r#r$r"Úspan_candidatesLs "  z6RealmReaderProjection.forward..span_candidatescSsd| |¡t |¡jS©Nçð?©ÚtyperJZfinfoÚmin©Úmaskr`rrr"Ú mask_to_scoreksz4RealmReaderProjection.forward..mask_to_scorer)r]r¨rr'r_) rJÚfloat32rÚchunkr)r rrÚsqueezer`)rwr¡Ú block_maskr+r3Zstart_projectionZend_projectionÚcandidate_startsÚcandidate_endsZcandidate_maskZcandidate_start_projectionsZcandidate_end_projectionsZcandidate_hiddenÚ reader_logitsrr*r"r…Ks    zRealmReaderProjection.forwardrrrrxr"rBs raH This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`RealmConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a5 Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. c@s2eZdZdZeZeZdZdgZ dd„Z dd„Z dS) ÚRealmPreTrainedModelz† An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. Úrealmr\cCs¤t|tjƒr:|jjjd|jjd|jdur |jj  ¡nft|tj ƒrz|jjjd|jjd|j dur |jj|j   ¡n&t|tj ƒr |jj  ¡|jj  d¡dS)zInitialize the weightsg)ÚmeanZstdNr-)r5rr•r&rKZnormal_rMZinitializer_ranger(Zzero_rdrXrkZfill_)rwrèrrr"Ú _init_weightsÍs      z"RealmPreTrainedModel._init_weightscGsTg}|D]F}|dur | d¡q|j}t|ƒdkrD| d|df¡}| |¡q|S)z.Flatten inputs' shape to (-1, input_shape[-1])Nr)r])r3rGrDr)rwrçZflattened_inputsr­r€rrr"Ú_flatten_inputsÝs   z$RealmPreTrainedModel._flatten_inputsN) r8r†r‡rˆrÚ config_classrVZload_tf_weightsZbase_model_prefixÚ_keys_to_ignore_on_load_missingr>r?rrrr"r;Âsr;c sDeZdZdZd‡fdd„ Zdd„Zdd„Zd d „Zdd d „Z‡Z S)ÚRealmBertModelz? Same as the original BertModel but remove docstrings. TcsDtƒ |¡||_t|ƒ|_t|ƒ|_|r2t|ƒnd|_|  ¡dSr») rbrcrMrWr„rÛÚencoderrõÚpoolerÚ post_init)rwrMZadd_pooling_layerrxrr"rcðs    zRealmBertModel.__init__cCs|jjSr»©r„rgr*rrr"Úget_input_embeddingsýsz#RealmBertModel.get_input_embeddingscCs ||j_dSr»rF©rwr˜rrr"Úset_input_embeddingssz#RealmBertModel.set_input_embeddingscCs*| ¡D]\}}|jj|j |¡qdS)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)ÚitemsrCrârÔrÇ)rwZheads_to_prunerârÅrrr"Ú _prune_headsszRealmBertModel._prune_headsNc CsR| dur | n|jj} | dur | n|jj} | dur4| n|jj} |jjrZ| durP| n|jj} nd} |durx|durxtdƒ‚n4|durŠ| ¡}n"|dur¤| ¡dd…}ntdƒ‚|\}}|durÂ|jn|j}| durâ| ddj dnd}|durt j |||f|d}|durZt |j dƒrH|j jdd…d|…f}| ||¡}|}nt j|t j|d }| ||¡}|jjr´|dur´| ¡\}}}||f}|dur¨t j ||d}| |¡}nd}| ||jj¡}|j |||||d }|j|||||| | | | | d }|d}|jdur| |¡nd}| s6||f|d d…St|||j|j|j|jd S)NFzDYou cannot specify both input_ids and inputs_embeds at the same timer]z5You have to specify either input_ids or inputs_embedsrr)r!r^r})ryr\r^rzr{) r¢r£r¤r¥rär³r§rårær)rìÚ pooler_outputrär¡rírî)rMr§råÚuse_return_dictršr³r’rtr~rGrJZonesrr„r^rrrsruZget_extended_attention_maskZinvert_attention_maskZ get_head_maskrárCrDr rär¡rírî)rwryr¢r^r\r£rzr¤r¥rär³r§rårær€Ú batch_sizerr~r{r‚rƒZextended_attention_maskZencoder_batch_sizeZencoder_sequence_lengthrÜZencoder_hidden_shapeZencoder_extended_attention_maskZembedding_outputZencoder_outputsrr÷rrr"r… s‚ÿ        ûö úzRealmBertModel.forward)T) NNNNNNNNNNNNN) r8r†r‡rˆrcrGrIrKr…rŒrrrxr"rBës&  òrBz`The embedder of REALM outputting projected score that will be used to calculate relevance score.cs¦eZdZdgZ‡fdd„Zdd„Zdd„Zee  d¡ƒe e e d de eje eje eje eje eje eje ee ee eeee fd œ d d „ƒƒZ‡ZS)r=zcls.predictions.decoder.biascs0tƒ |¡t|jƒ|_t|jƒ|_| ¡dSr»)rbrcrBrMr<rrrErvrxrr"rc€s   zRealmEmbedder.__init__cCs |jjjSr»©r<r„rgr*rrr"rG‡sz"RealmEmbedder.get_input_embeddingscCs||jj_dSr»rOrHrrr"rIŠsz"RealmEmbedder.set_input_embeddingsúbatch_size, sequence_length©Ú output_typer@N) ryr¢r^r\r£rzr§rårær|c Csn| dur | n|jj} |j||||||||| d } | d} | | ¡} | sX| f| dd…St| | j| jdSdS)a Returns: Example: ```python >>> from transformers import AutoTokenizer, RealmEmbedder >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("google/realm-cc-news-pretrained-embedder") >>> model = RealmEmbedder.from_pretrained("google/realm-cc-news-pretrained-embedder") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> projected_score = outputs.projected_score ``` N©r¢r^r\r£rzr§rårærr)é)rùr¡rí)rMrMr<rrør¡rí) rwryr¢r^r\r£rzr§råræZ realm_outputsrLrùrrr"r…s*!÷  ýzRealmEmbedder.forward) NNNNNNNNN)r8r†r‡rArcrGrIrÚREALM_INPUTS_DOCSTRINGÚformatrrøÚ_CONFIG_FOR_DOCrrJr‰rŠrµrrr…rŒrrrxr"r=ys6   ö õr=zoThe scorer of REALM outputting relevance scores representing the score of document candidates (before softmax).cs¶eZdZdZd ‡fdd„ Zee d¡ƒee e dd e e j e e je e j e e j e e j e e je e j e e je e je e je ee ee eeee fdœdd „ƒƒZ‡ZS) Ú RealmScorerz­ Args: query_embedder ([`RealmEmbedder`]): Embedder for input sequences. If not specified, it will use the same embedder as candidate sequences. Ncs8tƒ |¡t|jƒ|_|dur$|n|j|_| ¡dSr»)rbrcr=rMÚembedderÚquery_embedderrE)rwrMrZrxrr"rcÖs  zRealmScorer.__init__rPrQ)ryr¢r^r\Úcandidate_input_idsÚcandidate_attention_maskÚcandidate_token_type_idsÚcandidate_inputs_embedsr£rzr§rårær|c Csà| dur | n|jj} |dur,| dur,tdƒ‚|durD|durDtdƒ‚|j||||| | | | | d }| |||¡\}}}|j||||| || | | d }|d}|d}| d|jj|jj¡}t   d||¡}| sÒ|||fSt |||dS) a÷ candidate_input_ids (`torch.LongTensor` of shape `(batch_size, num_candidates, sequence_length)`): Indices of candidate input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) candidate_attention_mask (`torch.FloatTensor` of shape `(batch_size, num_candidates, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) candidate_token_type_ids (`torch.LongTensor` of shape `(batch_size, num_candidates, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) candidate_inputs_embeds (`torch.FloatTensor` of shape `(batch_size * num_candidates, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `candidate_input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *candidate_input_ids* indices into associated vectors than the model's internal embedding lookup matrix. Returns: Example: ```python >>> import torch >>> from transformers import AutoTokenizer, RealmScorer >>> tokenizer = AutoTokenizer.from_pretrained("google/realm-cc-news-pretrained-scorer") >>> model = RealmScorer.from_pretrained("google/realm-cc-news-pretrained-scorer", num_candidates=2) >>> # batch_size = 2, num_candidates = 2 >>> input_texts = ["How are you?", "What is the item in the picture?"] >>> candidates_texts = [["Hello world!", "Nice to meet you!"], ["A cute cat.", "An adorable dog."]] >>> inputs = tokenizer(input_texts, return_tensors="pt") >>> candidates_inputs = tokenizer.batch_encode_candidates(candidates_texts, max_length=10, return_tensors="pt") >>> outputs = model( ... **inputs, ... candidate_input_ids=candidates_inputs.input_ids, ... candidate_attention_mask=candidates_inputs.attention_mask, ... candidate_token_type_ids=candidates_inputs.token_type_ids, ... ) >>> relevance_score = outputs.relevance_score ```Nz5You have to specify either input_ids or input_embeds.zJYou have to specify either candidate_input_ids or candidate_inputs_embeds.rSrr]z bd,bnd->bn)rürýrþ) rMrMr’rZr?rYrÚnum_candidatesrrJr¯rû)rwryr¢r^r\r[r\r]r^r£rzr§råræZ query_outputsÚflattened_input_idsÚflattened_attention_maskÚflattened_token_type_idsZcandidate_outputsrýrþrürrr"r…ßsNI÷ ÿ ÷  ÿzRealmScorer.forward)N) NNNNNNNNNNNNN)r8r†r‡rˆrcrrUrVrrûrWrrJr‰rŠrµrrr…rŒrrrxr"rXËsB  ò ñrXzrThe knowledge-augmented encoder of REALM outputting masked language model logits and marginal log-likelihood loss.csÎeZdZdgZ‡fdd„Zdd„Zdd„Zdd „Zd d „Ze e   d ¡ƒe e ed deejeejeejeejeejeejeejeejeejeeeeeeeee fdœ dd„ƒƒZ‡ZS)r<zcls.predictions.decodercs0tƒ |¡t|jƒ|_t|jƒ|_| ¡dSr»)rbrcrBrMr<rrrErvrxrr"rcfs   z!RealmKnowledgeAugEncoder.__init__cCs |jjjSr»rOr*rrr"rGlsz-RealmKnowledgeAugEncoder.get_input_embeddingscCs||jj_dSr»rOrHrrr"rIosz-RealmKnowledgeAugEncoder.set_input_embeddingscCs |jjjSr»©rrrr*rrr"Úget_output_embeddingsrsz.RealmKnowledgeAugEncoder.get_output_embeddingscCs||jj_dSr»rc)rwZnew_embeddingsrrr"Úset_output_embeddingsusz.RealmKnowledgeAugEncoder.set_output_embeddingsz+batch_size, num_candidates, sequence_lengthrQN) ryr¢r^r\r£rzrüÚlabelsÚmlm_maskr§rårær|c  Csz| dur | n|jj} | |||¡\} }}|j| |||||| | | d }|d}| |¡}|}d}|dur6|durxtdƒ‚| ¡\}}| duržtj|tj d} n |   tj ¡} t dd}|  d|jj ¡}| d |jj¡  d¡}|||ƒ  ||jj|¡ }| d¡ d¡}||}| d ¡}t t || ¡t | ¡¡ }| sf|f|d d …}|durb|f|S|St|||j|jd S) aÕ relevance_score (`torch.FloatTensor` of shape `(batch_size, num_candidates)`, *optional*): Relevance score derived from RealmScorer, must be specified if you want to compute the masked language modeling loss. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` mlm_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid calculating joint loss on certain positions. If not specified, the loss will not be masked. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. Returns: Example: ```python >>> import torch >>> from transformers import AutoTokenizer, RealmKnowledgeAugEncoder >>> tokenizer = AutoTokenizer.from_pretrained("google/realm-cc-news-pretrained-encoder") >>> model = RealmKnowledgeAugEncoder.from_pretrained( ... "google/realm-cc-news-pretrained-encoder", num_candidates=2 ... ) >>> # batch_size = 2, num_candidates = 2 >>> text = [["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]] >>> inputs = tokenizer.batch_encode_candidates(text, max_length=10, return_tensors="pt") >>> outputs = model(**inputs) >>> logits = outputs.logits ```NrSrzZYou have to specify `relevance_score` when `labels` is specified in order to compute loss.r_Únone)Z reductionr]rr)rT)rÚlogitsr¡rí)rMrMr?r<rr’rtrJZ ones_liker4r/rrreZtiler_Z log_softmaxÚ unsqueezeÚ logsumexpZnansumÚsumr r¡rí)rwryr¢r^r\r£rzrürfrgr§rårær`rarbZ joint_outputsZ joint_outputrrþZmasked_lm_lossrNrZloss_fctZ mlm_logitsZ mlm_targetsZmasked_lm_log_probZcandidate_log_probZjoint_gold_log_probZmarginal_gold_log_probsrÁrrr"r…xs^9ÿ ÷   ÿ     ÿ  üz RealmKnowledgeAugEncoder.forward) NNNNNNNNNNNN)r8r†r‡rArcrGrIrdrerrUrVrr rWrrJr‰rŠrµrrr…rŒrrrxr"r<^sJ ÿ ó òr<zThe reader of REALM.csÀeZdZddgZ‡fdd„Zee d¡ƒee e dd e e j e e je e j e e j e e je e je e je e je e j e e j e e je ee ee eeee fdœd d „ƒƒZ‡ZS) r6rDrcs>tƒ |¡|j|_t|ƒ|_t|ƒ|_t|ƒ|_|  ¡dSr») rbrcZ num_labelsrBr<rrrÚ qa_outputsrErvrxrr"rcüs     zRealmReader.__init__z!reader_beam_size, sequence_lengthrQN)ryr¢r^r\r£rzrür7Ústart_positionsÚ end_positionsÚ has_answersr§rårær|c$ CsL|dur |n|jj}|dur$tdƒ‚|dur4tdƒ‚| d¡|jjkrNtdƒ‚|j||||||| | |d }|d}| ||d|jj…¡\}}}t  |d|jj…d¡}||7}t  tj |dd j ¡}t  tj |dd j ¡}tj |d|d }tj |d|d }d}d}d}d}d}| durì| durì| durìd d „}d d„}| d¡} |  d| ¡} |  d| ¡} | }t |¡}!|||| d|jj…| d|jj…d}t |¡}"|||ƒ}|| d¡| d¡ƒ}||! tj¡9}||" tj¡9}|| ¡}|s*||||f|dd…}#|dur&|||||f|#S|#St||||||||||j|jd S)ar relevance_score (`torch.FloatTensor` of shape `(searcher_beam_size,)`, *optional*): Relevance score, which must be specified if you want to compute the logits and marginal log loss. block_mask (`torch.BoolTensor` of shape `(searcher_beam_size, sequence_length)`, *optional*): The mask of the evidence block, which must be specified if you want to compute the logits and marginal log loss. start_positions (`torch.LongTensor` of shape `(searcher_beam_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(searcher_beam_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. has_answers (`torch.BoolTensor` of shape `(searcher_beam_size,)`, *optional*): Whether or not the evidence block has answer(s). Returns: NzCYou have to specify `relevance_score` to calculate logits and loss.zOYou have to specify `block_mask` to separate question block and evidence block.rzQThe input sequence length must be greater than or equal to config.max_span_width.rSrr]r¨r'cSs\t t t |d¡d¡t |d¡¡}t t t |d¡d¡t |d¡¡}t t ||¡d¡S)zCompute correct span.rr]r)rJÚeqrjr?Ú logical_and)r8r9Ú gold_startsÚ gold_endsZ is_gold_startZ is_gold_endrrr"Úcompute_correct_candidates[sÿÿz7RealmReader.forward..compute_correct_candidatescSs@tjfdd„}tj||||jddd}tj|dd}||S)z3Loss based on the negative marginal log-likelihood.cSsd| |¡t |¡jSr,r.r1rrr"r3kszERealmReader.forward..marginal_log_loss..mask_to_scorer_r]r¨)rJr4rkr`)riZ is_correctr3Z log_numeratorZlog_denominatorrrr"Úmarginal_log_losshsz.RealmReader.forward..marginal_log_loss)r8r9rsrtr)) rrrrrrrrrr¡rí)rMrMr’rtr(r<rmÚreader_beam_sizerJrjZargmaxÚmaxÚvaluesr)Úclampr?rr/r4r=rÿr¡rí)$rwryr¢r^r\r£rzrür7rnrorpr§rårær´rr:r8r9Zretriever_logitsZpredicted_block_indexZpredicted_candidateZpredicted_startZ predicted_endZ total_lossrrrrrurvZ ignored_indexZany_retriever_correctZany_reader_correctrÁrrr"r…s&÷ ÿ      ü   ÿýõzRealmReader.forward)NNNNNNNNNNNNNN)r8r†r‡Z"_keys_to_ignore_on_load_unexpectedrcrrUrVrrÿrWrrJr‰rŠr rµrrr…rŒrrrxr"r6øsF  ñ ðr6ay Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token (should not be used in this model by design). [What are token type IDs?](../glossary#token-type-ids) answer_ids (`list` of shape `(num_answers, answer_length)`, *optional*): Answer ids for computing the marginal log-likelihood loss. Indices should be in `[-1, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-1` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. z?`RealmForOpenQA` for end-to-end open domain question answering.c sŠeZdZd ‡fdd„ Zedd„ƒZdd„Zee  d¡ƒe e e d de eje eje eje eje eeee fd œd d „ƒƒZ‡ZS)r:Nc s`tƒ |¡t|ƒ|_t|ƒ|_| dt d¡j |j |j ftj t  d¡d¡||_| ¡dS)NÚ block_embrÚcpu)rtr`r~)rbrcr=rYr6rrprJrsZ new_emptyZnum_block_recordsrr4r~Ú retrieverrE)rwrMr}rxrr"rcÈs     ýþzRealmForOpenQA.__init__cCs|jr|jjS|jjSr»)rïrMÚsearcher_beam_sizerwr*rrr"r~Øsz!RealmForOpenQA.searcher_beam_sizecCs|j |¡|_dS)z´Send `self.block_emb` to a specific device. Args: device (`str` or `torch.device`): The device to which `self.block_emb` will be sent. N)r{r®)rwr~rrr"Úblock_embedding_toÞsz!RealmForOpenQA.block_embedding_toz1, sequence_lengthrQ)ryr¢r^Ú answer_idsrær|c Csà|dur |n|jj}|dur2|jddkr2tdƒ‚|j|||dd}|d}t d|j| |jj ¡¡}tj ||j dd \} } |   ¡} tj |jd| d } |j|  ¡|||jjd \} } }}| |jj ¡}|j tj¡j|jj d }| ¡ |j tj¡¡| durDtj| tj|jj d } tj| tj|jj d } tj|tj|jj d }t d|  ¡|  |jj ¡¡}|j|jd|jj…|jd|jj…|jd|jj…||| | |dd }|j|j}||j|jd…}|sÔ||fSt ||dS)a Returns: Example: ```python >>> import torch >>> from transformers import RealmForOpenQA, RealmRetriever, AutoTokenizer >>> retriever = RealmRetriever.from_pretrained("google/realm-orqa-nq-openqa") >>> tokenizer = AutoTokenizer.from_pretrained("google/realm-orqa-nq-openqa") >>> model = RealmForOpenQA.from_pretrained("google/realm-orqa-nq-openqa", retriever=retriever) >>> question = "Who is the pioneer in modern computer science?" >>> question_ids = tokenizer([question], return_tensors="pt") >>> answer_ids = tokenizer( ... ["alan mathison turing"], ... add_special_tokens=False, ... return_token_type_ids=False, ... return_attention_mask=False, ... ).input_ids >>> reader_output, predicted_answer_ids = model(**question_ids, answer_ids=answer_ids, return_dict=False) >>> predicted_answer = tokenizer.decode(predicted_answer_ids) >>> loss = reader_output.loss ```Nrrz'The batch_size of the inputs must be 1.T)ryr^r¢ræz BD,QD->QBr])Úkr©r')Ú max_lengthr!r}zD,BD->B) ryr¢r^rür7rprnroræ)r r )!rMrMrGr’rYrJr¯r{r®r~Ztopkr~r6r)r}r|Zreader_seq_lenrZspecial_tokens_maskr/rµZ logical_not_Z logical_and_r^r­ruryrwr¢rrrr )rwryr¢r^r€ræZquestion_outputsZquestion_projectionZ batch_scoresrÜZretrieved_block_idsZretrieved_block_embrprrZ concat_inputsr7Zretrieved_logitsr Zpredicted_blockr rrr"r…èsV%ÿÿ ÿ÷ þzRealmForOpenQA.forward)N)NNNN)r8r†r‡rcÚpropertyr~rrÚREALM_FOR_OPEN_QA_DOCSTRINGrVrr rWrrJr‰rŠrµrrr…rŒrrrxr"r:Ãs$   ú ùr:)Grˆr°r/Z dataclassesrÚtypingrrrrJrZtorch.nnrZ activationsr Zmodeling_outputsr r r r Zmodeling_utilsrZ pytorch_utilsrrrrñrrrrZconfiguration_realmrZ get_loggerr8r-Z_EMBEDDER_CHECKPOINT_FOR_DOCZ_ENCODER_CHECKPOINT_FOR_DOCZ_SCORER_CHECKPOINT_FOR_DOCrWZ#REALM_PRETRAINED_MODEL_ARCHIVE_LISTrVÚModulerWrr¶r¿rÉrÐrÑrÛrõrørûrÿr rrrrrZREALM_START_DOCSTRINGrUr;rBr=rXr<r6r„r:rrrr"Ús”        l?2Wc1  C 2)þNþý*!þ