a
    AdF                     @   s   d dl mZmZ d dlmZ d dlZd dlmZ d dl mZm	Z	 d dl
mZ G dd deZG dd	 d	eZG d
d deZG dd deZdS )    )BertPreTrainedModel
BertConfigN)XLMRobertaConfig)XLMRobertaModelXLMRobertaTokenizer)Optionalc                       s   e Zd Zd fdd	Z  ZS )BertSeriesConfig:w           gelu皙?      {Gz?-q=r   absoluteTNaverageFbertc                    sH   t  j|||||||||	|
||||||fi | || _|| _|| _d S )Nsuper__init__project_dim	pooler_fnlearn_encoder)self
vocab_sizehidden_sizenum_hidden_layersnum_attention_headsintermediate_size
hidden_acthidden_dropout_probattention_probs_dropout_probmax_position_embeddingstype_vocab_sizeinitializer_rangelayer_norm_epspad_token_idposition_embedding_type	use_cacheZclassifier_dropoutr   r   r   
model_typekwargs	__class__ 4/var/www/html/stable-diffusion-webui/modules/xlmr.pyr   	   s    2zBertSeriesConfig.__init__)r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   TNr   r   Fr   __name__
__module____qualname__r   __classcell__r0   r0   r.   r1   r      s   r   c                       s   e Zd Zd	 fdd	Z  ZS )
RobertaSeriesConfig   r   r   r   clsFc                    s0   t  jf |||d| || _|| _|| _d S )N)r)   bos_token_ideos_token_idr   )r   r)   r:   r;   r   r   r   r-   r.   r0   r1   r      s    zRobertaSeriesConfig.__init__)r8   r   r   r   r9   Fr2   r0   r0   r.   r1   r7      s   r7   c                       s   e Zd ZdgZddgZeZd fdd	Zdd Zde	e
j e	e
j e	e
j e	e
j e	e
j e	e
j e	e
j e	e
j e	e e	e e	e d	d
dZ  ZS )!BertSeriesModelWithTransformationpoolerposition_idszpredictions.decoder.biasNc                    s   |d u rt  }d|_d|_d|_d|_d|_d|_d|_d|_d|_	d	|_
d
|_d|_d|_d|_d|_d|_d|_d|_d|_d|_t | t|| _t|j|j| _tj|j|j	d| _td| _ dd | _!| "  d S )Nr   r   r   r   i   r   i   gh㈵>i        Tr8   r   i r
   F)epszxlm-roberta-largec                 S   s   | d d df S )Nr   r0   )xr0   r0   r1   <lambda>=       z<BertSeriesModelWithTransformation.__init__.<locals>.<lambda>)#r   r$   r:   r;   r"   r#   r   r'   r!   r(   r%   r    r   Zoutput_pastr)   r*   r&   r+   r   r   r   r   r   r   robertannLineartransformation	LayerNormpre_LNr   from_pretrained	tokenizerr=   	post_init)r   configkargsr.   r0   r1   r      s:    

z*BertSeriesModelWithTransformation.__init__c              	   C   sl   t |  j}| j|ddddddd}t|d ||d< t|d ||d< | f i |}|d	 S )
NTM   F
max_lengthpt)
truncationrQ   return_lengthreturn_overflowing_tokenspaddingreturn_tensors	input_idsattention_maskprojection_state)next
parametersdevicerL   torchtensorto)r   cr]   textfeaturesr0   r0   r1   encode@   s"    z(BertSeriesModelWithTransformation.encode)rX   rY   token_type_idsr>   	head_maskinputs_embedsencoder_hidden_statesencoder_attention_maskoutput_attentionsreturn_dictoutput_hidden_statesc                 C   s~   |
dur|
n| j j}
| j|||||||||	d|
d}|d }| |}| |}| |}| |j}||j|j|j||dS )z	
        NT)rX   rY   re   r>   rf   rg   rh   ri   rj   rl   rk   r   )pooler_outputlast_hidden_statehidden_states
attentionsrZ   Zsequence_out)	rN   use_return_dictrE   rJ   r=   rH   rn   ro   rp   )r   rX   rY   re   r>   rf   rg   rh   ri   rj   rk   rl   outputssequence_outputZsequence_output_lnrm   rZ   r0   r0   r1   forwardO   s4    


z)BertSeriesModelWithTransformation.forward)N)NNNNNNNNNNN)r3   r4   r5   "_keys_to_ignore_on_load_unexpected_keys_to_ignore_on_load_missingr   config_classr   rd   r   r^   Tensorboolrt   r6   r0   r0   r.   r1   r<      s:   "           r<   c                   @   s   e Zd ZdZeZdS )$RobertaSeriesModelWithTransformationrE   N)r3   r4   r5   base_model_prefixr7   rw   r0   r0   r0   r1   rz      s   rz   )transformersr   r   torch.nnrF   r^   Z9transformers.models.xlm_roberta.configuration_xlm_robertar   r   r   typingr   r   r7   r<   rz   r0   r0   r0   r1   <module>   s   o