a
    þdùÉ  ã                4   @   s  d Z ddlZddlmZmZmZ ddlmZmZm	Z	m
Z
mZmZmZ ddlmZmZmZ ddlmZ G dd	„ d	ƒZeed
œdd„ZG dd„ dƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZG dd„ deƒZ G dd „ d eƒZ!G d!d"„ d"eƒZ"G d#d$„ d$eƒZ#G d%d&„ d&e#ƒZ$G d'd(„ d(e#ƒZ%G d)d*„ d*e#ƒZ&G d+d,„ d,e#ƒZ'G d-d.„ d.e#ƒZ(G d/d0„ d0e#ƒZ)G d1d2„ d2e#ƒZ*G d3d4„ d4e#ƒZ+G d5d6„ d6e#ƒZ,G d7d8„ d8e#ƒZ-G d9d:„ d:e#ƒZ.G d;d<„ d<e#ƒZ/G d=d>„ d>e#ƒZ0G d?d@„ d@e#ƒZ1G dAdB„ dBeƒZ2G dCdD„ dDe#ƒZ3G dEdF„ dFeƒZ4G dGdH„ dHeƒZ5G dIdJ„ dJeƒZ6G dKdL„ dLe#ƒZ7G dMdN„ dNe#ƒZ8G dOdP„ dPeƒZ9e$e e%ee3e6e&e4eee"e'eeeeee$eeeeee e+e e ee9e(e)eee e*ee0ee-e.ee e!ee1e2e+e,ee7e8dQœ3Z:e	dRœdSdT„Z;dS )Uzù
Utilities to convert slow tokenizers in their fast tokenizers counterparts.

All the conversions are grouped here to gather SentencePiece dependencies outside of the fast tokenizers files and
allow to make our dependency on SentencePiece optional.
é    N)ÚDictÚListÚTuple)Ú
AddedTokenÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramÚ	WordPieceé   )Úrequires_backendsc                   @   sB   e Zd ZdZedœdd„Zd	eeeef e	e f dœdd„Z
dS )
ÚSentencePieceExtractorzl
    Extractor implementation for SentencePiece trained models. https://github.com/google/sentencepiece
    )Úmodelc                 C   s.   t | dƒ ddlm} |ƒ | _| j |¡ d S )NÚsentencepiecer   )ÚSentencePieceProcessor)r   r   r   ÚspÚLoad)Úselfr   r   © r   úl/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/transformers/convert_slow_tokenizer.pyÚ__init__$   s    
zSentencePieceExtractor.__init__N©Úreturnc           	         s¶   | j ‰ ‡ fdd„tˆ  ¡ ƒD ƒ}|dur8t|ƒd }}n
|d }}g }| ¡ D ]>}| ¡ D ]0}|› |› }| |d¡}|rZ||||fg7 }qZqNt|dd„ |d}d	d
„ |D ƒ}||fS )zÂ
        By default will return vocab and merges with respect to their order, by sending `vocab_scores` we're going to
        order the merges with respect to the piece scores instead.
        c                    s   i | ]}ˆ   |¡|“qS r   )Zid_to_piece)Ú.0Úindex©r   r   r   Ú
<dictcomp>1   ó    z2SentencePieceExtractor.extract.<locals>.<dictcomp>NTFc                 S   s   | d S )Né   r   )Úvalr   r   r   Ú<lambda>?   r!   z0SentencePieceExtractor.extract.<locals>.<lambda>)ÚkeyÚreversec                 S   s   g | ]}|d  |d f‘qS )r   r   r   )r   r#   r   r   r   Ú
<listcomp>@   r!   z2SentencePieceExtractor.extract.<locals>.<listcomp>)r   ÚrangeZGetPieceSizeÚdictÚkeysÚgetÚsorted)	r   Úvocab_scoresÚvocabr&   ÚmergesZpiece_lZpiece_rÚmergeZpiece_scorer   r   r   Úextract+   s    
zSentencePieceExtractor.extract)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ústrr   r   r   Úintr   r1   r   r   r   r   r      s   r   )Úpiecer   c                 C   s&   t | ƒdk p$| d dkp$| d  ¡  S )Nr"   éÿÿÿÿú,éþÿÿÿ)ÚlenÚisdigit)r8   r   r   r   Úcheck_number_commaD   s    r>   c                   @   s"   e Zd Zdd„ Zedœdd„ZdS )Ú	Converterc                 C   s
   || _ d S ©N)Úoriginal_tokenizer)r   rA   r   r   r   r   I   s    zConverter.__init__r   c                 C   s
   t ƒ ‚d S r@   )ÚNotImplementedError©r   r   r   r   Ú	convertedL   s    zConverter.convertedN)r2   r3   r4   r   r   rD   r   r   r   r   r?   H   s   r?   c                   @   s   e Zd Zedœdd„ZdS )ÚBertConverterr   c           
      C   sê   | j j}tt|t| j jƒdƒ}d}d}d}t| j dƒrV| j jj}| j jj	}| j jj
}tjd|||d|_t ¡ |_t| j jƒ}t| j jƒ}| j j}| j j}	tj|› d|› d|› d|› d|› d	||f||	fgd
|_tjdd|_|S )N©Ú	unk_tokenFÚbasic_tokenizerT©Z
clean_textZhandle_chinese_charsÚstrip_accentsÚ	lowercaseú:0 $A:0 ú:0ú:0 $B:1 ú:1©ÚsingleÚpairÚspecial_tokensú##©Úprefix©rA   r.   r   r   r6   rG   ÚhasattrrH   Útokenize_chinese_charsrJ   Údo_lower_caser	   ÚBertNormalizerÚ
normalizerr
   ÚBertPreTokenizerÚpre_tokenizerÚ	cls_tokenÚ	sep_tokenÚcls_token_idÚsep_token_idr   ÚTemplateProcessingÚpost_processorr   Údecoder©
r   r.   Ú	tokenizerrY   rJ   rZ   ÚclsÚsepra   rb   r   r   r   rD   Q   s:    


ü
þýzBertConverter.convertedN©r2   r3   r4   r   rD   r   r   r   r   rE   P   s   rE   c                   @   s   e Zd Zedœdd„ZdS )ÚSplinterConverterr   c              
   C   sZ  | j j}tt|t| j jƒdƒ}d}d}d}t| j dƒrV| j jj}| j jj	}| j jj
}tjd|||d|_t ¡ |_t| j jƒ}t| j jƒ}t| j jƒ}d}	| j j}
| j j}| j j}| j  d¡}| j jdkrð|› d|› d	|	› d	|› d
|› d
}n"|› d|› d
|› d	|	› d	|› d
}tj|› d|› d|||
f||f||f|	|fgd|_tjdd|_|S )NrF   FrH   TrI   Ú.ÚrightrL   ú rN   rO   rM   rP   rT   rU   )rA   r.   r   r   r6   rG   rX   rH   rY   rJ   rZ   r	   r[   r\   r
   r]   r^   r_   r`   Zquestion_tokenra   rb   Úquestion_token_idÚconvert_tokens_to_idsZpadding_sider   rc   rd   r   re   )r   r.   rg   rY   rJ   rZ   rh   ri   ZquestionÚdotra   rb   ro   Zdot_token_idrR   r   r   r   rD   x   sL    


ü
$"üý
zSplinterConverter.convertedNrj   r   r   r   r   rk   w   s   rk   c                   @   s   e Zd Zedœdd„ZdS )ÚFunnelConverterr   c           
      C   sê   | j j}tt|t| j jƒdƒ}d}d}d}t| j dƒrV| j jj}| j jj	}| j jj
}tjd|||d|_t ¡ |_t| j jƒ}t| j jƒ}| j j}| j j}	tj|› d|› d|› d|› d|› d	||f||	fgd
|_tjdd|_|S )NrF   FrH   TrI   z:2 $A:0 rM   rN   rO   rP   rT   rU   rW   rf   r   r   r   rD   ª   s:    


ü
þýzFunnelConverter.convertedNrj   r   r   r   r   rr   ©   s   rr   c                   @   s   e Zd Zedœdd„ZdS )ÚMPNetConverterr   c           
   
   C   sð   | j j}tt|t| j jƒdƒ}d}d}d}t| j dƒrV| j jj}| j jj	}| j jj
}tjd|||d|_t ¡ |_t| j jƒ}t| j jƒ}| j j}| j j}	tj|› d|› d|› d|› d|› d	|› d
||f||	fgd|_tjdd|_|S )NrF   FrH   TrI   rL   rM   ú:0 rN   rO   rP   rT   rU   rW   rf   r   r   r   rD   Ñ   s:    


ü
þýzMPNetConverter.convertedNrj   r   r   r   r   rs   Ð   s   rs   c                   @   s   e Zd Zedœdd„ZdS )ÚOpenAIGPTConverterr   c              	   C   sˆ   | j j}t| j j ¡ ƒ}| j j}tt||d t|ƒdddƒ}| 	t|ƒ¡d ur^| 
t|ƒg¡ tjdd|_t ¡ |_tjdd|_|S )Nú</w>F)r.   r/   ÚdropoutrG   Úend_of_word_suffixÚfuse_unkT)rK   ©Úsuffix)rA   ÚencoderÚlistÚ	bpe_ranksr*   rG   r   r   r6   Ztoken_to_idÚadd_special_tokensr	   r[   r\   r
   r]   r^   r   Ú
BPEDecoderre   ©r   r.   r/   rG   rg   r   r   r   rD   ø   s&    úÿ
zOpenAIGPTConverter.convertedNrj   r   r   r   r   ru   ÷   s   ru   c                   @   s   e Zd Zedœdd„ZdS )ÚGPT2Converterr   c              	   C   sœ   | j j}t| j j ¡ ƒ}tt||d ddddƒ}tj| j j	d|_
t ¡ |_| j jrŠ| j j}| j j}tj|› d|› d||fgd|_ntjdd|_|S )	NÚ F©r.   r/   rw   Úcontinuing_subword_prefixrx   ry   ©Úadd_prefix_spacez:0 $A:0z:0 $A:0 $B:1rP   )Útrim_offsets)rA   r|   r}   r~   r*   r   r   r
   Ú	ByteLevelr‡   r^   r   re   Úadd_bos_tokenÚ	bos_tokenÚbos_token_idr   rc   rd   )r   r.   r/   rg   ÚbosrŒ   r   r   r   rD     s2    úÿ
ÿý

zGPT2Converter.convertedNrj   r   r   r   r   r‚     s   r‚   c                   @   s   e Zd Zedœdd„ZdS )ÚHerbertConverterr   c                 C   sª   d}d}| j j}t| j j ¡ ƒ}||d d v r<|dd … }tt||d | j j|dƒ}tj	ddd|_
t ¡ |_tj|d|_tj| j j| j jf| j j| j jfd	|_|S )
Nz	#version:rv   r   r   )rw   rG   rx   F)rK   rJ   rz   )ri   rh   )rA   r|   r}   r~   r*   r   r   rG   r	   r[   r\   r
   r]   r^   r   r€   re   r   ZBertProcessingr`   rb   r_   ra   rd   )r   Ztokenizer_info_strZtoken_suffixr.   r/   rg   r   r   r   rD   6  s.    ûÿ

þzHerbertConverter.convertedNrj   r   r   r   r   rŽ   5  s   rŽ   c                   @   s   e Zd Zedœdd„ZdS )ÚRobertaConverterr   c              	   C   sv   | j }|j}t|j ¡ ƒ}tt||d ddddƒ}tj|j	d|_
t ¡ |_tj|j|jf|j|jf|j	dd|_|S )Nrƒ   Fr„   r†   T©ri   rh   r‡   rˆ   )rA   r|   r}   r~   r*   r   r   r
   r‰   r‡   r^   r   re   r   ÚRobertaProcessingr`   rb   r_   ra   rd   ©r   Úotr.   r/   rg   r   r   r   rD   U  s,    úÿ


üzRobertaConverter.convertedNrj   r   r   r   r   r   T  s   r   c                   @   s   e Zd Zedœdd„ZdS )ÚRoFormerConverterr   c           
      C   sð   ddl m} | jj}tt|t| jjƒdƒ}d}d}t| jdƒrT| jj	j
}| jj	j}tjdd||d|_tj ||ƒ¡|_t| jjƒ}t| jjƒ}| jj}| jj}	tj|› d|› d	|› d|› d
|› d||f||	fgd|_tjdd|_|S )Nr   )ÚJiebaPreTokenizerrF   FrH   TrI   rL   rM   rN   rO   rP   rT   rU   )Z"models.roformer.tokenization_utilsr•   rA   r.   r   r   r6   rG   rX   rH   rJ   rZ   r	   r[   r\   r
   ZPreTokenizerZcustomr^   r_   r`   ra   rb   r   rc   rd   r   re   )
r   r•   r.   rg   rJ   rZ   rh   ri   ra   rb   r   r   r   rD   r  s8    

üþýzRoFormerConverter.convertedNrj   r   r   r   r   r”   q  s   r”   c                   @   s   e Zd Zedœdd„ZdS )ÚDebertaConverterr   c              	   C   s~   | j }|j}t|j ¡ ƒ}tt||d ddddƒ}tj|j	d|_
t ¡ |_tjddd| j  d¡fd| j  d¡fgd	|_|S )
Nrƒ   Fr„   r†   ú[CLS]:0 $A:0 [SEP]:0ú![CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1ú[CLS]ú[SEP]rP   )rA   r|   r}   r~   r*   r   r   r
   r‰   r‡   r^   r   re   r   rc   rp   rd   r’   r   r   r   rD   ™  s.    úÿ
þý	zDebertaConverter.convertedNrj   r   r   r   r   r–   ˜  s   r–   c                       sb   e Zd Z‡ fdd„Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
edœdd„Z‡  ZS )ÚSpmConverterc                    s   t | dƒ tƒ j|Ž  ddlm} | ¡ }t| jjdƒ}| 	| 
¡ ¡ W d   ƒ n1 s\0    Y  || _| jjjrŒt| dd ƒsŒt d¡ d S )NZprotobufr   )Úsentencepiece_model_pb2ÚrbÚhandle_byte_fallbacka  The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.)r   Úsuperr   Úutilsrœ   Z
ModelProtoÚopenrA   Ú
vocab_fileZParseFromStringÚreadÚprotoÚtrainer_specÚbyte_fallbackÚgetattrÚwarningsÚwarn)r   ÚargsZ	model_pb2ÚmÚf©Ú	__class__r   r   r   ¸  s    
,
ÿzSpmConverter.__init__c                 C   s   dd„ |j D ƒS )Nc                 S   s   g | ]}|j |jf‘qS r   ©r8   Úscore©r   r8   r   r   r   r'   Î  r!   z&SpmConverter.vocab.<locals>.<listcomp>©Úpieces©r   r¤   r   r   r   r.   Í  s    zSpmConverter.vocabc                 C   s   |j jS r@   )r¥   Úunk_idr´   r   r   r   rµ   Ð  s    zSpmConverter.unk_idc           	      C   sˆ   |j j}|  |¡}|  |¡}|dkr4tt||ƒƒ}nP|dkr|t| jjƒ 	¡ \}}dd„ t
|ƒD ƒ}tt|||j jddƒ}ntdƒ‚|S )Nr   r"   c                 S   s   i | ]\}\}}||“qS r   r   )r   ÚiÚwordr°   r   r   r   r    Ü  r!   z*SpmConverter.tokenizer.<locals>.<dictcomp>T)rG   ry   ú]You're trying to run a `Unigram` model but you're file was trained with a different algorithm)r¥   Ú
model_typer.   rµ   r   r   r   rA   r¢   r1   Ú	enumerater   Ú	unk_pieceÚ	Exception)	r   r¤   r¹   r-   rµ   rg   Ú_r/   Ú	bpe_vocabr   r   r   rg   Ó  s(    

üÿ	ÿzSpmConverter.tokenizerc                 C   sH   |j j}|s$t t tdƒd¡g¡S t t |¡t tdƒd¡g¡S d S ©Nú {2,}rn   )Únormalizer_specÚprecompiled_charsmapr	   ÚSequenceÚReplacer   ÚPrecompiled)r   r¤   rÂ   r   r   r   r\   ì  s    ÿzSpmConverter.normalizerc                 C   s   t j||dS ©N©Úreplacementr‡   )r
   Ú	Metaspace©r   rÈ   r‡   r   r   r   r^   õ  s    zSpmConverter.pre_tokenizerc                 C   s   d S r@   r   rC   r   r   r   rd   ø  s    zSpmConverter.post_processorc                 C   s   t j||dS rÆ   )r   rÉ   rÊ   r   r   r   re   û  s    zSpmConverter.decoderr   c                 C   sl   |   | j¡}|  | j¡}|d ur&||_d}d}|  ||¡}|d urH||_|  ||¡|_|  ¡ }|rh||_|S )Nõ   â–T)rg   r¤   r\   r^   re   rd   )r   rg   r\   rÈ   r‡   r^   rd   r   r   r   rD   þ  s    zSpmConverter.converted)r2   r3   r4   r   r.   rµ   rg   r\   r^   rd   re   r   rD   Ú__classcell__r   r   r­   r   r›   ·  s   	r›   c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚAlbertConverterc                 C   s   dd„ |j D ƒS )Nc                 S   s2   g | ]*}t |jƒr|j|jfn|j|jd  f‘qS ©éd   ©r>   r8   r°   r±   r   r   r   r'     s   ÿz)AlbertConverter.vocab.<locals>.<listcomp>r²   r´   r   r   r   r.     s    þzAlbertConverter.vocabc                 C   sŠ   t  dd¡t  dd¡g}| jjs<| t  ¡ ¡ | t  ¡ ¡ | jjrR| t  ¡ ¡ |j	j
}| t  |¡¡ | t  tdƒd¡¡ t  |¡S ©Nz``ú"z''rÀ   rn   ©r	   rÄ   rA   Úkeep_accentsÚappendÚNFKDÚStripAccentsrZ   Ú	LowercaserÁ   rÂ   rÅ   r   rÃ   ©r   r¤   Zlist_normalizersrÂ   r   r   r   r\     s    

þzAlbertConverter.normalizerc                 C   s,   t jddd| j d¡fd| j d¡fgdS ©Nr—   r˜   r™   rš   rP   ©r   rc   rA   rp   rC   r   r   r   rd   +  s    þýzAlbertConverter.post_processorN©r2   r3   r4   r.   r\   rd   r   r   r   r   rÍ     s   rÍ   c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚBarthezConverterc                 C   s   d}|S ©Né   r   ©r   r¤   rµ   r   r   r   rµ   7  s    zBarthezConverter.unk_idc                 C   s,   t jddd| j d¡fd| j d¡fgdS ©Nz<s> $A </s>z<s> $A </s> </s> $B </s>ú<s>ú</s>rP   rÛ   rC   r   r   r   rd   ;  s    þýzBarthezConverter.post_processorN)r2   r3   r4   rµ   rd   r   r   r   r   rÝ   6  s   rÝ   c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚCamembertConverterc                 C   s2   g d¢}|dd„ |j dd … D ƒ7 }|dg7 }|S )N))z
<s>NOTUSEDç        ©z<pad>rå   )z</s>NOTUSEDrå   ©ú<unk>rå   )z<unk>NOTUSEDiœÿÿÿc                 S   s   g | ]}|j |jf‘qS r   r¯   r±   r   r   r   r'   P  r!   z,CamembertConverter.vocab.<locals>.<listcomp>r   ©z<mask>rå   r²   ©r   r¤   r.   r   r   r   r.   G  s    
zCamembertConverter.vocabc                 C   s   dS rÞ   r   r´   r   r   r   rµ   T  s    zCamembertConverter.unk_idc                 C   s,   t jddd| j d¡fd| j d¡fgdS rá   rÛ   rC   r   r   r   rd   X  s    þýz!CamembertConverter.post_processorN©r2   r3   r4   r.   rµ   rd   r   r   r   r   rä   F  s   rä   c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚDebertaV2Converterc                 C   s<   g }| j jr| tjdd¡ | tj||d¡ t |¡S )NÚisolated)ÚbehaviorrÇ   )rA   Zsplit_by_punctrÕ   r
   ZPunctuationrÉ   rÃ   )r   rÈ   r‡   Zlist_pretokenizersr   r   r   r^   d  s
    z DebertaV2Converter.pre_tokenizerc                 C   sd   g }| j jr| t ¡ ¡ | t ¡ ¡ |jj}|rD| t |¡¡ | t 	t
dƒd¡¡ t |¡S r¿   )rA   rZ   rÕ   r	   rØ   ÚStriprÁ   rÂ   rÅ   rÄ   r   rÃ   rÙ   r   r   r   r\   k  s    zDebertaV2Converter.normalizerc                 C   s,   t jddd| j d¡fd| j d¡fgdS rÚ   rÛ   rC   r   r   r   rd   x  s    þýz!DebertaV2Converter.post_processorN)r2   r3   r4   r^   r\   rd   r   r   r   r   rì   c  s   rì   c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚMBartConverterc                 C   s>   g d¢}|dd„ |j dd … D ƒ7 }|g d¢7 }|dg7 }|S )N©©râ   rå   ræ   ©rã   rå   rç   c                 S   s   g | ]}|j |jf‘qS r   r¯   r±   r   r   r   r'   ‹  r!   z(MBartConverter.vocab.<locals>.<listcomp>rß   )©Zar_ARrå   ©Úcs_CZrå   ©Úde_DErå   ©Úen_XXrå   ©Zes_XXrå   ©Úet_EErå   ©Úfi_FIrå   ©Zfr_XXrå   ©Úgu_INrå   ©Úhi_INrå   ©Úit_ITrå   ©Zja_XXrå   ©Úkk_KZrå   ©Úko_KRrå   ©Últ_LTrå   ©Úlv_LVrå   ©Zmy_MMrå   ©Úne_NPrå   ©Znl_XXrå   ©Úro_ROrå   ©Úru_RUrå   ©Úsi_LKrå   ©Útr_TRrå   ©Úvi_VNrå   ©Úzh_CNrå   ré   r²   rê   r   r   r   r.   „  s
    
zMBartConverter.vocabc                 C   s   dS rÞ   r   r´   r   r   r   rµ   ª  s    zMBartConverter.unk_idc                 C   s,   t jddd| j d¡fd| j d¡fgdS )Nz$A </s> en_XXz$A $B </s> en_XXrú   rã   rP   rÛ   rC   r   r   r   rd   ­  s    þýzMBartConverter.post_processorNrë   r   r   r   r   rð   ƒ  s   &rð   c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚMBart50Converterc                 C   s>   g d¢}|dd„ |j dd … D ƒ7 }|g d¢7 }|dg7 }|S )Nrñ   c                 S   s   g | ]}|j |jf‘qS r   r¯   r±   r   r   r   r'   À  r!   z*MBart50Converter.vocab.<locals>.<listcomp>rß   )4rô   rõ   r÷   rù   rû   rü   rþ   r   r  r  r  r  r  r
  r  r  r  r  r  r  r  r  r  r  r  )Úaf_ZArå   )Úaz_AZrå   )Úbn_INrå   )Úfa_IRrå   )Úhe_ILrå   )Úhr_HRrå   )Úid_IDrå   )Úka_GErå   )Zkm_KHrå   )Úmk_MKrå   )Úml_INrå   )Úmn_MNrå   )Úmr_INrå   )Úpl_PLrå   )Úps_AFrå   )Zpt_XXrå   )Úsv_SErå   )Úsw_KErå   )Úta_INrå   )Úte_INrå   )Úth_THrå   )Ztl_XXrå   )Úuk_UArå   )Úur_PKrå   )Úxh_ZArå   )Úgl_ESrå   )Úsl_SIrå   ré   r²   rê   r   r   r   r.   ¹  s
    
zMBart50Converter.vocabc                 C   s   dS rÞ   r   r´   r   r   r   rµ   Ç  s    zMBart50Converter.unk_idc                 C   s,   t jddd| j d¡fd| j d¡fgdS )Nzen_XX $A </s>zen_XX $A $B </s>rú   rã   rP   rÛ   rC   r   r   r   rd   Ê  s    þýzMBart50Converter.post_processorNrë   r   r   r   r   r   ¸  s   r   c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚNllbConverterc                 C   s>   g d¢}|dd„ |j dd … D ƒ7 }|g d¢7 }|dg7 }|S )Nrñ   c                 S   s   g | ]}|j |jf‘qS r   r¯   r±   r   r   r   r'   Ý  r!   z'NllbConverter.vocab.<locals>.<listcomp>rß   )Ê)Zace_Arabrå   )Zace_Latnrå   )Zacm_Arabrå   )Zacq_Arabrå   )Zaeb_Arabrå   )Zafr_Latnrå   )Zajp_Arabrå   )Zaka_Latnrå   )Zamh_Ethirå   )Zapc_Arabrå   )Zarb_Arabrå   )Zars_Arabrå   )Zary_Arabrå   )Zarz_Arabrå   )Zasm_Bengrå   )Zast_Latnrå   )Zawa_Devarå   )Zayr_Latnrå   )Zazb_Arabrå   )Zazj_Latnrå   )Zbak_Cyrlrå   )Zbam_Latnrå   )Zban_Latnrå   )Zbel_Cyrlrå   )Zbem_Latnrå   )Zben_Bengrå   )Zbho_Devarå   )Zbjn_Arabrå   )Zbjn_Latnrå   )Zbod_Tibtrå   )Zbos_Latnrå   )Zbug_Latnrå   )Zbul_Cyrlrå   )Zcat_Latnrå   )Zceb_Latnrå   )Zces_Latnrå   )Zcjk_Latnrå   )Zckb_Arabrå   )Zcrh_Latnrå   )Zcym_Latnrå   )Zdan_Latnrå   )Zdeu_Latnrå   )Zdik_Latnrå   )Zdyu_Latnrå   )Zdzo_Tibtrå   )Zell_Grekrå   )Úeng_Latnrå   )Zepo_Latnrå   )Zest_Latnrå   )Zeus_Latnrå   )Zewe_Latnrå   )Zfao_Latnrå   )Zpes_Arabrå   )Zfij_Latnrå   )Zfin_Latnrå   )Zfon_Latnrå   )Zfra_Latnrå   )Zfur_Latnrå   )Zfuv_Latnrå   )Zgla_Latnrå   )Zgle_Latnrå   )Zglg_Latnrå   )Zgrn_Latnrå   )Zguj_Gujrrå   )Zhat_Latnrå   )Zhau_Latnrå   )Zheb_Hebrrå   )Zhin_Devarå   )Zhne_Devarå   )Zhrv_Latnrå   )Zhun_Latnrå   )Zhye_Armnrå   )Zibo_Latnrå   )Zilo_Latnrå   )Zind_Latnrå   )Zisl_Latnrå   )Zita_Latnrå   )Zjav_Latnrå   )Zjpn_Jpanrå   )Zkab_Latnrå   )Zkac_Latnrå   )Zkam_Latnrå   )Zkan_Kndarå   )Zkas_Arabrå   )Zkas_Devarå   )Zkat_Georrå   )Zknc_Arabrå   )Zknc_Latnrå   )Zkaz_Cyrlrå   )Zkbp_Latnrå   )Zkea_Latnrå   )Zkhm_Khmrrå   )Zkik_Latnrå   )Zkin_Latnrå   )Zkir_Cyrlrå   )Zkmb_Latnrå   )Zkon_Latnrå   )Zkor_Hangrå   )Zkmr_Latnrå   )Zlao_Laoorå   )Zlvs_Latnrå   )Zlij_Latnrå   )Zlim_Latnrå   )Zlin_Latnrå   )Zlit_Latnrå   )Zlmo_Latnrå   )Zltg_Latnrå   )Zltz_Latnrå   )Zlua_Latnrå   )Zlug_Latnrå   )Zluo_Latnrå   )Zlus_Latnrå   )Zmag_Devarå   )Zmai_Devarå   )Zmal_Mlymrå   )Zmar_Devarå   )Zmin_Latnrå   )Zmkd_Cyrlrå   )Zplt_Latnrå   )Zmlt_Latnrå   )Zmni_Bengrå   )Zkhk_Cyrlrå   )Zmos_Latnrå   )Zmri_Latnrå   )Zzsm_Latnrå   )Zmya_Mymrrå   )Znld_Latnrå   )Znno_Latnrå   )Znob_Latnrå   )Znpi_Devarå   )Znso_Latnrå   )Znus_Latnrå   )Znya_Latnrå   )Zoci_Latnrå   )Zgaz_Latnrå   )Zory_Oryarå   )Zpag_Latnrå   )Zpan_Gururå   )Zpap_Latnrå   )Zpol_Latnrå   )Zpor_Latnrå   )Zprs_Arabrå   )Zpbt_Arabrå   )Zquy_Latnrå   )Zron_Latnrå   )Zrun_Latnrå   )Zrus_Cyrlrå   )Zsag_Latnrå   )Zsan_Devarå   )Zsat_Bengrå   )Zscn_Latnrå   )Zshn_Mymrrå   )Zsin_Sinhrå   )Zslk_Latnrå   )Zslv_Latnrå   )Zsmo_Latnrå   )Zsna_Latnrå   )Zsnd_Arabrå   )Zsom_Latnrå   )Zsot_Latnrå   )Zspa_Latnrå   )Zals_Latnrå   )Zsrd_Latnrå   )Zsrp_Cyrlrå   )Zssw_Latnrå   )Zsun_Latnrå   )Zswe_Latnrå   )Zswh_Latnrå   )Zszl_Latnrå   )Ztam_Tamlrå   )Ztat_Cyrlrå   )Ztel_Telurå   )Ztgk_Cyrlrå   )Ztgl_Latnrå   )Ztha_Thairå   )Ztir_Ethirå   )Ztaq_Latnrå   )Ztaq_Tfngrå   )Ztpi_Latnrå   )Ztsn_Latnrå   )Ztso_Latnrå   )Ztuk_Latnrå   )Ztum_Latnrå   )Ztur_Latnrå   )Ztwi_Latnrå   )Ztzm_Tfngrå   )Zuig_Arabrå   )Zukr_Cyrlrå   )Zumb_Latnrå   )Zurd_Arabrå   )Zuzn_Latnrå   )Zvec_Latnrå   )Zvie_Latnrå   )Zwar_Latnrå   )Zwol_Latnrå   )Zxho_Latnrå   )Zydd_Hebrrå   )Zyor_Latnrå   )Zyue_Hantrå   )Zzho_Hansrå   )Zzho_Hantrå   )Zzul_Latnrå   ré   r²   rê   r   r   r   r.   Ö  s
    
zNllbConverter.vocabc                 C   s   dS rÞ   r   r´   r   r   r   rµ   æ  s    zNllbConverter.unk_idc                 C   s,   t jddd| j d¡fd| j d¡fgdS )Nzeng_Latn $A </s>zeng_Latn $A $B </s>r:  rã   rP   rÛ   rC   r   r   r   rd   é  s    þýzNllbConverter.post_processorNrë   r   r   r   r   r9  Õ  s   r9  c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚXLMRobertaConverterc                 C   s2   g d¢}|dd„ |j dd … D ƒ7 }|dg7 }|S )Nrñ   c                 S   s   g | ]}|j |jf‘qS r   r¯   r±   r   r   r   r'   ü  r!   z-XLMRobertaConverter.vocab.<locals>.<listcomp>rß   ré   r²   rê   r   r   r   r.   õ  s    
zXLMRobertaConverter.vocabc                 C   s   d}|S rÞ   r   rà   r   r   r   rµ      s    zXLMRobertaConverter.unk_idc                 C   s,   t jddd| j d¡fd| j d¡fgdS rá   rÛ   rC   r   r   r   rd     s    þýz"XLMRobertaConverter.post_processorNrë   r   r   r   r   r;  ô  s   r;  c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚXLNetConverterc                 C   s   dd„ |j D ƒS )Nc                 S   s2   g | ]*}t |jƒr|j|jfn|j|jd  f‘qS rÎ   rÐ   r±   r   r   r   r'     s   ÿz(XLNetConverter.vocab.<locals>.<listcomp>r²   r´   r   r   r   r.     s    þzXLNetConverter.vocabc                 C   sŠ   t  dd¡t  dd¡g}| jjs<| t  ¡ ¡ | t  ¡ ¡ | jjrR| t  ¡ ¡ |j	j
}| t  |¡¡ | t  tdƒd¡¡ t  |¡S rÑ   rÓ   rÙ   r   r   r   r\     s    

þzXLNetConverter.normalizerc                 C   s,   t jddd| j d¡fd| j d¡fgdS )Nz$A:0 <sep>:0 <cls>:2z!$A:0 <sep>:0 $B:1 <sep>:1 <cls>:2z<sep>z<cls>rP   rÛ   rC   r   r   r   rd   &  s    þýzXLNetConverter.post_processorNrÜ   r   r   r   r   r<    s   r<  c                   @   s   e Zd ZdS )ÚReformerConverterN©r2   r3   r4   r   r   r   r   r=  1  s   r=  c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚRemBertConverterc                 C   s‚   t  dd¡t  dd¡t  tdƒd¡g}| jjsJ| t  ¡ ¡ | t  ¡ ¡ | jjr`| t  	¡ ¡ |j
j}| t  |¡¡ t  |¡S rÑ   )r	   rÄ   r   rA   rÔ   rÕ   rÖ   r×   rZ   rØ   rÁ   rÂ   rÅ   rÃ   rÙ   r   r   r   r\   7  s    

ýzRemBertConverter.normalizerc                 C   s,   t jddd| j d¡fd| j d¡fgdS rÚ   rÛ   rC   r   r   r   rd   G  s    þýzRemBertConverter.post_processorN)r2   r3   r4   r\   rd   r   r   r   r   r?  5  s   r?  c                   @   s   e Zd ZdS )ÚBertGenerationConverterNr>  r   r   r   r   r@  R  s   r@  c                   @   s,   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	S )
ÚPegasusConverterc                 C   s    | j jdf| j jdfg}| j jd ur6|| j jdfg7 }| j jd urd| j j| j jk rd|| j jdfg7 }|dd„ td| j jƒD ƒ7 }|dd„ |jdd … D ƒ7 }|S )Nrå   c                 S   s   g | ]}d |› ddf‘qS )z<unk_ú>g      YÀr   ©r   r¶   r   r   r   r'   f  r!   z*PegasusConverter.vocab.<locals>.<listcomp>r"   c                 S   s   g | ]}|j |jf‘qS r   r¯   r±   r   r   r   r'   g  r!   )	rA   Z	pad_tokenÚ	eos_tokenZmask_token_sentZ
mask_tokenZmask_token_idÚoffsetr(   r³   rê   r   r   r   r.   W  s    

þ
ÿþzPegasusConverter.vocabc                 C   s   |j j| jj S r@   )r¥   rµ   rA   rE  r´   r   r   r   rµ   j  s    zPegasusConverter.unk_idc                 C   s   t  t  ¡ t j||dg¡S rÆ   )r
   rÃ   ZWhitespaceSplitrÉ   rÊ   r   r   r   r^   m  s
    þÿzPegasusConverter.pre_tokenizerc                 C   s0   | j j}|| j jfg}tjd|gdd|g|dS )Nú$Aú$BrP   )rA   rD  Úeos_token_idr   rc   )r   ÚeosrS   r   r   r   rd   u  s    
ÿzPegasusConverter.post_processorN)r2   r3   r4   r.   rµ   r^   rd   r   r   r   r   rA  V  s   rA  c                   @   s   e Zd Zdd„ Zdd„ ZdS )ÚT5Converterc                 C   s:   | j j}dd„ |jD ƒ}|dd„ t|d ddƒD ƒ7 }|S )Nc                 S   s   g | ]}|j |jf‘qS r   r¯   r±   r   r   r   r'   €  r!   z%T5Converter.vocab.<locals>.<listcomp>c                 S   s   g | ]}d |› ddf‘qS )z
<extra_id_rB  rå   r   rC  r   r   r   r'     r!   r   r9   )rA   Z
_extra_idsr³   r(   )r   r¤   Znum_extra_idsr.   r   r   r   r.   ~  s    zT5Converter.vocabc                 C   s&   t jddgg d¢d| j d¡fgdS )NrF  rã   )rF  rã   rG  rã   rP   rÛ   rC   r   r   r   rd   „  s    ÿýzT5Converter.post_processorN)r2   r3   r4   r.   rd   r   r   r   r   rJ  }  s   rJ  c                   @   s   e Zd Zedœdd„ZdS )ÚWhisperConverterr   c           	   	   C   sÂ   | j j}t| j j ¡ ƒ}tt||d ddddƒ}tj| j j	d|_
t ¡ |_| j j}| j  |¡}| j j}| j j}d dd„ |D ƒ¡}tj|› d|› d	|› d
|› d||fgt||ƒ¢d|_|S )Nrƒ   Fr„   r†   rn   c                 S   s   g | ]}|› d ‘qS )rM   r   )r   Útokenr   r   r   r'   ¥  r!   z.WhisperConverter.converted.<locals>.<listcomp>z $A:0 rM   z $A:0 $B:1 rO   rP   )rA   r|   r}   r~   r*   r   r   r
   r‰   r‡   r^   r   re   Zprefix_tokensZconvert_ids_to_tokensrD  rH  Újoinr   rc   Úziprd   )	r   r.   r/   rg   Zprefix_token_idsÚprefixesrI  rH  Zprefix_templater   r   r   rD     s8    úÿ
ÿþý	zWhisperConverter.convertedNrj   r   r   r   r   rK  Ž  s   rK  c                   @   s   e Zd Zdd„ ZdS )ÚBigBirdConverterc                 C   s,   t jddd| j d¡fd| j d¡fgdS rÚ   rÛ   rC   r   r   r   rd   ³  s    þýzBigBirdConverter.post_processorN)r2   r3   r4   rd   r   r   r   r   rP  ²  s   rP  c                   @   s   e Zd Zedœdd„ZdS )ÚCLIPConverterr   c              
   C   sÆ   | j j}t| j j ¡ ƒ}| j j}tt||d dddt|ƒdƒ}t	 
t	 ¡ t	 tdƒd¡t	 ¡ g¡|_t 
tjtdƒdd	d
tjddg¡|_t ¡ |_tj| j j| j jf| j j| j jfddd|_|S )Nrƒ   rv   F©r.   r/   rw   r…   rx   ry   rG   z\s+rn   z9's|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+ÚremovedT)rî   Úinvertr†   r   )rA   r|   r}   r~   r*   rG   r   r   r6   r	   rÃ   ÚNFCrÄ   r   rØ   r\   r
   ZSplitr‰   r^   r   re   r   r‘   rD  rH  r‹   rŒ   rd   r   r   r   r   rD   ¿  sD    ùÿÿý
úÿ

üzCLIPConverter.convertedNrj   r   r   r   r   rQ  ¾  s   rQ  c                   @   s   e Zd Zedœdd„ZdS )ÚLayoutLMv2Converterr   c           
      C   sê   | j j}tt|t| j jƒdƒ}d}d}d}t| j dƒrV| j jj}| j jj	}| j jj
}tjd|||d|_t ¡ |_t| j jƒ}t| j jƒ}| j j}| j j}	tj|› d|› d|› d|› d|› d	||f||	fgd
|_tjdd|_|S )NrF   FTrH   rI   rL   rM   rN   rO   rP   rT   rU   rW   rf   r   r   r   rD   ê  s:    


ü
þýzLayoutLMv2Converter.convertedNrj   r   r   r   r   rV  é  s   rV  c                   @   s   e Zd Zedœdd„ZdS )ÚBlenderbotConverterr   c              	   C   st   | j }|j}t|j ¡ ƒ}tt||d ddddƒ}tj|j	d|_
t ¡ |_tjd|j› d|j|jfgd|_|S )Nrƒ   Fr„   r†   z$A:0 rM   )rQ   rS   )rA   r|   r}   r~   r*   r   r   r
   r‰   r‡   r^   r   re   r   rc   rD  rH  rd   r’   r   r   r   rD     s*    úÿ

ÿþzBlenderbotConverter.convertedNrj   r   r   r   r   rW    s   rW  c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚXGLMConverterc                 C   s4   g d¢}|dd„ |j dd … D ƒ7 }|g d¢7 }|S )Nrñ   c                 S   s   g | ]}|j |jf‘qS r   r¯   r±   r   r   r   r'   5  r!   z'XGLMConverter.vocab.<locals>.<listcomp>rß   ))z<madeupword0>rå   )z<madeupword1>rå   )z<madeupword2>rå   )z<madeupword3>rå   )z<madeupword4>rå   )z<madeupword5>rå   )z<madeupword6>rå   r²   rê   r   r   r   r.   .  s    zXGLMConverter.vocabc                 C   s   d}|S rÞ   r   rà   r   r   r   rµ   ;  s    zXGLMConverter.unk_idc                 C   s,   t jddd| j d¡fd| j d¡fgdS )Nz</s> $Az</s> $A </s> </s> $Brâ   rã   rP   rÛ   rC   r   r   r   rd   ?  s    þýzXGLMConverter.post_processorNrë   r   r   r   r   rX  -  s   rX  c                   @   sH   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zd
d„ Zdd„ Z	dd„ Z
dS )ÚLlamaConverterTc                 C   s(   g d¢}|dd„ |j dd … D ƒ7 }|S )N)rç   rò   ró   c                 S   s   g | ]}|j |jf‘qS r   r¯   r±   r   r   r   r'   S  r!   z(LlamaConverter.vocab.<locals>.<listcomp>rß   r²   rê   r   r   r   r.   M  s    zLlamaConverter.vocabc                 C   s   d}|S )Nr   r   rà   r   r   r   rµ   V  s    zLlamaConverter.unk_idc              	   C   s,   t  t  dd¡t  ¡ t  ¡ t jdddg¡S )NrË   rn   r   )ÚcontentÚleft)r   rÃ   rÄ   ZByteFallbackZFuserï   rÊ   r   r   r   re   Z  s    
üÿzLlamaConverter.decoderc                 C   s¤   |j j}|  |¡}|dkr$tdƒ‚n||dkr˜t| jjƒ |¡\}}dd„ t|ƒD ƒ}t	t
|||j jdddƒ}| tddd	td
dd	tddd	g¡ ntdƒ‚|S )Nr   z$Llama is supposed to be a BPE model!r"   c                 S   s   i | ]\}\}}||“qS r   r   )r   r¶   r·   Z_scorer   r   r   r    k  r!   z,LlamaConverter.tokenizer.<locals>.<dictcomp>T)rG   ry   r¦   rè   )Ú
normalizedrâ   rã   r¸   )r¥   r¹   r.   ÚRuntimeErrorr   rA   r¢   r1   rº   r   r   r»   r   r   r¼   )r   r¤   r¹   r-   r½   r/   r¾   rg   r   r   r   rg   d  s(    

ÿ


ýÿÿzLlamaConverter.tokenizerc                 C   s    t  t jddt jdddg¡S )NrË   )Úprependrn   )ÚpatternrZ  )r	   rÃ   ZPrependrÄ   r´   r   r   r   r\   }  s
    
þÿzLlamaConverter.normalizerc                 C   s   d S r@   r   rÊ   r   r   r   r^   …  s    zLlamaConverter.pre_tokenizerc           
      C   sÄ   | j j}| j j}|s|r¼| j j}| j j}| j j}| j j}|d | › dd| d | › }|› d| d | › dd| d | › }g }	|rš|	 ||f¡ |r¬|	 ||f¡ tj	|||	dS d S d S )Nrt   z$A:0rn   rM   rO   z $B:1rP   )
rA   rŠ   Zadd_eos_tokenr‹   rŒ   rD  rH  rÕ   r   rc   )
r   Zadd_bosZadd_eosr   rŒ   rI  rH  rQ   rR   rS   r   r   r   rd   ˆ  s     "*zLlamaConverter.post_processorN)r2   r3   r4   rž   r.   rµ   re   rg   r\   r^   rd   r   r   r   r   rY  J  s   	
rY  c                   @   s   e Zd Zedœdd„ZdS )ÚMarkupLMConverterr   c           	   
   C   s¶   | j }|j}t|j ¡ ƒ}tt||d ddd| j jdƒ}tj	|j
d|_t 	¡ |_t| j jƒ}t| j jƒ}| j j}| j j}tj|› d|› |› d|› d|› ||f||fgd|_|S )Nrƒ   FrR  r†   z $A z $B rP   )rA   r|   r}   r~   r*   r   r   rG   r
   r‰   r‡   r^   r   re   r6   r_   r`   ra   rb   r   rc   rd   )	r   r“   r.   r/   rg   rh   ri   ra   rb   r   r   r   rD   ¦  s8    ùÿ
þý	zMarkupLMConverter.convertedNrj   r   r   r   r   r`  ¥  s   r`  )3ZAlbertTokenizerZBartTokenizerZBarthezTokenizerZBertTokenizerZBigBirdTokenizerZBlenderbotTokenizerZCamembertTokenizerZCLIPTokenizerZCodeGenTokenizerZConvBertTokenizerZDebertaTokenizerZDebertaV2TokenizerZDistilBertTokenizerZDPRReaderTokenizerZDPRQuestionEncoderTokenizerZDPRContextEncoderTokenizerZElectraTokenizerZFNetTokenizerZFunnelTokenizerZGPT2TokenizerZHerbertTokenizerZLayoutLMTokenizerZLayoutLMv2TokenizerZLayoutLMv3TokenizerZLayoutXLMTokenizerZLongformerTokenizerZLEDTokenizerZLxmertTokenizerZMarkupLMTokenizerZMBartTokenizerZMBart50TokenizerZMPNetTokenizerZMobileBertTokenizerZMvpTokenizerZNllbTokenizerZOpenAIGPTTokenizerZPegasusTokenizerZRealmTokenizerZReformerTokenizerZRemBertTokenizerZRetriBertTokenizerZRobertaTokenizerZRoFormerTokenizerZSqueezeBertTokenizerZT5TokenizerZWhisperTokenizerZXLMRobertaTokenizerZXLNetTokenizerZSplinterTokenizerZXGLMTokenizerZLlamaTokenizerr   c                 C   s@   | j j}|tvr,td|› dtt ¡ ƒ› ƒ‚t| }|| ƒ ¡ S )aé  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    zAn instance of tokenizer class zv cannot be converted in a Fast tokenizer instance. No converter was found. Currently available slow->fast convertors: )r®   r2   ÚSLOW_TO_FAST_CONVERTERSÚ
ValueErrorr}   r*   rD   )Ztransformer_tokenizerZtokenizer_class_nameZconverter_classr   r   r   Úconvert_slow_tokenizer  s    
þÿrc  )<r5   r¨   Útypingr   r   r   Z
tokenizersr   r   r   r   r	   r
   r   Ztokenizers.modelsr   r   r   r    r   r   r6   Úboolr>   r?   rE   rk   rr   rs   ru   r‚   rŽ   r   r”   r–   r›   rÍ   rÝ   rä   rì   rð   r   r9  r;  r<  r=  r?  r@  rA  rJ  rK  rP  rQ  rV  rW  rX  rY  r`  ra  rc  r   r   r   r   Ú<module>   s¼   $%'2''#']" 5"'$+'['Í7