a  dS@s UdZddlZddlZddlZddlmZddlmZmZm Z m Z m Z ddl m Z ddlmZmZddlmZdd lmZdd lmZmZmZmZmZd d lmZd dlmZd dlm Z m!Z!m"Z"m#Z#m$Z$erddl%m&Z&ndZ&e'e(Z)ereZ*ee+e e e+e e+ffe,d< neder,dnder:dndffdderPdndffdderfdndertdndffddderdndffderdnddffd d!d"erd#nderd$ndffd%d&erd'ndffd(d)d*d+derdndffd,d-erd.ndffd/der&d0ndffd1d2erddnddffde rTdnde rbdndffde rvdnde rdndffde rdnde rdndffde rdnde rdndffde rdnde rdndffgZ*ee e*Z-dde .DZ/e+dddZ0d e e+ej1fe e e+ej1fe2e2e ee+e+fe e e2e+fe e+e2e+d ddZ3Gd d d Z4dS( z Auto Tokenizer class.N) OrderedDict) TYPE_CHECKINGDictOptionalTupleUnion)PretrainedConfig)get_class_from_dynamic_moduleresolve_trust_remote_code)PreTrainedTokenizer)TOKENIZER_CONFIG_FILE) cached_fileextract_commit_hashis_sentencepiece_availableis_tokenizers_availablelogging)EncoderDecoderConfig)_LazyAutoMapping)CONFIG_MAPPING_NAMES AutoConfigconfig_class_to_model_typemodel_type_to_module_name!replace_list_option_in_docstrings)PreTrainedTokenizerFastTOKENIZER_MAPPING_NAMESZalbertZAlbertTokenizerZAlbertTokenizerFastZalignZ BertTokenizerZBertTokenizerFast)Zbart)Z BartTokenizerZBartTokenizerFastZbarthezZBarthezTokenizerZBarthezTokenizerFast)Zbartpho)ZBartphoTokenizerNZbertzbert-generationZBertGenerationTokenizer)z bert-japanese)ZBertJapaneseTokenizerN)Zbertweet)ZBertweetTokenizerNZbig_birdZBigBirdTokenizerZBigBirdTokenizerFastZbigbird_pegasusZPegasusTokenizerZPegasusTokenizerFast)Zbiogpt)ZBioGptTokenizerN)Z blenderbot)ZBlenderbotTokenizerZBlenderbotTokenizerFast)zblenderbot-small)ZBlenderbotSmallTokenizerNZblipzblip-2Z GPT2TokenizerZGPT2TokenizerFastZbloomZBloomTokenizerFastZ bridgetowerZRobertaTokenizerZRobertaTokenizerFast)Zbyt5)Z ByT5TokenizerNZ camembertZCamembertTokenizerZCamembertTokenizerFast)Zcanine)ZCanineTokenizerNZ chinese_clipZclapZclipZ CLIPTokenizerZCLIPTokenizerFastZclipsegZcodegenZCodeGenTokenizerZCodeGenTokenizerFastZconvbertZConvBertTokenizerZConvBertTokenizerFastZcpmZ CpmTokenizerZCpmTokenizerFast)Zcpmant)ZCpmAntTokenizerN)Zctrl)Z CTRLTokenizerNz data2vec-textZdebertaZDebertaTokenizerZDebertaTokenizerFastz deberta-v2ZDebertaV2TokenizerZDebertaV2TokenizerFastZ distilbertZDistilBertTokenizerZDistilBertTokenizerFastZdprZDPRQuestionEncoderTokenizerZDPRQuestionEncoderTokenizerFastZelectraZElectraTokenizerZElectraTokenizerFastZernieZernie_mZErnieMTokenizer)Zesm)Z EsmTokenizerN)Zflaubert)ZFlaubertTokenizerNZfnetZ FNetTokenizerZFNetTokenizerFast)Zfsmt)Z FSMTTokenizerNZfunnelZFunnelTokenizerZFunnelTokenizerFastgitzgpt-sw3ZGPTSw3TokenizerZgpt2Z gpt_bigcodeZgpt_neoZgpt_neoxZGPTNeoXTokenizerFast)Zgpt_neox_japanese)ZGPTNeoXJapaneseTokenizerNZgptj)zgptsan-japanese)ZGPTSanJapaneseTokenizerNZgroupvitZherbertZHerbertTokenizerZHerbertTokenizerFast)ZhubertZWav2Vec2CTCTokenizerNZibert)Zjukebox)ZJukeboxTokenizerNZlayoutlmZLayoutLMTokenizerZLayoutLMTokenizerFastZ layoutlmv2ZLayoutLMv2TokenizerZLayoutLMv2TokenizerFastZ layoutlmv3ZLayoutLMv3TokenizerZLayoutLMv3TokenizerFastZ layoutxlmZLayoutXLMTokenizerZLayoutXLMTokenizerFastZledZ LEDTokenizerZLEDTokenizerFastZliltZllamaZLlamaTokenizerZLlamaTokenizerFastZ longformerZLongformerTokenizerZLongformerTokenizerFastZlongt5Z T5TokenizerZT5TokenizerFast)Zluke)Z LukeTokenizerNZlxmertZLxmertTokenizerZLxmertTokenizerFastZm2m_100ZM2M100TokenizerZmarianZMarianTokenizerZmbartZMBartTokenizerZMBartTokenizerFastZmbart50ZMBart50TokenizerZMBart50TokenizerFastZmegaz megatron-bert)zmgp-str)ZMgpstrTokenizerNZmlukeZMLukeTokenizerZ mobilebertZMobileBertTokenizerZMobileBertTokenizerFastZmpnetZMPNetTokenizerZMPNetTokenizerFastZmt5Z MT5TokenizerZMT5TokenizerFastZmvpZ MvpTokenizerZMvpTokenizerFastZnezhaZnllbZ NllbTokenizerZNllbTokenizerFastznllb-moeZ nystromformerZ oneformerz openai-gptZOpenAIGPTTokenizerZOpenAIGPTTokenizerFastoptZowlvitZpegasusZ pegasus_x)Z perceiver)ZPerceiverTokenizerN)Zphobert)ZPhobertTokenizerNZ pix2structZplbartZPLBartTokenizer)Z prophetnet)ZProphetNetTokenizerNZqdqbert)Zrag)Z RagTokenizerNrealmZRealmTokenizerZRealmTokenizerFastZreformerZReformerTokenizerZReformerTokenizerFastZrembertZRemBertTokenizerZRemBertTokenizerFastZ retribertZRetriBertTokenizerZRetriBertTokenizerFastZrobertazroberta-prelayernorm)Zroc_bert)ZRoCBertTokenizerNZroformerZRoFormerTokenizerZRoFormerTokenizerFastZrwkvZspeech_to_textZSpeech2TextTokenizer)Zspeech_to_text_2)ZSpeech2Text2TokenizerNZspeecht5ZSpeechT5Tokenizer)Zsplinter)ZSplinterTokenizerZSplinterTokenizerFastZ squeezebertZSqueezeBertTokenizerZSqueezeBertTokenizerFastZswitch_transformersZt5)Ztapas)ZTapasTokenizerN)Ztapex)ZTapexTokenizerN)z transfo-xl)ZTransfoXLTokenizerNZviltZ visual_bert)Zwav2vec2r)zwav2vec2-conformerr)Zwav2vec2_phoneme)ZWav2Vec2PhonemeCTCTokenizerNZwhisperZWhisperTokenizerZWhisperTokenizerFastZxclipZxglmZ XGLMTokenizerZXGLMTokenizerFast)Zxlm)Z XLMTokenizerNzxlm-prophetnetZXLMProphetNetTokenizerz xlm-robertaZXLMRobertaTokenizerZXLMRobertaTokenizerFastzxlm-roberta-xlZxlnetZXLNetTokenizerZXLNetTokenizerFastZxmodZyosocCsi|]\}}||qSr").0kvr"r"s/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/transformers/models/auto/tokenization_auto.py ~r') class_namec Cs|dkr tStD]R\}}||vrt|}td|d}zt||WStydYqYq0qtj D].\}}|D] }t|dd|kr~|Sq~qrtd}t ||rt||SdS)Nr.ztransformers.models__name__Z transformers) rritemsr importlib import_modulegetattrAttributeErrorTOKENIZER_MAPPING_extra_contenthasattr)r) module_nameZ tokenizersmoduleconfig tokenizerZ main_moduler"r"r&tokenizer_class_from_names$     r8F) pretrained_model_name_or_path cache_dirforce_downloadresume_downloadproxiesuse_auth_tokenrevisionlocal_files_only subfolderc Ks| dd} t|t||||||||dd| d } | durDtdiSt| | } t| dd} t| } Wdn1sz0Y| | d<| S)a Loads the tokenizer configuration from a pretrained model tokenizer configuration. Args: pretrained_model_name_or_path (`str` or `os.PathLike`): This can be either: - a string, the *model id* of a pretrained model configuration hosted inside a model repo on huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`. - a path to a *directory* containing a configuration file saved using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. cache_dir (`str` or `os.PathLike`, *optional*): Path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used. force_download (`bool`, *optional*, defaults to `False`): Whether or not to force to (re-)download the configuration files and override the cached versions if they exist. resume_download (`bool`, *optional*, defaults to `False`): Whether or not to delete incompletely received file. Attempts to resume the download if such a file exists. proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. use_auth_token (`str` or *bool*, *optional*): The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). revision (`str`, *optional*, defaults to `"main"`): The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git. local_files_only (`bool`, *optional*, defaults to `False`): If `True`, will only try to load the tokenizer configuration from local files. subfolder (`str`, *optional*, defaults to `""`): In case the tokenizer config is located inside a subfolder of the model repo on huggingface.co, you can specify the folder name here. Passing `use_auth_token=True` is required when you want to use a private model. Returns: `Dict`: The configuration of the tokenizer. Examples: ```python # Download configuration from huggingface.co and cache. tokenizer_config = get_tokenizer_config("bert-base-uncased") # This model does not have a tokenizer config so the result will be an empty dict. tokenizer_config = get_tokenizer_config("xlm-roberta-base") # Save a pretrained tokenizer locally and you can reload its config from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") tokenizer.save_pretrained("tokenizer-test") tokenizer_config = get_tokenizer_config("tokenizer-test") ``` _commit_hashNF) r;r<r=r>r?r@rArBZ%_raise_exceptions_for_missing_entriesZ'_raise_exceptions_for_connection_errorsrCz\Could not locate the tokenizer configuration file, will try to use the model config instead.zutf-8)encoding) getrr loggerinforopenjsonload)r:r;r<r=r>r?r@rArBkwargsZ commit_hashZresolved_config_filereaderresultr"r"r&get_tokenizer_configs0I   (rNc@s6eZdZdZddZeeeddZd ddZ dS) AutoTokenizera This is a generic tokenizer class that will be instantiated as one of the tokenizer classes of the library when created with the [`AutoTokenizer.from_pretrained`] class method. This class cannot be instantiated directly using `__init__()` (throws an error). cCs tddS)Nz}AutoTokenizer is designed to be instantiated using the `AutoTokenizer.from_pretrained(pretrained_model_name_or_path)` method.)EnvironmentError)selfr"r"r&__init__ szAutoTokenizer.__init__c Os|dd}d|d<|dd}|dd}|dd}|durd}t|d} | durtd|d d d d tDd | \} } |r| durt| }n td|durt| }|durtd| d|j |g|Ri|St |fi|} d| vr| d|d<| d} d}d| vrVt | dt t frF| d}n| ddd}| durt |tstj |fd|i|}|j} t|drd|jvr|jd}|du}| dupt|tv}t||||}|r>|r>|r|ddur|d}n|d}t||fi|}|dd}|j |g|Ri|S| durd}|rp| dsp| d}t|}|dur| }t|}|durtd|d|j |g|Ri|St |trt|jt|jurtd|jjd|jjd|j}tt|j}|durtt|\}}|rV|s>|durV|j |g|Ri|S|durx|j |g|Ri|Stdtd|jdd d d tDd dS)!a9 Instantiate one of the tokenizer classes of the library from a pretrained model vocabulary. The tokenizer class to instantiate is selected based on the `model_type` property of the config object (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by falling back to using pattern matching on `pretrained_model_name_or_path`: List options Params: pretrained_model_name_or_path (`str` or `os.PathLike`): Can be either: - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`. - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. - A path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (like Bert or XLNet), e.g.: `./my_model_directory/vocab.txt`. (Not applicable to all derived classes) inputs (additional positional arguments, *optional*): Will be passed along to the Tokenizer `__init__()` method. config ([`PretrainedConfig`], *optional*) The configuration object used to dertermine the tokenizer class to instantiate. cache_dir (`str` or `os.PathLike`, *optional*): Path to a directory in which a downloaded pretrained model configuration should be cached if the standard cache should not be used. force_download (`bool`, *optional*, defaults to `False`): Whether or not to force the (re-)download the model weights and configuration files and override the cached versions if they exist. resume_download (`bool`, *optional*, defaults to `False`): Whether or not to delete incompletely received files. Will attempt to resume the download if such a file exists. proxies (`Dict[str, str]`, *optional*): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. revision (`str`, *optional*, defaults to `"main"`): The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git. subfolder (`str`, *optional*): In case the relevant files are located inside a subfolder of the model repo on huggingface.co (e.g. for facebook/rag-token-base), specify it here. use_fast (`bool`, *optional*, defaults to `True`): Use a [fast Rust-based tokenizer](https://huggingface.co/docs/tokenizers/index) if it is supported for a given model. If a fast tokenizer is not available for a given model, a normal Python-based tokenizer is returned instead. tokenizer_type (`str`, *optional*): Tokenizer type to be loaded. trust_remote_code (`bool`, *optional*, defaults to `False`): Whether or not to allow for custom models defined on the Hub in their own modeling files. This option should only be set to `True` for repositories you trust and in which you have read the code, as it will execute code present on the Hub on your local machine. kwargs (additional keyword arguments, *optional*): Will be passed to the Tokenizer `__init__()` method. Can be used to set special tokens like `bos_token`, `eos_token`, `unk_token`, `sep_token`, `pad_token`, `cls_token`, `mask_token`, `additional_special_tokens`. See parameters in the `__init__()` for more details. Examples: ```python >>> from transformers import AutoTokenizer >>> # Download vocabulary from huggingface.co and cache. >>> tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") >>> # Download vocabulary from huggingface.co (user-uploaded) and cache. >>> tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-german-cased") >>> # If vocabulary files are in a directory (e.g. tokenizer was saved using *save_pretrained('./test/saved_model/')*) >>> # tokenizer = AutoTokenizer.from_pretrained("./test/bert_saved_model/") >>> # Download vocabulary from huggingface.co and define model-specific arguments >>> tokenizer = AutoTokenizer.from_pretrained("roberta-base", add_prefix_space=True) ```r6NTZ _from_autouse_fasttokenizer_typetrust_remote_codezPassed `tokenizer_type` z3 does not exist. `tokenizer_type` should be one of z, css|] }|VqdSNr"r#cr"r"r& mr(z0AutoTokenizer.from_pretrained..r*zt`use_fast` is set to `True` but the tokenizer class does not have a fast version. Falling back to the slow version.zTokenizer class z is not currently imported.rCtokenizer_classauto_maprOrrZ code_revisionZFastz- does not exist or is not currently imported.z The encoder model config class: z3 is different from the decoder model config class: z. It is not recommended to use the `AutoTokenizer.from_pretrained()` method in this case. Please use the encoder and decoder specific tokenizer classes.zzThis tokenizer cannot be instantiated. Please make sure you have `sentencepiece` installed in order to use this tokenizer.z!Unrecognized configuration class z8 to build an AutoTokenizer. Model type should be one of css|] }|jVqdSrV)r+rWr"r"r&rYr()poprrE ValueErrorjoinkeysr8rFwarningfrom_pretrainedrN isinstancetuplelistr rrZr3r[typer1r r endswithrdecoderencoder __class__rr+)clsr:inputsrKr6rSrTrUrZZtokenizer_class_tupleZtokenizer_class_nameZtokenizer_fast_class_nameZtokenizer_configZconfig_tokenizer_classZtokenizer_auto_mapZhas_remote_codeZhas_local_codeZ class_ref_Ztokenizer_class_candidateZ model_typeZtokenizer_class_pyZtokenizer_class_fastr"r"r&rasO                           zAutoTokenizer.from_pretrainedNcCs|dur|durtd|dur2t|tr2td|durLt|trLtd|dur|durt|tr|j|krtd|jd|d|tjvrt|\}}|dur|}|dur|}t|||fdS)a Register a new tokenizer in this mapping. Args: config_class ([`PretrainedConfig`]): The configuration corresponding to the model to register. slow_tokenizer_class ([`PretrainedTokenizer`], *optional*): The slow tokenizer to register. slow_tokenizer_class ([`PretrainedTokenizerFast`], *optional*): The fast tokenizer to register. NzKYou need to pass either a `slow_tokenizer_class` or a `fast_tokenizer_classz:You passed a fast tokenizer in the `slow_tokenizer_class`.z:You passed a slow tokenizer in the `fast_tokenizer_class`.zThe fast tokenizer class you are passing has a `slow_tokenizer_class` attribute that is not consistent with the slow tokenizer class you passed (fast tokenizer has z and you passed z!. Fix one of those so they match!)r] issubclassrr slow_tokenizer_classr1r2register)Z config_classrnZfast_tokenizer_classZ existing_slowZ existing_fastr"r"r&ros8   zAutoTokenizer.register)NN) r+ __module__ __qualname____doc__rR classmethodrrraror"r"r"r&rOs DrO)NFFNNNFr9)5rrr-rIos collectionsrtypingrrrrrZconfiguration_utilsr Zdynamic_module_utilsr r Ztokenization_utilsr Ztokenization_utils_baser utilsrrrrrZencoder_decoderrZ auto_factoryrZconfiguration_autorrrrrZtokenization_utils_fastrZ get_loggerr+rFrstr__annotations__r1r,ZCONFIG_TO_TYPEr8PathLikeboolrNrOr"r"r"r&s`        *J  d