a  d@sdZddlZddlmZmZmZddlmZddlm Z ddl m Z e e Zdd iZdd d d iZd d d ZddZGdddeZdS)zTokenization classes for ESM.N)ListOptionalUnion)PreTrainedTokenizer) AddedToken)logging vocab_file vocab.txtzGhttps://huggingface.co/facebook/esm2_t6_8M_UR50D/resolve/main/vocab.txtzIhttps://huggingface.co/facebook/esm2_t12_35M_UR50D/resolve/main/vocab.txt)zfacebook/esm2_t6_8M_UR50Dzfacebook/esm2_t12_35M_UR50DicCsHt|d*}|}dd|DWdS1s:0YdS)NrcSsg|] }|qS)strip).0lr r q/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/transformers/models/esm/tokenization_esm.py ,z#load_vocab_file..)openread splitlines)r flinesr r rload_vocab_file)s  rcseZdZdZeZeZeZ ddgZ d+fd d Z e e d d d Ze e dddZddZd,ddZddZe e dddZe e d ddZd-ee eee ee dddZd.eeeeee d d!d"Zd#d$Zee d%d&d'Zd/eee eefee d(fd)d* ZZ S)0 EsmTokenizerz& Constructs an ESM tokenizer. Z input_idsZattention_maskc sxtjfi|t||_tt|j|_ddt|jD|_||_||_ ||_ ||_ ||_ |j|_ ||j dS)NcSsi|]\}}||qSr r )rindtokr r r Frz)EsmTokenizer.__init__..)super__init__r all_tokensdict enumerate _id_to_token _token_to_id unk_token cls_token pad_token mask_token eos_tokenZunique_no_split_tokensZ _create_trie)selfr r)r*r+r,r-kwargs __class__r rr#9s  zEsmTokenizer.__init__)indexreturncCs|j||jSNr'getr)r.r2r r r_convert_id_to_tokenOsz!EsmTokenizer._convert_id_to_token)tokenr3cCs|j||j|jSr4r(r6r)r.r9r r r_convert_token_to_idRsz!EsmTokenizer._convert_token_to_idcKs|Sr4)split)r.textr/r r r _tokenizeUszEsmTokenizer._tokenizeFcCs t|jSr4)lenr')r.with_added_tokensr r rget_vocab_sizeXszEsmTokenizer.get_vocab_sizecCsddt|jDS)NcSsi|]\}}||qSr r )rir9r r rr!\rz*EsmTokenizer.get_vocab..)r&r$r.r r r get_vocab[szEsmTokenizer.get_vocabcCs|j||j|jSr4r:r;r r r token_to_id^szEsmTokenizer.token_to_idcCs|j||jSr4r5r7r r r id_to_tokenaszEsmTokenizer.id_to_tokenN) token_ids_0 token_ids_1r3cCs^|jg}|jg}|dur8|jdur*||S|||Sn|jdurJtd|||||S)Nz=Cannot tokenize multiple sequences when EOS token is not set!)Z cls_token_idZ eos_token_id ValueError)r.rHrIclssepr r r build_inputs_with_special_tokensds  z-EsmTokenizer.build_inputs_with_special_tokens)rHrIalready_has_special_tokensr3csd|r&|durtdfdd|DSdgdgt|dg}|dur`|dgt|dg7}|S)a Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` or `encode_plus` methods. Args: token_ids_0 (`List[int]`): List of ids of the first sequence. token_ids_1 (`List[int]`, *optional*): List of ids of the second sequence. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. NzYou should not supply a second sequence if the provided sequence of ids is already formatted with special tokens for the model.csg|]}|jvrdndqS)r)Zall_special_ids)rr9rDr rrrz8EsmTokenizer.get_special_tokens_mask..rOr)rJr@)r.rHrIrNmaskr rDrget_special_tokens_maskrsz$EsmTokenizer.get_special_tokens_maskcCs`tj||r|dndd}t|d"}|d|jWdn1sP0Y|fS)N-r w )ospathjoinrwriter$)r.Zsave_directoryZfilename_prefixr rr r rsave_vocabularys 0zEsmTokenizer.save_vocabulary)r3cCs |jddS)NF)rA)rBrDr r r vocab_sizeszEsmTokenizer.vocab_size) new_tokensspecial_tokensr3cstj|ddS)NT)r])r" _add_tokens)r.r\r]r0r rr^szEsmTokenizer._add_tokens)rrrrr)F)N)NF)F)!__name__ __module__ __qualname____doc__VOCAB_FILES_NAMESZvocab_files_namesPRETRAINED_VOCAB_FILES_MAPZpretrained_vocab_files_map&PRETRAINED_POSITIONAL_EMBEDDINGS_SIZESZmax_model_input_sizesZmodel_input_namesr#intstrr8r<r?rBrErFrGrrrMboolrQrZpropertyr[rrr^ __classcell__r r r0rr/s@    r)rbrVtypingrrrZtokenization_utilsrZtokenization_utils_baserutilsrZ get_loggerr_loggerrcrdrerrr r r rs"