a
    
d                     @   s|   d dl Z d dlZd dlZd dlmZ d dlZd dlZe dd Ze dd Z	dd Z
d	d
 Zdd ZG dd deZdS )    N)	lru_cachec                   C   s   t jt jt jtdS )Nzbpe_simple_vocab_16e6.txt.gz)ospathjoindirnameabspath__file__ r	   r	   ^/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/clip/simple_tokenizer.pydefault_bpe
   s    r   c                  C   s   t ttdtdd t ttdtdd  t ttdtdd  } | dd }d	}td
D ],}|| vrf| | |d
|  |d7 }qfdd |D }tt| |S )a9  
    Returns list of utf-8 byte and a corresponding list of unicode strings.
    The reversible bpe codes work on unicode strings.
    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
    This is a signficant percentage of your normal, say, 32K bpe vocab.
    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
    And avoids mapping to whitespace/control characters the bpe code barfs on.
    !~      ¡   ¬   ®   ÿNr      c                 S   s   g | ]}t |qS r	   )chr).0nr	   r	   r
   
<listcomp>"       z$bytes_to_unicode.<locals>.<listcomp>)listrangeordappenddictzip)bscsr   br	   r	   r
   bytes_to_unicode   s    N

r"   c                 C   s6   t  }| d }| dd D ]}|||f |}q|S )zReturn set of symbol pairs in a word.
    Word is represented as tuple of symbols (symbols being variable-length strings).
    r   r   N)setadd)wordpairsZ	prev_charcharr	   r	   r
   	get_pairs&   s    r(   c                 C   s"   t | } tt| } |  S N)ftfyZfix_texthtmlunescapestriptextr	   r	   r
   basic_clean2   s    
r0   c                 C   s   t dd| } |  } | S )Nz\s+ )resubr-   r.   r	   r	   r
   whitespace_clean8   s    r4   c                   @   s8   e Zd Ze fedddZdd Zdd Zdd	 Zd
S )SimpleTokenizer)bpe_pathc                 C   s   t  | _dd | j D | _t| dd}|dd }dd |D }t	t  
 }|d	d |D  }|D ]}|d
| qv|ddg tt|tt|| _dd | j D | _tt|tt|| _ddd| _tdtj| _d S )Nc                 S   s   i | ]\}}||qS r	   r	   r   kvr	   r	   r
   
<dictcomp>A   r   z,SimpleTokenizer.__init__.<locals>.<dictcomp>utf-8
r   i  c                 S   s   g | ]}t | qS r	   )tuplesplit)r   merger	   r	   r
   r   D   r   z,SimpleTokenizer.__init__.<locals>.<listcomp>c                 S   s   g | ]}|d  qS )</w>r	   )r   r9   r	   r	   r
   r   F   r    <|startoftext|><|endoftext|>c                 S   s   i | ]\}}||qS r	   r	   r7   r	   r	   r
   r:   K   r   )rB   rC   z[<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+)r"   byte_encoderitemsbyte_decodergzipopenreaddecoder>   r   valuesr   r   extendr   r   r   lenencoderdecoder	bpe_rankscacher2   compile
IGNORECASEpat)selfr6   ZmergesZvocabr?   r	   r	   r
   __init__?   s    zSimpleTokenizer.__init__c           
         sv  | j v r j | S t|d d |d d f }t|}|sF|d S t| fddd}| jvrhq^|\}}g }d}|t|k r4z&|||}	||||	  |	}W n$   |||d   Y q4Y n0 || |kr|t|d k r||d  |kr|||  |d7 }qx|||  |d7 }qxt|}|}t|dkrTq^qFt|}qFd		|}| j |< |S )
Nr@   c                    s    j | tdS )Ninf)rP   getfloat)pairrU   r	   r
   <lambda>Z   r   z%SimpleTokenizer.bpe.<locals>.<lambda>)keyr   r      r1   )
rQ   r=   r(   minrP   rM   indexrL   r   r   )
rU   tokenr%   r&   ZbigramfirstsecondZnew_wordijr	   r\   r
   bpeP   sB    


2




zSimpleTokenizer.bpec                    sn   g }t t| }t j|D ]F}d fdd|dD }| fdd 	|
dD  q"|S )NrA   c                 3   s   | ]} j | V  qd S r)   )rD   )r   r!   r\   r	   r
   	<genexpr>}   r   z)SimpleTokenizer.encode.<locals>.<genexpr>r;   c                 3   s   | ]} j | V  qd S r)   )rN   )r   Z	bpe_tokenr\   r	   r
   rh   ~   r   r1   )r4   r0   lowerr2   findallrT   r   encoderL   rg   r>   )rU   r/   Z
bpe_tokensrb   r	   r\   r
   rk   y   s    &zSimpleTokenizer.encodec                    sD   d  fdd|D }t fdd|D jddddd	}|S )
NrA   c                    s   g | ]} j | qS r	   )rO   )r   rb   r\   r	   r
   r      r   z*SimpleTokenizer.decode.<locals>.<listcomp>c                    s   g | ]} j | qS r	   )rF   )r   cr\   r	   r
   r      r   r;   replace)errorsr@   r1   )r   	bytearrayrJ   rm   )rU   tokensr/   r	   r\   r
   rJ      s    (zSimpleTokenizer.decodeN)	__name__
__module____qualname__r   strrV   rg   rk   rJ   r	   r	   r	   r
   r5   >   s   )r5   )rG   r+   r   	functoolsr   r*   Zregexr2   r   r"   r(   r0   r4   objectr5   r	   r	   r	   r
   <module>   s   

