a
    dK                     @   s  d dl mZmZ d dlZd dlmZ d dlmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZ d dlmZ er|ddlmZ ddlmZmZmZ dd	lmZmZm Z  dd
l!m"Z" d dl#m#Z# G dd deeZG dd deZ$G dd deZ%G dd deZ&edddZ'G dd de(Z)G dd dZ*G dd dZ+G dd dZ,dd Z-d d! Z.G d"d# d#Z/e(d$d%d&Z0G d'd( d(Z1G d)d* d*Z2ee)ge)f Z3G d+d, d,eZ4G d-d. d.e4Z5G d/d0 d0e4Z6dS )1    )abstractmethodABCN)suppress)TypeVarTypeListDictIterator
CollectionCallableOptional	FrozenSetAnyPatternClassVarTYPE_CHECKING)
ModuleType   )	LexerConf)classifyget_regexp_width	Serialize)UnexpectedCharactersLexErrorUnexpectedToken)TOKEN_DEFAULT_PRIORITY)copyc                   @   s   e Zd ZU eed< ee ed< ee ed< ee ed< deee ee dddd	Zd
d Z	dd Z
dd ZeedddZeeedddZeeedddZdd ZdS )r   valueflagsrawtype N)r   r   r   returnc                 C   s   || _ t|| _|| _d S N)r   	frozensetr   r   )selfr   r   r   r!   r!   S/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/lark/lexer.py__init__   s    
zPattern.__init__c                 C   s   t |  S r#   )repr	to_regexpr%   r!   r!   r&   __repr__"   s    zPattern.__repr__c                 C   s   t t| | j| jfS r#   )hashr    r   r   r*   r!   r!   r&   __hash__&   s    zPattern.__hash__c                 C   s(   t | t |ko&| j|jko&| j|jkS r#   )r    r   r   r%   otherr!   r!   r&   __eq__)   s    zPattern.__eq__r"   c                 C   s
   t  d S r#   NotImplementedErrorr*   r!   r!   r&   r)   ,   s    zPattern.to_regexpc                 C   s
   t  d S r#   r2   r*   r!   r!   r&   	min_width0   s    zPattern.min_widthc                 C   s
   t  d S r#   r2   r*   r!   r!   r&   	max_width5   s    zPattern.max_widthc                 C   s   | j D ]}d||f }q|S )Nz(?%s:%s))r   )r%   r   fr!   r!   r&   
_get_flags:   s    
zPattern._get_flags)r!   N)__name__
__module____qualname__str__annotations__r
   r   r   r'   r+   r-   r0   r   r)   propertyintr4   r5   r7   r!   r!   r!   r&   r      s"   
r   c                   @   sT   e Zd ZU dZdZee ed< edddZe	e
dddZe	e
dd	d
ZdS )
PatternStr)r   r   r;   r    r1   c                 C   s   |  t| jS r#   )r7   reescaper   r*   r!   r!   r&   r)   E   s    zPatternStr.to_regexpc                 C   s
   t | jS r#   lenr   r*   r!   r!   r&   r4   H   s    zPatternStr.min_widthc                 C   s
   t | jS r#   rB   r*   r!   r!   r&   r5   L   s    zPatternStr.max_widthN)r8   r9   r:   __serialize_fields__r    r   r;   r<   r)   r=   r>   r4   r5   r!   r!   r!   r&   r?   @   s   
r?   c                   @   s`   e Zd ZU dZdZee ed< edddZdZ	dd	 Z
eedd
dZeedddZdS )	PatternRE)r   r   _widthr@   r    r1   c                 C   s   |  | jS r#   )r7   r   r*   r!   r!   r&   r)   V   s    zPatternRE.to_regexpNc                 C   s   | j d u rt|  | _ | j S r#   )rF   r   r)   r*   r!   r!   r&   
_get_widthZ   s    
zPatternRE._get_widthc                 C   s   |   d S Nr   rG   r*   r!   r!   r&   r4   _   s    zPatternRE.min_widthc                 C   s   |   d S )Nr   rI   r*   r!   r!   r&   r5   c   s    zPatternRE.max_width)r8   r9   r:   rD   r    r   r;   r<   r)   rF   rG   r=   r>   r4   r5   r!   r!   r!   r&   rE   Q   s   
rE   c                   @   s`   e Zd ZU dZeefZeed< e	ed< e
ed< efee	e
ddddZd	d
 ZedddZdS )TerminalDef)namepatternpriorityrK   rL   rM   N)rK   rL   rM   r"   c                 C   s(   t |tsJ ||| _|| _|| _d S r#   )
isinstancer   rK   rL   rM   )r%   rK   rL   rM   r!   r!   r&   r'   p   s    zTerminalDef.__init__c                 C   s   dt | j| j| jf S )Nz
%s(%r, %r))r    r8   rK   rL   r*   r!   r!   r&   r+   v   s    zTerminalDef.__repr__r1   c                 C   s$   | j dr| jjp| j S | j S d S )N__)rK   
startswithrL   r   r*   r!   r!   r&   	user_repry   s    zTerminalDef.user_repr)r8   r9   r:   rD   r?   rE   Z__serialize_namespace__r;   r<   r   r>   r   r'   r+   rQ   r!   r!   r!   r&   rJ   h   s   
rJ   _TToken)boundc                       s   e Zd ZU dZdZeed< eed< eed< eed< eed< eed< eed	< eed
< d fdd	Z	de
e e
e d dddZeee eed edddZdd Zdd Zdd Zdd ZejZ  ZS )rS   a  A string with meta-information, that is produced by the lexer.

    When parsing text, the resulting chunks of the input that haven't been discarded,
    will end up in the tree as Token instances. The Token class inherits from Python's ``str``,
    so normal string comparisons and operations will work as expected.

    Attributes:
        type: Name of the token (as specified in grammar)
        value: Value of the token (redundant, as ``token.value == token`` will always be true)
        start_pos: The index of the token in the text
        line: The line of the token in the text (starting with 1)
        column: The column of the token in the text (starting with 1)
        end_line: The line where the token ends
        end_column: The next column after the end of the token. For example,
            if the token is a single character with a column value of 4,
            end_column will be 5.
        end_pos: the index where the token ends (basically ``start_pos + len(token)``)
    )r    	start_posr   linecolumnend_line
end_columnend_posr    rU   r   rV   rW   rX   rY   rZ   Nc	           
         sF   t t| | |}	||	_||	_||	_||	_||	_||	_||	_	||	_
|	S r#   )superrS   __new__r    rU   r   rV   rW   rX   rY   rZ   )
clstype_r   rU   rV   rW   rX   rY   rZ   inst	__class__r!   r&   r\      s    zToken.__new__)r^   r   r"   c                 C   s*   t |d ur|n| j|d ur |n| j| S r#   )rS   new_borrow_posr    r   )r%   r^   r   r!   r!   r&   update   s
    zToken.update)r]   r^   r   borrow_tr"   c              	   C   s"   | |||j |j|j|j|j|jS r#   )rU   rV   rW   rX   rY   rZ   )r]   r^   r   rd   r!   r!   r&   rb      s    zToken.new_borrow_posc                 C   s   | j | j| j| j| j| jffS r#   )ra   r    r   rU   rV   rW   r*   r!   r!   r&   
__reduce__   s    zToken.__reduce__c                 C   s   d| j | jf S )NzToken(%r, %r))r    r   r*   r!   r!   r&   r+      s    zToken.__repr__c                 C   s   t | j| j| j| j| jS r#   )rS   r    r   rU   rV   rW   )r%   memor!   r!   r&   __deepcopy__   s    zToken.__deepcopy__c                 C   s&   t |tr| j|jkrdS t| |S )NF)rN   rS   r    r;   r0   r.   r!   r!   r&   r0      s    zToken.__eq__)NNNNNN)NN)r8   r9   r:   __doc__	__slots__r;   r<   r>   r   r\   r   rc   classmethodr   rR   rb   re   r+   rg   r0   r-   __classcell__r!   r!   r`   r&   rS      s&   
c                   @   s0   e Zd ZdZdd Zdd Zdeddd	Zd
S )LineCounter)char_posrV   rW   line_start_posnewline_charc                 C   s"   || _ d| _d| _d| _d| _d S )Nr   r   )ro   rm   rV   rW   rn   )r%   ro   r!   r!   r&   r'      s
    zLineCounter.__init__c                 C   s&   t |tstS | j|jko$| j|jkS r#   )rN   rl   NotImplementedrm   ro   r.   r!   r!   r&   r0      s    
zLineCounter.__eq__T)tokenc                 C   sb   |r:| | j}|r:|  j|7  _| j|| j d | _|  jt|7  _| j| j d | _dS )zConsume a token and calculate the new line & column.

        As an optional optimization, set test_newline=False if token doesn't contain a newline.
        r   N)countro   rV   rm   rindexrn   rC   rW   )r%   rq   Ztest_newlinenewlinesr!   r!   r&   feed   s    zLineCounter.feedN)T)r8   r9   r:   ri   r'   r0   rS   ru   r!   r!   r!   r&   rl      s   rl   c                   @   s   e Zd Zdd Zdd ZdS )UnlessCallbackc                 C   s
   || _ d S r#   )scanner)r%   rw   r!   r!   r&   r'      s    zUnlessCallback.__init__c                 C   s"   | j |jd}|r|\}|_|S rH   )rw   matchr   r    )r%   tres_valuer!   r!   r&   __call__   s    
zUnlessCallback.__call__Nr8   r9   r:   r'   r|   r!   r!   r!   r&   rv      s   rv   c                   @   s   e Zd Zdd Zdd ZdS )	CallChainc                 C   s   || _ || _|| _d S r#   )	callback1	callback2cond)r%   r   r   r   r!   r!   r&   r'      s    zCallChain.__init__c                 C   s"   |  |}| |r| |S |S r#   )r   r   r   )r%   ry   t2r!   r!   r&   r|      s    
zCallChain.__call__Nr}   r!   r!   r!   r&   r~      s   r~   c                 C   s    |  |||}|r|dS d S rH   )rx   group)re_regexpsr   mr!   r!   r&   
_get_match   s    r   c              	      s   t | dd }t|dks&J | t  i }|tg D ]}g }|tg D ]V}|j|jkrbqP|jj	}	|	t
||j |	|krP|| |jj|jjkrP | qP|r<tt|||d|d||j< q< fdd| D }
|
|fS )Nc                 S   s
   t | jS r#   )r    rL   ry   r!   r!   r&   <lambda>      z _create_unless.<locals>.<lambda>   T)match_whole	use_bytesc                    s   g | ]}| vr|qS r!   r!   .0ry   Zembedded_strsr!   r&   
<listcomp>  r   z"_create_unless.<locals>.<listcomp>)r   rC   keyssetgetrE   r?   rM   rL   r   r   r)   appendr   addrv   ScannerrK   )	terminalsg_regex_flagsr   r   Ztokens_by_typecallbackZretokZunlessZstrtokr   Znew_terminalsr!   r   r&   _create_unless  s$    
r   c                   @   s&   e Zd Zd	ddZdd Zdd ZdS )
r   Fc                 C   sF   || _ || _|| _|| _|| _dd | j D | _| |t|| _d S )Nc                 S   s   h | ]
}|j qS r!   rK   r   r!   r!   r&   	<setcomp>   r   z#Scanner.__init__.<locals>.<setcomp>)	r   r   r   r   r   allowed_types_build_mresrC   _mres)r%   r   r   r   r   r   r!   r!   r&   r'     s    zScanner.__init__c                    s   | j r
dnd g }|rd fdd|d | D }| jrF|d}z| j|| j}W n" ty|   | ||d  Y S 0 |	|dd	 |j
 D f ||d  }q|S )
N$ |c                 3   s&   | ]}d |j |j   f V  qdS )z
(?P<%s>%s)N)rK   rL   r)   r   Zpostfixr!   r&   	<genexpr>+  r   z&Scanner._build_mres.<locals>.<genexpr>zlatin-1r   c                 S   s   i | ]\}}||qS r!   r!   )r   nir!   r!   r&   
<dictcomp>3  r   z'Scanner._build_mres.<locals>.<dictcomp>)r   joinr   encoder   compiler   AssertionErrorr   r   
groupindexitems)r%   r   max_sizeZmresrL   mrer!   r   r&   r   $  s     
zScanner._build_mresc                 C   s<   | j D ]0\}}|||}|r|d||j f  S qd S rH   )r   rx   r   	lastindex)r%   textposr   Ztype_from_indexr   r!   r!   r&   rx   7  s    zScanner.matchN)F)r8   r9   r:   r'   r   rx   r!   r!   r!   r&   r     s   
r   rc                 C   s0   d| v p.d| v p.d| v p.d| v p.d| v o.d| v S )zExpressions that may indicate newlines in a regexp:
        - newlines (\n)
        - escaped newline (\\n)
        - anything but ([^...])
        - any-char (.) when the flag (?s) exists
        - spaces (\s)
    
z\nz\sz[^z(?s.r!   r   r!   r!   r&   _regexp_has_newline>  s    r   c                   @   s.   e Zd ZdZdZd
ddZdd Zdd	 ZdS )
LexerStatezRepresents the current state of the lexer as it scans the text
    (Lexer objects are only instanciated per grammar, not per text)
    )r   line_ctr
last_tokenNc                 C   s,   || _ |ptt|trdnd| _|| _d S )N   
r   )r   rl   rN   bytesr   r   )r%   r   r   r   r!   r!   r&   r'   P  s    zLexerState.__init__c                 C   s2   t |tstS | j|ju o0| j|jko0| j|jkS r#   )rN   r   rp   r   r   r   r.   r!   r!   r&   r0   U  s    
zLexerState.__eq__c                 C   s   t | | jt| j| jS r#   )r    r   r   r   r   r*   r!   r!   r&   __copy__[  s    zLexerState.__copy__)NN)r8   r9   r:   rh   ri   r'   r0   r   r!   r!   r!   r&   r   I  s
   
r   c                   @   sH   e Zd ZdZdedddZededddZd	d
 Z	dd Z
eZdS )LexerThreadzTA thread that ties a lexer instance and a lexer state, to be used by the parser
    Lexer)lexerlexer_statec                 C   s   || _ || _d S r#   )r   state)r%   r   r   r!   r!   r&   r'   c  s    zLexerThread.__init__)r   r   c                 C   s   | |t |S r#   r   )r]   r   r   r!   r!   r&   	from_textg  s    zLexerThread.from_textc                 C   s   | j | j|S r#   )r   lexr   )r%   parser_stater!   r!   r&   r   k  s    zLexerThread.lexc                 C   s   t | | jt| jS r#   )r    r   r   r   r*   r!   r!   r&   r   n  s    zLexerThread.__copy__N)r8   r9   r:   rh   r   r'   rj   r;   r   r   r   rS   Z_Tokenr!   r!   r!   r&   r   _  s   r   c                   @   s2   e Zd ZdZeeeee dddZ	dd Z
dS )r   zlLexer interface

    Method Signatures:
        lex(self, lexer_state, parser_state) -> Iterator[Token]
    r   r   r"   c                 C   s   t S r#   )rp   )r%   r   r   r!   r!   r&   r   |  s    z	Lexer.lexc                 C   s   t |S )Z
Deprecatedr   )r%   r   r!   r!   r&   make_lexer_state  s    zLexer.make_lexer_stateN)r8   r9   r:   rh   r   r   r   r	   rS   r   r   r!   r!   r!   r&   r   v  s   r   c                   @   s   e Zd ZU ee ed< ee ed< ee ed< eee	f ed< eee	f ed< e
ed< ddd	d
dZdd Zedd Zdd Zeeee dddZdeeedddZdS )
BasicLexerr   ignore_typesnewline_typesuser_callbacksr   r@   r   N)confr"   c              	   C   s:  t |j}tdd |D s$J ||j| _|js|D ]j}z| j|j |j	 W n* | jj
y~   td|j|jf Y n0 |jjdkr6td|j|jf q6t|jdd |D kstdt|jd	d |D   td
d |D | _t|j| _|jdd d || _|j| _|j	| _	|j| _|j| _d | _d S )Nc                 s   s   | ]}t |tV  qd S r#   )rN   rJ   r   r!   r!   r&   r     r   z&BasicLexer.__init__.<locals>.<genexpr>zCannot compile token %s: %sr   z3Lexer does not allow zero-width terminals. (%s: %s)c                 S   s   h | ]
}|j qS r!   r   r   r!   r!   r&   r     r   z&BasicLexer.__init__.<locals>.<setcomp>z$Ignore terminals are not defined: %sc                 S   s   h | ]
}|j qS r!   r   r   r!   r!   r&   r     r   c                 s   s"   | ]}t |j r|jV  qd S r#   )r   rL   r)   rK   r   r!   r!   r&   r     r   c                 S   s"   | j  | jj t| jj | jfS r#   )rM   rL   r5   rC   r   rK   )xr!   r!   r&   r     r   z%BasicLexer.__init__.<locals>.<lambda>)key)listr   allZ	re_moduler@   Zskip_validationr   rL   r)   r   errorr   rK   r4   r   ignorer$   r   r   sort	callbacksr   r   terminals_by_name_scanner)r%   r   r   ry   r!   r!   r&   r'     s,    
 zBasicLexer.__init__c                    s   t | j| j| j| j\}| _t| j s.J | j	 D ]>\ } | jv rlt
| j  | fdd| j < q8|| j < q8t|| j| j| j| _d S )Nc                    s
   | j  kS r#   )r    r   r^   r!   r&   r     r   z+BasicLexer._build_scanner.<locals>.<lambda>)r   r   r   r@   r   r   r   valuesr   r   r~   r   r   )r%   r   r6   r!   r   r&   _build_scanner  s    
"zBasicLexer._build_scannerc                 C   s   | j d u r|   | j S r#   )r   r   r*   r!   r!   r&   rw     s    
zBasicLexer.scannerc                 C   s   | j ||S r#   )rw   rx   )r%   r   r   r!   r!   r&   rx     s    zBasicLexer.match)r   r   r"   c                 c   s<   t t  | ||V  q
W d    n1 s.0    Y  d S r#   )r   EOFError
next_token)r%   r   r   r!   r!   r&   r     s    
zBasicLexer.lex)	lex_stater   r"   c           
   
   C   sP  |j }|jt|jk rD| |j|j}|sp| jj| j }|sDdh}t|j|j|j	|j
||jod|jg|| jd|\}}|| jvr t|||j|j	|j
}|||| jv  |j	|_|j
|_|j|_|j| jv r| j|j |}t|tstd| ||_|S || jv r0t|||j|j	|j
}	| j| |	 |||| jv  qt| d S )Nz<END-OF-FILE>)allowedtoken_historyr   r   z+Callbacks must return a token (returned %r))r   rm   rC   r   rx   rw   r   r   r   rV   rW   r   r   rS   ru   r   rX   rY   rZ   r    r   rN   r   r   )
r%   r   r   r   rz   r   r   r^   ry   r   r!   r!   r&   r     s:    
zBasicLexer.next_token)N)r8   r9   r:   r
   rJ   r<   r   r;   r   	_Callbackr   r'   r   r=   rw   rx   r   r   r	   rS   r   r   r!   r!   r!   r&   r     s   
!
r   c                   @   sb   e Zd ZU eeef ed< eed< ddeeee f ee ddddZe	e
ee d	d
dZdS )ContextualLexerlexers
root_lexerr!   r   N)r   statesalways_acceptr"   c              	      s   t |j}|j t|}||_i }i | _| D ]\}}t|}	z||	 }
W nX ty   t|t|j	B t|B }t|} fdd|D |_t
|}
|
||	< Y n0 |
| j|< q0|j|u sJ t
|| _d S )Nc                    s   g | ]}| v r | qS r!   r!   )r   r   r   r!   r&   r     r   z,ContextualLexer.__init__.<locals>.<listcomp>)r   r   r   r   r   r   r$   KeyErrorr   r   r   r   )r%   r   r   r   r   Z	trad_confZlexer_by_tokensr   Zacceptsr   r   Z
lexer_confr!   r   r&   r'     s&    
zContextualLexer.__init__r   c                 c   s   z | j |j }|||V  qW n~ ty2   Y nn ty } zVz2|j}| j||}t||j||g| jj	dW n ty   |Y n0 W Y d }~n
d }~0 0 d S )N)r   r   r   )
r   positionr   r   r   r   r   r   r   r   )r%   r   r   r   er   rq   r!   r!   r&   r     s    zContextualLexer.lex)r!   )r8   r9   r:   r   r;   r   r<   r
   r'   r   r   r	   rS   r   r!   r!   r!   r&   r     s   
&r   )7abcr   r   r@   
contextlibr   typingr   r   r   r   r	   r
   r   r   r   r   r   Z	REPatternr   r   typesr   commonr   utilsr   r   r   
exceptionsr   r   r   Zgrammarr   r   r?   rE   rJ   rR   r;   rS   rl   rv   r~   r   r   r   r   r   r   r   r   r   r   r!   r!   r!   r&   <module>   s:   <*G&i