a
    d                  '   @   s.  d Z ddlZddlZddlZddlmZ ddlmZmZ ddl	Z	ddl
mZ ddlmZ ddlmZmZmZmZmZmZmZ dd	lmZmZmZmZmZmZmZ dd
lmZm Z m!Z!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z(m)Z) ddl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0 ddlm1Z1m2Z2 ddl3m4Z4m5Z5m6Z6m7Z7m8Z8 ddl9m:Z:m;Z< ddl=m>Z>m?Z?m@Z@mAZAmBZB e@ddZCejDEeFZGdgZHdZIdZJe/dZKdddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=$ZLd>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOeJ dPdQdRdSdTdUdVdWdXdYdZd[d\ZMd]gd^d_gg d`dagdbdcgdddcgdedcgdfdggdhgg didjdkgdlgdcdmgg dng dog dpdqgdfgdfdqgdrdsgdtgdugdvgdwdxgdydzgd{d|gd}d~gdgdgg dddgdgdgddgdgddgddgddgd&ZNdZOdZPG dd de>ZQeCG dd deAZRG dd de?ZSG dd de>ZTG dd deAZUG dd deAZVG dd deAZWdd ZXdd ZYdd ZZeCG dd deAZ[dd Z\G dd deBZ]G dd deAZ^dd Z_G dd dZ`eddZaG dd dZbebdeHZcdd Zddd ZeeCG dd deAZfdd Zgdd Zhddgfddgfdg dŢfdg dǢfddgfddgfdg d͢fdg dϢfddgfddgfddgfddgfddgfgZiddۄ ZjdddބZkdd Zldd Zmdeneneee8enf  dddZodddZpdd Zqdd ZrG dd dZsG dd dZtdd Zug fddZvdd ZwdS )z"Parses and creates Grammar objects    N)
namedtuple)copydeepcopy)literal_eval)suppress)ListTupleUnionCallableDictOptionalSequence   )bfsloggerclassify_boolis_id_continueis_id_startbfs_all_uniquesmall_factors)TokenTerminalDef
PatternStr	PatternRE)ParseTreeBuilder)ParsingFrontend)	LexerConf
ParserConf)RuleOptionsRuleTerminalNonTerminalSymbolTOKEN_DEFAULT_PRIORITY)classify
dedup_list)GrammarErrorUnexpectedCharactersUnexpectedToken
ParseErrorUnexpectedInput)TreeSlottedTree)TransformerVisitorv_argsTransformer_InPlaceTransformer_NonRecursiveT)inlineZgrammarsz.larkZimsluxZ	__empty__DOTCOMMACOLON	SEMICOLONPLUSMINUSSTARSLASH	BACKSLASHVBARZQMARKZBANGATHASHZDOLLARPERCENT
CIRCUMFLEXZ	AMPERSANDZ
UNDERSCOREZLESSTHANZMORETHANEQUALZDBLQUOTEZQUOTEZ	BACKQUOTETILDELPARRPARLBRACERBRACELSQBRSQBNEWLINECRLFZTABSPACE)$.,:;+-*/\|?!@#$%^&_<>="'`~(){}[]
z
	 z\(z\)z\[z\]z\{z\}z[+*]|[?](?![a-z])rN   rM   z\|z\.(?!\.)z\.\.re   z(!|![?]?|[?]!?)(?=[_a-z])z_?[a-z][_a-z0-9]*z_?[A-Z][_A-Z0-9]*z"(\\"|\\\\|[^"\n])*?"i?z/(?!/)(\\/|\\\\|[^/])*?/[%s]*z(\r?\n)+\s*z(\r?\n)+\s*\|z[ \t]+z\s*//[^\n]*z\\[ ]*\nz->z%ignorez	%overridez%declarez%extendz%importz[+-]?\d+)Z_LPARZ_RPARZ_LBRAZ_RBRAZ_LBRACEZ_RBRACEOPZ_COLONZ_COMMAZ_ORZ_DOTZ_DOTDOTrB   RULE_MODIFIERSRULETERMINALSTRINGREGEXP_NLZ_NL_ORWSCOMMENTr;   Z_TOZ_IGNOREZ	_OVERRIDEZ_DECLAREZ_EXTENDZ_IMPORTNUMBER_list_itemz_list _item)ruletermignoreimportdeclareoverrideextendru   zBrule_modifiers RULE template_params priority _COLON expansions _NLrp    z_DOT NUMBERz _LBRACE _template_params _RBRACErq   z_template_params _COMMA RULE_expansions)aliasz_expansions _OR aliasz_expansions _NL_OR aliaszexpansion _TO nonterminal	expansion
_expansionz_expansion expr)atomzatom OPzatom TILDE NUMBERz atom TILDE NUMBER _DOTDOT NUMBER)z_LPAR expansions _RPARmaybevalue)terminalnonterminalliteralrangetemplate_usagerr   r   r   z_LBRA expansions _RBRAzSTRING _DOTDOT STRINGz*nonterminal _LBRACE _template_args _RBRACEr   z_template_args _COMMA valuezTERMINAL _COLON expansions _NLz*TERMINAL _DOT NUMBER _COLON expansions _NLz_OVERRIDE rulez_OVERRIDE termz_EXTEND rulez_EXTEND termz_IGNORE expansions _NLz_DECLARE _declare_args _NL)z_IMPORT _import_path _NLz._IMPORT _import_path _LPAR name_list _RPAR _NLz!_IMPORT _import_path _TO name _NL
import_lib
import_rel_import_argsz_DOT _import_argsnamez_import_args _DOT name
_name_listz_name_list _COMMA namesymbolz_declare_args symbolrt   rs   )&startry   rz   r{   Zrule_modifierspriorityZtemplate_paramsZ_template_params
expansionsr   z?aliasr   r   z?exprz?atomr   r   r   z?namez?symbolr   r   r   Z_template_argsr|   r   r   r}   r   r~   Z_import_pathr   r   r   	name_listr   Z_declare_argsr      2   c                   @   s4   e Zd Zdd Zdd Zdd Zdd Zd	d
 ZdS )FindRuleSizec                 C   s
   || _ d S Nkeep_all_tokens)selfr    r   Z/var/www/html/stable-diffusion-webui/venv/lib/python3.9/site-packages/lark/load_grammar.py__init__   s    zFindRuleSize.__init__c                 C   sL   t |tr|jd S t |tr0| jp.|j S |tu r<dS dsHJ |d S )Nr^   F)
isinstancer!   r   
startswithr    r   
filter_out_EMPTY)r   symr   r   r   _will_not_get_removed   s    

z"FindRuleSize._will_not_get_removedc                 c   sH   |D ]>}t |tr|V  qt |tr:| |r2dndV  qdsJ qd S )Nr   r   F)r   intr"   r   )r   argsar   r   r   _args_as_int   s    

zFindRuleSize._args_as_intc                 C   s   t | |S r   )sumr   r   r   r   r   r   r      s    zFindRuleSize.expansionc                 C   s   t | |S r   )maxr   r   r   r   r   r      s    zFindRuleSize.expansionsN)__name__
__module____qualname__r   r   r   r   r   r   r   r   r   r      s
   		r   c                   @   sT   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dd Z
dd ZdS )EBNF_to_BNFc                 C   s"   g | _ i | _d| _d| _d | _d S )NZanonr   )	new_rulesrules_cacheprefixirule_optionsr   r   r   r   r      s
    zEBNF_to_BNF.__init__c                 C   s$   d| j || jf }|  jd7  _|S )Nz
__%s_%s_%dr   )r   r   )r   innernew_namer   r   r   
_name_rule   s    zEBNF_to_BNF._name_rulec                 C   s*   t |}| j||| jf || j|< |S r   )r!   r   appendr   r   )r   keyr   r   tr   r   r   	_add_rule   s    
zEBNF_to_BNF._add_rulec              
   C   sd   z| j | W S  ty^   | |}t|}tdtd|gtd||gg}| ||| Y S 0 d S )Nr   r   )r   KeyErrorr   r!   STr   )r   type_exprr   r   treer   r   r   _add_recurse_rule   s    

zEBNF_to_BNF._add_recurse_rulec              
   C   sr   ||||f}z| j | W S  tyl   | d||f }tdtd|g| |g|  g}| ||| Y S 0 dS )aN  Generate a rule that repeats target ``a`` times, and repeats atom ``b`` times.

        When called recursively (into target), it repeats atom for x(n) times, where:
            x(0) = 1
            x(n) = a(n) * x(n-1) + b

        Example rule when a=3, b=4:

            new_rule: target target target atom atom atom atom

        zrepeat_a%d_b%dr   r   N)r   r   r   r   r   )r   r   btargetr   r   r   r   r   r   r   _add_repeat_rule  s    "zEBNF_to_BNF._add_repeat_rulec           	   	      s    |df}z| j | W S  ty   | d |f }tdfddt D  fddt|D  }| ||| Y S 0 dS )a_  Creates a rule that matches atom 0 to (a*n+b)-1 times.

        When target matches n times atom, and target_opt 0 to n-1 times target_opt,

        First we generate target * i followed by target_opt, for i from 0 to a-1
        These match 0 to n*a - 1 times atom

        Then we generate target * a followed by atom * i, for i from 0 to b-1
        These match n*a to n*a + b-1 times atom

        The created rule will not have any shift/reduce conflicts so that it can be used with lalr

        Example rule when a=3, b=4:

            new_rule: target_opt
                    | target target_opt
                    | target target target_opt

                    | target target target
                    | target target target atom
                    | target target target atom atom
                    | target target target atom atom atom

        optzrepeat_a%d_b%d_optr   c                    s"   g | ]}t d  g| g qS r   r   .0r   )r   
target_optr   r   
<listcomp>3  s   z4EBNF_to_BNF._add_repeat_opt_rule.<locals>.<listcomp>c                    s&   g | ]}t d g  g|  qS r   r   r   )r   r   r   r   r   r   5  s   N)r   r   r   r   r   r   )	r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   _add_repeat_opt_rule  s    z EBNF_to_BNF._add_repeat_opt_rulec                    s   |t k r*td fddt||d D S  }t|tD ]\}}| ||| }q8||kr^|S || d }t|t} }	tdg }
|dd D ]*\}}| |||	|
 }
| |||	 }	q|d \}}| |||	|
 }
tdtd|g|
g gS )z\Generates a rule tree that repeats ``rule`` exactly between ``mn`` to ``mx`` times.
        r   c                    s   g | ]}t d  g| qS r   r   )r   nr{   r   r   r   ?      z1EBNF_to_BNF._generate_repeats.<locals>.<listcomp>r   r   N)REPEAT_BREAK_THRESHOLDr   r   r   SMALL_FACTOR_THRESHOLDr   r   )r   r{   mnmxZ	mn_targetr   r   diffZdiff_factorsZdiff_targetZdiff_opt_targetr   r   r   _generate_repeats:  s"    "

zEBNF_to_BNF._generate_repeatsc                 G   s   |j dkr"tdg }td||gS |j dkr8| d|S |j dkrb| d|}td|tdg gS |j dkrt|d	krt|d
  }}n0tt|\}}||k s|d
k rtd|||f | |||S dsJ |d S )NrV   r   r   rP   plusrR   Zstarre   r   r   'Bad Range for %s (%d..%d isn't allowed)F)r   r   r   lenr   mapr&   r   )r   r{   opr   emptyr   r   r   r   r   r   r   Y  s     




zEBNF_to_BNF.exprc                 C   s:   | j o| j j}t||}tdtg| }td||gS )Nr   r   )r   r   r   	transformr   r   )r   r{   r   Z	rule_sizer   r   r   r   r   v  s    zEBNF_to_BNF.maybeN)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r      s   %r   c                   @   s0   e Zd Zedd Zdd Zdd Zdd Zd	S )
SimplifyRule_Visitorc                 C   s   |  | jrq d S r   )Zexpand_kids_by_datadatar   r   r   r   _flatten  s    zSimplifyRule_Visitor._flattenc                    sh     tjD ]N\ }t|tr|jdkrd_ fddt|jD _   qdqd S )Nr   c              	      s2   g | ]*  td  fddtjD qS )r   c                    s    g | ]\}} |krn|qS r   r   )r   jother)r   optionr   r   r     s   z=SimplifyRule_Visitor.expansion.<locals>.<listcomp>.<listcomp>)visitr   	enumeratechildren)r   r   r   r   )r   r   r     s   z2SimplifyRule_Visitor.expansion.<locals>.<listcomp>)r   r   r   r   r+   r   r%   )r   r   childr   r   r   r     s    

zSimplifyRule_Visitor.expansionc                 C   sN   |j \}}|jdkrJg }|j d j D ]}|td||g q$d|_||_ d S )Nr   r   r   )r   r   r   r   )r   r   r{   Z
alias_namealiasesr   r   r   r   r     s    

zSimplifyRule_Visitor.aliasc                 C   s2   |  | tt|jt|jkr.t|j|_d S r   )r   r   setr   r%   )r   r   r   r   r   r     s    
zSimplifyRule_Visitor.expansionsN)r   r   r   staticmethodr   r   r   r   r   r   r   r   r   }  s
   
	r   c                   @   s$   e Zd Zdd Zdd Zdd ZdS )RuleTreeToTextc                 C   s   |S r   r   )r   xr   r   r   r     s    zRuleTreeToText.expansionsc                 C   s   |d fS r   r   )r   symbolsr   r   r   r     s    zRuleTreeToText.expansionc                 C   s.   |\\}}}|d u s$J ||d|f||j fS )NrQ   r   )r   r   r   _aliasr   r   r   r   r     s    zRuleTreeToText.aliasN)r   r   r   r   r   r   r   r   r   r   r     s   r   c                   @   s$   e Zd ZdZdd Zedd ZdS )PrepareAnonTerminalszfCreate a unique list of anonymous terminals. Attempt to give meaningful names to them when we add themc                 C   s8   || _ dd | j D | _dd |D | _d| _d | _d S )Nc                 S   s   h | ]
}|j qS r   r   r   tdr   r   r   	<setcomp>  r   z0PrepareAnonTerminals.__init__.<locals>.<setcomp>c                 S   s   i | ]}|j |qS r   )patternr   r   r   r   
<dictcomp>  r   z1PrepareAnonTerminals.__init__.<locals>.<dictcomp>r   )	terminalsterm_setterm_reverser   r   )r   r   r   r   r   r     s
    zPrepareAnonTerminals.__init__c                 C   s  |j }|| jv r0|j| j| jjkr0td| d }t|trz| j| j}W q ty   zt	| }W n@ ty   |rt
|rt|d r| | jvr| }Y n0 || jv rd }Y q0 n.t|tr|| jv r| j| j}ndsJ ||d u rd| j }|  jd7  _|| jvrX|| jvs,J | j| t||}|| j|< | j| | jrn| jjrndnt|t}t||dS )Nz+Conflicting flags for the same terminal: %sr   Fz	__ANON_%dr   r   )r   r   flagsr   r&   r   r   r   r   _TERMINAL_NAMESr   r   upperr   r   r   addr   r   r   r   r   r    )r   pr   Z	term_nameZtermdefr   r   r   r   r     s<    
&






 zPrepareAnonTerminals.patternN)r   r   r   __doc__r   inline_argsr   r   r   r   r   r     s   r   c                   @   s(   e Zd ZdZdd Zdd Zdd ZdS )	_ReplaceSymbolszHelper for ApplyTemplatesc                 C   s
   i | _ d S r   )namesr   r   r   r   r     s    z_ReplaceSymbols.__init__c                 C   sH   t |dkr:t|d tr:|d j| jv r:| j|d j S | d|d S )Nr   r   r   )r   r   r"   r   r	  __default__)r   cr   r   r   r     s    *z_ReplaceSymbols.valuec                 C   sD   |d j }|| jv r6| d| j| g|dd   d S | d|d S )Nr   r   r   )r   r	  r
  )r   r  r   r   r   r   r     s    

"z_ReplaceSymbols.template_usageN)r   r   r   r  r   r   r   r   r   r   r   r    s   r  c                   @   s    e Zd ZdZdd Zdd ZdS )ApplyTemplateszIApply the templates, creating new rules that represent the used templatesc                 C   s   || _ t | _t | _d S r   )	rule_defsr  replacerr   created_templates)r   r  r   r   r   r     s    zApplyTemplates.__init__c           	         s   |d j  |dd  }d ddd |D f }|| jvr| j|  fdd| jD \\}}}}t|t|ks~J |t|}tt||| j	_
| j	| | j|g |t|f t|S )Nr   r   z%s{%s}rM   c                 s   s   | ]}|j V  qd S r   r   )r   r   r   r   r   	<genexpr>  r   z0ApplyTemplates.template_usage.<locals>.<genexpr>c                 3   s   | ]}|d   kr|V  qdS )r   Nr   r   r   r   r   r   r    r   )r   joinr  r  r  r   r   dictzipr  r	  r   r   r!   )	r   r  r   Zresult_nameZ_nparamsr   optionsZresult_treer   r   r   r     s    

zApplyTemplates.template_usageN)r   r   r   r  r   r   r   r   r   r   r     s   r  c                    s   t  fdd|D S )Nc                 3   s   | ]}  |V  qd S r   )rfind)r   r  sr   r   r    r   z_rfind.<locals>.<genexpr>)r   )r  choicesr   r  r   _rfind  s    r  c              
   C   s   d}t | }|D ]j}||7 }|dkrzt|}W n tyN   td|  Y n0 |dkrb|d7 }n|dvrr|d7 }||7 }q|dddd	}d
| }zt|} W n. ty } zt| |W Y d }~n
d }~0 0 | S )Nr   rT   z/Literal ended unexpectedly (bad escaping): `%r`\\ZUuxnftrz\"rb   rc   z\'z	u'''%s''')iternextStopIterationr&   replacer   SyntaxError)r  wr   r   Zn2Zto_evaler   r   r   eval_escaping  s*    

 r$  c                 C   s:  t | tsJ | j}t|dd }|dks.J ||d  }tdd |D sTJ || jdkrnd|v rntd| jd	krd|v rd
|vrtd|d | }|d |d kr|d dv sJ |dd }t|}|dkrtd|  | jdkr|dd}t	||| jdS | jd	kr(t
||| jdS ds6J dd S )Nz/"r   r   c                 s   s   | ]}|t v V  qd S r   )	_RE_FLAGS)r   fr   r   r   r  ;  r   z&_literal_to_pattern.<locals>.<genexpr>rs   rl   z*You cannot put newlines in string literalsrt   r   zLYou can only use newlines in regular expressions with the `x` (verbose) flagr   z"/r   z$Empty terminals are not allowed (%s)r  rT   )rawFz:Invariant failed: literal.type not in ["STRING", "REGEXP"])r   r   r   r  alltyper&   r$  r   r   r   )r   vZ
flag_startr  r   r  r   r   r   _literal_to_pattern5  s,     r+  c                   @   s   e Zd Zdd Zdd ZdS )PrepareLiteralsc                 C   s   t dt|gS )Nr   )r   r+  )r   r   r   r   r   r   X  s    zPrepareLiterals.literalc                 C   s   |j |j   krdksn J |jdd }|jdd }tt|tt|  kr^dksdn J d||f }tdt|gS )Nrs   r   r   z[%s-%s]r   )r)  r   r   r$  r   r   )r   r   endregexpr   r   r   r   [  s    *zPrepareLiterals.rangeN)r   r   r   r   r   r   r   r   r   r,  V  s   r,  c                 C   s
   t | dS )Nr   )r   )r.  Z	flags_setr   r   r   _make_joined_patternd  s    r/  c                   @   sD   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd Zdd Z	dS )TerminalTreeToPatternc                 C   s
   |\}|S r   r   )r   Zpsr  r   r   r   r   h  s    zTerminalTreeToPattern.patternc                 C   sD   |sJ t |dkr|d S ddd |D }t|dd |D S )Nr   r   r   c                 s   s   | ]}|  V  qd S r   	to_regexpr   r   r   r   r  q  r   z2TerminalTreeToPattern.expansion.<locals>.<genexpr>c                 S   s   h | ]
}|j qS r   r  r   r   r   r   r   r  r   z2TerminalTreeToPattern.expansion.<locals>.<setcomp>)r   r  r/  )r   itemsr   r   r   r   r   l  s
    zTerminalTreeToPattern.expansionc                 C   sP   t |dkr|d S |jdd d dddd	 |D  }t|d
d |D S )Nr   r   c                 S   s   | j  | j t| j fS r   )	max_widthZ	min_widthr   r   r   r   r   r   <lambda>z  r   z2TerminalTreeToPattern.expansions.<locals>.<lambda>)r   z(?:%s)rU   c                 s   s   | ]}|  V  qd S r   r1  r   r   r   r   r  |  r   z3TerminalTreeToPattern.expansions.<locals>.<genexpr>c                 S   s   h | ]
}|j qS r   r3  r   r   r   r   r   }  r   z3TerminalTreeToPattern.expansions.<locals>.<setcomp>)r   sortr  r/  )r   Zexpsr   r   r   r   r   t  s
    z TerminalTreeToPattern.expansionsc                 C   s   |d d \}}|dkrtt |dkr6dt|d  }qtt|dd  \}}||k rftd|||f d||f }nt |dksJ td| |f |jS )N   re      z{%d}r   z{%d,%d}z(?:%s)%s)r   r   r   r&   r   r2  r  )r   r   r   r   r   r   r   r   r   r     s    zTerminalTreeToPattern.exprc                 C   s   |  |dg S )NrV   )r   )r   r   r   r   r   r     s    zTerminalTreeToPattern.maybec                 C   s   t dd S )NzBAliasing not allowed in terminals (You used -> in the wrong place))r&   )r   r   r   r   r   r     s    zTerminalTreeToPattern.aliasc                 C   s   |d S Nr   r   r   r*  r   r   r   r     s    zTerminalTreeToPattern.valueN)
r   r   r   r   r   r   r   r   r   r   r   r   r   r   r0  g  s   r0  c                   @   s   e Zd Zdd ZdS )ValidateSymbolsc                 C   s   |\}t |ttfsJ |S r   )r   r+   r"   r<  r   r   r   r     s    zValidateSymbols.valueN)r   r   r   r   r   r   r   r   r=    s   r=  c                 C   s   t d| S )z#Deepcopy tree `t` without recursionF)r1   r   r   r   r   r   nr_deepcopy_tree  s    r?  c                   @   s   e Zd ZU eeeeeef f  ed< eeeeedf ee	f  ed< ee ed< eeeeedf ee	f  eeeeeef f  ee ddddZ
d	d
 ZdS )Grammar	term_defs.r  r}   N)r  rA  r}   returnc                 C   s   || _ || _|| _d S r   )rA  r  r}   )r   r  rA  r}   r   r   r   r     s    zGrammar.__init__c           #         s  dd  j D }dd  jD }|D ]F\}\}}|d u r:q$t|d}t|dkr$|d js$td| q$t t  fdd|D }	t	|	}
t t
  |
 t|9 t }g }d}|t|k rD|| \}}}}|d7 }t|dkrq|r|jrtd	d
nd }||_||_||
_|}||}||||f q||j7 }t|tdd |D ksrJ dt }t }g }|D ]}|\}}}|| ||}t|D ]\}\}}|r|drtd||f dd |D }t|rt|pt }||_dd |D }n|}|D ]2}t|ts8J |jr$|r$|jr$d|_q$t t!|||||}|| qqtt"|t|krt#|dd }|$ D ]\}t|dkr|d j%rtdd&dd |D  tdd |D t|ksJ qtt"|}t|} dd |D dd |D O t'|fdd\}}!|!D ]}"t()d|" qVt|| krqqdkrdd |D t'|	 fd d\}	}!|!rt()d!d"d |!D  |	| j*fS )#Nc                 S   s$   g | ]\}\}}|t ||ffqS r   r?  )r   r   r   r  r   r   r   r     r   z#Grammar.compile.<locals>.<listcomp>c                 S   s$   g | ]\}}}}||t ||fqS r   rC  )r   r   r  r   or   r   r   r     r   r   r   r   zTerminals cannot be empty (%s)c                    s*   g | ]"\}\}}|rt | ||qS r   )r   r   )r   r   	term_treer   )transformerr   r   r     s   Tr   c                 S   s   h | ]\}}}|qS r   r   )r   r   Z_tZ_or   r   r   r     r   z"Grammar.compile.<locals>.<setcomp>zWhoops, name collisionr^   zkRule %s is marked for expansion (it starts with an underscore) and isn't allowed to have aliases (alias=%s)c                 S   s   g | ]}|t kqS r   r   r   r   r   r   r   r     r   c                 S   s   g | ]}|t kr|qS r   rG  rH  r   r   r   r     r   Fc                 S   s   | S r   r   r6  r   r   r   r7    r   z!Grammar.compile.<locals>.<lambda>zXRules defined twice: %s

(Might happen due to colliding expansion of optionals: [] or ?)r   c                 s   s   | ]}d | V  qdS )z
  * %sNr   r   r   r   r   r  	  r   z"Grammar.compile.<locals>.<genexpr>c                 S   s   h | ]}|j |j|jfqS r   )r   orderr  )r   rr   r   r   r     r   c                 S   s0   h | ](}|j D ]}t|tr||jkr|qqS r   )r   r   r!   origin)r   rJ  r  r   r   r   r     s   


c                 S   s   h | ]}t |qS r   )r!   r   r  r   r   r   r     r   c                    s
   | j  v S r   )rK  )rJ  )
used_rulesr   r   r7    r   zUnused rule: %srR   c                 S   s(   h | ] }|j D ]}t|tr|jqqS r   )r   r   r    r   )r   rJ  r   r   r   r   r   !  s   

c                    s    | j v p| j  jv p| j v S r   )r   r}   r>  )r   terminals_to_keep
used_termsr   r   r7  $  r   zUnused terminals: %sc                 S   s   g | ]
}|j qS r   r   r  r   r   r   r   &  r   )+rA  r  list	find_datar   r   r&   r,  r0  r   r=  r  r   r   r   r   r   r   r   r   r   r   r   r   r   anyr   empty_indicesr   r"   is_termr   r   r!   r   r$   valuesr   r  r   r   debugr}   )#r   r   rN  rA  r  r   rE  r   r   r   Zanon_tokens_transfZebnf_to_bnfrulesr   r  Z	rule_treer  r   r   resZrule_tree_to_textZsimplify_ruleZcompiled_rulesZrule_contentr   r   rS  Zexp_optionsr   r{   
duplicatesZdupsr  ZunusedrJ  r   )r   rN  rF  rM  rO  r   compile  s    



$




$
zGrammar.compile)r   r   r   r   r   strr+   r   __annotations__r   r   rZ  r   r   r   r   r@    s
   
 Dr@  PackageResourcezpkg_name pathc                   @   sj   e Zd ZU dZeed< ee ed< deee ddddZd	d
 Ze	dee
f eee
ef dddZdS )FromPackageLoadera  
    Provides a simple way of creating custom import loaders that load from packages via ``pkgutil.get_data`` instead of using `open`.
    This allows them to be compatible even from within zip files.

    Relative imports are handled, so you can just freely use them.

    pkg_name: The name of the package. You can probably provide `__name__` most of the time
    search_paths: All the path that will be search on absolute imports.
    pkg_namesearch_pathsr   N)r_  r`  rB  c                 C   s   || _ || _d S r   )r_  r`  )r   r_  r`  r   r   r   r   <  s    zFromPackageLoader.__init__c                 C   s   dt | j| j| jf S )Nz
%s(%r, %r))r)  r   r_  r`  r   r   r   r   __repr__@  s    zFromPackageLoader.__repr__)	base_pathgrammar_pathrB  c           	      C   s   |d u r| j }n$t|tr&|j| jkr,t |jg}d }|D ]z}tj||}zt	| j|}W n4 ty } z|}W Y d }~q<W Y d }~q<d }~0 0 t| j||r|
 ndf  S q<td|d S )Nr   z"Cannot find grammar in given paths)r`  r   r]  r_  IOErrorpathosr  pkgutilget_datadecode)	r   rc  rd  to_tryerrrf  	full_pathtextr#  r   r   r   __call__C  s    ""zFromPackageLoader.__call__)ra  )r   r   r   r  r[  r\  r   r   rb  r	   r]  r   ro  r   r   r   r   r^  .  s   

r^  Zlarkc           
   
   C   s  d}|   D ]\}}|d u rq|dD ]}|j\}t|trPtd||f q(t|trz| |j }W n  ty   td|j Y n0 |d usJ ||jd< d}q(t|t	s(J q(q|s qq |   D ]B\}}|r|jD ].}dd |
 D }	t||	v rtd	| qqd S )
NFr   z0Rules aren't allowed inside terminals (%s in %s)z!Terminal used but not defined: %sr   Tc                 S   s   g | ]}t |qS r   )idrH  r   r   r   r   {  r   z+resolve_term_references.<locals>.<listcomp>zNRecursion in terminal '%s' (recursion is only allowed in rules, not terminals))r4  rQ  r   r   r!   r&   r    r   r   r+   iter_subtreesrp  )
Z	term_dictchangedr   Z
token_treeexpitemZ
term_valuer|   r   idsr   r   r   resolve_term_references_  s2    



rv  c                 C   s0   t | tsJ |  r(t| | ddS t| S Nr^   r   )r   r[  isupperr    r   r!   r  r   r   r   symbol_from_strcase  s    ry  c                   @   s   e Zd Zdd Zdd ZdS )PrepareGrammarc                 C   s   t t||ddS rw  )r    r[  r   r   r   r   r   r   r     s    zPrepareGrammar.terminalc                 C   s
   t |jS r   )r!   r   r{  r   r   r   r     s    zPrepareGrammar.nonterminalN)r   r   r   r   r   r   r   r   r   rz    s   rz  c                 C   s"   | j dksJ dd | dD S )Nr   c                 S   s&   h | ]}| d d D ]
}|jqqS )c                 S   s
   t | tS r   )r   r"   r>  r   r   r   r7    r   z._find_used_symbols.<locals>.<setcomp>.<lambda>)Zscan_valuesr   )r   r   r   r   r   r   r     s   z%_find_used_symbols.<locals>.<setcomp>r   )r   rQ  r   r   r   r   _find_used_symbols  s    r|  c                  C   s   zt jW S  ty   dd t D } dd t D }dd |D }t|t }dd l	}t
| |g d}t||dg}d|_d	|_t||d t _t j Y S 0 d S )
Nc                 S   s   g | ]\}}t |t|qS r   )r   r   )r   r   r   r   r   r   r     r   z_get_parser.<locals>.<listcomp>c                 S   s,   g | ]$\}}| d |t|d dfqS )rV   )expand1)lstripr   r   )r   r   r   r   r   r   r     s   c              	   S   sF   g | ]>\}}}t |D ]*\}}tt|d d | D |d|qqS )c                 S   s   g | ]}t |qS r   )ry  rL  r   r   r   r     r   z*_get_parser.<locals>.<listcomp>.<listcomp>N)r   r   r!   split)r   rJ  xsrD  r   r   r   r   r   r     s   r   )rv   rw   r;   r   basicZlalr)_get_parsercacheAttributeError	TERMINALSr4  RULESr   r   Zcreate_callbackrer   r   Z
lexer_typeZparser_typer   )r   rW  callbackr  Z
lexer_confZparser_confr   r   r   r    s$    r  zIncorrect type of valueza: 1
zUnclosed parenthesisza: (
zUnmatched closing parenthesis)za: )
za: [)
za: (]
z5Expecting rule or terminal definition (missing colon))za
zA
za->
zA->
za A
z#Illegal name for rules or terminalszAa:
zAlias expects lowercase namez
a: -> "a"
zUnexpected colon)za::
za: b:
za: B:
za: "a":
zMisplaced operator)za: b??za: b(?)za:+
za:?
za:*
za:|*
z;Expecting option ("|") or a new rule or terminal definitionza:a
()
z"Terminal names cannot contain dotszA.B
z%Expecting rule or terminal definitionz"a"
z%import expects a namez%import "a"
z%ignore expects a valuez%ignore %import
c                 C   s*   |j | tdd}|r|S d|jv r&dS d S )NT)Zuse_acceptsrs   zExpecting a value)Zmatch_examplesGRAMMAR_ERRORSexpected)parser#  errorr   r   r   _translate_parser_exception  s
    
r  r   c              
   C   s   zt  | d |}W n ty\ } z.|| }td|j|j||f W Y d }~nbd }~0  ty } zB|| }tt  j|}|rtd||j|j|f  W Y d }~n
d }~0 0 t	 
|S )Nrl   z1Unexpected input at line %d column %d in %s: 

%sz%s, at line %s column %s

%s)r  r  r'   Zget_contextr&   linecolumnr(   r  rz  r   )rn  r   r   r   r#  contextr  r   r   r   _parse_grammar  s    

r  c                 C   sP   t | trDtt j| }|r |S d| jp.| j}dt| j	|f S t| S d S )Nz, z*Unexpected token %r. Expected one of: {%s})
r   r(   r  r  r  r  Zacceptsr  r[  token)r  Zerror2r  r   r   r   _error_repr  s    
r  c                 C   s:   dd }t d| fg|D ]\}}||r||f  S qd S )Nc              	   s   sV   | \}}|  D ]@}t|d}z||}W n ty>   Y q0 ||f |fV  qd S )Nr   )r  r   
feed_tokenr)   )noderf  r  choicer   Znew_pr   r   r   expand  s    
z*_search_interactive_parser.<locals>.expandr   )r   )interactive_parser	predicater  rf  r  r   r   r   _search_interactive_parser  s    r  )rn  r   rB  c                    s^   g   fdd}t  j| d ||d}t dd }dd | D   D ]}d |d	 _qJ S )
Nc                    s\     | t| f t| j dd \}}|D ]}| jt|d q.| jtdd dS )Nc                 S   s   d|   v S )Nru   )r  )r  r   r   r   r7    r   z7find_grammar_errors.<locals>.on_error.<locals>.<lambda>r   ru   rl   T)r   r  r  r  Zas_immutabler  r   )r#  Z
token_pathr^   
token_typeerrorsr   r   on_error  s    z%find_grammar_errors.<locals>.on_errorrl   )r  c                 S   s
   | d j S r;  )r  )r#  r   r   r   r7    r   z%find_grammar_errors.<locals>.<lambda>c                 S   s   g | ]}|d  qS )r   r   )r   elr   r   r   r     r   z'find_grammar_errors.<locals>.<listcomp>r   )r  r  r$   rU  r  )rn  r   r  Z_treeZerrors_by_liner#  r   r  r   find_grammar_errors  s    
r  c                    s    fdd}|S )Nc                    sT   |  v r |  } n.| d dkr4d| dd  f } nd| f } d urP| } | S )Nr   r^   z_%s__%sr   z%s__%sr   r  r   base_mangler   r   r   mangle  s    
z_get_mangle.<locals>.mangler   )r   r   r  r  r   r  r   _get_mangle   s    r  c                 C   sT   |d u r| S t | } |  D ]2}t|jD ]"\}}t|tr*|||j|< q*q| S r   )r   rq  r   r   r   r"   Zrenamed)rs  r  r   r   r  r   r   r   _mangle_definition_tree  s    
r  c           
   	   C   s|   | j r | j \}d|v }d|v }nd}d}|j r@|j \}t|}	nd }	|d ur\dd |j D }|||t|||	|rr|nd dfS )NrV   rW   Fc                 S   s   g | ]
}|j qS r   )r   r  r   r   r   r   )  r   z$_make_rule_tuple.<locals>.<listcomp>)r   Ztemplate_source)r   r   r   )
Zmodifiers_treer   r  Zpriority_treer   mr}  r   r  r   r   r   r   _make_rule_tuple  s    


r  c                   @   s   e Zd ZdddZdS )
Definitionr   Nc                 C   s    || _ || _t|| _|| _d S r   )rT  r   tupler  r  )r   rT  r   r  r  r   r   r   r   0  s    
zDefinition.__init__)r   N)r   r   r   r   r   r   r   r   r  /  s   r  c                   @   sL  e Zd ZU eed< eeeef  ed< e	eef ed< e	ee
f ed< ee ed< d*eeeeeef   ee	eef  ddd	d
Zdd Zdd Zd+ddddZd,ddZdd Zdd Zdd Zd-eeeeegef  ddddZdd  Zd.eed!f ee e	eef eeegef  dd"d#d$Zdd%d&d'Zed%d(d)ZdS )/GrammarBuilderglobal_keep_all_tokensimport_paths
used_files_definitions_ignore_namesFN)r  r  r  rB  c                 C   s*   || _ |pg | _|pi | _i | _g | _d S r   )r  r  r  r  r  )r   r  r  r  r   r   r   r   ?  s
    

zGrammarBuilder.__init__c           	      G   sv   i }t |ddD ]L\}}|dkr$dnt|}||d| < d|  |d| < }| |d| < qt|jf i |d S )Nr   )r   r   r   )r{   r   r)  Type)r   r[  titler&   format)	r   rT  msgr	  r   r   r   ZpostfixZlowercase_typer   r   r   _grammar_errorG  s    zGrammarBuilder._grammar_errorc                 C   sb   |r0|d u rd}q^t |ts^tdt|f n.|d u r@t }nt |tsRtd| jr^d|_|S )Nr   zBTerminal require a single int as 'options' (e.g. priority), got %sz1Rules require a RuleOptions instance as 'options'T)r   r   r&   r)  r   r  r   )r   rT  r  r   r   r   _check_optionsP  s    

zGrammarBuilder._check_optionsr   )r   c                C   sh   || j v r|s0| |d| n|r0| |d| |drH| |d| t|||| ||| j |< d S )Nz&{Type} '{name}' defined more than oncez+Cannot override a nonexisting {type} {name}__zDNames starting with double-underscore are reserved (Error at {name}))r  r  r   r  r  )r   r   rT  rs  r  r  r   r   r   r   _define`  s    

zGrammarBuilder._definec                 C   s   || j vr| |d| | j | }||jkr:| |d| t||jkrV| |d| |jd u rn| |d| |j}t|tr|jdksJ |j	
d| d S )Nz6Can't extend {type} {name} as it wasn't defined beforezHCannot extend {type} {name} - one is a terminal, while the other is not.z6Cannot extend {type} with different parameters: {name}z,Can't extend {type} {name} - it is abstract.r   r   )r  r  rT  r  r  r   r   r+   r   r   insert)r   r   rT  rs  r  r  dbaser   r   r   _extendl  s    



zGrammarBuilder._extendc                 C   s   t |tr| j| nt |ts&J |}|jdkrt|jdkr|j\}|jdkrt|jdkr|j\}|jdkr|j\}t |tr| j|j	 d S dt| j }| j| t
d|td| j|< d S )Nr   r   r   r   z__IGNORE_%dT)r  )r   r[  r  r   r+   r   r   r   r    r   r  r#   r  )r   Zexp_or_namer   t2rt  r   r   r   r   _ignore  s     


zGrammarBuilder._ignorec                 C   s<  t |jdkr|j\}}n|j\}d }t|trPt|j}|j}tt||}nDt|jd d }|sz|j\}td| |jd }|j|p|ji}|j	dkrd }	n|dkrzt
jtjd j}
W q ty   d }
Y q0 n|}
|
r"t|
trt|
jt
j|
jd }	nt
j|
d }	nt
jt
jj}	||	|fS )Nr   r   z&Nothing was imported from grammar `%s`r   <string>__main__r   )r   r   r   r+   r  r  r  r&   r   r   rg  rf  abspathsysmodules__file__r  r]  r_  r  curdir)r   stmtgrammar_nameZ	path_nodeZarg1dotted_pathr	  r   r   rc  Z	base_filer   r   r   _unpack_import  s8    



zGrammarBuilder._unpack_importc                    s   |j dkr"t|j \}}}}d}n>|jd j}d}t|jdkrNt|jd nt}|jd }d} d urt fd	d
|D } |}t| }|||||fS )Nr{   Fr   r   r:  r   r   Tc                 3   s   | ]} |V  qd S r   r   )r   r  r  r   r   r    r   z4GrammarBuilder._unpack_definition.<locals>.<genexpr>)	r   r  r   r   r   r   r#   r  r  )r   r   r  r   r  rs  optsrT  r   r  r   _unpack_definition  s    
 

z!GrammarBuilder._unpack_definition<?>)grammar_textr  r  rB  c              	   C   s  t ||}i }|jD ]t}|jdkr| ||\}}}	z4|| \}
}||
ks\J dd| ||	 W q ty   ||	f||< Y q0 q| D ]\}\}}	| |||	| q|jD ]}|jdv r| j	| 
||  q|jdkr|j\}| j	| 
||ddi q|jdkr0|j\}| j| 
||  q|jdkrT|d u r| j|j  q|jd	kr|jD ]N}t|ts~J |t|t}|d u r|j}n
||j}| 	||d  qfq|jdkrqd
sJ |qdd | j D }t| d S )Nr~   zInconsistent base_path for %s.rL   )r|   r{   r   Tr   r}   r   Fc                 S   s   i | ]\}}|j r||jqS r   )rT  r   )r   r   r  r   r   r   r     s   z/GrammarBuilder.load_grammar.<locals>.<dictcomp>)r  r   r   r  r  updater   r4  	do_importr  r  r  r  r   r"   r    r   r  rv  )r   r  r  r  r   Zimportsr  r  rc  r   Zimport_base_pathZimport_aliasesrJ  r   rT  r   rA  r   r   r   load_grammar  sP    








zGrammarBuilder.load_grammarc                    s8   fdd}t t||  fddj D _d S )Nc                    sD   z j |  }W n ty$   g  Y S 0 |jr0g S t|jt|j S r   )r  r   rT  r|  r   r   r  )r   r  r   r   r   rule_dependencies  s    
z8GrammarBuilder._remove_unused.<locals>.rule_dependenciesc                    s   i | ]\}}| v r||qS r   r   )r   kr*  )_usedr   r   r     r   z1GrammarBuilder._remove_unused.<locals>.<dictcomp>)r   r   r  r4  )r   usedr  r   )r  r   r   _remove_unused  s    	zGrammarBuilder._remove_unused.)r  rc  r   r  rB  c              	   C   s  |sJ t d|||}tjj| t }| j|d ur<|gng  tg }|D ]&}z^t|rl|||\}	}
nBtj||}	t|	dd}|	 }
W d    n1 s0    Y  W n t
y   Y qLY qL0 t|
d }| j|	||krtd|| j|	< t| j| j| j}||
|	| |t|| |jD ]"}|| jv r6td||f q6| jjf i |j  qqLt|dd dsJ d|f d S )Nr  utf8encodingz)Grammar file was changed during importingz5Cannot import '%s' from '%s': Symbol already defined.FzjCouldn't import grammar %s, but a corresponding file was found at a place where lark doesn't search for it)r  r  rg  rf  EXTr  stdlib_loadercallableopenreadre  hashlibmd5encode	hexdigestr  getRuntimeErrorr  r  r  r  r   r  r&   r  )r   r  rc  r   r  r  rd  rk  sourceZjoined_pathrn  r&  hgbr   r   r   r   r    s6    
*


zGrammarBuilder.do_import)rB  c                 C   s  | j  D ]>\}}|j}|j}t|D ]B\}}|| j v rJtd||f ||d | v r(td||f q(|d u rvq
|dD ]}|jd j}|jdd  }	||vr|| j vr| 	|j
d| | t|	t| j | jkrt| j | jt|	 }
}| 	|j
d|
|f || qt|D ].}|| j vr||vr| 	|j
d|| qq
t| j | js|td	t| jt| j   d S )
Nz:Template Parameter conflicts with rule %s (in template %s)z0Duplicate Template Parameter %s (in template %s)r   r   r   z5Template '%s' used but not defined (in {type} {name})z]Wrong number of template arguments used for {name} (expected %s, got %s) (in {type2} {name2})z9{Type} '{name}' used but not defined (in {type2} {name2})z8Terminals %s were marked to ignore but were not defined!)r  r4  r  r   r   r&   rQ  r   r   r  rT  r   r|  r   
issupersetr  )r   r   r  r  rs  r   r  tempr   r   r  actualr   r   r   validate7  s8    


zGrammarBuilder.validatec                 C   s   |    g }g }| j D ]Z\}}|j|j|j  }}}|jrbt|dksNJ ||||ff q|||||f qt	||| j
S r;  )r  r  r4  r  r   r  rT  r   r   r@  r  )r   r  rA  r   r  r  rs  r  r   r   r   buildW  s    zGrammarBuilder.build)FNN)r   N)r   N)r  N)N)r   r   r   boolr\  r   r	   r[  r
   r   r  r   r   r  r  r  r  r  r  r  r  r  r   r  r  r@  r  r   r   r   r   r  6  s$   
2	
'$5:$ r  c              	   C   s   |   D ]\}}d }t|tr`tj|r`t|dd}| }W d    q1 sT0    Y  nBt|trt	t
  tj| d}W d    n1 s0    Y  |d u rqt|  }||krtd|   dS qdS )Nr  r  zutf-8z"File %r changed, rebuilding ParserFT)r4  r   r[  rg  rf  existsr  r  r]  r   re  rh  ri  rj  r  r  r  r  r   info)Zfile_hashesrf  oldrn  r&  currentr   r   r   verify_used_filesf  s    (

.r  c                 C   s$   t d|}|| d t|j S )zXReturns a list of paths to the lark grammars imported by the given grammar (recursively)Fr  )r  r  rP  r  keys)grammarr  builderr   r   r   list_grammar_importsx  s    
r  c                 C   s$   t ||}|| | | |jfS r   )r  r  r  r  )r  r  r  r  r  r   r   r   r  ~  s    
r  )r   )r   )N)xr  r  os.pathrg  r  collectionsr   r   r   rh  astr   
contextlibr   typingr   r   r	   r
   r   r   r   utilsr   r   r   r   r   r   r   lexerr   r   r   r   Zparse_tree_builderr   Zparser_frontendsr   commonr   r   r  r   r   r    r!   r"   r#   r$   r%   
exceptionsr&   r'   r(   r)   r*   r   r+   r,   r   Zvisitorsr-   r.   r/   r0   r1   r  rf  dirnamer  __path__ZIMPORT_PATHSr  r%  r   r  r  r  r   r   r   r   r   r   r   r  r  r  r$  r+  r,  r/  r0  r=  r?  r@  r]  r^  r  rv  ry  rz  r|  r  r  r  r  r  r  r[  r  r  r  r  r  r  r  r  r  r   r   r   r   <module>   s  $$ 
)"P -7!0 	
-
"





 
  2