
    [                        d dl mZmZ d dlZd dlmZ d dlmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZ d dlmZ d dlZ	 d dlZerddlmZ ddlmZmZmZmZ dd	lm Z m!Z!m"Z" dd
l#m$Z$ d dl%m%Z% 	  e&e      Z' G d dee      Z) G d de)      Z* G d de)      Z+ G d de      Z, edd      Z- G d de.      Z/ G d d      Z0 G d d      Z1 G d d      Z2d Z3d  Z4 G d! d"      Z5d#e.fd$Z6 G d% d&      Z7 G d' d(      Z8ee/ge/f   Z9 G d) d*e      Z:d1d+e	e,e.f   fd,Z; G d- d.e:      Z< G d/ d0e:      Z=y# e$ r Y w xY w# e($ r dZ'Y w xY w)2    )abstractmethodABCN)suppress)TypeVarTypeDictIterator
CollectionCallableOptional	FrozenSetAnyClassVarTYPE_CHECKINGoverload)
ModuleType   )	LexerConf)classifyget_regexp_width	Serializelogger)UnexpectedCharactersLexErrorUnexpectedToken)TOKEN_DEFAULT_PRIORITY)copyFc            	           e Zd ZU eed<   ee   ed<   ee   ed<   ee   ed<   ddedee   dee   ddfdZd Z	d	 Z
d
 Zedefd       Zeedefd              Zeedefd              Zd Zy)PatternvalueflagsrawtypeNreturnc                 @    || _         t        |      | _        || _        y N)r    	frozensetr!   r"   )selfr    r!   r"   s       lib/third_party/lark/lexer.py__init__zPattern.__init__'   s    
u%
    c                 4    t        | j                               S r&   )repr	to_regexpr(   s    r)   __repr__zPattern.__repr__,   s    DNN$%%r+   c                 X    t        t        |       | j                  | j                  f      S r&   )hashr#   r    r!   r/   s    r)   __hash__zPattern.__hash__0   s     T$ZTZZ899r+   c                     t        |       t        |      k(  xr4 | j                  |j                  k(  xr | j                  |j                  k(  S r&   )r#   r    r!   r(   others     r)   __eq__zPattern.__eq__3   s<    DzT%[(dTZZ5;;-Fd4::Y^YdYdKddr+   c                     t               r&   NotImplementedErrorr/   s    r)   r.   zPattern.to_regexp6   s    !##r+   c                     t               r&   r9   r/   s    r)   	min_widthzPattern.min_width:        "##r+   c                     t               r&   r9   r/   s    r)   	max_widthzPattern.max_width?   r=   r+   c                 :    | j                   D ]  }d|d|d} |S )Nz(?:))r!   )r(   r    fs      r)   
_get_flagszPattern._get_flagsD   s    A#$e,E r+   ) N)__name__
__module____qualname__str__annotations__r
   r   r   r*   r0   r3   r7   r   r.   propertyintr<   r?   rD   rE   r+   r)   r   r       s    Jc?	#
3-c *S/ Xc] ^b 
&:e $3 $ $ $3 $  $ $3 $  $r+   r   c                   ^    e Zd ZU dZdZee   ed<   defdZe	de
fd       Ze	de
fd       Zy)	
PatternStr)r    r!   r"   rI   r#   r$   c                 ^    | j                  t        j                  | j                              S r&   )rD   reescaper    r/   s    r)   r.   zPatternStr.to_regexpO   s    ryy455r+   c                 ,    t        | j                        S r&   lenr    r/   s    r)   r<   zPatternStr.min_widthR       4::r+   c                 ,    t        | j                        S r&   rS   r/   s    r)   r?   zPatternStr.max_widthV   rU   r+   N)rF   rG   rH   __serialize_fields__r#   r   rI   rJ   r.   rK   rL   r<   r?   rE   r+   r)   rN   rN   J   sX    2D(3-63 6 3   3  r+   rN   c                   h    e Zd ZU dZdZee   ed<   defdZdZ	d Z
edefd       Zedefd	       Zy)
	PatternRE)r    r!   r"   _widthrP   r#   r$   c                 8    | j                  | j                        S r&   )rD   r    r/   s    r)   r.   zPatternRE.to_regexp`   s    tzz**r+   Nc                 n    | j                   t        | j                               | _         | j                   S r&   )rZ   r   r.   r/   s    r)   
_get_widthzPatternRE._get_widthd   s)    ;;*4>>+;<DK{{r+   c                 (    | j                         d   S Nr   r]   r/   s    r)   r<   zPatternRE.min_widthi        ##r+   c                 (    | j                         d   S )Nr   r`   r/   s    r)   r?   zPatternRE.max_widthm   ra   r+   )rF   rG   rH   rW   r#   r   rI   rJ   r.   rZ   r]   rK   rL   r<   r?   rE   r+   r)   rY   rY   [   sd    <D(3-+3 + F
 $3 $ $ $3 $ $r+   rY   c            	       f    e Zd ZU dZeefZeed<   e	ed<   e
ed<   efdede	de
ddfdZd Zdefd	Zy)
TerminalDef)namepatternpriorityre   rf   rg   r$   Nc                 \    t        |t              sJ |       || _        || _        || _        y r&   )
isinstancer   re   rf   rg   )r(   re   rf   rg   s       r)   r*   zTerminalDef.__init__z   s,    '7+4W4+	 r+   c                 f    t        |       j                  d| j                  d| j                  dS )N(, rB   )r#   rF   re   rf   r/   s    r)   r0   zTerminalDef.__repr__   s!    #Dz22DIIt||LLr+   c                     | j                   j                  d      r$| j                  j                  xs | j                   S | j                   S )N__)re   
startswithrf   r"   r/   s    r)   	user_reprzTerminalDef.user_repr   s7    99%<<##0tyy099r+   )rF   rG   rH   rW   rN   rY   __serialize_namespace__rI   rJ   r   rL   r   r*   r0   rp   rE   r+   r)   rd   rd   r   sZ    8()3
IMDZ !S !7 !c !_c !M3 r+   rd   _TToken)boundc                   ^    e Zd ZU dZdZdZeed<   ee	   ed<   e
ed<   ee	   ed<   ee	   ed<   ee	   ed	<   ee	   ed
<   ee	   ed<   e	 	 	 	 	 	 ddede
dee	   dee	   dee	   d	ee	   d
ee	   dee	   dd fd       Ze	 	 	 	 	 	 ddede
dee	   dee	   dee	   d	ee	   d
ee	   dee	   dd fd       Zd Zed fd	       Zeddee   dee
   dd fd       Zeddee   dee
   dd fd       Zd Zddee   dee
   dd fdZedee   dede
dd def
d       Zd Zd Zd Zd Zej0                  Z xZS )rs   a  A string with meta-information, that is produced by the lexer.

    When parsing text, the resulting chunks of the input that haven't been discarded,
    will end up in the tree as Token instances. The Token class inherits from Python's ``str``,
    so normal string comparisons and operations will work as expected.

    Attributes:
        type: Name of the token (as specified in grammar)
        value: Value of the token (redundant, as ``token.value == token`` will always be true)
        start_pos: The index of the token in the text
        line: The line of the token in the text (starting with 1)
        column: The column of the token in the text (starting with 1)
        end_line: The line where the token ends
        end_column: The next column after the end of the token. For example,
            if the token is a single character with a column value of 4,
            end_column will be 5.
        end_pos: the index where the token ends (basically ``start_pos + len(token)``)
    )r#   	start_posr    linecolumnend_line
end_columnend_posr#   r    r#   rv   r    rw   rx   ry   rz   r{   r$   c	                      y r&   rE   )	clsr#   r    rv   rw   rx   ry   rz   r{   s	            r)   __new__zToken.__new__   s     	r+   type_c	                      y r&   rE   )	r~   r   r    rv   rw   rx   ry   rz   r{   s	            r)   r   zToken.__new__   s     r+   c                     d|v r=t        j                  dt               d|v rt        d      |j	                  d      |d<    | j
                  |i |S Nr   z(`type_` is deprecated use `type` insteadr#   zAError: using both 'type' and the deprecated 'type_' as arguments.)warningswarnDeprecationWarning	TypeErrorpop_future_new)r~   argskwargss      r)   r   zToken.__new__   sV    fMMDFXY cdd#ZZ0F6Ns///r+   c	                     t         t        |   | |      }	||	_        ||	_        ||	_        ||	_        ||	_        ||	_        ||	_	        ||	_
        |	S r&   )superrs   r   r#   rv   r    rw   rx   ry   rz   r{   )r~   r#   r    rv   rw   rx   ry   rz   r{   inst	__class__s             r)   r   zToken._future_new   sS    UC(e4	"
	 $r+   c                      y r&   rE   r(   r#   r    s      r)   updatezToken.update       r+   c                      y r&   rE   )r(   r   r    s      r)   r   zToken.update   r   r+   c                     d|v r=t        j                  dt               d|v rt        d      |j	                  d      |d<    | j
                  |i |S r   )r   r   r   r   r   _future_update)r(   r   r   s      r)   r   zToken.update   sX    fMMDFXY cdd#ZZ0F6N"t""D3F33r+   c                 r    t         j                  ||n| j                  |||       S | j                  |       S r&   )rs   new_borrow_posr#   r    r   s      r)   r   zToken._future_update   sB    ##$D$))&E
 	
,0JJ
 	
r+   r~   borrow_tc           
           | |||j                   |j                  |j                  |j                  |j                  |j
                        S r&   )rv   rw   rx   ry   rz   r{   )r~   r   r    r   s       r)   r   zToken.new_borrow_pos   sP    5%!3!3X]]HOOU]UfUfhph{h{  ~F  ~N  ~N  O  	Or+   c                     | j                   | j                  | j                  | j                  | j                  | j
                  ffS r&   )r   r#   r    rv   rw   rx   r/   s    r)   
__reduce__zToken.__reduce__   s3    DJJ		SWS^S^ _``r+   c                 <    d| j                   d| j                  dS )NzToken(rl   rB   r|   r/   s    r)   r0   zToken.__repr__   s    "&))TZZ88r+   c                     t        | j                  | j                  | j                  | j                  | j
                        S r&   )rs   r#   r    rv   rw   rx   )r(   memos     r)   __deepcopy__zToken.__deepcopy__  s)    TYY

DNNDIIt{{SSr+   c                     t        |t              r| j                  |j                  k7  ryt        j	                  | |      S )NF)ri   rs   r#   rI   r7   r5   s     r)   r7   zToken.__eq__  s/    eU#		UZZ(?zz$&&r+   )NNNNNNNN)rF   rG   rH   __doc__	__slots____match_args__rI   rJ   r   rL   r   r   r   classmethodr   r   r   r   rr   r   r   r0   r   r7   r3   __classcell__)r   s   @r)   rs   rs      s   $ fI&N
I}J
3-SMsmc] 
 (,"&$(&*(,%)   }	
 3- SM sm ! c] 
  
 (,"&$(&*(,%)

 
  }	

 3-
 SM
 sm
 !
 c]
 

 
0   8C=  QX   HSM # RY  4
8C= 
 
Y` 
 ODH OS O O OTV O Oa9T' ||Hr+   c                   *    e Zd ZdZd Zd ZddefdZy)LineCounter)char_posrw   rx   line_start_posnewline_charc                 J    || _         d| _        d| _        d| _        d| _        y )Nr   r   )r   r   rw   rx   r   )r(   r   s     r)   r*   zLineCounter.__init__  s'    (	r+   c                     t        |t              st        S | j                  |j                  k(  xr | j                  |j                  k(  S r&   )ri   r   NotImplementedr   r   r5   s     r)   r7   zLineCounter.__eq__  s;    %-!!}}.Z43D3DHZHZ3ZZr+   tokenc                 J   |rb|j                  | j                        }|rE| xj                  |z  c_        | j                  |j	                  | j                        z   dz   | _        | xj                  t        |      z  c_        | j                  | j
                  z
  dz   | _        y)zConsume a token and calculate the new line & column.

        As an optional optimization, set test_newline=False if token doesn't contain a newline.
        r   N)countr   rw   r   rindexr   rT   rx   )r(   r   test_newlinenewliness       r)   feedzLineCounter.feed  s    
 {{4#4#45H		X%	&*mmell4CTCT6U&UXY&Y#U#mmd&9&99A=r+   N)T)rF   rG   rH   r   r*   r7   rs   r   rE   r+   r)   r   r     s    NI [>% >r+   r   c                       e Zd Zd Zd Zy)UnlessCallbackc                     || _         y r&   )scanner)r(   r   s     r)   r*   zUnlessCallback.__init__/  s	    r+   c                 j    | j                   j                  |j                  d      }|r
|\  }|_        |S r_   )r   matchr    r#   )r(   tres_values       r)   __call__zUnlessCallback.__call__2  s/    ll  !, NFAFr+   NrF   rG   rH   r*   r   rE   r+   r)   r   r   .  s    r+   r   c                       e Zd Zd Zd Zy)	CallChainc                 .    || _         || _        || _        y r&   )	callback1	callback2cond)r(   r   r   r   s       r)   r*   zCallChain.__init__:  s    ""	r+   c                 l    | j                  |      }| j                  |      r| j                  |      S |S r&   )r   r   r   )r(   r   t2s      r)   r   zCallChain.__call__?  s.    ^^A$(IIbMt~~a 9r9r+   Nr   rE   r+   r)   r   r   9  s    
:r+   r   c                 P    | j                  |||      }|r|j                  d      S y r_   )r   group)re_regexpsr!   ms        r)   
_get_matchr   D  s*    		&!U#Awwqz 	r+   c           
         t        | d       }t        |      dk  sJ |j                                t               }i }|j	                  t
        g       D ]  }g }|j	                  t        g       D ]  }	|	j                  |j                  k7  r|	j                  j                  }
|
t        ||j                  j                         |
|      k(  s]|j                  |	       |	j                  j                  |j                  j                  k  s|j                  |	        |st        t!        |||d|            ||j"                  <    | D cg c]	  }||vs| }}||fS c c}w )Nc                 ,    t        | j                        S r&   )r#   rf   r   s    r)   <lambda>z _create_unless.<locals>.<lambda>J  s    4		?r+      T)match_whole	use_bytes)r   rT   keyssetgetrY   rN   rg   rf   r    r   r.   appendr!   addr   Scannerre   )	terminalsg_regex_flagsr   r   tokens_by_typeembedded_strscallbackretokunlessstrtokr   r   new_terminalss                r)   _create_unlessr   I  s=   i)BCN~!#:^%8%8%::#EMH##Ir2$((R8F%..0$$AJsEMM$;$;$=q-PPf%>>''5==+>+>>!%%f- 9 #1'&-QTbfr{2|#}HUZZ  3 !*D	1Qm-CQ	MD("" Es   	E E c                        e Zd ZddZd Zd Zy)r   c                     || _         || _        || _        || _        || _        | j                   D ch c]  }|j
                   c}| _        | j                  |t        |            | _	        y c c}w r&   )
r   r   r   r   r   re   allowed_types_build_mresrT   _mres)r(   r   r   r   r   r   r   s          r)   r*   zScanner.__init__`  sc    "*"&.2nn=naffn=%%iY@
 >s   A.c                 l   | j                   rdndg }|rzdj                  fd|d | D              }| j                  r|j                  d      }	 | j                  j                  || j                        }|j                  |       ||d  }|rz|S # t        $ r | j                  ||dz        cY S w xY w)N$ |c              3   z   K   | ]2  }d |j                   d|j                  j                         z   d 4 yw)z(?P<>rB   N)re   rf   r.   ).0r   postfixs     r)   	<genexpr>z&Scanner._build_mres.<locals>.<genexpr>r  s1     xcw^_9L9L9NQX9X Ycws   8;zlatin-1r   )
r   joinr   encoder   compiler   AssertionErrorr   r   )r(   r   max_sizemresrf   mrer   s         @r)   r   zScanner._build_mresk  s     ))#riixclmvnvcwxxG~~!..3Bhh&&w0B0BC KK!(),I   " B''	8q=AABs   &B B32B3c                     | j                   D ]5  }|j                  ||      }|s|j                  d      |j                  fc S  y r_   )r   r   r   	lastgroup)r(   textposr   r   s        r)   r   zScanner.match~  s:    ::C		$$Awwqz1;;.. r+   N)F)rF   rG   rH   r*   r   r   rE   r+   r)   r   r   _  s    	A&/r+   r   rc                 F    d| v xs d| v xs d| v xs d| v xs
 d| v xr d| v S )zExpressions that may indicate newlines in a regexp:
        - newlines (\n)
        - escaped newline (\\n)
        - anything but ([^...])
        - any-char (.) when the flag (?s) exists
        - spaces (\s)
    
z\nz\sz[^z(?s.rE   )r  s    r)   _regexp_has_newliner    s>     19Z
ZeqjZDAIZ%1*BYQTXYQYZr+   c                   h    e Zd ZU dZdZeed<   eed<   ee	   ed<   d
dedee   dee	   fdZ
d Zd	 Zy)
LexerStatezRepresents the current state of the lexer as it scans the text
    (Lexer objects are only instantiated per grammar, not per text)
    )r   line_ctr
last_tokenr   r  r	  Nc                 l    || _         |xs t        t        |t              rdnd      | _        || _        y )N   
r  )r   r   ri   bytesr  r	  )r(   r   r  r	  s       r)   r*   zLexerState.__init__  s-    	 [KD%9PVZ$[$r+   c                     t        |t              st        S | j                  |j                  u xr4 | j                  |j                  k(  xr | j
                  |j
                  k(  S r&   )ri   r  r   r   r  r	  r5   s     r)   r7   zLexerState.__eq__  sL    %,!!yyEJJ&r4==ENN+JrtbgbrbrOrrr+   c                 v     t        |       | j                  t        | j                        | j                        S r&   )r#   r   r   r  r	  r/   s    r)   __copy__zLexerState.__copy__  s(    tDz$))T$--%8$//JJr+   r   )rF   rG   rH   r   r   rI   rJ   r   r   rs   r*   r7   r  rE   r+   r)   r  r    sU     1I
I%S %H[,A %T\]bTc %
sKr+   r  c                   J    e Zd ZdZdddefdZedddefd       Zd Z	d	 Z
eZy
)LexerThreadzTA thread that ties a lexer instance and a lexer state, to be used by the parser
    lexerLexerlexer_statec                      || _         || _        y r&   )r  state)r(   r  r  s      r)   r*   zLexerThread.__init__  s    
 
r+   r   c                 &     | |t        |            S r&   r  )r~   r  r   s      r)   	from_textzLexerThread.from_text  s    5*T*++r+   c                 N    | j                   j                  | j                  |      S r&   )r  lexr  )r(   parser_states     r)   r  zLexerThread.lex  s    zz~~djj,77r+   c                 `     t        |       | j                  t        | j                              S r&   )r#   r  r   r  r/   s    r)   r  zLexerThread.__copy__  s"    tDz$**d4::&677r+   N)rF   rG   rH   r   r  r*   r   rI   r  r  r  rs   _TokenrE   r+   r)   r  r    sM    !g !J ! ,g ,S , ,88 Fr+   r  c                   :    e Zd ZdZedededee   fd       Z	d Z
y)r  zlLexer interface

    Method Signatures:
        lex(self, lexer_state, parser_state) -> Iterator[Token]
    r  r  r$   c                     t         S r&   )r   )r(   r  r  s      r)   r  z	Lexer.lex  s    r+   c                     t        |      S )
Deprecatedr  )r(   r   s     r)   make_lexer_statezLexer.make_lexer_state  s    $r+   N)rF   rG   rH   r   r   r  r   r	   rs   r  r#  rE   r+   r)   r  r    s9    
 z  %   r+   r  terminal_to_regexpc                    |st         j                  j                  |       }|rdnd}|j                         |k\  ry t	        | d       j                         D ]  }|j                  |d      D ]  \  }}|j                  |j                  k(  sJ |j                  ||       d|j                   d|j                   d}	 |j                  |||      j                         }	|rt        | d
|	       t        j                  d||	       |j                         |k\  st        j                  d         y   y # t        $ r d	}	Y fw xY w)Nr   g?c                     | j                   S r&   )rg   r   s    r)   r   z)_check_regex_collisions.<locals>.<lambda>  s    

r+   T)skip_markedzCollision between Terminals z and z. zONo example could be found fast enough. However, the collision does still existsr  z5%s The lexer will choose between them arbitrarily.
%sz2Found 8 regex collisions, will not check for more.)interegular
Comparatorfrom_regexescount_marked_pairsr   valuescheckrg   markre   get_example_overlapformat_multiline
ValueErrorr   r   warning)
r$  
comparatorstrict_modemax_collisions_to_showmax_timer   abmessageexamples
             r)   _check_regex_collisionsr;    sF    ++889KL
  qSH $$&*@@,.BCJJL$$U$=DAq::+++OOAq! 5QVVHE!&&LGl$88AxHYY[ '"WI677NNSU\^ef,,.2HHST# > M  lkls   >!D88EEc                       e Zd ZU ee   ed<   ee   ed<   ee   ed<   eee	f   ed<   eee	f   ed<   e
ed<   ddd	Zd
 Zed        Zd Zdededee   fdZddededefdZy)
BasicLexerr   ignore_typesnewline_typesuser_callbacksr   rP   Nr$   c                    t        |j                        }t        d |D              sJ |       |j                  | _        |j
                  sIi }|D ]  }|j                  j                         }	 | j                  j                  ||j                         |j                  j                  dk(  r&t        d|j                  d|j                  d      |j                  j                  dk(  s|||<    t        |j                         |D ch c]  }|j                   c}k  s<t        dt        |j                         |D ch c]  }|j                   c}z
  z        t"        rt%        |||j&                         n|j&                  rt        d	      t)        d
 |D              | _        t)        |j                         | _        |j/                  d        || _        |j0                  | _        |j                  | _	        |j4                  | _        |j6                  | _        d | _        y # | j                  j                  $ r& t        d|j                  d|j                        w xY wc c}w c c}w )Nc              3   <   K   | ]  }t        |t                y wr&   )ri   rd   r   r   s     r)   r   z&BasicLexer.__init__.<locals>.<genexpr>  s     Ay!:a-ys   zCannot compile token z: r   z,Lexer does not allow zero-width terminals. (rB   rP   z$Ignore terminals are not defined: %szUinteregular must be installed for strict mode. Use `pip install 'lark[interegular]'`.c              3   |   K   | ]4  }t        |j                  j                               s'|j                   6 y wr&   )r  rf   r.   re   rC  s     r)   r   z&BasicLexer.__init__.<locals>.<genexpr>  s-     &oy!DWXYXaXaXkXkXmDnqvvys   (<<c                     | j                    | j                  j                   t        | j                  j                         | j
                  fS r&   )rg   rf   r?   rT   r    re   )xs    r)   r   z%BasicLexer.__init__.<locals>.<lambda>  s7    qzzkAII4G4G3G#aiiooJ^I^`a`f`f%gr+   )key)listr   all	re_modulerP   skip_validationrf   r.   r   r   errorr   re   r<   r#   r   ignorehas_interegularr;  strictr'   r?  r>  sort	callbacksr@  r   terminals_by_name_scanner)r(   confr3  r   r$  r   r   s          r)   r*   zBasicLexer.__init__  s   (	AyAAL9LA..##!#,,.XGGOOFD,>,>? 99&&!+"\]\b\bdedmdm#noo99>>T),2&q)  $(CA(CCET[[IYmv\wmvhi]^]c]cmv\wIwxyy'(:JTvww '&oy&oo%dkk2gh""nn!//!%!7!77 ww}} X"AFFAII#VWWX )D\ws   /&H-I/I4-?I,c                    t        | j                  | j                  | j                  | j                        \  }| _        t        | j
                  j                               sJ | j                  j                         D ]M  \  }| j
                  v r+t        | j
                     |fd      | j
                  <   ?|| j
                  <   O t        || j                  | j                  | j                        | _        y )Nc                 "    | j                   k(  S r&   )r#   )r   r   s    r)   r   z+BasicLexer._build_scanner.<locals>.<lambda>)  s    TUTZTZ^cTcr+   )r   r   r   rP   r   r   rI  r,  r@  itemsr   r   rS  )r(   r   rC   r   s      @r)   _build_scannerzBasicLexer._build_scanner"  s    #1$..$BTBTVZV]V]_c_m_m#n 	4=4=='')***++113HE1%'0u1EqJc'de$'(e$ 4  	4+=+=twwWr+   c                 R    | j                   | j                          | j                   S r&   )rS  rX  r/   s    r)   r   zBasicLexer.scanner/  s"    == !}}r+   c                 :    | j                   j                  ||      S r&   )r   r   )r(   r   r   s      r)   r   zBasicLexer.match5  s    ||!!$,,r+   r  r  c              #   n   K   t        t              5  	 | j                  ||       # 1 sw Y   y xY wwr&   )r   EOFError
next_token)r(   r  r  s      r)   r  zBasicLexer.lex8  s/     hooe\::   s   5)25	lex_statec           
         |j                   }|j                  t        |j                        k  r/| j	                  |j                  |j                        }|s| j
                  j                  | j                  z
  }|sdh}t        |j                  |j                  |j                  |j                  ||j                  xr |j                  g|| j                        |\  }}|| j                  vrt        |||j                  |j                  |j                        }|j                  ||| j                  v        |j                  |_        |j                  |_        |j                  |_        |j&                  | j(                  v r= | j(                  |j&                     |      }t+        |t              st-        d|z        ||_        |S || j(                  v rBt        |||j                  |j                  |j                        }	 | j(                  |   |	       |j                  ||| j                  v        |j                  t        |j                        k  r/t/        |       )Nz<END-OF-FILE>)allowedtoken_historyr  rR  z+Callbacks must return a token (returned %r))r  r   rT   r   r   r   r   r>  r   rw   rx   r	  rR  rs   r   r?  ry   rz   r{   r#   r   ri   r   r\  )
r(   r^  r  r  r   r`  r    r   r   r   s
             r)   r]  zBasicLexer.next_token=  s   %%#inn"55**Y^^X->->?C,,44t7H7HH./G*9>>8;L;Lhmm]e]l]l3:)J^J^Jydmdxdxcy1=QUQgQgi i LE5D---%(9(98==(//ZeUd.@.@%@A%]]
'$--	66T]]*-aff-a0A%a/&'TWX'XYY'(	$DMM)ueX->->x_B(DMM%(,eUd.@.@%@A9 #inn"55> tnr+   r&   )rT  r   r$   N)rF   rG   rH   r
   rd   rJ   r   rI   r   	_Callbackr   r*   rX  rK   r   r   r  r   r	   rs   r  r]  rE   r+   r)   r=  r=    s    +&&C. S>!i((3	>""N(TX  
-; ;3 ;8E? ;
!J !c !U !r+   r=  c            	       v    e Zd ZU eeef   ed<   eed<   ddddeeee   f   dee   ddfd	Zd
e	de
dee   fdZy)ContextualLexerlexers
root_lexerrT  r   statesalways_acceptr$   Nc                    t        |j                        }|j                  }t        |      }||_        t        rS|j
                  sGt        j                  j                  |D ci c]  }||j                  j                          c}      }nd }i }	i | _        |j                         D ]%  \  }
}t        |      }	 |	|   }|| j                  |
<   ' |j                  |u sJ d|_        t#        ||      | _        y c c}w # t        $ ro t        |      t        |j                         z  t        |      z  }t        |      }|D cg c]  }||v s||    nc c}w c}|_        t#        ||      }||	|<   Y w xY w)NT)rH  r   rR  r   rN  rK  r(  r)  r*  rf   r.   re  rW  r'   KeyErrorr   rM  r=  rf  )r(   rT  rg  rh  r   rR  	trad_confr   r3  lexer_by_tokensr  acceptsrG  r  
lexer_confns                   r)   r*   zContextualLexer.__init__f  sj   (	 22J	'	4#7#7$//<<`i=j`i[\aATATAV>V`i=jkJJ<>$llnNE7G$C-', "'DKK - ""i///$(	!$Y
;) >k  -g,T[[)99C<NN!)_
FM'hgQRVgQg(9!(<g'h
$":z:',$-s+   #"C9:C>>AE6	EEE65E6r  r  c              #   n  K   	 	 | j                   |j                     }|j                  ||       .# t        $ r Y y t        $ rl}	 |j
                  }| j                  j                  ||      }t        ||j                  ||g| j                  j                        # t        $ r |w xY wd }~ww xY ww)N)r  ra  rR  )
re  positionr]  r\  r   r	  rf  r   r`  rR  )r(   r  r  r  er	  r   s          r)   r  zContextualLexer.lex  s     	L$9$9:&&{LAA   	# 	(33
22;M%eQYYl[eZfz~  {J  {J  {\  {\  ]  ]' 	s7   B5/3 	B2B5B2ABB**B--B22B5)rE   )rF   rG   rH   r   rI   r=  rJ   r
   r*   r  r   r	   rs   r  rE   r+   r)   rd  rd  a  sl    j!!<[ <$sJsO7K2L <]ghk]l <tx <<z  % r+   rd  )   )>abcr   r   rP   
contextlibr   typingr   r   r   r	   r
   r   r   r   r   r   r   r   typesr   r   r(  ImportErrorcommonr   utilsr   r   r   r   
exceptionsr   r   r   grammarr   r   boolrN  	NameErrorr   rN   rY   rd   rr   rI   rs   r   r   r   r   r   r   r  r  r  rb  r  r;  r=  rd  rE   r+   r)   <module>r     s   $ 	      	 ! @ @ G G + ;'O'i 'T "$ $.) . T!AC AH> >> : :
#,#/ #/L[3 [K K4 * eWe^$	 C  [#5E0F @n nb2e 2g  		  Os#   D1 )D< 1D98D9<EE