| Server IP : 121.121.20.254 / Your IP : 216.73.217.141 Web Server : Microsoft-IIS/10.0 System : Windows NT WEB-SERVER 10.0 build 20348 (Windows Server 2022) AMD64 User : IUSR ( 0) PHP Version : 8.3.28 Disable Function : NONE MySQL : ON | cURL : ON | WGET : OFF | Perl : OFF | Python : OFF | Sudo : OFF | Pkexec : OFF Directory : C:/Python313/Lib/test/__pycache__/ |
Upload File : |
�
��i�� � �� � S SK r S SKrS SKrS SKrS SKrS SKJrJr S SKJ r S SKJ
r
J r S SK J
r
S SKJr S SKJrJrJr S SKJrJr S r " S
S \
5 r " S S
\5 rS r " S S\
5 r " S S\
5 r " S S\
5 r " S S\
5 r " S S\
5 r " S S\
5 r S r! " S S \
5 r" " S S \
5 r# " S S!\
5 r$ " S" S#\R 5 r%\&S$:X a \RN " 5 gg)%� N)�BytesIO�StringIO)�dedent)�TestCase�mock)�support)� os_helper)�run_test_script� make_script�run_python_until_end)�VALID_UNDERSCORE_LITERALS�INVALID_UNDERSCORE_LITERALSc �F � / n[ UR 5 5 nUS S; nU Hw u pVpxn U[ R :X a U$ U( a U[ R :X a US U:X a MD [ R
U nUR
SUS SU<S SU SU 35 My U$ )N���z
r � �10� �13)�len�
splitlines�tokenize� ENDMARKER�NEWLINE�tok_name�append)
�token_generator�
source_string�result� num_lines�missing_trailing_nl�type�token�start�end�lines
�'C:\Python313\\Lib\test\test_tokenize.py� stringify_tokens_from_sourcer' s� � �
�F��M�,�,�.�/�I�'��+�6�9��)8�%��U�� �8�%�%� %� � �M� �4�8�+;�+;�#;��A��)�@S� �� � ��&���
�
��T�"�I�Q�u�l�r�l�!�E�7�!�C�5�A�B� *9� �M� c � � \ rS rSrS rS rS rS rS rS r S r
S r S
r S r
S rS
rS rS rS rS rS rS rS rS rS rS rS rS rS r Sr g ) � TokenizeTest�& c �� � [ UR S5 5 n[ [ R " UR 5 U5 nU R US/UR
5 R 5 - 5 g )N�utf-8z* ENCODING 'utf-8' (0, 0) (0, 0))r �encoder' r �readline� assertEqual�rstripr ��self�s�expected�fr s r& �check_tokenize�TokenizeTest.check_tokenize- s` � �
�A�H�H�W�%� &��-�h�.?�.?��
�
�.K�Q�O�� ����F�G�!� � �*�5�5�7�8� 9r( c �D � S nU R [ 5 [ [ R " U" 5 R 5 5 S S S 5 S nU R [ 5 [ [ R
" U" 5 R 5 5 S S S 5 S nU R [ 5 [ [ R
" U" 5 R 5 5 S S S 5 g ! , ( d f N�= f! , ( d f No= f! , ( d f g = f)Nc 3 � # � Sv � Sv � g 7f)N�sdfosdg� r< r( r& �gen�/TokenizeTest.test_invalid_readline.<locals>.gen7 s � � ��O��O�� �
c 3 � # � Sv � Sv � g 7f)Ns sdfosdgr< r< r( r& r= r> = s � � � � � � �r? c 3 � # � Sv � SS- g 7f)Nr; � r r< r< r( r& r= r> C s � � ��O�
�a�C�s �
)� assertRaises� TypeError�listr �__next__�generate_tokens�ZeroDivisionError)r3 r= s r& �test_invalid_readline�"TokenizeTest.test_invalid_readline6 s� � � � �
�
�y�
)� ��"�"�3�5�>�>�2� 3� *� � �
�
�y�
)� ��)�)�#�%�.�.�9� :� *� � �
�
� 0�
1� ��)�)�#�%�.�.�9� :� 2�
1� *�
)�� *�
)�� 2�
1�s# �/C/�(/D �7/D�/
C=�
D�
D c �> � [ SR S5 5 n[ [ R " UR 5 5 nU R US R [ R 5 U R US R [ R 5 g )N�xr- ���r ) r r. rE r r/ r0 r! r r )r3 r6 �tokenss r& �test_implicit_newline�"TokenizeTest.test_implicit_newlineI sl � �
�C�J�J�w�'� (���h�'�'��
�
�3�4�� ������ � �(�*:�*:�;� ������ � �(�*<�*<�=r( c �r � U R SS5 U R SS5 U R SS5 U R SS5 S n[ U5 R nU R [ S
5 n[
R
" U5 H nM S S S 5 U R
WR R S 5 U R
UR R S 5 U R
UR R S 5 U R
UR R S 5 U R
UR R S
5 U R
UR R S
5 U R
UR R S5 g ! , ( d f GN= f)Nz1 + 1z� NUMBER '1' (1, 0) (1, 1)
OP '+' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
z3if False:
# NL
True = False # NEWLINE
af NAME 'if' (1, 0) (1, 2)
NAME 'False' (1, 3) (1, 8)
OP ':' (1, 8) (1, 9)
NEWLINE '\n' (1, 9) (1, 10)
COMMENT '# NL' (2, 4) (2, 8)
NL '\n' (2, 8) (2, 9)
NL '\n' (3, 4) (3, 5)
INDENT ' ' (4, 0) (4, 4)
NAME 'True' (4, 4) (4, 8)
OP '=' (4, 9) (4, 10)
NAME 'False' (4, 11) (4, 16)
COMMENT '# NEWLINE' (4, 17) (4, 26)
NEWLINE '\n' (4, 26) (4, 27)
DEDENT '' (5, 0) (5, 0)
z%if True:
# NL
foo='bar'
a@ NAME 'if' (1, 0) (1, 2)
NAME 'True' (1, 3) (1, 7)
OP ':' (1, 7) (1, 8)
NEWLINE '\r\n' (1, 8) (1, 10)
COMMENT '# NL' (2, 4) (2, 8)
NL '\r\n' (2, 8) (2, 10)
INDENT ' ' (3, 0) (3, 4)
NAME 'foo' (3, 4) (3, 7)
OP '=' (3, 7) (3, 8)
STRING "'bar'" (3, 8) (3, 13)
NEWLINE '\r\n' (3, 13) (3, 15)
NL '\r\n' (4, 0) (4, 2)
DEDENT '' (5, 0) (5, 0)
zx = 1 + \
1
a NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '+' (1, 6) (1, 7)
NUMBER '1' (2, 0) (2, 1)
NEWLINE '\r\n' (2, 1) (2, 3)
s def k(x):
x += 2
x += 5
z3unindent does not match any outer indentation level� �<string>� z x += 5)r7 r r/ �assertRaisesRegex�IndentationErrorr r0 � exception�lineno�filename�
end_lineno�
end_offset�msg�offset�text)r3 �indent_error_filer/ �e�toks r& �
test_basic�TokenizeTest.test_basicQ s{ � � ���G� &� �
��� ;�=� �(
��� K� N� �
��� 1� 4� � ��
�,�-�6�6��
�
#�
#�$4�%>�?�BC� �(�(��2��� 3�?�
������+�+�Q�/� ������-�-�z�:� ������/�/��6� ������/�/��6� ���
�K�K�O�O� A� C�
������+�+�Q�/� ������)�)�:�6�?� ?�s �6 F'�'
F6c �H � U R SS5 U R SS5 U R SS5 U R SS5 U R S S
5 U R S S 5 U R S
S5 U R SS5 U R SS5 g )N� 0xff <= 255z� NUMBER '0xff' (1, 0) (1, 4)
OP '<=' (1, 5) (1, 7)
NUMBER '255' (1, 8) (1, 11)
� 0b10 <= 255z� NUMBER '0b10' (1, 0) (1, 4)
OP '<=' (1, 5) (1, 7)
NUMBER '255' (1, 8) (1, 11)
�0o123 <= 0O123z� NUMBER '0o123' (1, 0) (1, 5)
OP '<=' (1, 6) (1, 8)
NUMBER '0O123' (1, 9) (1, 14)
�1234567 > ~0x15z� NUMBER '1234567' (1, 0) (1, 7)
OP '>' (1, 8) (1, 9)
OP '~' (1, 10) (1, 11)
NUMBER '0x15' (1, 11) (1, 15)
�2134568 != 1231515z� NUMBER '2134568' (1, 0) (1, 7)
OP '!=' (1, 8) (1, 10)
NUMBER '1231515' (1, 11) (1, 18)
�(-124561-1) & 200000000ac OP '(' (1, 0) (1, 1)
OP '-' (1, 1) (1, 2)
NUMBER '124561' (1, 2) (1, 8)
OP '-' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
OP ')' (1, 10) (1, 11)
OP '&' (1, 12) (1, 13)
NUMBER '200000000' (1, 14) (1, 23)
�0xdeadbeef != -1z� NUMBER '0xdeadbeef' (1, 0) (1, 10)
OP '!=' (1, 11) (1, 13)
OP '-' (1, 14) (1, 15)
NUMBER '1' (1, 15) (1, 16)
�0xdeadc0de & 12345z� NUMBER '0xdeadc0de' (1, 0) (1, 10)
OP '&' (1, 11) (1, 12)
NUMBER '12345' (1, 13) (1, 18)
�0xFF & 0x15 | 1234z� NUMBER '0xFF' (1, 0) (1, 4)
OP '&' (1, 5) (1, 6)
NUMBER '0x15' (1, 7) (1, 11)
OP '|' (1, 12) (1, 13)
NUMBER '1234' (1, 14) (1, 18)
�r7 �r3 s r& �test_int�TokenizeTest.test_int� s� � � ���M� ,� �
���M� ,� �
��� ,� /� �
��� -� 0� �
��� 0� 3� �
��� 5� 8� �
��� .� 1� �
��� 0� 3� �
��� 0� 3� r( c � � U R SS5 U R SS5 U R SS5 U R SS5 g ) Nzx = 0z� NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '0' (1, 4) (1, 5)
zx = 0xfffffffffffz� NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '0xfffffffffff' (1, 4) (1, 17)
zx = 123141242151251616110z� NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '123141242151251616110' (1, 4) (1, 25)
zx = -15921590215012591z� NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
OP '-' (1, 4) (1, 5)
NUMBER '15921590215012591' (1, 5) (1, 22)
rn ro s r& � test_long�TokenizeTest.test_long� s_ � � ���G� &� �
��� /� 2� �
��� 7� :� �
��� 4� 7� r( c � � U R SS5 U R SS5 U R SS5 U R SS5 U R S S
5 U R S S 5 U R S
S5 g )N� x = 3.14159z� NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3.14159' (1, 4) (1, 11)
� x = 314159.z� NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '314159.' (1, 4) (1, 11)
� x = .314159z� NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '.314159' (1, 4) (1, 11)
� x = 3e14159z� NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3e14159' (1, 4) (1, 11)
� x = 3E123z� NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3E123' (1, 4) (1, 9)
�
x+y = 3e-1230z� NAME 'x' (1, 0) (1, 1)
OP '+' (1, 1) (1, 2)
NAME 'y' (1, 2) (1, 3)
OP '=' (1, 4) (1, 5)
NUMBER '3e-1230' (1, 6) (1, 13)
� x = 3.14e159z� NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '3.14e159' (1, 4) (1, 12)
rn ro s r& �
test_float�TokenizeTest.test_float� s� � � ���M� ,� �
���M� ,� �
���M� ,� �
���M� ,� �
���K� *� �
���O� .� �
���N� -� r( c � � S n[ H# nSU; a M U R U" U5 U5 M% SS1n[ H+ nX#; a M
U" U5 U R U" U5 U5 M- g ! [ R a MG f = f)Nc � � [ U R S5 5 n[ R " UR 5 H u p#pEnU[ R :X d M Us $ g)Nr- z
invalid token)r r. r r/ �NUMBER)r4 r6 �toktyper" r# r$ r% s r& � number_token�;TokenizeTest.test_underscore_literals.<locals>.number_token sL � ������)�*�A�4<�4E�4E�a�j�j�4Q�0���D��h�o�o�-� �L� 5R� #r( �(�0_7�09_99)r
r0 r r �
TokenError�assertNotEqual)r3 r� �lit�extra_valid_casess r& �test_underscore_literals�%TokenizeTest.test_underscore_literals s� � � #� -�C��c�z�� � � �\�#� .�� 4� -� #�G� ,��.�C��'��
� �S�!�
� � � �S� 1�3� 7� /��
�&�&�
��
�s �A+�+B�Bc � � U R SS5 U R SS5 U R SS5 U R SS5 U R S S
5 U R S S 5 U R S
S5 U R SS5 U R SS5 U R SS5 U R SS5 U R SS5 U R SS5 U R SS 5 U R S S 5 U R S S 5 U R S!S"5 U R S#S$5 U R S%S&5 U R S'S(5 U R S)S*5 U R S+S,5 U R S-S.5 U R S/S05 U R S1S25 U R S3S45 U R S5S65 U R S7S85 U R S9S:5 U R S;S<5 U R S=S>5 U R S?S@5 U R SASB5 U R SCSD5 U R SESF5 U R SGSH5 g )IN�x = ''; y = ""a5 NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING "''" (1, 4) (1, 6)
OP ';' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
OP '=' (1, 10) (1, 11)
STRING '""' (1, 12) (1, 14)
�x = '"'; y = "'"a6 NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING '\'"\'' (1, 4) (1, 7)
OP ';' (1, 7) (1, 8)
NAME 'y' (1, 9) (1, 10)
OP '=' (1, 11) (1, 12)
STRING '"\'"' (1, 13) (1, 16)
� x = "doesn't "shrink", does it"z� NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING '"doesn\'t "' (1, 4) (1, 14)
NAME 'shrink' (1, 14) (1, 20)
STRING '", does it"' (1, 20) (1, 31)
�x = 'abc' + 'ABC'z� NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING "'abc'" (1, 4) (1, 9)
OP '+' (1, 10) (1, 11)
STRING "'ABC'" (1, 12) (1, 17)
�y = "ABC" + "ABC"z� NAME 'y' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING '"ABC"' (1, 4) (1, 9)
OP '+' (1, 10) (1, 11)
STRING '"ABC"' (1, 12) (1, 17)
�%x = r'abc' + r'ABC' + R'ABC' + R'ABC'a� NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING "r'abc'" (1, 4) (1, 10)
OP '+' (1, 11) (1, 12)
STRING "r'ABC'" (1, 13) (1, 19)
OP '+' (1, 20) (1, 21)
STRING "R'ABC'" (1, 22) (1, 28)
OP '+' (1, 29) (1, 30)
STRING "R'ABC'" (1, 31) (1, 37)
�%y = r"abc" + r"ABC" + R"ABC" + R"ABC"a� NAME 'y' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
STRING 'r"abc"' (1, 4) (1, 10)
OP '+' (1, 11) (1, 12)
STRING 'r"ABC"' (1, 13) (1, 19)
OP '+' (1, 20) (1, 21)
STRING 'R"ABC"' (1, 22) (1, 28)
OP '+' (1, 29) (1, 30)
STRING 'R"ABC"' (1, 31) (1, 37)
�u'abc' + U'abc'z� STRING "u'abc'" (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING "U'abc'" (1, 9) (1, 15)
�u"abc" + U"abc"z� STRING 'u"abc"' (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING 'U"abc"' (1, 9) (1, 15)
�b'abc' + B'abc'z� STRING "b'abc'" (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING "B'abc'" (1, 9) (1, 15)
�b"abc" + B"abc"z� STRING 'b"abc"' (1, 0) (1, 6)
OP '+' (1, 7) (1, 8)
STRING 'B"abc"' (1, 9) (1, 15)
�%br'abc' + bR'abc' + Br'abc' + BR'abc'a; STRING "br'abc'" (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING "bR'abc'" (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING "Br'abc'" (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING "BR'abc'" (1, 30) (1, 37)
�%br"abc" + bR"abc" + Br"abc" + BR"abc"a; STRING 'br"abc"' (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING 'bR"abc"' (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING 'Br"abc"' (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING 'BR"abc"' (1, 30) (1, 37)
�%rb'abc' + rB'abc' + Rb'abc' + RB'abc'a; STRING "rb'abc'" (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING "rB'abc'" (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING "Rb'abc'" (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING "RB'abc'" (1, 30) (1, 37)
�%rb"abc" + rB"abc" + Rb"abc" + RB"abc"a; STRING 'rb"abc"' (1, 0) (1, 7)
OP '+' (1, 8) (1, 9)
STRING 'rB"abc"' (1, 10) (1, 17)
OP '+' (1, 18) (1, 19)
STRING 'Rb"abc"' (1, 20) (1, 27)
OP '+' (1, 28) (1, 29)
STRING 'RB"abc"' (1, 30) (1, 37)
� "a\
de\
fg"�3 STRING '"a\\\nde\\\nfg"' (1, 0) (3, 3)
�u"a\
de"�/ STRING 'u"a\\\nde"' (1, 0) (2, 3)
�rb"a\
d"�/ STRING 'rb"a\\\nd"' (1, 0) (2, 2)
�
"""a\
b"""�0 STRING '"""a\\\nb"""' (1, 0) (2, 4)
� u"""a\
b"""�1 STRING 'u"""a\\\nb"""' (1, 0) (2, 4)
�rb"""a\
b\
c"""�7 STRING 'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4)
�f"abc"�� FSTRING_START 'f"' (1, 0) (1, 2)
FSTRING_MIDDLE 'abc' (1, 2) (1, 5)
FSTRING_END '"' (1, 5) (1, 6)
� fR"a{b}c"a= FSTRING_START 'fR"' (1, 0) (1, 3)
FSTRING_MIDDLE 'a' (1, 3) (1, 4)
OP '{' (1, 4) (1, 5)
NAME 'b' (1, 5) (1, 6)
OP '}' (1, 6) (1, 7)
FSTRING_MIDDLE 'c' (1, 7) (1, 8)
FSTRING_END '"' (1, 8) (1, 9)
zfR"a{{{b!r}}}c"a� FSTRING_START 'fR"' (1, 0) (1, 3)
FSTRING_MIDDLE 'a{' (1, 3) (1, 5)
OP '{' (1, 6) (1, 7)
NAME 'b' (1, 7) (1, 8)
OP '!' (1, 8) (1, 9)
NAME 'r' (1, 9) (1, 10)
OP '}' (1, 10) (1, 11)
FSTRING_MIDDLE '}' (1, 11) (1, 12)
FSTRING_MIDDLE 'c' (1, 13) (1, 14)
FSTRING_END '"' (1, 14) (1, 15)
z f"{{{1+1}}}"a� FSTRING_START 'f"' (1, 0) (1, 2)
FSTRING_MIDDLE '{' (1, 2) (1, 3)
OP '{' (1, 4) (1, 5)
NUMBER '1' (1, 5) (1, 6)
OP '+' (1, 6) (1, 7)
NUMBER '1' (1, 7) (1, 8)
OP '}' (1, 8) (1, 9)
FSTRING_MIDDLE '}' (1, 9) (1, 10)
FSTRING_END '"' (1, 11) (1, 12)
z f"""{f'''{f'{f"{1+1}"}'}'''}"""ad FSTRING_START 'f"""' (1, 0) (1, 4)
OP '{' (1, 4) (1, 5)
FSTRING_START "f'''" (1, 5) (1, 9)
OP '{' (1, 9) (1, 10)
FSTRING_START "f'" (1, 10) (1, 12)
OP '{' (1, 12) (1, 13)
FSTRING_START 'f"' (1, 13) (1, 15)
OP '{' (1, 15) (1, 16)
NUMBER '1' (1, 16) (1, 17)
OP '+' (1, 17) (1, 18)
NUMBER '1' (1, 18) (1, 19)
OP '}' (1, 19) (1, 20)
FSTRING_END '"' (1, 20) (1, 21)
OP '}' (1, 21) (1, 22)
FSTRING_END "'" (1, 22) (1, 23)
OP '}' (1, 23) (1, 24)
FSTRING_END "'''" (1, 24) (1, 27)
OP '}' (1, 27) (1, 28)
FSTRING_END '"""' (1, 28) (1, 31)
z.f""" x
str(data, encoding={invalid!r})
"""a� FSTRING_START 'f"""' (1, 0) (1, 4)
FSTRING_MIDDLE ' x\nstr(data, encoding=' (1, 4) (2, 19)
OP '{' (2, 19) (2, 20)
NAME 'invalid' (2, 20) (2, 27)
OP '!' (2, 27) (2, 28)
NAME 'r' (2, 28) (2, 29)
OP '}' (2, 29) (2, 30)
FSTRING_MIDDLE ')\n' (2, 30) (3, 0)
FSTRING_END '"""' (3, 0) (3, 3)
z#f"""123456789
something{None}bad"""aO FSTRING_START 'f"""' (1, 0) (1, 4)
FSTRING_MIDDLE '123456789\nsomething' (1, 4) (2, 9)
OP '{' (2, 9) (2, 10)
NAME 'None' (2, 10) (2, 14)
OP '}' (2, 14) (2, 15)
FSTRING_MIDDLE 'bad' (2, 15) (2, 18)
FSTRING_END '"""' (2, 18) (2, 21)
�
f"""abc"""�� FSTRING_START 'f"""' (1, 0) (1, 4)
FSTRING_MIDDLE 'abc' (1, 4) (1, 7)
FSTRING_END '"""' (1, 7) (1, 10)
� f"abc\
def"�� FSTRING_START 'f"' (1, 0) (1, 2)
FSTRING_MIDDLE 'abc\\\ndef' (1, 2) (2, 3)
FSTRING_END '"' (2, 3) (2, 4)
� Rf"abc\
def"�� FSTRING_START 'Rf"' (1, 0) (1, 3)
FSTRING_MIDDLE 'abc\\\ndef' (1, 3) (2, 3)
FSTRING_END '"' (2, 3) (2, 4)
z5f'some words {a+b:.3f} more words {c+d=} final words'aA FSTRING_START "f'" (1, 0) (1, 2)
FSTRING_MIDDLE 'some words ' (1, 2) (1, 13)
OP '{' (1, 13) (1, 14)
NAME 'a' (1, 14) (1, 15)
OP '+' (1, 15) (1, 16)
NAME 'b' (1, 16) (1, 17)
OP ':' (1, 17) (1, 18)
FSTRING_MIDDLE '.3f' (1, 18) (1, 21)
OP '}' (1, 21) (1, 22)
FSTRING_MIDDLE ' more words ' (1, 22) (1, 34)
OP '{' (1, 34) (1, 35)
NAME 'c' (1, 35) (1, 36)
OP '+' (1, 36) (1, 37)
NAME 'd' (1, 37) (1, 38)
OP '=' (1, 38) (1, 39)
OP '}' (1, 39) (1, 40)
FSTRING_MIDDLE ' final words' (1, 40) (1, 52)
FSTRING_END "'" (1, 52) (1, 53)
z
f'''{
3
=}'''a` FSTRING_START "f'''" (1, 0) (1, 4)
OP '{' (1, 4) (1, 5)
NL '\n' (1, 5) (1, 6)
NUMBER '3' (2, 0) (2, 1)
NL '\n' (2, 1) (2, 2)
OP '=' (3, 0) (3, 1)
OP '}' (3, 1) (3, 2)
FSTRING_END "'''" (3, 2) (3, 5)
�f'''__{
x:a
}__'''a� FSTRING_START "f'''" (1, 0) (1, 4)
FSTRING_MIDDLE '__' (1, 4) (1, 6)
OP '{' (1, 6) (1, 7)
NL '\n' (1, 7) (1, 8)
NAME 'x' (2, 4) (2, 5)
OP ':' (2, 5) (2, 6)
FSTRING_MIDDLE 'a\n' (2, 6) (3, 0)
OP '}' (3, 0) (3, 1)
FSTRING_MIDDLE '__' (3, 1) (3, 3)
FSTRING_END "'''" (3, 3) (3, 6)
�+f'''__{
x:a
b
c
d
}__'''a� FSTRING_START "f'''" (1, 0) (1, 4)
FSTRING_MIDDLE '__' (1, 4) (1, 6)
OP '{' (1, 6) (1, 7)
NL '\n' (1, 7) (1, 8)
NAME 'x' (2, 4) (2, 5)
OP ':' (2, 5) (2, 6)
FSTRING_MIDDLE 'a\n b\n c\n d\n' (2, 6) (6, 0)
OP '}' (6, 0) (6, 1)
FSTRING_MIDDLE '__' (6, 1) (6, 3)
FSTRING_END "'''" (6, 3) (6, 6)
uz '''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli
aktualni pracownicy, obecni pracownicy'''
u INDENT ' ' (1, 0) (1, 4)
STRING "'''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli\n aktualni pracownicy, obecni pracownicy'''" (1, 4) (2, 45)
NEWLINE '\n' (2, 45) (2, 46)
DEDENT '' (3, 0) (3, 0)
rn ro s r& � test_string�TokenizeTest.test_string* s � � ��� .� 1� �
��� 1� 4� �
��� A� D� �
��� /� 2� �
��� /� 2� �
��� C�
F�
�
��� C�
F�
�
��� -� 0� �
��� -� 0� �
��� -� 0� �
��� -� 0� �
��� C� F� �
��� C� F� �
��� C� F� �
��� C� F� �
��� �� �
��� �� �
��� �� �
��� �� �
��� �� �
��� �� �
���H� '� �
���K� *� �
��� -� 0� �
���N�
-�
�
��� E� H� �*
��� N�
Q�
�
��� B� E� �
���L� +� �
��� �� �
��� �� �
��� S� V� �(
��� � � �
���
� � �
���
� �
�&
��� �� r( c �p � U R SS5 U R SS5 U R SS5 g )N�!def d22(a, b, c=2, d=2, *k): passa NAME 'def' (1, 0) (1, 3)
NAME 'd22' (1, 4) (1, 7)
OP '(' (1, 7) (1, 8)
NAME 'a' (1, 8) (1, 9)
OP ',' (1, 9) (1, 10)
NAME 'b' (1, 11) (1, 12)
OP ',' (1, 12) (1, 13)
NAME 'c' (1, 14) (1, 15)
OP '=' (1, 15) (1, 16)
NUMBER '2' (1, 16) (1, 17)
OP ',' (1, 17) (1, 18)
NAME 'd' (1, 19) (1, 20)
OP '=' (1, 20) (1, 21)
NUMBER '2' (1, 21) (1, 22)
OP ',' (1, 22) (1, 23)
OP '*' (1, 24) (1, 25)
NAME 'k' (1, 25) (1, 26)
OP ')' (1, 26) (1, 27)
OP ':' (1, 27) (1, 28)
NAME 'pass' (1, 29) (1, 33)
� def d01v_(a=1, *k, **w): passa� NAME 'def' (1, 0) (1, 3)
NAME 'd01v_' (1, 4) (1, 9)
OP '(' (1, 9) (1, 10)
NAME 'a' (1, 10) (1, 11)
OP '=' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
OP ',' (1, 13) (1, 14)
OP '*' (1, 15) (1, 16)
NAME 'k' (1, 16) (1, 17)
OP ',' (1, 17) (1, 18)
OP '**' (1, 19) (1, 21)
NAME 'w' (1, 21) (1, 22)
OP ')' (1, 22) (1, 23)
OP ':' (1, 23) (1, 24)
NAME 'pass' (1, 25) (1, 29)
�&def d23(a: str, b: int=3) -> int: passa� NAME 'def' (1, 0) (1, 3)
NAME 'd23' (1, 4) (1, 7)
OP '(' (1, 7) (1, 8)
NAME 'a' (1, 8) (1, 9)
OP ':' (1, 9) (1, 10)
NAME 'str' (1, 11) (1, 14)
OP ',' (1, 14) (1, 15)
NAME 'b' (1, 16) (1, 17)
OP ':' (1, 17) (1, 18)
NAME 'int' (1, 19) (1, 22)
OP '=' (1, 22) (1, 23)
NUMBER '3' (1, 23) (1, 24)
OP ')' (1, 24) (1, 25)
OP '->' (1, 26) (1, 28)
NAME 'int' (1, 29) (1, 32)
OP ':' (1, 32) (1, 33)
NAME 'pass' (1, 34) (1, 38)
rn ro s r& �
test_function�TokenizeTest.test_functionj sJ � � ��� ?� B� �,
��� ;� >� �"
��� D� G� r( c �( � U R SS5 g )N�Xif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa� NAME 'if' (1, 0) (1, 2)
NUMBER '1' (1, 3) (1, 4)
OP '<' (1, 5) (1, 6)
NUMBER '1' (1, 7) (1, 8)
OP '>' (1, 9) (1, 10)
NUMBER '1' (1, 11) (1, 12)
OP '==' (1, 13) (1, 15)
NUMBER '1' (1, 16) (1, 17)
OP '>=' (1, 18) (1, 20)
NUMBER '5' (1, 21) (1, 22)
OP '<=' (1, 23) (1, 25)
NUMBER '0x15' (1, 26) (1, 30)
OP '<=' (1, 31) (1, 33)
NUMBER '0x12' (1, 34) (1, 38)
OP '!=' (1, 39) (1, 41)
NUMBER '1' (1, 42) (1, 43)
NAME 'and' (1, 44) (1, 47)
NUMBER '5' (1, 48) (1, 49)
NAME 'in' (1, 50) (1, 52)
NUMBER '1' (1, 53) (1, 54)
NAME 'not' (1, 55) (1, 58)
NAME 'in' (1, 59) (1, 61)
NUMBER '1' (1, 62) (1, 63)
NAME 'is' (1, 64) (1, 66)
NUMBER '1' (1, 67) (1, 68)
NAME 'or' (1, 69) (1, 71)
NUMBER '5' (1, 72) (1, 73)
NAME 'is' (1, 74) (1, 76)
NAME 'not' (1, 77) (1, 80)
NUMBER '1' (1, 81) (1, 82)
OP ':' (1, 82) (1, 83)
NAME 'pass' (1, 84) (1, 88)
rn ro s r& �test_comparison� TokenizeTest.test_comparison� � � � ��� M�!O�" r( c �( � U R SS5 g )Nzx = 1 << 1 >> 5a6 NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '<<' (1, 6) (1, 8)
NUMBER '1' (1, 9) (1, 10)
OP '>>' (1, 11) (1, 13)
NUMBER '5' (1, 14) (1, 15)
rn ro s r& �
test_shift�TokenizeTest.test_shift� s � � ��� -� 0� r( c �( � U R SS5 g )N�%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$ NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '-' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
OP '+' (1, 10) (1, 11)
NUMBER '15' (1, 12) (1, 14)
OP '-' (1, 15) (1, 16)
NUMBER '1' (1, 17) (1, 18)
OP '+' (1, 19) (1, 20)
NUMBER '0x124' (1, 21) (1, 26)
OP '+' (1, 27) (1, 28)
NAME 'z' (1, 29) (1, 30)
OP '+' (1, 31) (1, 32)
NAME 'a' (1, 33) (1, 34)
OP '[' (1, 34) (1, 35)
NUMBER '5' (1, 35) (1, 36)
OP ']' (1, 36) (1, 37)
rn ro s r& �
test_additive�TokenizeTest.test_additive� � � � ��� C� F� r( c �( � U R SS5 g )N�x = 1//1*1/5*12%0x12@42a� NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
OP '//' (1, 5) (1, 7)
NUMBER '1' (1, 7) (1, 8)
OP '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
OP '/' (1, 10) (1, 11)
NUMBER '5' (1, 11) (1, 12)
OP '*' (1, 12) (1, 13)
NUMBER '12' (1, 13) (1, 15)
OP '%' (1, 15) (1, 16)
NUMBER '0x12' (1, 16) (1, 20)
OP '@' (1, 20) (1, 21)
NUMBER '42' (1, 21) (1, 23)
rn ro s r& �test_multiplicative� TokenizeTest.test_multiplicative� � � � ��� 5� 8� r( c �L � U R SS5 U R SS5 g )N�~1 ^ 1 & 1 |1 ^ -1a� OP '~' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
OP '^' (1, 3) (1, 4)
NUMBER '1' (1, 5) (1, 6)
OP '&' (1, 7) (1, 8)
NUMBER '1' (1, 9) (1, 10)
OP '|' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
OP '^' (1, 14) (1, 15)
OP '-' (1, 16) (1, 17)
NUMBER '1' (1, 17) (1, 18)
�-1*1/1+1*1//1 - ---1**1aH OP '-' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
OP '*' (1, 2) (1, 3)
NUMBER '1' (1, 3) (1, 4)
OP '/' (1, 4) (1, 5)
NUMBER '1' (1, 5) (1, 6)
OP '+' (1, 6) (1, 7)
NUMBER '1' (1, 7) (1, 8)
OP '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
OP '//' (1, 10) (1, 12)
NUMBER '1' (1, 12) (1, 13)
OP '-' (1, 14) (1, 15)
OP '-' (1, 16) (1, 17)
OP '-' (1, 17) (1, 18)
OP '-' (1, 18) (1, 19)
NUMBER '1' (1, 19) (1, 20)
OP '**' (1, 20) (1, 22)
NUMBER '1' (1, 22) (1, 23)
rn ro s r& �
test_unary�TokenizeTest.test_unary s0 � � ��� 0� 3� �
��� 5� 8� r( c �( � U R SS5 g )N�/import sys, time
x = sys.modules['time'].time()a� NAME 'import' (1, 0) (1, 6)
NAME 'sys' (1, 7) (1, 10)
OP ',' (1, 10) (1, 11)
NAME 'time' (1, 12) (1, 16)
NEWLINE '\n' (1, 16) (1, 17)
NAME 'x' (2, 0) (2, 1)
OP '=' (2, 2) (2, 3)
NAME 'sys' (2, 4) (2, 7)
OP '.' (2, 7) (2, 8)
NAME 'modules' (2, 8) (2, 15)
OP '[' (2, 15) (2, 16)
STRING "'time'" (2, 16) (2, 22)
OP ']' (2, 22) (2, 23)
OP '.' (2, 23) (2, 24)
NAME 'time' (2, 24) (2, 28)
OP '(' (2, 28) (2, 29)
OP ')' (2, 29) (2, 30)
rn ro s r& �
test_selector�TokenizeTest.test_selector( � � � ��� N� Q� r( c �( � U R SS5 g )N� @staticmethod
def foo(x,y): passa OP '@' (1, 0) (1, 1)
NAME 'staticmethod' (1, 1) (1, 13)
NEWLINE '\n' (1, 13) (1, 14)
NAME 'def' (2, 0) (2, 3)
NAME 'foo' (2, 4) (2, 7)
OP '(' (2, 7) (2, 8)
NAME 'x' (2, 8) (2, 9)
OP ',' (2, 9) (2, 10)
NAME 'y' (2, 10) (2, 11)
OP ')' (2, 11) (2, 12)
OP ':' (2, 12) (2, 13)
NAME 'pass' (2, 14) (2, 18)
rn ro s r& � test_method�TokenizeTest.test_method> � � � ��� ?�
B�
r( c �( � U R SS5 g )Nz def f():
if x
passa_ NAME 'def' (1, 0) (1, 3)
NAME 'f' (1, 4) (1, 5)
OP '(' (1, 5) (1, 6)
OP ')' (1, 6) (1, 7)
OP ':' (1, 7) (1, 8)
NEWLINE '\n' (1, 8) (1, 9)
INDENT '\t' (2, 0) (2, 1)
NAME 'if' (2, 1) (2, 3)
NAME 'x' (2, 4) (2, 5)
NEWLINE '\n' (2, 5) (2, 6)
INDENT ' \t' (3, 0) (3, 9)
NAME 'pass' (3, 9) (3, 13)
DEDENT '' (4, 0) (4, 0)
DEDENT '' (4, 0) (4, 0)
rn ro s r& � test_tabs�TokenizeTest.test_tabsO s � � ��� -�/� r( c �( � U R SS5 g )Nu! Örter = 'places'
grün = 'green'u7 NAME 'Örter' (1, 0) (1, 5)
OP '=' (1, 6) (1, 7)
STRING "'places'" (1, 8) (1, 16)
NEWLINE '\n' (1, 16) (1, 17)
NAME 'grün' (2, 0) (2, 4)
OP '=' (2, 5) (2, 6)
STRING "'green'" (2, 7) (2, 14)
rn ro s r& �test_non_ascii_identifiers�'TokenizeTest.test_non_ascii_identifiersd s � � ��� @� C� r( c �( � U R SS5 g )N�# Örter = u'places'
grün = U'green'u7 NAME 'Örter' (1, 0) (1, 5)
OP '=' (1, 6) (1, 7)
STRING "u'places'" (1, 8) (1, 17)
NEWLINE '\n' (1, 17) (1, 18)
NAME 'grün' (2, 0) (2, 4)
OP '=' (2, 5) (2, 6)
STRING "U'green'" (2, 7) (2, 15)
rn ro s r& � test_unicode�TokenizeTest.test_unicodep � � � ��� B� E� r( c � � U R SS5 U R SS5 U R SS5 U R SS5 U R S S
5 U R S S 5 U R S
S5 U R SS5 U R SS5 U R SS5 U R SS5 U R SS5 U R SS5 U R SS 5 U R S S 5 U R S S 5 U R S!S"5 U R S#S$5 U R S%S&5 U R S'S(5 U R S)S*5 g )+N� async = 1z� NAME 'async' (1, 0) (1, 5)
OP '=' (1, 6) (1, 7)
NUMBER '1' (1, 8) (1, 9)
�a = (async = 1)a8 NAME 'a' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
OP '(' (1, 4) (1, 5)
NAME 'async' (1, 5) (1, 10)
OP '=' (1, 11) (1, 12)
NUMBER '1' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
�async()z� NAME 'async' (1, 0) (1, 5)
OP '(' (1, 5) (1, 6)
OP ')' (1, 6) (1, 7)
�class async(Bar):passa< NAME 'class' (1, 0) (1, 5)
NAME 'async' (1, 6) (1, 11)
OP '(' (1, 11) (1, 12)
NAME 'Bar' (1, 12) (1, 15)
OP ')' (1, 15) (1, 16)
OP ':' (1, 16) (1, 17)
NAME 'pass' (1, 17) (1, 21)
�class async:passz� NAME 'class' (1, 0) (1, 5)
NAME 'async' (1, 6) (1, 11)
OP ':' (1, 11) (1, 12)
NAME 'pass' (1, 12) (1, 16)
� await = 1z� NAME 'await' (1, 0) (1, 5)
OP '=' (1, 6) (1, 7)
NUMBER '1' (1, 8) (1, 9)
� foo.asyncz� NAME 'foo' (1, 0) (1, 3)
OP '.' (1, 3) (1, 4)
NAME 'async' (1, 4) (1, 9)
�async for a in b: passa; NAME 'async' (1, 0) (1, 5)
NAME 'for' (1, 6) (1, 9)
NAME 'a' (1, 10) (1, 11)
NAME 'in' (1, 12) (1, 14)
NAME 'b' (1, 15) (1, 16)
OP ':' (1, 16) (1, 17)
NAME 'pass' (1, 18) (1, 22)
�async with a as b: passa< NAME 'async' (1, 0) (1, 5)
NAME 'with' (1, 6) (1, 10)
NAME 'a' (1, 11) (1, 12)
NAME 'as' (1, 13) (1, 15)
NAME 'b' (1, 16) (1, 17)
OP ':' (1, 17) (1, 18)
NAME 'pass' (1, 19) (1, 23)
� async.fooz� NAME 'async' (1, 0) (1, 5)
OP '.' (1, 5) (1, 6)
NAME 'foo' (1, 6) (1, 9)
�async�/ NAME 'async' (1, 0) (1, 5)
�async
#comment
awaitz� NAME 'async' (1, 0) (1, 5)
NEWLINE '\n' (1, 5) (1, 6)
COMMENT '#comment' (2, 0) (2, 8)
NL '\n' (2, 8) (2, 9)
NAME 'await' (3, 0) (3, 5)
�async
...
awaitz� NAME 'async' (1, 0) (1, 5)
NEWLINE '\n' (1, 5) (1, 6)
OP '...' (2, 0) (2, 3)
NEWLINE '\n' (2, 3) (2, 4)
NAME 'await' (3, 0) (3, 5)
� async
awaitz� NAME 'async' (1, 0) (1, 5)
NEWLINE '\n' (1, 5) (1, 6)
NAME 'await' (2, 0) (2, 5)
�
foo.async + 1z� NAME 'foo' (1, 0) (1, 3)
OP '.' (1, 3) (1, 4)
NAME 'async' (1, 4) (1, 9)
OP '+' (1, 10) (1, 11)
NUMBER '1' (1, 12) (1, 13)
�async def foo(): passa; NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
OP '(' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
OP ':' (1, 15) (1, 16)
NAME 'pass' (1, 17) (1, 21)
�Nasync def foo():
def foo(await):
await = 1
if 1:
await
async += 1
a� NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
OP '(' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
OP ':' (1, 15) (1, 16)
NEWLINE '\n' (1, 16) (1, 17)
INDENT ' ' (2, 0) (2, 2)
NAME 'def' (2, 2) (2, 5)
NAME 'foo' (2, 6) (2, 9)
OP '(' (2, 9) (2, 10)
NAME 'await' (2, 10) (2, 15)
OP ')' (2, 15) (2, 16)
OP ':' (2, 16) (2, 17)
NEWLINE '\n' (2, 17) (2, 18)
INDENT ' ' (3, 0) (3, 4)
NAME 'await' (3, 4) (3, 9)
OP '=' (3, 10) (3, 11)
NUMBER '1' (3, 12) (3, 13)
NEWLINE '\n' (3, 13) (3, 14)
DEDENT '' (4, 2) (4, 2)
NAME 'if' (4, 2) (4, 4)
NUMBER '1' (4, 5) (4, 6)
OP ':' (4, 6) (4, 7)
NEWLINE '\n' (4, 7) (4, 8)
INDENT ' ' (5, 0) (5, 4)
NAME 'await' (5, 4) (5, 9)
NEWLINE '\n' (5, 9) (5, 10)
DEDENT '' (6, 0) (6, 0)
DEDENT '' (6, 0) (6, 0)
NAME 'async' (6, 0) (6, 5)
OP '+=' (6, 6) (6, 8)
NUMBER '1' (6, 9) (6, 10)
NEWLINE '\n' (6, 10) (6, 11)
�)async def foo():
async for i in 1: passa� NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
OP '(' (1, 13) (1, 14)
OP ')' (1, 14) (1, 15)
OP ':' (1, 15) (1, 16)
NEWLINE '\n' (1, 16) (1, 17)
INDENT ' ' (2, 0) (2, 2)
NAME 'async' (2, 2) (2, 7)
NAME 'for' (2, 8) (2, 11)
NAME 'i' (2, 12) (2, 13)
NAME 'in' (2, 14) (2, 16)
NUMBER '1' (2, 17) (2, 18)
OP ':' (2, 18) (2, 19)
NAME 'pass' (2, 20) (2, 24)
DEDENT '' (3, 0) (3, 0)
�async def foo(async): awaitah NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
OP '(' (1, 13) (1, 14)
NAME 'async' (1, 14) (1, 19)
OP ')' (1, 19) (1, 20)
OP ':' (1, 20) (1, 21)
NAME 'await' (1, 22) (1, 27)
�@def f():
def baz(): pass
async def bar(): pass
await = 2a� NAME 'def' (1, 0) (1, 3)
NAME 'f' (1, 4) (1, 5)
OP '(' (1, 5) (1, 6)
OP ')' (1, 6) (1, 7)
OP ':' (1, 7) (1, 8)
NEWLINE '\n' (1, 8) (1, 9)
NL '\n' (2, 0) (2, 1)
INDENT ' ' (3, 0) (3, 2)
NAME 'def' (3, 2) (3, 5)
NAME 'baz' (3, 6) (3, 9)
OP '(' (3, 9) (3, 10)
OP ')' (3, 10) (3, 11)
OP ':' (3, 11) (3, 12)
NAME 'pass' (3, 13) (3, 17)
NEWLINE '\n' (3, 17) (3, 18)
NAME 'async' (4, 2) (4, 7)
NAME 'def' (4, 8) (4, 11)
NAME 'bar' (4, 12) (4, 15)
OP '(' (4, 15) (4, 16)
OP ')' (4, 16) (4, 17)
OP ':' (4, 17) (4, 18)
NAME 'pass' (4, 19) (4, 23)
NEWLINE '\n' (4, 23) (4, 24)
NL '\n' (5, 0) (5, 1)
NAME 'await' (6, 2) (6, 7)
OP '=' (6, 8) (6, 9)
NUMBER '2' (6, 10) (6, 11)
DEDENT '' (7, 0) (7, 0)
�Fasync def f():
def baz(): pass
async def bar(): pass
await = 2a NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'f' (1, 10) (1, 11)
OP '(' (1, 11) (1, 12)
OP ')' (1, 12) (1, 13)
OP ':' (1, 13) (1, 14)
NEWLINE '\n' (1, 14) (1, 15)
NL '\n' (2, 0) (2, 1)
INDENT ' ' (3, 0) (3, 2)
NAME 'def' (3, 2) (3, 5)
NAME 'baz' (3, 6) (3, 9)
OP '(' (3, 9) (3, 10)
OP ')' (3, 10) (3, 11)
OP ':' (3, 11) (3, 12)
NAME 'pass' (3, 13) (3, 17)
NEWLINE '\n' (3, 17) (3, 18)
NAME 'async' (4, 2) (4, 7)
NAME 'def' (4, 8) (4, 11)
NAME 'bar' (4, 12) (4, 15)
OP '(' (4, 15) (4, 16)
OP ')' (4, 16) (4, 17)
OP ':' (4, 17) (4, 18)
NAME 'pass' (4, 19) (4, 23)
NEWLINE '\n' (4, 23) (4, 24)
NL '\n' (5, 0) (5, 1)
NAME 'await' (6, 2) (6, 7)
OP '=' (6, 8) (6, 9)
NUMBER '2' (6, 10) (6, 11)
DEDENT '' (7, 0) (7, 0)
rn ro s r& �
test_async�TokenizeTest.test_async| s� � � ���K� *� �
��� -� 0� �
���I� (� �
��� 3� 6� �
��� .� 1� �
���K� *� �
���K� *� �
��� 4� 7� �
��� 5� 8� �
���K� *� �
���G� &� �
��� 4� 7� �
��� /� 2� �
���N� -� �
���O� .� �
��� 3� 6� �
��� �#�* �X
��� � � �*
��� =� @� �
��� � �
# �J
��� � �
$ r( c �( � U R SS5 g )Nz [
# A comment here
1
]
ad OP '[' (1, 0) (1, 1)
NL '\n' (1, 1) (1, 2)
COMMENT '# A comment here' (2, 4) (2, 20)
NL '\n' (2, 20) (2, 21)
NUMBER '1' (3, 4) (3, 5)
NL '\n' (3, 5) (3, 6)
OP ']' (4, 0) (4, 1)
NEWLINE '\n' (4, 1) (4, 2)
rn ro s r& �3test_newline_after_parenthesized_block_with_comment�@TokenizeTest.test_newline_after_parenthesized_block_with_comment� s � � ��� �
� r( c �( � U R SS5 g )Nz); xz� OP ')' (1, 0) (1, 1)
OP ';' (1, 1) (1, 2)
NAME 'x' (1, 3) (1, 4)
rn ro s r& �,test_closing_parenthesis_from_different_line�9TokenizeTest.test_closing_parenthesis_from_different_line� s � � ���F� %� r( c �( � U R SS5 g )NuT a = f'''
Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli'''u% NAME 'a' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
FSTRING_START "f'''" (1, 4) (1, 8)
FSTRING_MIDDLE '\n Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli' (1, 8) (2, 68)
FSTRING_END "'''" (2, 68) (2, 71)
rn ro s r& � test_multiline_non_ascii_fstring�-TokenizeTest.test_multiline_non_ascii_fstring� s � � ��� O�Q� r( c �L � U R SS5 U R SS5 g )Nu4 f'''
🔗 This is a test {test_arg1}🔗
🔗'''uZ FSTRING_START "f'''" (1, 0) (1, 4)
FSTRING_MIDDLE '\n 🔗 This is a test ' (1, 4) (2, 21)
OP '{' (2, 21) (2, 22)
NAME 'test_arg1' (2, 22) (2, 31)
OP '}' (2, 31) (2, 32)
FSTRING_MIDDLE '🔗\n🔗' (2, 32) (3, 1)
FSTRING_END "'''" (3, 1) (3, 4)
u f"{f(a=lambda: 'à'
)}"u? FSTRING_START 'f"' (1, 0) (1, 2)
OP '{' (1, 2) (1, 3)
NAME 'f' (1, 3) (1, 4)
OP '(' (1, 4) (1, 5)
NAME 'a' (1, 5) (1, 6)
OP '=' (1, 6) (1, 7)
NAME 'lambda' (1, 7) (1, 13)
OP ':' (1, 13) (1, 14)
STRING "'à'" (1, 15) (1, 18)
NL '\n' (1, 18) (1, 19)
OP ')' (2, 0) (2, 1)
OP '}' (2, 1) (2, 2)
FSTRING_END '"' (2, 2) (2, 3)
rn ro s r& �*test_multiline_non_ascii_fstring_with_expr�7TokenizeTest.test_multiline_non_ascii_fstring_with_expr� s2 � � ��� �
� �
��� :� =� r( r< N) �__name__�
__module__� __qualname__�__firstlineno__r7 rI rO rb rp rs r} r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r r �__static_attributes__r< r( r& r* r* & s� � �9�;�&>�F7�P7 �r �0& �P8�0~ �@
: �x$ �L
� �. �(# �J �, �" �*
�
�P �d �" � � r( r* c � � \ rS rSrS rSrg)�GenerateTokensTesti� c �� � [ U5 n[ [ R " UR 5 U5 nU R XBR
5 R 5 5 g �N)r r' r rG r/ r0 r1 r r2 s r& r7 �!GenerateTokensTest.check_tokenize� sG � �
�Q�K��-�h�.F�.F�q�z�z�.R�TU�V�� ���� � �!2�!=�!=�!?�@r( r< N)r r r r r7 r r< r( r& r r � s � �Ar( r c � � / n[ R " [ U R S5 5 R 5 nU H� u p4 nU[ R :X af SU; a` UR [ R S4[ R S4[ R [ U5 4[ R S4/5 M� UR X445 M� [ R " U5 R S5 R 5 $ )Nr- �.�Decimalr� �))r r r. r/ r� �extend�NAME�OP�STRING�reprr �
untokenize�decode�strip)r4 r �g�toknum�tokval�_s r& �decistmtr( � s� � �
�F����'�!�(�(�7�"3�4�=�=�>�A�$%� ���1�a� �X�_�_� $��� � �M�M�� � � �*����c�"�� � �$�v�,�/����c�"� �
�
�M�M�6�*� +� %&� � � �v� &� -� -�g� 6� <� <� >�>r( c � � \ rS rSrS rS rSrg)�TestMisci� c �� � SSK Jn SnU R [ U5 S5 U R [ [
U5 5 S5 U R [
[ U5 5 U" S5 5 g )Nr )r z+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)�decimalr r0 r( � assertRegexr �eval)r3 r r4 s r& �
test_decistmt�TestMisc.test_decistmt� sd � � $� "�� ����!��S� U�
����d�1�g� �(B�C�
����h�q�k�*� �!D�E� Gr( c �n � [ R / SQ- nU R [ R U5 g )N)� TokenInfor� rG �detect_encodingr! �openr )r" �__all__�assertCountEqualr )r3 r5 s r& � test___all__�TestMisc.test___all__� s- � ��=�=� $
�
��
� � �h� .� .��9r( r< N)r r r r r/ r7 r r< r( r& r* r* � s
� �G�(:r( r* c �< � \ rS rSrSrS rS rS rS rS r S r
S r g
) � TestTokenizerAdheresToPep0263i zM
Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263.
c � � [ R R [ R R [ 5 SU5 n[ US5 n[ R X5 S S S 5 g ! , ( d f g = f)N� tokenizedata�rb)�os�path�join�dirname�__file__r4 �
TestRoundtrip�check_roundtrip)r3 rY r? r6 s r& � _testFile�'TestTokenizerAdheresToPep0263._testFile sK � ��w�w�|�|�B�G�G�O�O�H� 5�~�x�P��
�$��
�� � )� )�$� 2� �
�
�s �A.�.
A<c �* � SnU R U5 g )Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt�rE �r3 r6 s r& �'test_utf8_coding_cookie_and_no_utf8_bom�ETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bom � � � G�� ���q�r( c �J � SnU R [ U R U5 g)aV
As per PEP 0263, if a file starts with a utf-8 BOM signature, the only
allowed encoding for the comment is 'utf-8'. The text file used in
this test starts with a BOM signature, but specifies latin1 as the
coding, so verify that a SyntaxError is raised, which matches the
behaviour of the interpreter when it encounters a similar condition.
z8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txtN�rC � SyntaxErrorrE rI s r& �&test_latin1_coding_cookie_and_utf8_bom�DTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bom s � �
G�� ���+�t�~�~�q�9r( c �* � SnU R U5 g )Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txtrH rI s r& �"test_no_coding_cookie_and_utf8_bom�@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bom rL r( c �* � SnU R U5 g )Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txtrH rI s r& �$test_utf8_coding_cookie_and_utf8_bom�BTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bom# s � � D�� ���q�r( c � � U R [ U R S5 U R [ U R S5 g )Nz
bad_coding.pyzbad_coding2.pyrN ro s r& �test_bad_coding_cookie�4TestTokenizerAdheresToPep0263.test_bad_coding_cookie' s/ � � ���+�t�~�~��G� ���+�t�~�~�7G�Hr( r< N) r r r r �__doc__rE rJ rP rS rV rY r r< r( r& r: r: s& � ��3�
� :���Ir( r: c � � \ rS rSrS rSrg)�
Test_Tokenizei, c � ^^� SnUR S5 mSmUU4S jn[ [ R " U" 5 R SSS95 S S n[ R
" SSS S
S5 /nU R
X4S 5 g ) Nu "ЉЊЈЁЂ"r- Fc 3 �2 ># � T ( d Sm Tv � g Sv � g 7f)NTr( r< )�firstr% s ��r& r/ �NTest_Tokenize.test__tokenize_decodes_with_specified_encoding.<locals>.readline2 s �� � �� ���
�� �s �T)�encoding� extra_tokensrM rR �rB r �rB � z bytes not decoded with encoding)r. rE r �!_generate_tokens_from_c_tokenizerrF r2 r0 )r3 �literalr/ rN �expected_tokensr` r% s @@r& �.test__tokenize_decodes_with_specified_encoding�<Test_Tokenize.test__tokenize_decodes_with_specified_encoding. s� �� � ���~�~�g�&���� � �h�@�@���AT�AT�JQ�NR�T� U�UX�VX�Z�� $�-�-�a� ���Q_�`�a�� ����:� <r( r< N)r r r r rj r r< r( r& r] r] , s � �<r( r] c � � \ rS rSrS rS rS rS rS rS r S r
S r S
r S r
S rS
rS rS rS rS rS rS rS rS rS rS rS rS rS r S r S r S r g ) �TestDetectEncodingiC c � ^^� SmUU4S jnU$ )Nr c �F >� T[ T5 :X a [ eTT n TS-
mU $ �NrB )r �
StopIteration)r% �index�liness ��r& r/ �1TestDetectEncoding.get_readline.<locals>.readlineG s, �� ���E�
�"�#�#���<�D� �Q�J�E��Kr( r< )r3 rs r/ rr s ` @r& � get_readline� TestDetectEncoding.get_readlineE s �� ��� � �r( c � � Sn[ R " U R U5 5 u p#U R US5 U R U[ US S 5 5 g )N)� #!/home/ä/bin/python
s # something €
� print(something)
� do_something(else)
r- � �r r3 ru r0 rE �r3 rs rb �consumed_liness r& � test_no_bom_no_encoding_cookie�1TestDetectEncoding.test_no_bom_no_encoding_cookieP sS � �
�� $,�#;�#;�D�<M�<M�e�<T�#U� �� ����7�+� ��� ��e�B�Q�i� �9r( c � � SnU R [ 5 [ R " U R U5 5 S S S 5 g ! , ( d f g = f)N)s #!/home/�/bin/python
ry rz �rC rO r r3 ru �r3 rs s r& �/test_no_bom_no_encoding_cookie_first_line_error�BTestDetectEncoding.test_no_bom_no_encoding_cookie_first_line_error[ �> � �
��
�
�
�{�
+� � $� $�T�%6�%6�u�%=� >� ,�
+�
+�� �&A�
Ac � � SnU R [ 5 [ R " U R U5 5 S S S 5 g ! , ( d f g = f)N)� #!/usr/bin/python
� # something �
ry rz r� r� s r& �0test_no_bom_no_encoding_cookie_second_line_error�CTestDetectEncoding.test_no_bom_no_encoding_cookie_second_line_errord �> � �
�� �
�
�{�
+� � $� $�T�%6�%6�u�%=� >� ,�
+�
+�r� c � � Sn[ R " U R U5 5 u p#U R US5 U R USS/5 g )N)s #!/home/ä/bin/python
ry rz � utf-8-sigrx ry �r r3 ru r0 r} s r& �test_bom_no_cookie�%TestDetectEncoding.test_bom_no_cookien sS � �
��
$,�#;�#;�D�<M�<M�e�<T�#U� �� ����;�/� ��� �:�<Q�R� Tr( c � � SnU R [ 5 [ R " U R U5 5 S S S 5 g ! , ( d f g = f)N)s #!/home/�/bin/python
ry rz r� r� s r& �#test_bom_no_cookie_first_line_error�6TestDetectEncoding.test_bom_no_cookie_first_line_errory r� r� c � � SnU R [ 5 [ R " U R U5 5 S S S 5 g ! , ( d f g = f)N)s #!/usr/bin/python
r� ry rz r� r� s r& �$test_bom_no_cookie_second_line_error�7TestDetectEncoding.test_bom_no_cookie_second_line_error� r� r� c � � Sn[ R " U R U5 5 u p#U R US5 U R US/5 g )N)� # -*- coding: latin-1 -*-
ry rz �
iso-8859-1r� r� r} s r& � test_cookie_first_line_no_bom�0TestDetectEncoding.test_cookie_first_line_no_bom� sM � �
��
$,�#;�#;�D�<M�<M�e�<T�#U� �� ����<�0� ��� �*H�)I�Jr( c � � Sn[ R " U R U5 5 u p#U R US5 U R US/5 g )N)s # coding=utf-8
ry rz r� s # coding=utf-8
r� r} s r& �&test_matched_bom_and_cookie_first_line�9TestDetectEncoding.test_matched_bom_and_cookie_first_line� sM � �
��
$,�#;�#;�D�<M�<M�e�<T�#U� �� ����;�/� ��� �*=�)>�?r( c �t � SnU R U5 nU R [ [ R U5 g )N)s# # vim: set fileencoding=ascii :
ry rz �ru rC rO r r3 �r3 rs r/ s r& �<test_mismatched_bom_and_cookie_first_line_raises_syntaxerror�OTestDetectEncoding.test_mismatched_bom_and_cookie_first_line_raises_syntaxerror� s3 � �
��
�$�$�U�+�� ���+�x�'?�'?��Jr( c � � Sn[ R " U R U5 5 u p#U R US5 SS/nU R X45 g )N)�
#! something
� # vim: set fileencoding=ascii :
ry rz �asciir� r� r� �r3 rs rb r~ r5 s r& � test_cookie_second_line_no_bom�1TestDetectEncoding.test_cookie_second_line_no_bom� sR � �
�� $,�#;�#;�D�<M�<M�e�<T�#U� �� ����7�+�%�'K�L�� ��� �2r( c � � Sn[ R " U R U5 5 u p#U R US5 U R USS/5 g )N)� #! something
� f# coding=utf-8
ry rz r� r� r� r� r} s r& �'test_matched_bom_and_cookie_second_line�:TestDetectEncoding.test_matched_bom_and_cookie_second_line� sS � �
�� $,�#;�#;�D�<M�<M�e�<T�#U� �� ����;�/� ��� �+�-A�B� Dr( c �t � SnU R U5 nU R [ [ R U5 g )N)r� r� ry rz r� r� s r& �=test_mismatched_bom_and_cookie_second_line_raises_syntaxerror�PTestDetectEncoding.test_mismatched_bom_and_cookie_second_line_raises_syntaxerror� s3 � �
�� �$�$�U�+�� ���+�x�'?�'?��Jr( c � � Sn[ R " U R U5 5 u p#U R US5 S/nU R X45 g )N)� print('£')
�% # vim: set fileencoding=iso8859-15 :
�
print('€')
r- r� r� r� s r& �/test_cookie_second_line_noncommented_first_line�BTestDetectEncoding.test_cookie_second_line_noncommented_first_line� sO � �
��
$,�#;�#;�D�<M�<M�e�<T�#U� �� ����7�+�*�+�� ��� �2r( c � � Sn[ R " U R U5 5 u p#U R US5 SS/nU R X45 g )N)�
r� r� �
iso8859-15r� r� r� r� s r& �(test_cookie_second_line_empty_first_line�;TestDetectEncoding.test_cookie_second_line_empty_first_line� sR � �
��
$,�#;�#;�D�<M�<M�e�<T�#U� �� ����<�0��D�E�� ��� �2r( c � � Sn[ R " U R U5 5 u p#U R US5 U R U[ US S 5 5 g )N)rx s # something
r� ry rz r- r{ r| r} s r& �test_cookie_third_line�)TestDetectEncoding.test_cookie_third_line� sS � �
�� $,�#;�#;�D�<M�<M�e�<T�#U� �� ����7�+� ��� ��e�B�Q�i� �9r( c � � Sn[ R " U R U5 5 u p#U R US5 U R U[ US S 5 5 g )N)s #coding:iso8859-15
� #coding:latin1
ry r� rB r| r} s r& �test_double_coding_line�*TestDetectEncoding.test_double_coding_line� sS � �
��
$,�#;�#;�D�<M�<M�e�<T�#U� �� ����<�0� ��� ��e�B�Q�i� �9r( c � � Sn[ R " U R U5 5 u p#U R US5 U R U[ US S 5 5 g )N)s! #coding:iso8859-15 coding:latin1
ry r� rB r| r} s r& � test_double_coding_same_line�/TestDetectEncoding.test_double_coding_same_line� sS � �
�� $,�#;�#;�D�<M�<M�e�<T�#U� �� ����<�0� ��� ��e�B�Q�i� �9r( c � � Sn[ R " U R U5 5 u p#U R US5 U R U[ US S 5 5 g )N)s #coding:utf-8
r� ry r- rB r| r} s r& �test_double_coding_utf8�*TestDetectEncoding.test_double_coding_utf8 sS � �
��
$,�#;�#;�D�<M�<M�e�<T�#U� �� ����7�+� ��� ��e�B�Q�i� �9r( c � � SnU Ht nS Hk nUR SU5 nSSUR S5 - S- SS 4nU R U5 n[ R " U5 u pxU R US
5 Mm Mv g ) N)�latin-1r� z iso-latin-1z latin-1-unixziso-8859-1-unixziso-latin-1-mac��-r' r� r� �
# coding: r� r� s print(things)
s do_something += 4
r� ��replacer. ru r r3 r0 � r3 � encodingsrb �rep�encrs �rl�foundr~ s r& �test_latin1_normalization�,TestDetectEncoding.test_latin1_normalization s� � �;� �!�H�!�� �&�&�s�C�0��/�&����G�)<�<�u�D�+�/�1�� �&�&�u�-��(0�(@�(@��(D�%��� � �� �5� "� "r( c �t � SnU R U5 nU R [ [ R U5 g )N�s
print("�")r� r� s r& �test_syntaxerror_latin1�*TestDetectEncoding.test_syntaxerror_latin1 s3 � ��� �$�$�U�+�� ���+�x�'?�'?��Jr( c � � SnU Hs nS Hj nUR SU5 nSSUR S5 - S- S4nU R U5 n[ R " U5 u pxU R US 5 Ml Mu g )
N)r- z utf-8-macz
utf-8-unixr� r� r� r� r� r� s 1 + 3
r- r� r� s r& �test_utf8_normalization�*TestDetectEncoding.test_utf8_normalization# s� � �8� �!�H�!�� �&�&�s�C�0��/�&����G�)<�<�u�D�#�%�� �&�&�u�-��(0�(@�(@��(D�%��� � ���0� "� "r( c �� � U R S5 n[ R " U5 u p#U R US5 U R US/5 [ R " U R S5 5 u p#U R US5 U R U/ 5 U R S5 n[ R " U5 u p#U R US5 U R US/5 U R S5 n[ R " U5 u p#U R US5 U R U/ 5 U R S5 nU R [
[ R U5 g ) N)ry r- ry r< )s print(something)
r� )� )s # coding: bad
)ru r r3 r0 rC rO �r3 r/ rb r~ s r& �test_short_files�#TestDetectEncoding.test_short_files0 s= � ��$�$�%=�>��#+�#;�#;�H�#E� �� ����7�+� ��� �*?�)@�A�#+�#;�#;�D�<M�<M�b�<Q�#R� �� ����7�+� ��� ��,��$�$�%I�J��#+�#;�#;�H�#E� �� ����;�/� ��� �*?�)@�A��$�$�%7�8��#+�#;�#;�H�#E� �� ����;�/� ��� ��,��$�$�%:�;�� ���+�x�'?�'?��Jr( c � � U R S5 n[ R " U5 u p#U R US5 U R US/5 g )N)� print("#coding=fake")r- r� )ru r r3 r0 r� s r& �test_false_encoding�&TestDetectEncoding.test_false_encodingG sJ � ��$�$�%@�A��#+�#;�#;�H�#E� �� ����7�+� ��� �*B�)C�Dr( c � � [ R S- nU R [ R U5 S H� n[ USUS9 n[ SU- US9 [ SUS9 S S S 5 [ R " U5 nU R UR U5 U R UR S5 S S S 5 M� [ USS S9 n[ SUS9 S S S 5 [ R " U5 nU R UR S 5 U R UR S5 S S S 5 g ! , ( d f N�= f! , ( d f GM = f! , ( d f N�= f! , ( d f g = f)
Nz.py)z iso-8859-15r- �w�rb z # coding: %s��fileu print('euro:€')�rr� )
r �TESTFN�
addCleanup�unlinkr4 �printr r0 rb �mode)r3 rY rb �fps r& � test_open� TestDetectEncoding.test_openN s � � �#�#�e�+�� ��� �(�(�(�3� 1�H��h��h�7�2��n�x�/�b�9��,�2�6� 8� � � �x�(�B�� � ����h�7�� � ����#�.� )�(� 1� �(�C�+�
6�"� �(�r� 2� 7�
�]�]�8�
$�� � � �R�[�[�+� 6� � � �R�W�W�c� *� %�
$� 8�7�� )�(��
7�
6��
$�
$�s0 �D5�:9E�
E�39E*�5
E �
E �
E'�*
E8c � ^� SnSm " U4S jS5 nU R [ 5 U" TU5 nU?[ R " UR
5 S S S 5 U R
[ SR U5 5 U" TU5 n[ R " UR
5 S S S 5 g ! , ( d f Nf= f! , ( d f g = f)N�some_file_pathr� c �( >� \ rS rSrS rU 4S jrSrg)�;TestDetectEncoding.test_filename_in_exception.<locals>.Bunkih c �* � X l Xl SU l g )Nr )�name�_lines�_index)r3 rs r? s r& �__init__�DTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.__init__i s � � � � #� � �� r( c � >� U R [ T5 :X a [ eTU R nU =R S-
sl U$ rp )r r rq )r3 r% rs s �r& r/ �DTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.readlinen s: �� ��;�;�#�e�*�,�'�'� �T�[�[�)��� � �q� � �� r( )r r r� N)r r r r r r/ r )rs s �r& �Bunkr� h s �� �
�
�
r( r z.*{})rC rO r� r r3 r/ rU �format)r3 r? r �insrs s @r& �test_filename_in_exception�-TestDetectEncoding.test_filename_in_exceptionb s� �� � ���� � � �
�
�{�
+��u�d�#�C��� � $� $�S�\�\� 2� ,�
�
#�
#�K�� � �t�1D�
E��u�d�#�C� � $� $�S�\�\� 2� F�
E� ,�
+��
F�
E�s �,B4�*C�4
C�
Cc � � [ S5 n[ R " SUS9 U R [ [
R S5 S S S 5 U R UR 5 g ! , ( d f N*= f)Ns #coding:xxxztokenize._builtin_open)� return_value�foobar) r r �patchrC rO r r4 �
assertTrue�closed)r3 �ms r& �test_open_error�"TestDetectEncoding.test_open_error~ sQ � � �N� #��
�Z�Z�0�q�
A� � � �k�8�=�=�(� C� B� ������!� B�
A�s �&A+�+
A9r< N) r r r r ru r r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r r r r< r( r& rm rm C s� � � � :�?�?� T�?�?�K�@�K�
3�
D�K� 3� 3�
:� :�:�:�
6� K� 1�K�.E�+�(3�8"r( rm c �D � \ rS rSrS rS rS rS rS rS r S r
S r S
r g ) � TestTokenizei� c �V ^^ ^
� SS K nSm S m
U 4S jnU
4S jnSmU4S jnUR nUR nX!l X1l [ R " U5 nU R [ U5 SS / SQ5 XQl Xal U R T
T 5 g ! XQl Xal f = f)Nr r- c � >� TSS/4$ )N� first� secondr< )r/ rb s �r& �mock_detect_encoding�8TestTokenize.test_tokenize.<locals>.mock_detect_encoding� s �� ��h� � 2�2� 2r( c �z >� Um/ n U " 5 nU( a UR U5 M# U$ ! [ a Us $ f = fr )rq r )r/ rb �kwargs�out� next_line�
encoding_useds �r& �mock__tokenize�2TestTokenize.test_tokenize.<locals>.mock__tokenize� sK �� � $�M��C�� � (�
�I� ��J�J�y�)� ��
�� %� � �J� �s �+ � :�:c �N >� T S-
m T S:X a g[ T 5 R 5 $ )NrB � r( )�strr. )�counters �r& �
mock_readline�1TestTokenize.test_tokenize.<locals>.mock_readline� s) �� � �q�L�G��!�|���w�<�&�&�(� (r( rB )r r � 1� 2� 3� 4)r r3 rg r0 rE ) r3 �tokenize_moduler r! r' �orig_detect_encoding� orig_c_token�resultsr& rb r s @@@r& �
test_tokenize�TestTokenize.test_tokenize� s� �� �*��� �
� 3� � �� )� /� >� >� �&�H�H� �*>�'�<J�9� M� �'�'�
�6�G� � � �T�'�]�1�2� .� J�
L� /C� +�@L� =� ��� ��1�� /C� +�@L� =�s �6B �B(c �l � / n[ S5 H! nUR SR US95 M# UR S5 SR U5 n[ [
R
" [
UR S5 5 R 5 5 nU R US R S5 g )Ni� zdef i{i}(): return {i})�i�OK�
r- ���) �ranger r r@ rE r r r. r/ r0 �string)r3 �bufr4 �tokss r& �test_oneline_defs� TestTokenize.test_oneline_defs� s� � ����s��A� �J�J�/�6�6��6�;� <� � �
�
�4���i�i��n�� �H�%�%�g�c�j�j��.A�&B�&K�&K�L�M�� ����b�� � �$�/r( c �l � [ [ R " [ UR S5 5 R 5 5 n[ U5 nU R
[ U5 SU- 5 U R
[ R US R [ R [ R 5 [ U5 HH nU R
[ R X5S- R [ R X% 5 MJ U R
[ R USU- R [ R [ R 5 U R
[ R USU- R [ R [ R 5 g )Nr- rR r rB r{ )rE r r r. r/ r r0 r �
exact_type�ENCODINGr8 r" r r )r3 �opstr�optypesrN � num_optypesr4 s r& �assertExactTypeEqual�!TestTokenize.assertExactTypeEqual� sE � ��h�'�'��� � �W�0E�(F�(O�(O�P�Q���'�l� � ����V��a�+�o�6� ����*�*�6�!�9�+?�+?�@�!�*�*�8�+<�+<�=� ?��{�#�A� � � �X� .� .�v�!�e�}�/G�/G� H� %� .� .�w�z� :�
<� $�
����*�*�6�!�k�/�+B�+M�+M�N�!�*�*�5�=�=�9� ;� ����*�*�6�!�k�/�+B�+M�+M�N�!�*�*�5�?�?�;� =r( c �� � U R S[ R [ R 5 U R S[ R [ R
5 U R S[ R 5 U R S[ R 5 U R S[ R 5 U R S[ R 5 U R S[ R 5 U R S[ R 5 U R S [ R 5 U R S
[ R 5 U R S [ R 5 U R S [ R 5 U R S
[ R 5 U R S[ R" 5 U R S[ R$ 5 U R S[ R& 5 U R S[ R( [ R* 5 U R S[ R, 5 U R S[ R. 5 U R S[ R0 5 U R S[ R2 5 U R S[ R4 5 U R S[ R6 5 U R S[ R8 5 U R S[ R: 5 U R S[ R< 5 U R S[ R> 5 U R S [ R@ 5 U R S [ RB 5 U R S [ RD 5 U R S [ RF 5 U R S [ RH 5 U R S![ RJ 5 U R S"[ RL 5 U R S"[ RL 5 U R S#[ RN 5 U R S$[ RP 5 U R S%[ RR 5 U R S&[ RT 5 U R S'[ RV 5 U R S([ RX 5 U R S)[ RZ 5 U R S*[ R\ 5 U R S+[ R^ 5 U R S,[ R` 5 U R S-[b Rd [ R< [b Rf [ R [b Rd [ R< [b Rf [ R, [b Rd [ R< [b Rf 5 U R S.[ R( [ Rf [ R [ Rf [ R [ Rf [ R* 5 U R S/[ R6 [ R [ Rd [ R [ Rf [ R 5 g )0Nz()z[]�:�,�;�+r� �*�/�|�&�<�>�=r �%z{}z==z!=z<=z>=�~�^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z:=z...z->�@z@=za**2+b**2==c**2z {1, 2, 3}z
^(x & 0x1))4rD r" �LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENT�LBRACE�RBRACE�EQEQUAL�NOTEQUAL� LESSEQUAL� GREATEREQUAL�TILDE�
CIRCUMFLEX� LEFTSHIFT�
RIGHTSHIFT�
DOUBLESTAR� PLUSEQUAL�MINEQUAL� STAREQUAL�
SLASHEQUAL� PERCENTEQUAL�
AMPEREQUAL� VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL� DOUBLESLASH�DOUBLESLASHEQUAL�
COLONEQUAL�ELLIPSIS�RARROW�AT�ATEQUALr r r� ro s r& �test_exact_type� TestTokenize.test_exact_type� s� � � �!�!�$��
�
�E�J�J�?� �!�!�$��
�
�E�J�J�?� �!�!�#�u�{�{�3� �!�!�#�u�{�{�3� �!�!�#�u�z�z�2� �!�!�#�u�z�z�2� �!�!�#�u�{�{�3� �!�!�#�u�z�z�2� �!�!�#�u�{�{�3� �!�!�#�u�z�z�2� �!�!�#�u�{�{�3� �!�!�#�u�z�z�2� �!�!�#�u�}�}�5� �!�!�#�u�{�{�3� �!�!�#�u�y�y�1� �!�!�#�u�}�}�5� �!�!�$�� � �e�l�l�C� �!�!�$��
�
�6� �!�!�$����7� �!�!�$����8� �!�!�$��(:�(:�;� �!�!�#�u�{�{�3� �!�!�#�u�'7�'7�8� �!�!�$����8� �!�!�$��(8�(8�9� �!�!�$��(8�(8�9� �!�!�$����8� �!�!�$����7� �!�!�$����8� �!�!�$��(8�(8�9� �!�!�$��(:�(:�;� �!�!�$��(8�(8�9� �!�!�$����8� �!�!�$��(=�(=�>� �!�!�$��(=�(=�>� �!�!�%��)=�)=�>� �!�!�%��)>�)>�?� �!�!�%��)>�)>�?� �!�!�$��(9�(9�:� �!�!�%��)?�)?�@� �!�!�$��(8�(8�9� �!�!�%�� � �8� �!�!�$�� � �5� �!�!�#�u�x�x�0� �!�!�$��
�
�6� �!�!�"3�"*�-�-��1A�1A�8�?�?�"'�*�*�"*�-�-��1A�1A�8�?�?�"'�-�-�"*�-�-��1A�1A�8�?�?� T�
�!�!�+�"'�,�,�"'�,�,�� � �"'�,�,�� � �"'�,�,�"'�,�,� 0�
�!�!�,�"'�"2�"2�"'�*�*�"'�*�*�e�k�k�5�<�<�"'�*�*� .r( c �D � U R S[ R 5 g )Nz @ )rD r" r� ro s r& �%test_pathological_trailing_whitespace�2TestTokenize.test_pathological_trailing_whitespace s � � �!�!�-����:r( c �J � Sn[ R " [ R SSSSS9[ R " [ R SSSS S9[ R " [ R
S
S S S S9[ R " [ R S
SSS S9[ R " [ R SSSS S9[ R " [ R SSSSS9[ R " [ R SSSSS9[ R " [ R SSSSS9[ R " [ R SSSSS9/ n[ [ R " [ UR S5 5 R 5 5 nU R X25 g )Nz b = 1
#testr- �r r � �r! r9 r# r$ r% �brd �rB rB zb = 1
rQ �rB r{ �rB rR �1�rB � �rB r$ r6 �rB � �r{ r �r{ rB z#test�rR r )rR r$ )rR r� )r� r )r r2 r" r@ r r r� r �NL�COMMENTr rE r r. r/ r0 �r3 �sourceri rN s r& �5test_comment_at_the_end_of_the_source_without_newline�BTestTokenize.test_comment_at_the_end_of_the_source_without_newline s? � �!�� � � �E�N�N�7�&�V\�ce� f� � � �E�J�J�s�&�f�[d� e� � � �E�H�H�S��F�Yb� c� � � �E�L�L��F�PV�]f� g� � � �E�M�M�$�f�RX�_h� i� � � �E�H�H�T��V�Z^� _� � � �E�M�M�'��U[�bi� j� � � �E�H�H�R�v�6�X_� `� � � �E�O�O�B�f�RX�_a� b�
�� �h�'�'���
�
�g�0F�(G�(P�(P�Q�R�� ����1r( c �" � Sn[ R " [ R SSSSS9[ R " [ R SSSS S9[ R " [ R
S
SS S S9[ R " [ R SS S
SS9[ R " [ R SSSSS9/n[ [ R " [ UR S5 5 R 5 5 nU R X25 g )Nza
r- r� r� )r9 r# r$ r% �ard r� za
r6 r� r� )r{ r{ r r� )
r r2 r" r@ r r r� r rE r r. r/ r0 r� s r& �?test_newline_and_space_at_the_end_of_the_source_without_newline�LTestTokenize.test_newline_and_space_at_the_end_of_the_source_without_newline$ s� � ��� � � �u�~�~�g�V�QW�^`� a� � � �u�z�z�#�V��V[� \� � � �u�}�}�T��V�Z_� `� � � �u�x�x��&�f�SV� W� � � �u���r��V�Z\� ]�
�� �h�'�'���
�
�g�0F�(G�(P�(P�Q�R�� ����1r( c �f � Sn[ R " 5 n[ R R US5 n[ US5 nUR
U5 S S S 5 [ U5 u pVU R SUR 5 S S S 5 g ! , ( d f N@= f! , ( d f g = f)Ns F"""
�"""z script.py�wbs SyntaxError)
r �temp_dirr> r? r@ r4 �writer �assertIn�err)r3 �scriptr� rY r� �rsr' s r& �(test_invalid_character_in_fstring_middle�5TestTokenize.test_invalid_character_in_fstring_middle2 s� � ��� �
�
�
!�X��w�w�|�|�H�k�:�H��h��%���
�
�6�"� &�(��2�E�B� �M�M�.�"�&�&� 1� "�
!�%�%�� "�
!�s# �-B"�B�1B"�
B �B"�"
B0r< N)
r r r r r1 r< rD r� r� r� r� r� r r< r( r& r r � s, � �)2�V 0� =� ?.�B;�2�$ 2�
2r( r c �& � \ rS rSrS rS rS rSrg)�UntokenizeTesti? c �^ � [ R " 5 nSUl SUl U R [
5 nUR
S5 S S S 5 U R WR R S S5 U R [
UR S5 g ! , ( d f NY= f)Nr{ r� r z'start (1,3) precedes previous end (2,2)r� )
r � Untokenizer�prev_row�prev_colrC �
ValueError�add_whitespacer0 rW �args)r3 �u�cms r& �test_bad_input_order�#UntokenizeTest.test_bad_input_orderA s� � � � � � "����
���
�
�
�
�z�
*�b�
� � �U� #� +� ���� � �*�*�1�-�9� ;�
���*�a�&6�&6��>� +�
*�s �B �
B,c �L � [ R " 5 nSUl SUl / Ul UR S5 U R
UR S/5 SUl UR S5 U R
UR / SQ5 [ R U S5 g )NrB r� �\
r{ )r� r� )r� z\
\
r za
b
c
\
c
) r r� r� r� rN r� r0 rC rD )r3 r� s r& �test_backslash_continuation�*UntokenizeTest.test_backslash_continuationM s� � � � � � "����
���
���� ���� � ������F�8�,���
� ���� � ������#?�@��%�%�d�,H�Ir( c �~ � [ R " 5 n[ R S4n[ R S4U/nUR U[ / 5 5 U R
UR S/5 [ R " 5 nU R
UR [ U/5 5 S5 [ R " 5 nU R
UR [ U5 5 S5 U R
UR S5 U R
[ R " [ U5 5 S5 g )N�Hellor- zHello s Hello )
r r� r r@ �compat�iterr0 rN r! rb )r3 r� r" rN s r& �test_iter_compat� UntokenizeTest.test_iter_compatZ s� � � � � � "��� � ��(���$�$�g�.��6�� �����R��!� ������H�:�.� � � � "�� ���� � �d�E�7�m�4�h�?� � � � "�� ���� � �d�6�l�3�X�>� ������W�-� ����,�,�T�&�\�:�I�Fr( r< N)r r r r r� r� r� r r< r( r& r� r� ? s � �
?� J� Gr( r� c �T � [ R " S5 nUR U 5 SL$ )a Return `True` if the source contains a backslash on a
line by itself. For example:
a = (1
\
)
Code like this cannot be untokenized exactly. This is because
the tokenizer does not produce any tokens for the line containing
the backslash and so there is no way to know its indent.
s \n\s*\\\r?\nN)�re�compile�search)r� �patterns r& � contains_ambiguous_backslashr� h s' � � �j�j�)�*�G� �>�>�&� !�� -�-r( c �J � \ rS rSrS rS rS rS rS rS r S r
S r S
r S r
g )
rC ix c � � [ U[ 5 ( a UR S5 nOUR 5 n[ UR SS95 R n[ [ R " U5 5 nU Vs/ s H oUSS PM nn[ R " U5 n[ UR SS95 R n[ R " U5 Vs/ s H oUSS PM n nU R X�5 [ R " U5 n
[ U
R SS95 R n [ R " U 5 Vs/ s H oUSS PM n nU R X�5 [ U5 ( do UR S5 n
[ U
R SS95 R n[ R " [ R " U5 5 nU R X�5 ggs snf s snf s snf )aR
Test roundtrip for `untokenize`. `f` is an open file or a string.
The source code in f is tokenized to both 5- and 2-tuples.
Both sequences are converted back to source code via
tokenize.untokenize(), and the latter tokenized again to 2-tuples.
The test fails if the 3 pair tokenizations do not match.
If the source code can be untokenized unambiguously, the
untokenized code must match the original code exactly.
When untokenize bugs are fixed, untokenize with 5-tuples should
reproduce code that does not contain a backslash continuation
following spaces. A proper test should test this.
r- T��keependsNr{ r� )
�
isinstancer% r. �readr� r rF rE r r! r0 r� � removeprefix)r3 r6 �coder/ �tokens5ra �tokens2� bytes_from2� readline2�
tokens2_from2� bytes_from5� readline5�
tokens2_from5�code_without_bom�untokenized_codes r& rD � TestRoundtrip.check_roundtripz s� � � �a�� � ��8�8�G�$�D��6�6�8�D�������6�7�@�@���x�(�(��2�3��&-�.�g�s�r��7�g��.� �)�)�'�2� ���/�/��/�>�?�H�H� �,4�,=�,=�i�,H�I�,H�S�R�a��,H�
�I� ��� �0� �)�)�'�2� ���/�/��/�>�?�H�H� �,4�,=�,=�i�,H�I�,H�S�R�a��,H�
�I� ��� �0�+�D�1�1� #� 0� 0� � A� �� ,� 7� 7�� 7� F�G�P�P�H� '� 2� 2�8�3D�3D�X�3N� O� � � � � -� @� 2�� /�� J��
Js �=G/� G4�G9c �� � [ U[ 5 ( a UR S5 nOUR 5 n[ UR SS95 R n[ R " U5 Hv nUR [ R [ R 1; a M3 U R UR UR UR S UR S 5 Mx g )Nr- Tr� rB )r� r% r. r� r� r rF r r! r@ r r0 r9 r% r# r$ )r3 r6 r� r/ ra s r& �check_line_extraction�#TestRoundtrip.check_line_extraction� s� � � �a�� � ��8�8�G�$�D��6�6�8�D�������6�7�@�@���$�$�X�.�C��x�x�X� .� .��0B�0B� C�C�� � � �S�Z�Z����#�)�)�A�,�����
�)K� L� /r( c �� � U R S5 U R S5 U R S5 [ R " SSS9n[ US5 nU R U5 S S S 5 U R S5 U R S 5 U R S
5 U R S 5 U R S 5 U R S
5 U R S5 U R S5 U R S5 U R S5 U R S5 U R S5 U R S5 U R S5 U R S5 U R S5 U R S5 U R S5 U R S5 U R S5 U R S 5 U R S 5 U R S 5 U R S 5 U R S 5 U R S!5 U R S"5 U R S#5 U R S$5 U R S%5 U R S&5 U R S'5 U R S(5 U R S)5 U R S*5 / S+QnU H nU R U5 M g ! , ( d f GN�= f),Nzif x == 1:
print(x)
z # This is a comment
# This also
zif x == 1 :
print(x)
ztokenize_tests.txtr< )�subdirr= zoif x == 1:
# A comment by itself.
print(x) # Comment here, too.
# Another comment.
after_if = True
zPif (x # The comments need to go in the right place
== 1):
print('x==1')
z�class Test: # A comment here
# A comment with weird indent
after_com = 5
def x(m): return m*5 # a one liner
def y(m): # A whitespace after the colon
return y*4 # 3-space indent
zttry: import somemodule
except ImportError: # comment
print('Can not import' # comment2
)else: print('Loaded')
zf'\N{EXCLAMATION MARK}'z
f'\\N{SNAKE}'zf'\\N{{SNAKE}}'z f'\N{SNAKE}'zf'\\\N{SNAKE}'zf'\\\\\N{SNAKE}'zf'\\\\\\\N{SNAKE}'z f'\\N{1}'z f'\\\\N{2}'z
f'\\\\\\N{3}'zf'\\\\\\\\N{4}'zf'\\N{{'z
f'\\\\N{{'z f'\\\\\\N{{'zf'\\\\\\\\N{{'z f'\n{{foo}}'z
f'\\n{{foo}}'zf'\\\n{{foo}}'zf'\\\\n{{foo}}'z f'\t{{foo}}'z
f'\\t{{foo}}'zf'\\\t{{foo}}'zf'\\\\t{{foo}}'z
rf'\t{{foo}}'zrf'\\t{{foo}}'zrf'\\\t{{foo}}'zrf'\\\\t{{foo}}'z rf'\{{foo}}'z f'\\{{foo}}'zrf'\\\{{foo}}'zf'\\\\{{foo}}')z
if 1:
"foo"
"bar"
z
if 1:
("foo"
"bar")
z
if 1:
"foo"
"bar"
)rD r �findfiler4 )r3 �fnr6 �cases�cases r& �test_roundtrip� TestRoundtrip.test_roundtrip� s� � �
� � � .� /� � � � -� .�
� � � ,� -�
�
�
� 2�>�
J��
�"�d�^�q� � � �� #� � � � � 1� 2�
� � � 3� 4�
� � � B� C�
� � � 9� :�
� � � 7�8� � � � -�.� � � � /�0� � � �_�-� � � � .�/� � � � 0�1� � � � 2�3� � � �\�*� � � �^�,� � � � -�.� � � � /�0� � � �[�)� � � �]�+� � � �_�-� � � � .�/� � � �_�-� � � � -�.� � � � .�/� � � � /�0� � � �_�-� � � � -�.� � � � .�/� � � � /�0� � � � -�.� � � � .�/� � � � /�0� � � � 0�1� � � �_�-� � � �_�-� � � � .�/� � � � .�/��� �D� � � �� &� �Y �^�s �K!�!
K0c �& � U R S5 g )Nzna = (3,4,
5,6)
y = [3, 4,
5]
z = {'a': 5,
'b':15, 'c':True}
x = len(y) + 5 - a[
3] - a[2]
+ len(z) - z[
'b']
�rD ro s r& �test_continuation� TestRoundtrip.test_continuation s � � � � � &� 'r( c �H � U R S5 U R S5 g )Nz)x=1+\
1
# This is a comment\
# This also
z# Comment \
x = 0r� ro s r& r� �)TestRoundtrip.test_backslash_continuation s* � � � � � -� .�
� � � %� &r( c �& � U R S5 g )Nz'' ''r� ro s r& �test_string_concatenation�'TestRoundtrip.test_string_concatenation$ s � � � � �W�%r( c � � SS K nSS Kn[ R R [
5 =( d [ R nUR [ R R UR U5 S5 5 n[ R " S5 ( d UR US5 nU Hr n[ R S:� a [ SU5 [ US5 nU R US9 U R! U5 U R# U5 S S S 5 S S S 5 Mt g ! , ( d f N= f! , ( d f M� = f) Nr ztest*.py�cpu�
r{ r r= r� )�glob�randomr> r? rA rB �curdirr@ �escaper �is_resource_enabled�sample�verboser� r4 �subTestrD r� )r3 r� r� �tempdir� testfiles�testfiler6 s r& �test_random_files� TestRoundtrip.test_random_files( s� � � ��'�'�/�/�(�+�8�r�y�y���I�I�b�g�g�l�l�4�;�;�w�+?�� L�M� ��*�*�5�1�1� �
�
�i��4�I�!�H����!�#��j�(�+��h��%���\�\�x�\�0��(�(��+��.�.�q�1� 1� &�%� "� 1�0�� &�%�s$ � D8�/#D'�D8�'
D5 �1D8�8
E c �� � [ U[ 5 ( a UR S5 n[ R " [ R " [ U5 R 5 5 R S5 $ )Nr- )r� r% r. r r! r r/ r" )r3 r� s r& � roundtrip�TestRoundtrip.roundtrip< sO � � �d�C� � ��;�;�w�'�D��"�"�8�#4�#4�W�T�]�5K�5K�#L�M�T�T�U\�]�]r( c � � SnU R U5 R S5 nU R US US 5 U R U5 g)z~
Ensure that although whitespace might be mutated in a roundtrip,
the semantic meaning of the indentation remains consistent.
zif False:
x=3
x=3
r6 rB r{ N)r �splitr0 rD )r3 r� � codeliness r& �#test_indentation_semantics_retained�1TestRoundtrip.test_indentation_semantics_retainedA sH � �
+���N�N�4�(�.�.�t�4� � ����1� �y��|�4� � � �T�"r( r< N)r r r r rD r� r� r� r� r� r r r r r< r( r& rC rC x s5 � �'A�R M�]'�@ '�&�&�2�(^�
#r( rC c � � \ rS rSrS rS rSrg)�InvalidPythonTestsiL c � � Sn[ R " [ R SSSSS9[ R " [ R SSSSS9[ R " [ R
SSS SS9[ R " [ R S
S S SS9[ R " [ R
S S S
SS9[ R " [ R SS
SSS9[ R " [ R SSSSS9/n[ [ R " [ U5 R 5 5 nU R X25 g )Nz2sin(x)�2rd r� r� �sinr� r� r� rL r� r re r� )rB � r� )
r r2 r" r� r r r r rE rG r r/ r0 r� s r& � test_number_followed_by_name�/InvalidPythonTests.test_number_followed_by_nameM s� � ��� � � �E�L�L��F�PV�]f� g� � � �E�J�J�u�F�PV�]f� g� � � �E�H�H�S��F�Yb� c� � � �E�J�J�s�&�f�[d� e� � � �E�H�H�S��F�Yb� c� � � �E�M�M�"�F�PV�]f� g� � � �E�O�O�B�f�RX�_a� b�
�� �h�.�.�x��/?�/H�/H�I�J�� ����1r( c
�p � Sn[ R " [ R SSSSS9[ R " [ R SSSSS9[ R " [ R
SSSSS9/n[
[ R " [ U5 R 5 5 nU R X25 g )N�01234rd r� r� r� r� r� ) r r2 r" r� r r rE rG r r/ r0 r� s r& � test_number_starting_with_zero�1InvalidPythonTests.test_number_starting_with_zero] s� � ��� � � �E�L�L���TZ�ah� i� � � �E�M�M�"�F�PV�]d� e� � � �E�O�O�B�f�RX�_a� b�
�� �h�.�.�x��/?�/H�/H�I�J�� ����1r( r< N)r r r r r r r r< r( r& r r L s � �2� 2r( r c � � \ rS rSrS rS rS rS rS rS r S r
S r S
r S r
S rS
rS rS rS rS rS rS rSrg)�
CTokenizeTestih c � � [ U5 nU R US9 [ [ R " UR
5 U5 nU R
XBR 5 R 5 5 S S S 5 g ! , ( d f g = f)N)r� ) r r� r' r rg r/ r0 r1 r r2 s r& r7 � CTokenizeTest.check_tokenizei se � �
�Q�K��
�\�\��\�
#�1��:�:�1�:�:�F���F�
� � �V�_�_�%6�%A�%A�%C� D� $�
#�
#�s �AA<�<
B
c �f � S n[ R " [ R SSSSS9[ R " [ R SSSSS9[ R " [ R SSS SS9[ R " [ R S
S S SS9[ R " [ R
S
S S S
S9/nS
H[ nU R
US9 [ [ R " U" U5 R SUS95 nU R XB5 S S S 5 M] g ! , ( d f Mo = f)Nc 3 �2 # � SR U 5 v � g 7f)N�1+1)r. r� s r& r/ �-CTokenizeTest.test_encoding.<locals>.readlinet s � � ��,�,�x�(� (�� �r� rd r� r r� rJ r� r� r� r� r� )r- r� zutf-16r� T)rc rb ) r r2 r� r r r r� rE rg rF r0 )r3 r/ r5 rb rN s r& �
test_encoding�CTokenizeTest.test_encodings s � � )�
� � �H�O�O�C�v�SY�`e� f� � � �H�K�K��6�v�\a� b� � � �H�O�O�C�v�SY�`e� f� � � �H�$4�$4�R�v�SY�`e� f� � � �H�$6�$6�r��U[�bd� e�
�� 7�H�� � �x� �0� �h� H� H� �X�&�/�/�!%� %� � ��
� � ��2�
1�0� 7�0�0�s
�AD!�!
D0 c �H � U R SS5 U R SS5 U R SS5 U R SS5 U R S S
5 U R S S 5 U R S
S5 U R SS5 U R SS5 g )Nre z� NUMBER '0xff' (1, 0) (1, 4)
LESSEQUAL '<=' (1, 5) (1, 7)
NUMBER '255' (1, 8) (1, 11)
rf z� NUMBER '0b10' (1, 0) (1, 4)
LESSEQUAL '<=' (1, 5) (1, 7)
NUMBER '255' (1, 8) (1, 11)
rg z� NUMBER '0o123' (1, 0) (1, 5)
LESSEQUAL '<=' (1, 6) (1, 8)
NUMBER '0O123' (1, 9) (1, 14)
rh z� NUMBER '1234567' (1, 0) (1, 7)
GREATER '>' (1, 8) (1, 9)
TILDE '~' (1, 10) (1, 11)
NUMBER '0x15' (1, 11) (1, 15)
ri z� NUMBER '2134568' (1, 0) (1, 7)
NOTEQUAL '!=' (1, 8) (1, 10)
NUMBER '1231515' (1, 11) (1, 18)
rj ac LPAR '(' (1, 0) (1, 1)
MINUS '-' (1, 1) (1, 2)
NUMBER '124561' (1, 2) (1, 8)
MINUS '-' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
RPAR ')' (1, 10) (1, 11)
AMPER '&' (1, 12) (1, 13)
NUMBER '200000000' (1, 14) (1, 23)
rk z� NUMBER '0xdeadbeef' (1, 0) (1, 10)
NOTEQUAL '!=' (1, 11) (1, 13)
MINUS '-' (1, 14) (1, 15)
NUMBER '1' (1, 15) (1, 16)
rl z� NUMBER '0xdeadc0de' (1, 0) (1, 10)
AMPER '&' (1, 11) (1, 12)
NUMBER '12345' (1, 13) (1, 18)
rm z� NUMBER '0xFF' (1, 0) (1, 4)
AMPER '&' (1, 5) (1, 6)
NUMBER '0x15' (1, 7) (1, 11)
VBAR '|' (1, 12) (1, 13)
NUMBER '1234' (1, 14) (1, 18)
rn ro s r& rp �CTokenizeTest.test_int� s� � � ���M� ,� �
���M� ,� �
��� ,� /� �
��� -� 0� �
��� 0� 3� �
��� 5� 8� �
��� .� 1� �
��� 0� 3� �
��� 0� 3� r( c � � U R SS5 U R SS5 U R SS5 U R SS5 U R S S
5 U R S S 5 U R S
S5 g )Nrv z� NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '3.14159' (1, 4) (1, 11)
rw z� NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '314159.' (1, 4) (1, 11)
rx z� NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '.314159' (1, 4) (1, 11)
ry z� NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '3e14159' (1, 4) (1, 11)
rz z� NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '3E123' (1, 4) (1, 9)
r{ z� NAME 'x' (1, 0) (1, 1)
PLUS '+' (1, 1) (1, 2)
NAME 'y' (1, 2) (1, 3)
EQUAL '=' (1, 4) (1, 5)
NUMBER '3e-1230' (1, 6) (1, 13)
r| z� NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '3.14e159' (1, 4) (1, 12)
rn ro s r& r} �CTokenizeTest.test_float� s� � � ���M� ,� �
���M� ,� �
���M� ,� �
���M� ,� �
���K� *� �
���O� .� �
���N� -� r( c �` � U R SS5 U R SS5 U R SS5 U R SS5 U R S S
5 U R S S 5 U R S
S5 U R SS5 U R SS5 U R SS5 U R SS5 U R SS5 U R SS5 U R SS 5 U R S S 5 U R S S 5 U R S!S"5 U R S#S$5 U R S%S&5 U R S'S(5 U R S)S*5 U R S+S,5 U R S-S.5 U R S/S05 U R S1S25 U R S3S45 U R S5S65 U R S7S85 U R S9S:5 U R S;S<5 U R S=S>5 g )?Nr� a5 NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING "''" (1, 4) (1, 6)
SEMI ';' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
EQUAL '=' (1, 10) (1, 11)
STRING '""' (1, 12) (1, 14)
r� a6 NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING '\'"\'' (1, 4) (1, 7)
SEMI ';' (1, 7) (1, 8)
NAME 'y' (1, 9) (1, 10)
EQUAL '=' (1, 11) (1, 12)
STRING '"\'"' (1, 13) (1, 16)
r� z� NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING '"doesn\'t "' (1, 4) (1, 14)
NAME 'shrink' (1, 14) (1, 20)
STRING '", does it"' (1, 20) (1, 31)
r� z� NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING "'abc'" (1, 4) (1, 9)
PLUS '+' (1, 10) (1, 11)
STRING "'ABC'" (1, 12) (1, 17)
r� z� NAME 'y' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING '"ABC"' (1, 4) (1, 9)
PLUS '+' (1, 10) (1, 11)
STRING '"ABC"' (1, 12) (1, 17)
r� a� NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING "r'abc'" (1, 4) (1, 10)
PLUS '+' (1, 11) (1, 12)
STRING "r'ABC'" (1, 13) (1, 19)
PLUS '+' (1, 20) (1, 21)
STRING "R'ABC'" (1, 22) (1, 28)
PLUS '+' (1, 29) (1, 30)
STRING "R'ABC'" (1, 31) (1, 37)
r� a� NAME 'y' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
STRING 'r"abc"' (1, 4) (1, 10)
PLUS '+' (1, 11) (1, 12)
STRING 'r"ABC"' (1, 13) (1, 19)
PLUS '+' (1, 20) (1, 21)
STRING 'R"ABC"' (1, 22) (1, 28)
PLUS '+' (1, 29) (1, 30)
STRING 'R"ABC"' (1, 31) (1, 37)
r� z� STRING "u'abc'" (1, 0) (1, 6)
PLUS '+' (1, 7) (1, 8)
STRING "U'abc'" (1, 9) (1, 15)
r� z� STRING 'u"abc"' (1, 0) (1, 6)
PLUS '+' (1, 7) (1, 8)
STRING 'U"abc"' (1, 9) (1, 15)
r� z� STRING "b'abc'" (1, 0) (1, 6)
PLUS '+' (1, 7) (1, 8)
STRING "B'abc'" (1, 9) (1, 15)
r� z� STRING 'b"abc"' (1, 0) (1, 6)
PLUS '+' (1, 7) (1, 8)
STRING 'B"abc"' (1, 9) (1, 15)
r� a; STRING "br'abc'" (1, 0) (1, 7)
PLUS '+' (1, 8) (1, 9)
STRING "bR'abc'" (1, 10) (1, 17)
PLUS '+' (1, 18) (1, 19)
STRING "Br'abc'" (1, 20) (1, 27)
PLUS '+' (1, 28) (1, 29)
STRING "BR'abc'" (1, 30) (1, 37)
r� a; STRING 'br"abc"' (1, 0) (1, 7)
PLUS '+' (1, 8) (1, 9)
STRING 'bR"abc"' (1, 10) (1, 17)
PLUS '+' (1, 18) (1, 19)
STRING 'Br"abc"' (1, 20) (1, 27)
PLUS '+' (1, 28) (1, 29)
STRING 'BR"abc"' (1, 30) (1, 37)
r� a; STRING "rb'abc'" (1, 0) (1, 7)
PLUS '+' (1, 8) (1, 9)
STRING "rB'abc'" (1, 10) (1, 17)
PLUS '+' (1, 18) (1, 19)
STRING "Rb'abc'" (1, 20) (1, 27)
PLUS '+' (1, 28) (1, 29)
STRING "RB'abc'" (1, 30) (1, 37)
r� a; STRING 'rb"abc"' (1, 0) (1, 7)
PLUS '+' (1, 8) (1, 9)
STRING 'rB"abc"' (1, 10) (1, 17)
PLUS '+' (1, 18) (1, 19)
STRING 'Rb"abc"' (1, 20) (1, 27)
PLUS '+' (1, 28) (1, 29)
STRING 'RB"abc"' (1, 30) (1, 37)
r� r� r� r� r� r� r� r� r� r� r� r� z"hola\\\r\ndfgf"z9 STRING '"hola\\\\\\r\\ndfgf"' (1, 0) (1, 16)
r� r� r� a= FSTRING_START 'fR"' (1, 0) (1, 3)
FSTRING_MIDDLE 'a' (1, 3) (1, 4)
LBRACE '{' (1, 4) (1, 5)
NAME 'b' (1, 5) (1, 6)
RBRACE '}' (1, 6) (1, 7)
FSTRING_MIDDLE 'c' (1, 7) (1, 8)
FSTRING_END '"' (1, 8) (1, 9)
r� r� r� r� zf"{
a}"z� FSTRING_START 'f"' (1, 0) (1, 2)
LBRACE '{' (1, 2) (1, 3)
NAME 'a' (2, 0) (2, 1)
RBRACE '}' (2, 1) (2, 2)
FSTRING_END '"' (2, 2) (2, 3)
r� r� zf"hola\\\r\ndfgf"z� FSTRING_START 'f"' (1, 0) (1, 2)
FSTRING_MIDDLE 'hola\\\\\\r\\ndfgf' (1, 2) (1, 16)
FSTRING_END '"' (1, 16) (1, 17)
r� a� FSTRING_START "f'''" (1, 0) (1, 4)
FSTRING_MIDDLE '__' (1, 4) (1, 6)
LBRACE '{' (1, 6) (1, 7)
NAME 'x' (2, 4) (2, 5)
COLON ':' (2, 5) (2, 6)
FSTRING_MIDDLE 'a\n' (2, 6) (3, 0)
RBRACE '}' (3, 0) (3, 1)
FSTRING_MIDDLE '__' (3, 1) (3, 3)
FSTRING_END "'''" (3, 3) (3, 6)
r� a� FSTRING_START "f'''" (1, 0) (1, 4)
FSTRING_MIDDLE '__' (1, 4) (1, 6)
LBRACE '{' (1, 6) (1, 7)
NAME 'x' (2, 4) (2, 5)
COLON ':' (2, 5) (2, 6)
FSTRING_MIDDLE 'a\n b\n c\n d\n' (2, 6) (6, 0)
RBRACE '}' (6, 0) (6, 1)
FSTRING_MIDDLE '__' (6, 1) (6, 3)
FSTRING_END "'''" (6, 3) (6, 6)
rn ro s r& r� �CTokenizeTest.test_string� s� � � ��� .� 1� �
��� 1� 4� �
��� >� A� �
��� /� 2� �
��� /� 2� �
��� C�
F�
�
��� C�
F�
�
��� -� 0� �
��� -� 0� �
��� -� 0� �
��� -� 0� �
��� C� F� �
��� C� F� �
��� C� F� �
��� C� F� �
��� -� 0� �
���L� +� �
���L� +� �
��� �� �
��� �� �
��� �� �
��� /� 2� �
���H� '� �
���K� *� �
���L� +� �
��� �� �
��� � � �
��� �� �
��� 0� 3� �
���
�
�
�
���
�
�
r( c �p � U R SS5 U R SS5 U R SS5 g )Nr� a NAME 'def' (1, 0) (1, 3)
NAME 'd22' (1, 4) (1, 7)
LPAR '(' (1, 7) (1, 8)
NAME 'a' (1, 8) (1, 9)
COMMA ',' (1, 9) (1, 10)
NAME 'b' (1, 11) (1, 12)
COMMA ',' (1, 12) (1, 13)
NAME 'c' (1, 14) (1, 15)
EQUAL '=' (1, 15) (1, 16)
NUMBER '2' (1, 16) (1, 17)
COMMA ',' (1, 17) (1, 18)
NAME 'd' (1, 19) (1, 20)
EQUAL '=' (1, 20) (1, 21)
NUMBER '2' (1, 21) (1, 22)
COMMA ',' (1, 22) (1, 23)
STAR '*' (1, 24) (1, 25)
NAME 'k' (1, 25) (1, 26)
RPAR ')' (1, 26) (1, 27)
COLON ':' (1, 27) (1, 28)
NAME 'pass' (1, 29) (1, 33)
r� a� NAME 'def' (1, 0) (1, 3)
NAME 'd01v_' (1, 4) (1, 9)
LPAR '(' (1, 9) (1, 10)
NAME 'a' (1, 10) (1, 11)
EQUAL '=' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
COMMA ',' (1, 13) (1, 14)
STAR '*' (1, 15) (1, 16)
NAME 'k' (1, 16) (1, 17)
COMMA ',' (1, 17) (1, 18)
DOUBLESTAR '**' (1, 19) (1, 21)
NAME 'w' (1, 21) (1, 22)
RPAR ')' (1, 22) (1, 23)
COLON ':' (1, 23) (1, 24)
NAME 'pass' (1, 25) (1, 29)
r� a� NAME 'def' (1, 0) (1, 3)
NAME 'd23' (1, 4) (1, 7)
LPAR '(' (1, 7) (1, 8)
NAME 'a' (1, 8) (1, 9)
COLON ':' (1, 9) (1, 10)
NAME 'str' (1, 11) (1, 14)
COMMA ',' (1, 14) (1, 15)
NAME 'b' (1, 16) (1, 17)
COLON ':' (1, 17) (1, 18)
NAME 'int' (1, 19) (1, 22)
EQUAL '=' (1, 22) (1, 23)
NUMBER '3' (1, 23) (1, 24)
RPAR ')' (1, 24) (1, 25)
RARROW '->' (1, 26) (1, 28)
NAME 'int' (1, 29) (1, 32)
COLON ':' (1, 32) (1, 33)
NAME 'pass' (1, 34) (1, 38)
rn ro s r& r� �CTokenizeTest.test_function� sJ � � ��� ?� B� �.
��� ;� >� �$
��� D� G� r( c �( � U R SS5 g )Nr� a� NAME 'if' (1, 0) (1, 2)
NUMBER '1' (1, 3) (1, 4)
LESS '<' (1, 5) (1, 6)
NUMBER '1' (1, 7) (1, 8)
GREATER '>' (1, 9) (1, 10)
NUMBER '1' (1, 11) (1, 12)
EQEQUAL '==' (1, 13) (1, 15)
NUMBER '1' (1, 16) (1, 17)
GREATEREQUAL '>=' (1, 18) (1, 20)
NUMBER '5' (1, 21) (1, 22)
LESSEQUAL '<=' (1, 23) (1, 25)
NUMBER '0x15' (1, 26) (1, 30)
LESSEQUAL '<=' (1, 31) (1, 33)
NUMBER '0x12' (1, 34) (1, 38)
NOTEQUAL '!=' (1, 39) (1, 41)
NUMBER '1' (1, 42) (1, 43)
NAME 'and' (1, 44) (1, 47)
NUMBER '5' (1, 48) (1, 49)
NAME 'in' (1, 50) (1, 52)
NUMBER '1' (1, 53) (1, 54)
NAME 'not' (1, 55) (1, 58)
NAME 'in' (1, 59) (1, 61)
NUMBER '1' (1, 62) (1, 63)
NAME 'is' (1, 64) (1, 66)
NUMBER '1' (1, 67) (1, 68)
NAME 'or' (1, 69) (1, 71)
NUMBER '5' (1, 72) (1, 73)
NAME 'is' (1, 74) (1, 76)
NAME 'not' (1, 77) (1, 80)
NUMBER '1' (1, 81) (1, 82)
COLON ':' (1, 82) (1, 83)
NAME 'pass' (1, 84) (1, 88)
rn ro s r& r� � CTokenizeTest.test_comparison.
r� r( c �( � U R SS5 g )Nr� a$ NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
MINUS '-' (1, 6) (1, 7)
NAME 'y' (1, 8) (1, 9)
PLUS '+' (1, 10) (1, 11)
NUMBER '15' (1, 12) (1, 14)
MINUS '-' (1, 15) (1, 16)
NUMBER '1' (1, 17) (1, 18)
PLUS '+' (1, 19) (1, 20)
NUMBER '0x124' (1, 21) (1, 26)
PLUS '+' (1, 27) (1, 28)
NAME 'z' (1, 29) (1, 30)
PLUS '+' (1, 31) (1, 32)
NAME 'a' (1, 33) (1, 34)
LSQB '[' (1, 34) (1, 35)
NUMBER '5' (1, 35) (1, 36)
RSQB ']' (1, 36) (1, 37)
rn ro s r& r� �CTokenizeTest.test_additiveT
r� r( c �( � U R SS5 g )Nr� a� NAME 'x' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
DOUBLESLASH '//' (1, 5) (1, 7)
NUMBER '1' (1, 7) (1, 8)
STAR '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
SLASH '/' (1, 10) (1, 11)
NUMBER '5' (1, 11) (1, 12)
STAR '*' (1, 12) (1, 13)
NUMBER '12' (1, 13) (1, 15)
PERCENT '%' (1, 15) (1, 16)
NUMBER '0x12' (1, 16) (1, 20)
AT '@' (1, 20) (1, 21)
NUMBER '42' (1, 21) (1, 23)
rn ro s r& r� �!CTokenizeTest.test_multiplicativek
r� r( c �L � U R SS5 U R SS5 g )Nr� a� TILDE '~' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
CIRCUMFLEX '^' (1, 3) (1, 4)
NUMBER '1' (1, 5) (1, 6)
AMPER '&' (1, 7) (1, 8)
NUMBER '1' (1, 9) (1, 10)
VBAR '|' (1, 11) (1, 12)
NUMBER '1' (1, 12) (1, 13)
CIRCUMFLEX '^' (1, 14) (1, 15)
MINUS '-' (1, 16) (1, 17)
NUMBER '1' (1, 17) (1, 18)
r� aI MINUS '-' (1, 0) (1, 1)
NUMBER '1' (1, 1) (1, 2)
STAR '*' (1, 2) (1, 3)
NUMBER '1' (1, 3) (1, 4)
SLASH '/' (1, 4) (1, 5)
NUMBER '1' (1, 5) (1, 6)
PLUS '+' (1, 6) (1, 7)
NUMBER '1' (1, 7) (1, 8)
STAR '*' (1, 8) (1, 9)
NUMBER '1' (1, 9) (1, 10)
DOUBLESLASH '//' (1, 10) (1, 12)
NUMBER '1' (1, 12) (1, 13)
MINUS '-' (1, 14) (1, 15)
MINUS '-' (1, 16) (1, 17)
MINUS '-' (1, 17) (1, 18)
MINUS '-' (1, 18) (1, 19)
NUMBER '1' (1, 19) (1, 20)
DOUBLESTAR '**' (1, 20) (1, 22)
NUMBER '1' (1, 22) (1, 23)
rn ro s r& r� �CTokenizeTest.test_unary
s0 � � ��� 0� 3� �
��� 5� 8� r( c �( � U R SS5 g )Nr� a� NAME 'import' (1, 0) (1, 6)
NAME 'sys' (1, 7) (1, 10)
COMMA ',' (1, 10) (1, 11)
NAME 'time' (1, 12) (1, 16)
NEWLINE '' (1, 16) (1, 16)
NAME 'x' (2, 0) (2, 1)
EQUAL '=' (2, 2) (2, 3)
NAME 'sys' (2, 4) (2, 7)
DOT '.' (2, 7) (2, 8)
NAME 'modules' (2, 8) (2, 15)
LSQB '[' (2, 15) (2, 16)
STRING "'time'" (2, 16) (2, 22)
RSQB ']' (2, 22) (2, 23)
DOT '.' (2, 23) (2, 24)
NAME 'time' (2, 24) (2, 28)
LPAR '(' (2, 28) (2, 29)
RPAR ')' (2, 29) (2, 30)
rn ro s r& r� �CTokenizeTest.test_selector�
r� r( c �( � U R SS5 g �Nr� a AT '@' (1, 0) (1, 1)
NAME 'staticmethod' (1, 1) (1, 13)
NEWLINE '' (1, 13) (1, 13)
NAME 'def' (2, 0) (2, 3)
NAME 'foo' (2, 4) (2, 7)
LPAR '(' (2, 7) (2, 8)
NAME 'x' (2, 8) (2, 9)
COMMA ',' (2, 9) (2, 10)
NAME 'y' (2, 10) (2, 11)
RPAR ')' (2, 11) (2, 12)
COLON ':' (2, 12) (2, 13)
NAME 'pass' (2, 14) (2, 18)
rn ro s r& r� �CTokenizeTest.test_method�
r� r( c �( � U R SS5 g r5 rn ro s r& r� �CTokenizeTest.test_tabs�
r� r( c � � U R SS5 U R SS5 U R SS5 U R SS5 U R S S
5 U R S S 5 U R S
S5 U R SS5 U R SS5 U R SS5 U R SS5 U R SS5 U R SS5 U R SS 5 U R S S 5 U R S S 5 U R S!S"5 U R S#S$5 U R S%S&5 U R S'S(5 U R S)S*5 g )+Nr� z� NAME 'async' (1, 0) (1, 5)
EQUAL '=' (1, 6) (1, 7)
NUMBER '1' (1, 8) (1, 9)
r� a8 NAME 'a' (1, 0) (1, 1)
EQUAL '=' (1, 2) (1, 3)
LPAR '(' (1, 4) (1, 5)
NAME 'async' (1, 5) (1, 10)
EQUAL '=' (1, 11) (1, 12)
NUMBER '1' (1, 13) (1, 14)
RPAR ')' (1, 14) (1, 15)
r� z� NAME 'async' (1, 0) (1, 5)
LPAR '(' (1, 5) (1, 6)
RPAR ')' (1, 6) (1, 7)
r� a< NAME 'class' (1, 0) (1, 5)
NAME 'async' (1, 6) (1, 11)
LPAR '(' (1, 11) (1, 12)
NAME 'Bar' (1, 12) (1, 15)
RPAR ')' (1, 15) (1, 16)
COLON ':' (1, 16) (1, 17)
NAME 'pass' (1, 17) (1, 21)
r� z� NAME 'class' (1, 0) (1, 5)
NAME 'async' (1, 6) (1, 11)
COLON ':' (1, 11) (1, 12)
NAME 'pass' (1, 12) (1, 16)
r� z� NAME 'await' (1, 0) (1, 5)
EQUAL '=' (1, 6) (1, 7)
NUMBER '1' (1, 8) (1, 9)
r� z� NAME 'foo' (1, 0) (1, 3)
DOT '.' (1, 3) (1, 4)
NAME 'async' (1, 4) (1, 9)
r� a; NAME 'async' (1, 0) (1, 5)
NAME 'for' (1, 6) (1, 9)
NAME 'a' (1, 10) (1, 11)
NAME 'in' (1, 12) (1, 14)
NAME 'b' (1, 15) (1, 16)
COLON ':' (1, 16) (1, 17)
NAME 'pass' (1, 18) (1, 22)
r� a< NAME 'async' (1, 0) (1, 5)
NAME 'with' (1, 6) (1, 10)
NAME 'a' (1, 11) (1, 12)
NAME 'as' (1, 13) (1, 15)
NAME 'b' (1, 16) (1, 17)
COLON ':' (1, 17) (1, 18)
NAME 'pass' (1, 19) (1, 23)
r� z� NAME 'async' (1, 0) (1, 5)
DOT '.' (1, 5) (1, 6)
NAME 'foo' (1, 6) (1, 9)
r� r� r� z� NAME 'async' (1, 0) (1, 5)
NEWLINE '' (1, 5) (1, 5)
NAME 'await' (3, 0) (3, 5)
r� z� NAME 'async' (1, 0) (1, 5)
NEWLINE '' (1, 5) (1, 5)
ELLIPSIS '...' (2, 0) (2, 3)
NEWLINE '' (2, 3) (2, 3)
NAME 'await' (3, 0) (3, 5)
r� z� NAME 'async' (1, 0) (1, 5)
NEWLINE '' (1, 5) (1, 5)
NAME 'await' (2, 0) (2, 5)
r� z� NAME 'foo' (1, 0) (1, 3)
DOT '.' (1, 3) (1, 4)
NAME 'async' (1, 4) (1, 9)
PLUS '+' (1, 10) (1, 11)
NUMBER '1' (1, 12) (1, 13)
r� a; NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
LPAR '(' (1, 13) (1, 14)
RPAR ')' (1, 14) (1, 15)
COLON ':' (1, 15) (1, 16)
NAME 'pass' (1, 17) (1, 21)
r� a� NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
LPAR '(' (1, 13) (1, 14)
RPAR ')' (1, 14) (1, 15)
COLON ':' (1, 15) (1, 16)
NEWLINE '' (1, 16) (1, 16)
INDENT '' (2, -1) (2, -1)
NAME 'def' (2, 2) (2, 5)
NAME 'foo' (2, 6) (2, 9)
LPAR '(' (2, 9) (2, 10)
NAME 'await' (2, 10) (2, 15)
RPAR ')' (2, 15) (2, 16)
COLON ':' (2, 16) (2, 17)
NEWLINE '' (2, 17) (2, 17)
INDENT '' (3, -1) (3, -1)
NAME 'await' (3, 4) (3, 9)
EQUAL '=' (3, 10) (3, 11)
NUMBER '1' (3, 12) (3, 13)
NEWLINE '' (3, 13) (3, 13)
DEDENT '' (4, -1) (4, -1)
NAME 'if' (4, 2) (4, 4)
NUMBER '1' (4, 5) (4, 6)
COLON ':' (4, 6) (4, 7)
NEWLINE '' (4, 7) (4, 7)
INDENT '' (5, -1) (5, -1)
NAME 'await' (5, 4) (5, 9)
NEWLINE '' (5, 9) (5, 9)
DEDENT '' (6, -1) (6, -1)
DEDENT '' (6, -1) (6, -1)
NAME 'async' (6, 0) (6, 5)
PLUSEQUAL '+=' (6, 6) (6, 8)
NUMBER '1' (6, 9) (6, 10)
NEWLINE '' (6, 10) (6, 10)
r� a� NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
LPAR '(' (1, 13) (1, 14)
RPAR ')' (1, 14) (1, 15)
COLON ':' (1, 15) (1, 16)
NEWLINE '' (1, 16) (1, 16)
INDENT '' (2, -1) (2, -1)
NAME 'async' (2, 2) (2, 7)
NAME 'for' (2, 8) (2, 11)
NAME 'i' (2, 12) (2, 13)
NAME 'in' (2, 14) (2, 16)
NUMBER '1' (2, 17) (2, 18)
COLON ':' (2, 18) (2, 19)
NAME 'pass' (2, 20) (2, 24)
DEDENT '' (2, -1) (2, -1)
r� ah NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'foo' (1, 10) (1, 13)
LPAR '(' (1, 13) (1, 14)
NAME 'async' (1, 14) (1, 19)
RPAR ')' (1, 19) (1, 20)
COLON ':' (1, 20) (1, 21)
NAME 'await' (1, 22) (1, 27)
r� a~ NAME 'def' (1, 0) (1, 3)
NAME 'f' (1, 4) (1, 5)
LPAR '(' (1, 5) (1, 6)
RPAR ')' (1, 6) (1, 7)
COLON ':' (1, 7) (1, 8)
NEWLINE '' (1, 8) (1, 8)
INDENT '' (3, -1) (3, -1)
NAME 'def' (3, 2) (3, 5)
NAME 'baz' (3, 6) (3, 9)
LPAR '(' (3, 9) (3, 10)
RPAR ')' (3, 10) (3, 11)
COLON ':' (3, 11) (3, 12)
NAME 'pass' (3, 13) (3, 17)
NEWLINE '' (3, 17) (3, 17)
NAME 'async' (4, 2) (4, 7)
NAME 'def' (4, 8) (4, 11)
NAME 'bar' (4, 12) (4, 15)
LPAR '(' (4, 15) (4, 16)
RPAR ')' (4, 16) (4, 17)
COLON ':' (4, 17) (4, 18)
NAME 'pass' (4, 19) (4, 23)
NEWLINE '' (4, 23) (4, 23)
NAME 'await' (6, 2) (6, 7)
EQUAL '=' (6, 8) (6, 9)
NUMBER '2' (6, 10) (6, 11)
DEDENT '' (6, -1) (6, -1)
r� a� NAME 'async' (1, 0) (1, 5)
NAME 'def' (1, 6) (1, 9)
NAME 'f' (1, 10) (1, 11)
LPAR '(' (1, 11) (1, 12)
RPAR ')' (1, 12) (1, 13)
COLON ':' (1, 13) (1, 14)
NEWLINE '' (1, 14) (1, 14)
INDENT '' (3, -1) (3, -1)
NAME 'def' (3, 2) (3, 5)
NAME 'baz' (3, 6) (3, 9)
LPAR '(' (3, 9) (3, 10)
RPAR ')' (3, 10) (3, 11)
COLON ':' (3, 11) (3, 12)
NAME 'pass' (3, 13) (3, 17)
NEWLINE '' (3, 17) (3, 17)
NAME 'async' (4, 2) (4, 7)
NAME 'def' (4, 8) (4, 11)
NAME 'bar' (4, 12) (4, 15)
LPAR '(' (4, 15) (4, 16)
RPAR ')' (4, 16) (4, 17)
COLON ':' (4, 17) (4, 18)
NAME 'pass' (4, 19) (4, 23)
NEWLINE '' (4, 23) (4, 23)
NAME 'await' (6, 2) (6, 7)
EQUAL '=' (6, 8) (6, 9)
NUMBER '2' (6, 10) (6, 11)
DEDENT '' (6, -1) (6, -1)
rn ro s r& r �CTokenizeTest.test_async�
s� � � ���K� *� �
��� -� 0� �
���I� (� �
��� 3� 6� �
��� .� 1� �
���K� *� �
���K� *� �
��� 4� 7� �
��� 5� 8� �
���K� *� �
���G� &� �
��� 4� 7� �
��� /� 2� �
���N� -� �
���O� .� �
��� 3� 6� �
��� �#�* �X
��� H� K� �&
��� 9� <� �
��� � �
! �F
��� � �
" r( c �( � U R SS5 g )Nr� u7 NAME 'Örter' (1, 0) (1, 5)
EQUAL '=' (1, 6) (1, 7)
STRING "u'places'" (1, 8) (1, 17)
NEWLINE '' (1, 17) (1, 17)
NAME 'grün' (2, 0) (2, 4)
EQUAL '=' (2, 5) (2, 6)
STRING "U'green'" (2, 7) (2, 15)
rn ro s r& r� �CTokenizeTest.test_unicode� r� r( c � � S nS H; nU R US9 U R [ R X5 S S S 5 M= g ! , ( d f MO = f)Nc �j � [ U 5 n[ [ R " UR 5 5 $ r )r rE r rg r/ )r9 �
the_strings r& �
get_tokens�5CTokenizeTest.test_invalid_syntax.<locals>.get_tokens� s( � �!�&�)�J���B�B�:�CV�CV�W�X� Xr( ) z(1+2]z(1+2}z{1+2]�1_�1.2_�1e2_�1e+� u €�0b12�0b1_2�0b2�0b1_�0b�0o18�0o1_8�0o8�0o1_�0o�0x1_�0xrB �012rC rD rE z'sdfsdfz '''sdfsdf''a� ((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((a))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))�]z6 f'__{
x:d
}__'z a
)r� )r� rC r r� )r3 r@ r� s r& �test_invalid_syntax�!CTokenizeTest.test_invalid_syntax� sN � � Y�%
�D�L � � �4� �(��!�!�(�"5�"5�z�H� )�(�M%
�L )�(�s �!A�
A c �� ^� SnS nU" US-
5 n[ U5 m[ [ R " TR 5 5 nU R US R [ R 5 U R US R [ R 5 [ USS5 U" U5 n[ U5 mU R [ U4S j5 U R [ [ USS5 g ) N�d c �^ � SR S [ U 5 5 5 nUSU - S- -
nU$ )Nr� c 3 �2 # � U H
nS U- S- v � M g7f)� z if True:
Nr< )�.0rL s r& � <genexpr>�ICTokenizeTest.test_max_indent.<locals>.generate_source.<locals>.<genexpr>% s � � � O��1�d�Q�h�,� 6��r r[ zpass
)r@ r8 )�indentsr� s r& �generate_source�6CTokenizeTest.test_max_indent.<locals>.generate_source$ s2 � ��W�W� O��g�� O�O�F� �d�W�n�x�/� /�F��Mr( rB rM r rS �execc �V >� [ [ R " T R 5 5 $ r )rE r rg r/ )� the_inputs �r& �<lambda>�/CTokenizeTest.test_max_indent.<locals>.<lambda>2 s �� �D��9c�9c�dm�dv�dv�9w�4xr( ) r rE r rg r/ r0 r! �DEDENTr r� rC rV )r3 � MAXINDENTr` �validrN �invalidrd s @r& �test_max_indent� CTokenizeTest.test_max_indent! s� �� �� � �
� �A�
�.�� �U�O� ��h�@�@��AS�AS�T�U�� ������ � �(�/�/�:� ������ � �(�*<�*<�=���z�6�*�!�)�,�� �W�%� � ���*�,x�y� ��� �g�w�
�F�
r( c � � S n[ S5 nU R US5 [ S5 nU R U" U5 U" U5 5 [ S5 nU R US5 [ S5 nU R U" U5 U" U5 5 [ S5 nU R US 5 [ S
5 nU R U" U5 U" U5 5 g ) Nc � � [ U 5 n[ R " UR 5 VV Vs/ s H tp o2U 4PM
snn n$ s snn nf r )r r rg r/ )r9 r? �kindr' s r& r@ �ECTokenizeTest.test_continuation_lines_indentation.<locals>.get_tokens8 sL � �!�&�)�J� �A�A�*�BU�BU�V�X�V� (:��q�6�N�V�X�
X�� Xs �Az�
def fib(n):
\
'''Print a Fibonacci series up to n.'''
\
a, b = 0, 1
ae NAME 'def' (2, 0) (2, 3)
NAME 'fib' (2, 4) (2, 7)
LPAR '(' (2, 7) (2, 8)
NAME 'n' (2, 8) (2, 9)
RPAR ')' (2, 9) (2, 10)
COLON ':' (2, 10) (2, 11)
NEWLINE '' (2, 11) (2, 11)
INDENT '' (4, -1) (4, -1)
STRING "'''Print a Fibonacci series up to n.'''" (4, 0) (4, 39)
NEWLINE '' (4, 39) (4, 39)
NAME 'a' (6, 0) (6, 1)
COMMA ',' (6, 1) (6, 2)
NAME 'b' (6, 3) (6, 4)
EQUAL '=' (6, 5) (6, 6)
NUMBER '0' (6, 7) (6, 8)
COMMA ',' (6, 8) (6, 9)
NUMBER '1' (6, 10) (6, 11)
NEWLINE '' (6, 11) (6, 11)
DEDENT '' (6, -1) (6, -1)
zu
def fib(n):
'''Print a Fibonacci series up to n.'''
a, b = 0, 1
z>
pass
\
pass
z� NAME 'pass' (2, 0) (2, 4)
NEWLINE '' (2, 4) (2, 4)
NAME 'pass' (5, 0) (5, 4)
NEWLINE '' (5, 4) (5, 4)
z+
pass
pass
z�
if x:
y = 1
\
\
\
\
foo = 1
aj NAME 'if' (2, 0) (2, 2)
NAME 'x' (2, 3) (2, 4)
COLON ':' (2, 4) (2, 5)
NEWLINE '' (2, 5) (2, 5)
INDENT '' (3, -1) (3, -1)
NAME 'y' (3, 4) (3, 5)
EQUAL '=' (3, 6) (3, 7)
NUMBER '1' (3, 8) (3, 9)
NEWLINE '' (3, 9) (3, 9)
NAME 'foo' (8, 4) (8, 7)
EQUAL '=' (8, 8) (8, 9)
NUMBER '1' (8, 10) (8, 11)
NEWLINE '' (8, 11) (8, 11)
DEDENT '' (8, -1) (8, -1)
zI
if x:
y = 1
foo = 1
)r r7 r0 )r3 r@ r� � code_no_conts r& �#test_continuation_lines_indentation�1CTokenizeTest.test_continuation_lines_indentation7 s � � X�
� �
��
���D� # �
�, � �
� �
����D�)�:�l�+C�D�� �
��
���D� # �
� � �
� �
����D�)�:�l�+C�D�� �
��
���D� # �
�" � �
� �
����D�)�:�l�+C�Dr( r< N)r r r r r7 r rp r} r� r� r� r� r� r� r� r� r� r r� rU rk rr r r< r( r& r r h sj � �E�3�(? �B, �\w �r= �~$ �L �. �($ �L �, �" �"H �T
�,I�\
�,bEr( r c � � \ rS rSrS rSrg)�CTokenizerBufferTestsi� c � � SSS- SSS- 3n[ R " 5 n[ USU5 n[ U5 S S S 5 g ! , ( d f g = f)Nz" #coding: latin-1
#r� i' z
#i' �foo)r r� r r
)r3 � test_scriptr� � file_names r& �!test_newline_at_the_end_of_buffer�7CTokenizerBufferTests.test_newline_at_the_end_of_buffer� sW � �
�
�e�)��
�
�e�)��� � �
�
�
!�X�#�H�e�[�A�I� �I� &� "�
!�
!�s �A�
Ar< N)r r r r rz r r< r( r& ru ru � s � �
'r( ru �__main__)(r> r� r" r �unittest�ior r �textwrapr r r �testr � test.supportr �test.support.script_helperr
r r �test.support.numbersr
r r' r* r r( r* r: r] rm r r� r� rC r r ru r �mainr< r( r& �<module>r� s �� � � � � � � � #� � "� Y� Y��� h �8� h �T%A� � A�
?� :�x� :�<#I�H� #I�L<�H� <�.@"�� @"�F
v2�8� v2�r&G�X� &G�R
.� Q#�H� Q#�h2�� 2�8qE�H� qE�h! '�H� -� -� '� �z�� �M�M�O� r(