bpo-30377: Simplify handling of COMMENT and NL in tokenize.py (#1607)
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 6b57777..dcaf58f 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -39,6 +39,7 @@
""")
self.check_tokenize("if False:\n"
" # NL\n"
+ " \n"
" True = False # NEWLINE\n", """\
NAME 'if' (1, 0) (1, 2)
NAME 'False' (1, 3) (1, 8)
@@ -46,13 +47,14 @@
NEWLINE '\\n' (1, 9) (1, 10)
COMMENT '# NL' (2, 4) (2, 8)
NL '\\n' (2, 8) (2, 9)
- INDENT ' ' (3, 0) (3, 4)
- NAME 'True' (3, 4) (3, 8)
- OP '=' (3, 9) (3, 10)
- NAME 'False' (3, 11) (3, 16)
- COMMENT '# NEWLINE' (3, 17) (3, 26)
- NEWLINE '\\n' (3, 26) (3, 27)
- DEDENT '' (4, 0) (4, 0)
+ NL '\\n' (3, 4) (3, 5)
+ INDENT ' ' (4, 0) (4, 4)
+ NAME 'True' (4, 4) (4, 8)
+ OP '=' (4, 9) (4, 10)
+ NAME 'False' (4, 11) (4, 16)
+ COMMENT '# NEWLINE' (4, 17) (4, 26)
+ NEWLINE '\\n' (4, 26) (4, 27)
+ DEDENT '' (5, 0) (5, 0)
""")
indent_error_file = b"""\
def k(x):