bpo-40939: Remove the old parser (GH-20768)
This commit removes the old parser, the deprecated parser module, the old parser compatibility flags and environment variables and all associated support code and documentation.
diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index 13600c2..86fdf27 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -326,7 +326,7 @@
if dev_mode:
args.extend(('-X', 'dev'))
for opt in ('faulthandler', 'tracemalloc', 'importtime',
- 'showrefcount', 'utf8', 'oldparser'):
+ 'showrefcount', 'utf8'):
if opt in xoptions:
value = xoptions[opt]
if value is True:
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index f078ad7..498da64 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -1958,13 +1958,3 @@
# sanity check: it should not fail in practice
if pid2 != pid:
raise AssertionError(f"pid {pid2} != pid {pid}")
-
-
-def use_old_parser():
- import _testinternalcapi
- config = _testinternalcapi.get_configs()
- return (config['config']['_use_peg_parser'] == 0)
-
-
-def skip_if_new_parser(msg):
- return unittest.skipIf(not use_old_parser(), msg)
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index b7b7058..fe47289 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -347,7 +347,6 @@
'isolated': 0,
'use_environment': 1,
'dev_mode': 0,
- '_use_peg_parser': 1,
'install_signal_handlers': 1,
'use_hash_seed': 0,
@@ -733,7 +732,6 @@
'import_time': 1,
'show_ref_count': 1,
'malloc_stats': 1,
- '_use_peg_parser': 0,
'stdio_encoding': 'iso8859-1',
'stdio_errors': 'replace',
@@ -807,7 +805,6 @@
'warnoptions': ['EnvVar'],
'platlibdir': 'env_platlibdir',
'module_search_paths': self.IGNORE_CONFIG,
- '_use_peg_parser': 0,
}
self.check_all_configs("test_init_compat_env", config, preconfig,
api=API_COMPAT)
@@ -837,7 +834,6 @@
'warnoptions': ['EnvVar'],
'platlibdir': 'env_platlibdir',
'module_search_paths': self.IGNORE_CONFIG,
- '_use_peg_parser': 0,
}
self.check_all_configs("test_init_python_env", config, preconfig,
api=API_PYTHON)
diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index 196f31e..feae31b 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -251,9 +251,9 @@
check('def f():\n x, y: int', 2, 3)
check('[*x for x in xs]', 1, 2)
check('foo(x for x in range(10), 100)', 1, 5)
- check('(yield i) = 2', 1, 1 if support.use_old_parser() else 2)
- check('def f(*):\n pass', 1, 7 if support.use_old_parser() else 8)
- check('for 1 in []: pass', 1, 5 if support.use_old_parser() else 7)
+ check('(yield i) = 2', 1, 2)
+ check('def f(*):\n pass', 1, 8)
+ check('for 1 in []: pass', 1, 7)
@cpython_only
def testSettingException(self):
diff --git a/Lib/test/test_flufl.py b/Lib/test/test_flufl.py
index 2228585..0ff54aa 100644
--- a/Lib/test/test_flufl.py
+++ b/Lib/test/test_flufl.py
@@ -20,7 +20,7 @@
self.assertTrue(cm.exception.lineno, 2)
# The old parser reports the end of the token and the new
# parser reports the start of the token
- self.assertEqual(cm.exception.offset, 4 if support.use_old_parser() else 3)
+ self.assertEqual(cm.exception.offset, 3)
def test_guido_as_bdfl(self):
code = '2 {0} 3'
@@ -33,7 +33,7 @@
self.assertEqual(cm.exception.lineno, 1)
# The old parser reports the end of the token and the new
# parser reports the start of the token
- self.assertEqual(cm.exception.offset, 4 if support.use_old_parser() else 3)
+ self.assertEqual(cm.exception.offset, 3)
if __name__ == '__main__':
diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
index 9048e89..9eb7ebe 100644
--- a/Lib/test/test_fstring.py
+++ b/Lib/test/test_fstring.py
@@ -12,7 +12,7 @@
import types
import decimal
import unittest
-from test.support import temp_cwd, use_old_parser
+from test.support import temp_cwd
from test.support.script_helper import assert_python_failure
a_global = 'global variable'
@@ -1049,7 +1049,6 @@
r"f'{1000:j}'",
])
- @unittest.skipIf(use_old_parser(), "The old parser only supports <fstring> as the filename")
def test_filename_in_syntaxerror(self):
# see issue 38964
with temp_cwd() as cwd:
diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index e1a402e..ef7d1a1 100644
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -1,7 +1,7 @@
# Python test set -- part 1, grammar.
# This just tests whether the parser accepts them all.
-from test.support import check_syntax_error, check_syntax_warning, use_old_parser
+from test.support import check_syntax_error, check_syntax_warning
import inspect
import unittest
import sys
@@ -1714,69 +1714,53 @@
with manager() as x, manager():
pass
- if not use_old_parser():
- test_cases = [
- """if 1:
- with (
- manager()
- ):
- pass
- """,
- """if 1:
- with (
- manager() as x
- ):
- pass
- """,
- """if 1:
- with (
- manager() as (x, y),
- manager() as z,
- ):
- pass
- """,
- """if 1:
- with (
- manager(),
- manager()
- ):
- pass
- """,
- """if 1:
- with (
- manager() as x,
- manager() as y
- ):
- pass
- """,
- """if 1:
- with (
- manager() as x,
- manager()
- ):
- pass
- """,
- """if 1:
- with (
- manager() as x,
- manager() as y,
- manager() as z,
- ):
- pass
- """,
- """if 1:
- with (
- manager() as x,
- manager() as y,
- manager(),
- ):
- pass
- """,
- ]
- for case in test_cases:
- with self.subTest(case=case):
- compile(case, "<string>", "exec")
+ with (
+ manager()
+ ):
+ pass
+ with (
+ manager() as x
+ ):
+ pass
+
+ with (
+ manager() as (x, y),
+ manager() as z,
+ ):
+ pass
+
+ with (
+ manager(),
+ manager()
+ ):
+ pass
+
+ with (
+ manager() as x,
+ manager() as y
+ ):
+ pass
+
+ with (
+ manager() as x,
+ manager()
+ ):
+ pass
+
+ with (
+ manager() as x,
+ manager() as y,
+ manager() as z,
+ ):
+ pass
+
+ with (
+ manager() as x,
+ manager() as y,
+ manager(),
+ ):
+ pass
def test_if_else_expr(self):
# Test ifelse expressions in various cases
diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py
deleted file mode 100644
index a4d2cdc..0000000
--- a/Lib/test/test_parser.py
+++ /dev/null
@@ -1,1046 +0,0 @@
-import copy
-import warnings
-with warnings.catch_warnings():
- warnings.filterwarnings('ignore', 'The parser module is deprecated',
- DeprecationWarning)
- import parser
-import pickle
-import unittest
-import operator
-import struct
-from test import support
-from test.support.script_helper import assert_python_failure
-from test.support.script_helper import assert_python_ok
-
-#
-# First, we test that we can generate trees from valid source fragments,
-# and that these valid trees are indeed allowed by the tree-loading side
-# of the parser module.
-#
-
-class RoundtripLegalSyntaxTestCase(unittest.TestCase):
-
- def roundtrip(self, f, s):
- st1 = f(s)
- t = st1.totuple()
- try:
- st2 = parser.sequence2st(t)
- except parser.ParserError as why:
- self.fail("could not roundtrip %r: %s" % (s, why))
-
- self.assertEqual(t, st2.totuple(),
- "could not re-generate syntax tree")
-
- def check_expr(self, s):
- self.roundtrip(parser.expr, s)
-
- def test_flags_passed(self):
- # The unicode literals flags has to be passed from the parser to AST
- # generation.
- suite = parser.suite("from __future__ import unicode_literals; x = ''")
- code = suite.compile()
- scope = {}
- exec(code, {}, scope)
- self.assertIsInstance(scope["x"], str)
-
- def check_suite(self, s):
- self.roundtrip(parser.suite, s)
-
- def test_yield_statement(self):
- self.check_suite("def f(): yield 1")
- self.check_suite("def f(): yield")
- self.check_suite("def f(): x += yield")
- self.check_suite("def f(): x = yield 1")
- self.check_suite("def f(): x = y = yield 1")
- self.check_suite("def f(): x = yield")
- self.check_suite("def f(): x = y = yield")
- self.check_suite("def f(): 1 + (yield)*2")
- self.check_suite("def f(): (yield 1)*2")
- self.check_suite("def f(): return; yield 1")
- self.check_suite("def f(): yield 1; return")
- self.check_suite("def f(): yield from 1")
- self.check_suite("def f(): x = yield from 1")
- self.check_suite("def f(): f((yield from 1))")
- self.check_suite("def f(): yield 1; return 1")
- self.check_suite("def f():\n"
- " for x in range(30):\n"
- " yield x\n")
- self.check_suite("def f():\n"
- " if (yield):\n"
- " yield x\n")
-
- def test_await_statement(self):
- self.check_suite("async def f():\n await smth()")
- self.check_suite("async def f():\n foo = await smth()")
- self.check_suite("async def f():\n foo, bar = await smth()")
- self.check_suite("async def f():\n (await smth())")
- self.check_suite("async def f():\n foo((await smth()))")
- self.check_suite("async def f():\n await foo(); return 42")
-
- def test_async_with_statement(self):
- self.check_suite("async def f():\n async with 1: pass")
- self.check_suite("async def f():\n async with a as b, c as d: pass")
-
- def test_async_for_statement(self):
- self.check_suite("async def f():\n async for i in (): pass")
- self.check_suite("async def f():\n async for i, b in (): pass")
-
- def test_nonlocal_statement(self):
- self.check_suite("def f():\n"
- " x = 0\n"
- " def g():\n"
- " nonlocal x\n")
- self.check_suite("def f():\n"
- " x = y = 0\n"
- " def g():\n"
- " nonlocal x, y\n")
-
- def test_expressions(self):
- self.check_expr("foo(1)")
- self.check_expr("[1, 2, 3]")
- self.check_expr("[x**3 for x in range(20)]")
- self.check_expr("[x**3 for x in range(20) if x % 3]")
- self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]")
- self.check_expr("list(x**3 for x in range(20))")
- self.check_expr("list(x**3 for x in range(20) if x % 3)")
- self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)")
- self.check_expr("foo(*args)")
- self.check_expr("foo(*args, **kw)")
- self.check_expr("foo(**kw)")
- self.check_expr("foo(key=value)")
- self.check_expr("foo(key=value, *args)")
- self.check_expr("foo(key=value, *args, **kw)")
- self.check_expr("foo(key=value, **kw)")
- self.check_expr("foo(a, b, c, *args)")
- self.check_expr("foo(a, b, c, *args, **kw)")
- self.check_expr("foo(a, b, c, **kw)")
- self.check_expr("foo(a, *args, keyword=23)")
- self.check_expr("foo + bar")
- self.check_expr("foo - bar")
- self.check_expr("foo * bar")
- self.check_expr("foo / bar")
- self.check_expr("foo // bar")
- self.check_expr("(foo := 1)")
- self.check_expr("lambda: 0")
- self.check_expr("lambda x: 0")
- self.check_expr("lambda *y: 0")
- self.check_expr("lambda *y, **z: 0")
- self.check_expr("lambda **z: 0")
- self.check_expr("lambda x, y: 0")
- self.check_expr("lambda foo=bar: 0")
- self.check_expr("lambda foo=bar, spaz=nifty+spit: 0")
- self.check_expr("lambda foo=bar, **z: 0")
- self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0")
- self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0")
- self.check_expr("lambda x, *y, **z: 0")
- self.check_expr("(x for x in range(10))")
- self.check_expr("foo(x for x in range(10))")
- self.check_expr("...")
- self.check_expr("a[...]")
-
- def test_simple_expression(self):
- # expr_stmt
- self.check_suite("a")
-
- def test_simple_assignments(self):
- self.check_suite("a = b")
- self.check_suite("a = b = c = d = e")
-
- def test_var_annot(self):
- self.check_suite("x: int = 5")
- self.check_suite("y: List[T] = []; z: [list] = fun()")
- self.check_suite("x: tuple = (1, 2)")
- self.check_suite("d[f()]: int = 42")
- self.check_suite("f(d[x]): str = 'abc'")
- self.check_suite("x.y.z.w: complex = 42j")
- self.check_suite("x: int")
- self.check_suite("def f():\n"
- " x: str\n"
- " y: int = 5\n")
- self.check_suite("class C:\n"
- " x: str\n"
- " y: int = 5\n")
- self.check_suite("class C:\n"
- " def __init__(self, x: int) -> None:\n"
- " self.x: int = x\n")
- # double check for nonsense
- with self.assertRaises(SyntaxError):
- exec("2+2: int", {}, {})
- with self.assertRaises(SyntaxError):
- exec("[]: int = 5", {}, {})
- with self.assertRaises(SyntaxError):
- exec("x, *y, z: int = range(5)", {}, {})
- with self.assertRaises(SyntaxError):
- exec("x: int = 1, y = 2", {}, {})
- with self.assertRaises(SyntaxError):
- exec("u = v: int", {}, {})
- with self.assertRaises(SyntaxError):
- exec("False: int", {}, {})
- with self.assertRaises(SyntaxError):
- exec("x.False: int", {}, {})
- with self.assertRaises(SyntaxError):
- exec("x.y,: int", {}, {})
- with self.assertRaises(SyntaxError):
- exec("[0]: int", {}, {})
- with self.assertRaises(SyntaxError):
- exec("f(): int", {}, {})
-
- def test_simple_augmented_assignments(self):
- self.check_suite("a += b")
- self.check_suite("a -= b")
- self.check_suite("a *= b")
- self.check_suite("a /= b")
- self.check_suite("a //= b")
- self.check_suite("a %= b")
- self.check_suite("a &= b")
- self.check_suite("a |= b")
- self.check_suite("a ^= b")
- self.check_suite("a <<= b")
- self.check_suite("a >>= b")
- self.check_suite("a **= b")
-
- def test_function_defs(self):
- self.check_suite("def f(): pass")
- self.check_suite("def f(*args): pass")
- self.check_suite("def f(*args, **kw): pass")
- self.check_suite("def f(**kw): pass")
- self.check_suite("def f(foo=bar): pass")
- self.check_suite("def f(foo=bar, *args): pass")
- self.check_suite("def f(foo=bar, *args, **kw): pass")
- self.check_suite("def f(foo=bar, **kw): pass")
-
- self.check_suite("def f(a, b): pass")
- self.check_suite("def f(a, b, *args): pass")
- self.check_suite("def f(a, b, *args, **kw): pass")
- self.check_suite("def f(a, b, **kw): pass")
- self.check_suite("def f(a, b, foo=bar): pass")
- self.check_suite("def f(a, b, foo=bar, *args): pass")
- self.check_suite("def f(a, b, foo=bar, *args, **kw): pass")
- self.check_suite("def f(a, b, foo=bar, **kw): pass")
-
- self.check_suite("@staticmethod\n"
- "def f(): pass")
- self.check_suite("@staticmethod\n"
- "@funcattrs(x, y)\n"
- "def f(): pass")
- self.check_suite("@funcattrs()\n"
- "def f(): pass")
-
- self.check_suite("@False or x\n"
- "def f(): pass")
- self.check_suite("@d := x\n"
- "def f(): pass")
- self.check_suite("@lambda f: x(f)\n"
- "def f(): pass")
- self.check_suite("@[..., x, ...][1]\n"
- "def f(): pass")
- self.check_suite("@x(x)(x)\n"
- "def f(): pass")
- self.check_suite("@(x, x)\n"
- "def f(): pass")
- self.check_suite("@...\n"
- "def f(): pass")
- self.check_suite("@None\n"
- "def f(): pass")
- self.check_suite("@w @(x @y) @(z)\n"
- "def f(): pass")
- self.check_suite("@w[x].y.z\n"
- "def f(): pass")
-
- # keyword-only arguments
- self.check_suite("def f(*, a): pass")
- self.check_suite("def f(*, a = 5): pass")
- self.check_suite("def f(*, a = 5, b): pass")
- self.check_suite("def f(*, a, b = 5): pass")
- self.check_suite("def f(*, a, b = 5, **kwds): pass")
- self.check_suite("def f(*args, a): pass")
- self.check_suite("def f(*args, a = 5): pass")
- self.check_suite("def f(*args, a = 5, b): pass")
- self.check_suite("def f(*args, a, b = 5): pass")
- self.check_suite("def f(*args, a, b = 5, **kwds): pass")
-
- # positional-only arguments
- self.check_suite("def f(a, /): pass")
- self.check_suite("def f(a, /,): pass")
- self.check_suite("def f(a, b, /): pass")
- self.check_suite("def f(a, b, /, c): pass")
- self.check_suite("def f(a, b, /, c = 6): pass")
- self.check_suite("def f(a, b, /, c, *, d): pass")
- self.check_suite("def f(a, b, /, c = 1, *, d): pass")
- self.check_suite("def f(a, b, /, c, *, d = 1): pass")
- self.check_suite("def f(a, b=1, /, c=2, *, d = 3): pass")
- self.check_suite("def f(a=0, b=1, /, c=2, *, d = 3): pass")
-
- # function annotations
- self.check_suite("def f(a: int): pass")
- self.check_suite("def f(a: int = 5): pass")
- self.check_suite("def f(*args: list): pass")
- self.check_suite("def f(**kwds: dict): pass")
- self.check_suite("def f(*, a: int): pass")
- self.check_suite("def f(*, a: int = 5): pass")
- self.check_suite("def f() -> int: pass")
-
- def test_class_defs(self):
- self.check_suite("class foo():pass")
- self.check_suite("class foo(object):pass")
- self.check_suite("@class_decorator\n"
- "class foo():pass")
- self.check_suite("@class_decorator(arg)\n"
- "class foo():pass")
- self.check_suite("@decorator1\n"
- "@decorator2\n"
- "class foo():pass")
-
- self.check_suite("@False or x\n"
- "class C: pass")
- self.check_suite("@d := x\n"
- "class C: pass")
- self.check_suite("@lambda f: x(f)\n"
- "class C: pass")
- self.check_suite("@[..., x, ...][1]\n"
- "class C: pass")
- self.check_suite("@x(x)(x)\n"
- "class C: pass")
- self.check_suite("@(x, x)\n"
- "class C: pass")
- self.check_suite("@...\n"
- "class C: pass")
- self.check_suite("@None\n"
- "class C: pass")
- self.check_suite("@w @(x @y) @(z)\n"
- "class C: pass")
- self.check_suite("@w[x].y.z\n"
- "class C: pass")
-
- def test_import_from_statement(self):
- self.check_suite("from sys.path import *")
- self.check_suite("from sys.path import dirname")
- self.check_suite("from sys.path import (dirname)")
- self.check_suite("from sys.path import (dirname,)")
- self.check_suite("from sys.path import dirname as my_dirname")
- self.check_suite("from sys.path import (dirname as my_dirname)")
- self.check_suite("from sys.path import (dirname as my_dirname,)")
- self.check_suite("from sys.path import dirname, basename")
- self.check_suite("from sys.path import (dirname, basename)")
- self.check_suite("from sys.path import (dirname, basename,)")
- self.check_suite(
- "from sys.path import dirname as my_dirname, basename")
- self.check_suite(
- "from sys.path import (dirname as my_dirname, basename)")
- self.check_suite(
- "from sys.path import (dirname as my_dirname, basename,)")
- self.check_suite(
- "from sys.path import dirname, basename as my_basename")
- self.check_suite(
- "from sys.path import (dirname, basename as my_basename)")
- self.check_suite(
- "from sys.path import (dirname, basename as my_basename,)")
- self.check_suite("from .bogus import x")
-
- def test_basic_import_statement(self):
- self.check_suite("import sys")
- self.check_suite("import sys as system")
- self.check_suite("import sys, math")
- self.check_suite("import sys as system, math")
- self.check_suite("import sys, math as my_math")
-
- def test_relative_imports(self):
- self.check_suite("from . import name")
- self.check_suite("from .. import name")
- # check all the way up to '....', since '...' is tokenized
- # differently from '.' (it's an ellipsis token).
- self.check_suite("from ... import name")
- self.check_suite("from .... import name")
- self.check_suite("from .pkg import name")
- self.check_suite("from ..pkg import name")
- self.check_suite("from ...pkg import name")
- self.check_suite("from ....pkg import name")
-
- def test_pep263(self):
- self.check_suite("# -*- coding: iso-8859-1 -*-\n"
- "pass\n")
-
- def test_assert(self):
- self.check_suite("assert alo < ahi and blo < bhi\n")
-
- def test_with(self):
- self.check_suite("with open('x'): pass\n")
- self.check_suite("with open('x') as f: pass\n")
- self.check_suite("with open('x') as f, open('y') as g: pass\n")
-
- def test_try_stmt(self):
- self.check_suite("try: pass\nexcept: pass\n")
- self.check_suite("try: pass\nfinally: pass\n")
- self.check_suite("try: pass\nexcept A: pass\nfinally: pass\n")
- self.check_suite("try: pass\nexcept A: pass\nexcept: pass\n"
- "finally: pass\n")
- self.check_suite("try: pass\nexcept: pass\nelse: pass\n")
- self.check_suite("try: pass\nexcept: pass\nelse: pass\n"
- "finally: pass\n")
-
- def test_if_stmt(self):
- self.check_suite("if True:\n pass\nelse:\n pass\n")
- self.check_suite("if True:\n pass\nelif True:\n pass\nelse:\n pass\n")
-
- def test_position(self):
- # An absolutely minimal test of position information. Better
- # tests would be a big project.
- code = "def f(x):\n return x + 1"
- st = parser.suite(code)
-
- def walk(tree):
- node_type = tree[0]
- next = tree[1]
- if isinstance(next, (tuple, list)):
- for elt in tree[1:]:
- for x in walk(elt):
- yield x
- else:
- yield tree
-
- expected = [
- (1, 'def', 1, 0),
- (1, 'f', 1, 4),
- (7, '(', 1, 5),
- (1, 'x', 1, 6),
- (8, ')', 1, 7),
- (11, ':', 1, 8),
- (4, '', 1, 9),
- (5, '', 2, -1),
- (1, 'return', 2, 4),
- (1, 'x', 2, 11),
- (14, '+', 2, 13),
- (2, '1', 2, 15),
- (4, '', 2, 16),
- (6, '', 2, -1),
- (4, '', 2, -1),
- (0, '', 2, -1),
- ]
-
- self.assertEqual(list(walk(st.totuple(line_info=True, col_info=True))),
- expected)
- self.assertEqual(list(walk(st.totuple())),
- [(t, n) for t, n, l, c in expected])
- self.assertEqual(list(walk(st.totuple(line_info=True))),
- [(t, n, l) for t, n, l, c in expected])
- self.assertEqual(list(walk(st.totuple(col_info=True))),
- [(t, n, c) for t, n, l, c in expected])
- self.assertEqual(list(walk(st.tolist(line_info=True, col_info=True))),
- [list(x) for x in expected])
- self.assertEqual(list(walk(parser.st2tuple(st, line_info=True,
- col_info=True))),
- expected)
- self.assertEqual(list(walk(parser.st2list(st, line_info=True,
- col_info=True))),
- [list(x) for x in expected])
-
- def test_extended_unpacking(self):
- self.check_suite("*a = y")
- self.check_suite("x, *b, = m")
- self.check_suite("[*a, *b] = y")
- self.check_suite("for [*x, b] in x: pass")
-
- def test_raise_statement(self):
- self.check_suite("raise\n")
- self.check_suite("raise e\n")
- self.check_suite("try:\n"
- " suite\n"
- "except Exception as e:\n"
- " raise ValueError from e\n")
-
- def test_list_displays(self):
- self.check_expr('[]')
- self.check_expr('[*{2}, 3, *[4]]')
-
- def test_set_displays(self):
- self.check_expr('{*{2}, 3, *[4]}')
- self.check_expr('{2}')
- self.check_expr('{2,}')
- self.check_expr('{2, 3}')
- self.check_expr('{2, 3,}')
-
- def test_dict_displays(self):
- self.check_expr('{}')
- self.check_expr('{a:b}')
- self.check_expr('{a:b,}')
- self.check_expr('{a:b, c:d}')
- self.check_expr('{a:b, c:d,}')
- self.check_expr('{**{}}')
- self.check_expr('{**{}, 3:4, **{5:6, 7:8}}')
-
- def test_argument_unpacking(self):
- self.check_expr("f(*a, **b)")
- self.check_expr('f(a, *b, *c, *d)')
- self.check_expr('f(**a, **b)')
- self.check_expr('f(2, *a, *b, **b, **c, **d)')
- self.check_expr("f(*b, *() or () and (), **{} and {}, **() or {})")
-
- def test_set_comprehensions(self):
- self.check_expr('{x for x in seq}')
- self.check_expr('{f(x) for x in seq}')
- self.check_expr('{f(x) for x in seq if condition(x)}')
-
- def test_dict_comprehensions(self):
- self.check_expr('{x:x for x in seq}')
- self.check_expr('{x**2:x[3] for x in seq if condition(x)}')
- self.check_expr('{x:x for x in seq1 for y in seq2 if condition(x, y)}')
-
- def test_named_expressions(self):
- self.check_suite("(a := 1)")
- self.check_suite("(a := a)")
- self.check_suite("if (match := pattern.search(data)) is None: pass")
- self.check_suite("while match := pattern.search(f.read()): pass")
- self.check_suite("[y := f(x), y**2, y**3]")
- self.check_suite("filtered_data = [y for x in data if (y := f(x)) is None]")
- self.check_suite("(y := f(x))")
- self.check_suite("y0 = (y1 := f(x))")
- self.check_suite("foo(x=(y := f(x)))")
- self.check_suite("def foo(answer=(p := 42)): pass")
- self.check_suite("def foo(answer: (p := 42) = 5): pass")
- self.check_suite("lambda: (x := 1)")
- self.check_suite("(x := lambda: 1)")
- self.check_suite("(x := lambda: (y := 1))") # not in PEP
- self.check_suite("lambda line: (m := re.match(pattern, line)) and m.group(1)")
- self.check_suite("x = (y := 0)")
- self.check_suite("(z:=(y:=(x:=0)))")
- self.check_suite("(info := (name, phone, *rest))")
- self.check_suite("(x:=1,2)")
- self.check_suite("(total := total + tax)")
- self.check_suite("len(lines := f.readlines())")
- self.check_suite("foo(x := 3, cat='vector')")
- self.check_suite("foo(cat=(category := 'vector'))")
- self.check_suite("if any(len(longline := l) >= 100 for l in lines): print(longline)")
- self.check_suite(
- "if env_base := os.environ.get('PYTHONUSERBASE', None): return env_base"
- )
- self.check_suite(
- "if self._is_special and (ans := self._check_nans(context=context)): return ans"
- )
- self.check_suite("foo(b := 2, a=1)")
- self.check_suite("foo(b := 2, a=1)")
- self.check_suite("foo((b := 2), a=1)")
- self.check_suite("foo(c=(b := 2), a=1)")
- self.check_suite("{(x := C(i)).q: x for i in y}")
-
-
-#
-# Second, we take *invalid* trees and make sure we get ParserError
-# rejections for them.
-#
-
-class IllegalSyntaxTestCase(unittest.TestCase):
-
- def check_bad_tree(self, tree, label):
- try:
- parser.sequence2st(tree)
- except parser.ParserError:
- pass
- else:
- self.fail("did not detect invalid tree for %r" % label)
-
- def test_junk(self):
- # not even remotely valid:
- self.check_bad_tree((1, 2, 3), "<junk>")
-
- def test_illegal_terminal(self):
- tree = \
- (257,
- (269,
- (270,
- (271,
- (277,
- (1,))),
- (4, ''))),
- (4, ''),
- (0, ''))
- self.check_bad_tree(tree, "too small items in terminal node")
- tree = \
- (257,
- (269,
- (270,
- (271,
- (277,
- (1, b'pass'))),
- (4, ''))),
- (4, ''),
- (0, ''))
- self.check_bad_tree(tree, "non-string second item in terminal node")
- tree = \
- (257,
- (269,
- (270,
- (271,
- (277,
- (1, 'pass', '0', 0))),
- (4, ''))),
- (4, ''),
- (0, ''))
- self.check_bad_tree(tree, "non-integer third item in terminal node")
- tree = \
- (257,
- (269,
- (270,
- (271,
- (277,
- (1, 'pass', 0, 0))),
- (4, ''))),
- (4, ''),
- (0, ''))
- self.check_bad_tree(tree, "too many items in terminal node")
-
- def test_illegal_yield_1(self):
- # Illegal yield statement: def f(): return 1; yield 1
- tree = \
- (257,
- (264,
- (285,
- (259,
- (1, 'def'),
- (1, 'f'),
- (260, (7, '('), (8, ')')),
- (11, ':'),
- (291,
- (4, ''),
- (5, ''),
- (264,
- (265,
- (266,
- (272,
- (275,
- (1, 'return'),
- (313,
- (292,
- (293,
- (294,
- (295,
- (297,
- (298,
- (299,
- (300,
- (301,
- (302, (303, (304, (305, (2, '1')))))))))))))))))),
- (264,
- (265,
- (266,
- (272,
- (276,
- (1, 'yield'),
- (313,
- (292,
- (293,
- (294,
- (295,
- (297,
- (298,
- (299,
- (300,
- (301,
- (302,
- (303, (304, (305, (2, '1')))))))))))))))))),
- (4, ''))),
- (6, ''))))),
- (4, ''),
- (0, ''))))
- self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
-
- def test_illegal_yield_2(self):
- # Illegal return in generator: def f(): return 1; yield 1
- tree = \
- (257,
- (264,
- (265,
- (266,
- (278,
- (1, 'from'),
- (281, (1, '__future__')),
- (1, 'import'),
- (279, (1, 'generators')))),
- (4, ''))),
- (264,
- (285,
- (259,
- (1, 'def'),
- (1, 'f'),
- (260, (7, '('), (8, ')')),
- (11, ':'),
- (291,
- (4, ''),
- (5, ''),
- (264,
- (265,
- (266,
- (272,
- (275,
- (1, 'return'),
- (313,
- (292,
- (293,
- (294,
- (295,
- (297,
- (298,
- (299,
- (300,
- (301,
- (302, (303, (304, (305, (2, '1')))))))))))))))))),
- (264,
- (265,
- (266,
- (272,
- (276,
- (1, 'yield'),
- (313,
- (292,
- (293,
- (294,
- (295,
- (297,
- (298,
- (299,
- (300,
- (301,
- (302,
- (303, (304, (305, (2, '1')))))))))))))))))),
- (4, ''))),
- (6, ''))))),
- (4, ''),
- (0, ''))))
- self.check_bad_tree(tree, "def f():\n return 1\n yield 1")
-
- def test_a_comma_comma_c(self):
- # Illegal input: a,,c
- tree = \
- (258,
- (311,
- (290,
- (291,
- (292,
- (293,
- (295,
- (296,
- (297,
- (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))),
- (12, ','),
- (12, ','),
- (290,
- (291,
- (292,
- (293,
- (295,
- (296,
- (297,
- (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))),
- (4, ''),
- (0, ''))
- self.check_bad_tree(tree, "a,,c")
-
- def test_illegal_operator(self):
- # Illegal input: a $= b
- tree = \
- (257,
- (264,
- (265,
- (266,
- (267,
- (312,
- (291,
- (292,
- (293,
- (294,
- (296,
- (297,
- (298,
- (299,
- (300, (301, (302, (303, (304, (1, 'a'))))))))))))))),
- (268, (37, '$=')),
- (312,
- (291,
- (292,
- (293,
- (294,
- (296,
- (297,
- (298,
- (299,
- (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))),
- (4, ''))),
- (0, ''))
- self.check_bad_tree(tree, "a $= b")
-
- def test_malformed_global(self):
- #doesn't have global keyword in ast
- tree = (257,
- (264,
- (265,
- (266,
- (282, (1, 'foo'))), (4, ''))),
- (4, ''),
- (0, ''))
- self.check_bad_tree(tree, "malformed global ast")
-
- def test_missing_import_source(self):
- # from import fred
- tree = \
- (257,
- (268,
- (269,
- (270,
- (282,
- (284, (1, 'from'), (1, 'import'),
- (287, (285, (1, 'fred')))))),
- (4, ''))),
- (4, ''), (0, ''))
- self.check_bad_tree(tree, "from import fred")
-
- def test_illegal_encoding(self):
- # Illegal encoding declaration
- tree = \
- (341,
- (257, (0, '')))
- self.check_bad_tree(tree, "missed encoding")
- tree = \
- (341,
- (257, (0, '')),
- b'iso-8859-1')
- self.check_bad_tree(tree, "non-string encoding")
- tree = \
- (341,
- (257, (0, '')),
- '\udcff')
- with self.assertRaises(UnicodeEncodeError):
- parser.sequence2st(tree)
-
- def test_invalid_node_id(self):
- tree = (257, (269, (-7, '')))
- self.check_bad_tree(tree, "negative node id")
- tree = (257, (269, (99, '')))
- self.check_bad_tree(tree, "invalid token id")
- tree = (257, (269, (9999, (0, ''))))
- self.check_bad_tree(tree, "invalid symbol id")
-
- def test_ParserError_message(self):
- try:
- parser.sequence2st((257,(269,(257,(0,'')))))
- except parser.ParserError as why:
- self.assertIn("compound_stmt", str(why)) # Expected
- self.assertIn("file_input", str(why)) # Got
-
-
-
-class CompileTestCase(unittest.TestCase):
-
- # These tests are very minimal. :-(
-
- def test_compile_expr(self):
- st = parser.expr('2 + 3')
- code = parser.compilest(st)
- self.assertEqual(eval(code), 5)
-
- def test_compile_suite(self):
- st = parser.suite('x = 2; y = x + 3')
- code = parser.compilest(st)
- globs = {}
- exec(code, globs)
- self.assertEqual(globs['y'], 5)
-
- def test_compile_error(self):
- st = parser.suite('1 = 3 + 4')
- self.assertRaises(SyntaxError, parser.compilest, st)
-
- def test_compile_badunicode(self):
- st = parser.suite('a = "\\U12345678"')
- self.assertRaises(SyntaxError, parser.compilest, st)
- st = parser.suite('a = "\\u1"')
- self.assertRaises(SyntaxError, parser.compilest, st)
-
- def test_issue_9011(self):
- # Issue 9011: compilation of an unary minus expression changed
- # the meaning of the ST, so that a second compilation produced
- # incorrect results.
- st = parser.expr('-3')
- code1 = parser.compilest(st)
- self.assertEqual(eval(code1), -3)
- code2 = parser.compilest(st)
- self.assertEqual(eval(code2), -3)
-
- def test_compile_filename(self):
- st = parser.expr('a + 5')
- code = parser.compilest(st)
- self.assertEqual(code.co_filename, '<syntax-tree>')
- code = st.compile()
- self.assertEqual(code.co_filename, '<syntax-tree>')
- for filename in 'file.py', b'file.py':
- code = parser.compilest(st, filename)
- self.assertEqual(code.co_filename, 'file.py')
- code = st.compile(filename)
- self.assertEqual(code.co_filename, 'file.py')
- for filename in bytearray(b'file.py'), memoryview(b'file.py'):
- with self.assertWarns(DeprecationWarning):
- code = parser.compilest(st, filename)
- self.assertEqual(code.co_filename, 'file.py')
- with self.assertWarns(DeprecationWarning):
- code = st.compile(filename)
- self.assertEqual(code.co_filename, 'file.py')
- self.assertRaises(TypeError, parser.compilest, st, list(b'file.py'))
- self.assertRaises(TypeError, st.compile, list(b'file.py'))
-
-
-class ParserStackLimitTestCase(unittest.TestCase):
- """try to push the parser to/over its limits.
- see http://bugs.python.org/issue1881 for a discussion
- """
- def _nested_expression(self, level):
- return "["*level+"]"*level
-
- def test_deeply_nested_list(self):
- # This has fluctuated between 99 levels in 2.x, down to 93 levels in
- # 3.7.X and back up to 99 in 3.8.X. Related to MAXSTACK size in Parser.h
- e = self._nested_expression(99)
- st = parser.expr(e)
- st.compile()
-
- def test_trigger_memory_error(self):
- e = self._nested_expression(100)
- rc, out, err = assert_python_failure('-Xoldparser', '-c', e)
- # parsing the expression will result in an error message
- # followed by a MemoryError (see #11963)
- self.assertIn(b's_push: parser stack overflow', err)
- self.assertIn(b'MemoryError', err)
-
-class STObjectTestCase(unittest.TestCase):
- """Test operations on ST objects themselves"""
-
- def test_comparisons(self):
- # ST objects should support order and equality comparisons
- st1 = parser.expr('2 + 3')
- st2 = parser.suite('x = 2; y = x + 3')
- st3 = parser.expr('list(x**3 for x in range(20))')
- st1_copy = parser.expr('2 + 3')
- st2_copy = parser.suite('x = 2; y = x + 3')
- st3_copy = parser.expr('list(x**3 for x in range(20))')
-
- # exercise fast path for object identity
- self.assertEqual(st1 == st1, True)
- self.assertEqual(st2 == st2, True)
- self.assertEqual(st3 == st3, True)
- # slow path equality
- self.assertEqual(st1, st1_copy)
- self.assertEqual(st2, st2_copy)
- self.assertEqual(st3, st3_copy)
- self.assertEqual(st1 == st2, False)
- self.assertEqual(st1 == st3, False)
- self.assertEqual(st2 == st3, False)
- self.assertEqual(st1 != st1, False)
- self.assertEqual(st2 != st2, False)
- self.assertEqual(st3 != st3, False)
- self.assertEqual(st1 != st1_copy, False)
- self.assertEqual(st2 != st2_copy, False)
- self.assertEqual(st3 != st3_copy, False)
- self.assertEqual(st2 != st1, True)
- self.assertEqual(st1 != st3, True)
- self.assertEqual(st3 != st2, True)
- # we don't particularly care what the ordering is; just that
- # it's usable and self-consistent
- self.assertEqual(st1 < st2, not (st2 <= st1))
- self.assertEqual(st1 < st3, not (st3 <= st1))
- self.assertEqual(st2 < st3, not (st3 <= st2))
- self.assertEqual(st1 < st2, st2 > st1)
- self.assertEqual(st1 < st3, st3 > st1)
- self.assertEqual(st2 < st3, st3 > st2)
- self.assertEqual(st1 <= st2, st2 >= st1)
- self.assertEqual(st3 <= st1, st1 >= st3)
- self.assertEqual(st2 <= st3, st3 >= st2)
- # transitivity
- bottom = min(st1, st2, st3)
- top = max(st1, st2, st3)
- mid = sorted([st1, st2, st3])[1]
- self.assertTrue(bottom < mid)
- self.assertTrue(bottom < top)
- self.assertTrue(mid < top)
- self.assertTrue(bottom <= mid)
- self.assertTrue(bottom <= top)
- self.assertTrue(mid <= top)
- self.assertTrue(bottom <= bottom)
- self.assertTrue(mid <= mid)
- self.assertTrue(top <= top)
- # interaction with other types
- self.assertEqual(st1 == 1588.602459, False)
- self.assertEqual('spanish armada' != st2, True)
- self.assertRaises(TypeError, operator.ge, st3, None)
- self.assertRaises(TypeError, operator.le, False, st1)
- self.assertRaises(TypeError, operator.lt, st1, 1815)
- self.assertRaises(TypeError, operator.gt, b'waterloo', st2)
-
- def test_copy_pickle(self):
- sts = [
- parser.expr('2 + 3'),
- parser.suite('x = 2; y = x + 3'),
- parser.expr('list(x**3 for x in range(20))')
- ]
- for st in sts:
- st_copy = copy.copy(st)
- self.assertEqual(st_copy.totuple(), st.totuple())
- st_copy = copy.deepcopy(st)
- self.assertEqual(st_copy.totuple(), st.totuple())
- for proto in range(pickle.HIGHEST_PROTOCOL+1):
- st_copy = pickle.loads(pickle.dumps(st, proto))
- self.assertEqual(st_copy.totuple(), st.totuple())
-
- check_sizeof = support.check_sizeof
-
- @support.cpython_only
- def test_sizeof(self):
- def XXXROUNDUP(n):
- if n <= 1:
- return n
- if n <= 128:
- return (n + 3) & ~3
- return 1 << (n - 1).bit_length()
-
- basesize = support.calcobjsize('Piii')
- nodesize = struct.calcsize('hP3iP0h2i')
- def sizeofchildren(node):
- if node is None:
- return 0
- res = 0
- hasstr = len(node) > 1 and isinstance(node[-1], str)
- if hasstr:
- res += len(node[-1]) + 1
- children = node[1:-1] if hasstr else node[1:]
- if children:
- res += XXXROUNDUP(len(children)) * nodesize
- for child in children:
- res += sizeofchildren(child)
- return res
-
- def check_st_sizeof(st):
- self.check_sizeof(st, basesize + nodesize +
- sizeofchildren(st.totuple()))
-
- check_st_sizeof(parser.expr('2 + 3'))
- check_st_sizeof(parser.expr('2 + 3 + 4'))
- check_st_sizeof(parser.suite('x = 2 + 3'))
- check_st_sizeof(parser.suite(''))
- check_st_sizeof(parser.suite('# -*- coding: utf-8 -*-'))
- check_st_sizeof(parser.expr('[' + '2,' * 1000 + ']'))
-
-
- # XXX tests for pickling and unpickling of ST objects should go here
-
-class OtherParserCase(unittest.TestCase):
-
- def test_two_args_to_expr(self):
- # See bug #12264
- with self.assertRaises(TypeError):
- parser.expr("a", "b")
-
-
-class TestDeprecation(unittest.TestCase):
- def test_deprecation_message(self):
- code = "def f():\n import parser\n\nf()"
- rc, out, err = assert_python_ok('-c', code)
- self.assertIn(b'<string>:2: DeprecationWarning', err)
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/Lib/test/test_peg_parser.py b/Lib/test/test_peg_parser.py
deleted file mode 100644
index fae85e3..0000000
--- a/Lib/test/test_peg_parser.py
+++ /dev/null
@@ -1,803 +0,0 @@
-import ast
-import _peg_parser as peg_parser
-import unittest
-from typing import Any, Union, Iterable, Tuple
-from textwrap import dedent
-from test import support
-
-
-TEST_CASES = [
- ('annotated_assignment', 'x: int = 42'),
- ('annotated_assignment_with_tuple', 'x: tuple = 1, 2'),
- ('annotated_assignment_with_parens', '(paren): int = 3+2'),
- ('annotated_assignment_with_yield', 'x: int = yield 42'),
- ('annotated_no_assignment', 'x: int'),
- ('annotation_with_multiple_parens', '((parens)): int'),
- ('annotation_with_parens', '(parens): int'),
- ('annotated_assignment_with_attr', 'a.b: int'),
- ('annotated_assignment_with_subscript', 'a[b]: int'),
- ('annotated_assignment_with_attr_and_parens', '(a.b): int'),
- ('annotated_assignment_with_subscript_and_parens', '(a[b]): int'),
- ('assert', 'assert a'),
- ('assert_message', 'assert a, b'),
- ('assignment_false', 'a = False'),
- ('assignment_none', 'a = None'),
- ('assignment_true', 'a = True'),
- ('assignment_paren', '(a) = 42'),
- ('assignment_paren_multiple', '(a, b) = (0, 1)'),
- ('asyncfor',
- '''
- async for i in a:
- pass
- '''),
- ('attribute_call', 'a.b()'),
- ('attribute_multiple_names', 'abcd.efg.hij'),
- ('attribute_simple', 'a.b'),
- ('attributes_subscript', 'a.b[0]'),
- ('augmented_assignment', 'x += 42'),
- ('augmented_assignment_attribute', 'a.b.c += 42'),
- ('augmented_assignment_paren', '(x) += 42'),
- ('augmented_assignment_paren_subscript', '(x[0]) -= 42'),
- ('binop_add', '1 + 1'),
- ('binop_add_multiple', '1 + 1 + 1 + 1'),
- ('binop_all', '1 + 2 * 5 + 3 ** 2 - -3'),
- ('binop_boolop_comp', '1 + 1 == 2 or 1 + 1 == 3 and not b'),
- ('boolop_or', 'a or b'),
- ('boolop_or_multiple', 'a or b or c'),
- ('class_def_bases',
- '''
- class C(A, B):
- pass
- '''),
- ('class_def_decorators',
- '''
- @a
- class C:
- pass
- '''),
- ('class_def_decorator_with_expression',
- '''
- @lambda x: 42
- class C:
- pass
- '''),
- ('class_def_decorator_with_expression_and_walrus',
- '''
- @x:=lambda x: 42
- class C:
- pass
- '''),
-
- ('class_def_keywords',
- '''
- class C(keyword=a+b, **c):
- pass
- '''),
- ('class_def_mixed',
- '''
- class C(A, B, keyword=0, **a):
- pass
- '''),
- ('class_def_simple',
- '''
- class C:
- pass
- '''),
- ('class_def_starred_and_kwarg',
- '''
- class C(A, B, *x, **y):
- pass
- '''),
- ('class_def_starred_in_kwargs',
- '''
- class C(A, x=2, *[B, C], y=3):
- pass
- '''),
- ('call_attribute', 'f().b'),
- ('call_genexp', 'f(i for i in a)'),
- ('call_mixed_args', 'f(a, b, *c, **d)'),
- ('call_mixed_args_named', 'f(a, b, *c, d=4, **v)'),
- ('call_one_arg', 'f(a)'),
- ('call_posarg_genexp', 'f(a, (i for i in a))'),
- ('call_simple', 'f()'),
- ('call_subscript', 'f()[0]'),
- ('comp', 'a == b'),
- ('comp_multiple', 'a == b == c'),
- ('comp_paren_end', 'a == (b-1)'),
- ('comp_paren_start', '(a-1) == b'),
- ('decorator',
- '''
- @a
- def f():
- pass
- '''),
- ('decorator_async',
- '''
- @a
- async def d():
- pass
- '''),
- ('decorator_with_expression',
- '''
- @lambda x: 42
- def f():
- pass
- '''),
- ('decorator_with_expression_and_walrus',
- '''
- @x:=lambda x: 42
- def f():
- pass
- '''),
- ('del_attribute', 'del a.b'),
- ('del_call_attribute', 'del a().c'),
- ('del_call_genexp_attribute', 'del a(i for i in b).c'),
- ('del_empty', 'del()'),
- ('del_list', 'del a, [b, c]'),
- ('del_mixed', 'del a[0].b().c'),
- ('del_multiple', 'del a, b'),
- ('del_multiple_calls_attribute', 'del a()().b'),
- ('del_paren', 'del(a,b)'),
- ('del_paren_single_target', 'del(a)'),
- ('del_subscript_attribute', 'del a[0].b'),
- ('del_tuple', 'del a, (b, c)'),
- ('delete', 'del a'),
- ('dict',
- '''
- {
- a: 1,
- b: 2,
- c: 3
- }
- '''),
- ('dict_comp', '{x:1 for x in a}'),
- ('dict_comp_if', '{x:1+2 for x in a if b}'),
- ('dict_empty', '{}'),
- ('empty_line_after_linecont',
- r'''
- pass
- \
-
- pass
- '''),
- ('for',
- '''
- for i in a:
- pass
- '''),
- ('for_else',
- '''
- for i in a:
- pass
- else:
- pass
- '''),
- ('for_star_target_in_paren', 'for (a) in b: pass'),
- ('for_star_targets_attribute', 'for a.b in c: pass'),
- ('for_star_targets_call_attribute', 'for a().c in b: pass'),
- ('for_star_targets_empty', 'for () in a: pass'),
- ('for_star_targets_mixed', 'for a[0].b().c in d: pass'),
- ('for_star_targets_mixed_starred',
- '''
- for a, *b, (c, d) in e:
- pass
- '''),
- ('for_star_targets_multiple', 'for a, b in c: pass'),
- ('for_star_targets_nested_starred', 'for *[*a] in b: pass'),
- ('for_star_targets_starred', 'for *a in b: pass'),
- ('for_star_targets_subscript_attribute', 'for a[0].b in c: pass'),
- ('for_star_targets_trailing_comma',
- '''
- for a, (b, c), in d:
- pass
- '''),
- ('for_star_targets_tuple', 'for a, (b, c) in d: pass'),
- ('for_underscore',
- '''
- for _ in a:
- pass
- '''),
- ('function_return_type',
- '''
- def f() -> Any:
- pass
- '''),
- ('f-string_slice', "f'{x[2]}'"),
- ('f-string_slice_upper', "f'{x[2:3]}'"),
- ('f-string_slice_step', "f'{x[2:3:-2]}'"),
- ('f-string_constant', "f'{42}'"),
- ('f-string_boolop', "f'{x and y}'"),
- ('f-string_named_expr', "f'{(x:=42)}'"),
- ('f-string_binop', "f'{x+y}'"),
- ('f-string_unaryop', "f'{not x}'"),
- ('f-string_lambda', "f'{(lambda x, /, y, y2=42 , *z, k1, k2=34, **k3: 42)}'"),
- ('f-string_lambda_call', "f'{(lambda: 2)(2)}'"),
- ('f-string_ifexpr', "f'{x if y else z}'"),
- ('f-string_dict', "f'{ {2:34, 3:34} }'"),
- ('f-string_set', "f'{ {2,-45} }'"),
- ('f-string_list', "f'{ [2,-45] }'"),
- ('f-string_tuple', "f'{ (2,-45) }'"),
- ('f-string_listcomp', "f'{[x for x in y if z]}'"),
- ('f-string_setcomp', "f'{ {x for x in y if z} }'"),
- ('f-string_dictcomp', "f'{ {x:x for x in y if z} }'"),
- ('f-string_genexpr', "f'{ (x for x in y if z) }'"),
- ('f-string_yield', "f'{ (yield x) }'"),
- ('f-string_yieldfrom', "f'{ (yield from x) }'"),
- ('f-string_await', "f'{ await x }'"),
- ('f-string_compare', "f'{ x == y }'"),
- ('f-string_call', "f'{ f(x,y,z) }'"),
- ('f-string_attribute', "f'{ f.x.y.z }'"),
- ('f-string_starred', "f'{ *x, }'"),
- ('f-string_doublestarred', "f'{ {**x} }'"),
- ('f-string_escape_brace', "f'{{Escape'"),
- ('f-string_escape_closing_brace', "f'Escape}}'"),
- ('f-string_repr', "f'{a!r}'"),
- ('f-string_str', "f'{a!s}'"),
- ('f-string_ascii', "f'{a!a}'"),
- ('f-string_debug', "f'{a=}'"),
- ('f-string_padding', "f'{a:03d}'"),
- ('f-string_multiline',
- """
- f'''
- {hello}
- '''
- """),
- ('f-string_multiline_in_expr',
- """
- f'''
- {
- hello
- }
- '''
- """),
- ('f-string_multiline_in_call',
- """
- f'''
- {f(
- a, b, c
- )}
- '''
- """),
- ('global', 'global a, b'),
- ('group', '(yield a)'),
- ('if_elif',
- '''
- if a:
- pass
- elif b:
- pass
- '''),
- ('if_elif_elif',
- '''
- if a:
- pass
- elif b:
- pass
- elif c:
- pass
- '''),
- ('if_elif_else',
- '''
- if a:
- pass
- elif b:
- pass
- else:
- pass
- '''),
- ('if_else',
- '''
- if a:
- pass
- else:
- pass
- '''),
- ('if_simple', 'if a: pass'),
- ('import', 'import a'),
- ('import_alias', 'import a as b'),
- ('import_dotted', 'import a.b'),
- ('import_dotted_alias', 'import a.b as c'),
- ('import_dotted_multichar', 'import ab.cd'),
- ('import_from', 'from a import b'),
- ('import_from_alias', 'from a import b as c'),
- ('import_from_dotted', 'from a.b import c'),
- ('import_from_dotted_alias', 'from a.b import c as d'),
- ('import_from_multiple_aliases', 'from a import b as c, d as e'),
- ('import_from_one_dot', 'from .a import b'),
- ('import_from_one_dot_alias', 'from .a import b as c'),
- ('import_from_star', 'from a import *'),
- ('import_from_three_dots', 'from ...a import b'),
- ('import_from_trailing_comma', 'from a import (b,)'),
- ('kwarg',
- '''
- def f(**a):
- pass
- '''),
- ('kwonly_args',
- '''
- def f(*, a, b):
- pass
- '''),
- ('kwonly_args_with_default',
- '''
- def f(*, a=2, b):
- pass
- '''),
- ('lambda_kwarg', 'lambda **a: 42'),
- ('lambda_kwonly_args', 'lambda *, a, b: 42'),
- ('lambda_kwonly_args_with_default', 'lambda *, a=2, b: 42'),
- ('lambda_mixed_args', 'lambda a, /, b, *, c: 42'),
- ('lambda_mixed_args_with_default', 'lambda a, b=2, /, c=3, *e, f, **g: 42'),
- ('lambda_no_args', 'lambda: 42'),
- ('lambda_pos_args', 'lambda a,b: 42'),
- ('lambda_pos_args_with_default', 'lambda a, b=2: 42'),
- ('lambda_pos_only_args', 'lambda a, /: 42'),
- ('lambda_pos_only_args_with_default', 'lambda a=0, /: 42'),
- ('lambda_pos_posonly_args', 'lambda a, b, /, c, d: 42'),
- ('lambda_pos_posonly_args_with_default', 'lambda a, b=0, /, c=2: 42'),
- ('lambda_vararg', 'lambda *a: 42'),
- ('lambda_vararg_kwonly_args', 'lambda *a, b: 42'),
- ('list', '[1, 2, a]'),
- ('list_comp', '[i for i in a]'),
- ('list_comp_if', '[i for i in a if b]'),
- ('list_trailing_comma', '[1+2, a, 3+4,]'),
- ('mixed_args',
- '''
- def f(a, /, b, *, c):
- pass
- '''),
- ('mixed_args_with_default',
- '''
- def f(a, b=2, /, c=3, *e, f, **g):
- pass
- '''),
- ('multipart_string_bytes', 'b"Hola" b"Hello" b"Bye"'),
- ('multipart_string_triple', '"""Something here""" "and now"'),
- ('multipart_string_different_prefixes', 'u"Something" "Other thing" r"last thing"'),
- ('multiple_assignments', 'x = y = z = 42'),
- ('multiple_assignments_with_yield', 'x = y = z = yield 42'),
- ('multiple_pass',
- '''
- pass; pass
- pass
- '''),
- ('namedexpr', '(x := [1, 2, 3])'),
- ('namedexpr_false', '(x := False)'),
- ('namedexpr_none', '(x := None)'),
- ('namedexpr_true', '(x := True)'),
- ('nonlocal', 'nonlocal a, b'),
- ('number_complex', '-2.234+1j'),
- ('number_float', '-34.2333'),
- ('number_imaginary_literal', '1.1234j'),
- ('number_integer', '-234'),
- ('number_underscores', '1_234_567'),
- ('pass', 'pass'),
- ('pos_args',
- '''
- def f(a, b):
- pass
- '''),
- ('pos_args_with_default',
- '''
- def f(a, b=2):
- pass
- '''),
- ('pos_only_args',
- '''
- def f(a, /):
- pass
- '''),
- ('pos_only_args_with_default',
- '''
- def f(a=0, /):
- pass
- '''),
- ('pos_posonly_args',
- '''
- def f(a, b, /, c, d):
- pass
- '''),
- ('pos_posonly_args_with_default',
- '''
- def f(a, b=0, /, c=2):
- pass
- '''),
- ('primary_mixed', 'a.b.c().d[0]'),
- ('raise', 'raise'),
- ('raise_ellipsis', 'raise ...'),
- ('raise_expr', 'raise a'),
- ('raise_from', 'raise a from b'),
- ('return', 'return'),
- ('return_expr', 'return a'),
- ('set', '{1, 2+4, 3+5}'),
- ('set_comp', '{i for i in a}'),
- ('set_trailing_comma', '{1, 2, 3,}'),
- ('simple_assignment', 'x = 42'),
- ('simple_assignment_with_yield', 'x = yield 42'),
- ('string_bytes', 'b"hello"'),
- ('string_concatenation_bytes', 'b"hello" b"world"'),
- ('string_concatenation_simple', '"abcd" "efgh"'),
- ('string_format_simple', 'f"hello"'),
- ('string_format_with_formatted_value', 'f"hello {world}"'),
- ('string_simple', '"hello"'),
- ('string_unicode', 'u"hello"'),
- ('subscript_attribute', 'a[0].b'),
- ('subscript_call', 'a[b]()'),
- ('subscript_multiple_slices', 'a[0:a:2, 1]'),
- ('subscript_simple', 'a[0]'),
- ('subscript_single_element_tuple', 'a[0,]'),
- ('subscript_trailing_comma', 'a[0, 1, 2,]'),
- ('subscript_tuple', 'a[0, 1, 2]'),
- ('subscript_whole_slice', 'a[0+1:b:c]'),
- ('try_except',
- '''
- try:
- pass
- except:
- pass
- '''),
- ('try_except_else',
- '''
- try:
- pass
- except:
- pass
- else:
- pass
- '''),
- ('try_except_else_finally',
- '''
- try:
- pass
- except:
- pass
- else:
- pass
- finally:
- pass
- '''),
- ('try_except_expr',
- '''
- try:
- pass
- except a:
- pass
- '''),
- ('try_except_expr_target',
- '''
- try:
- pass
- except a as b:
- pass
- '''),
- ('try_except_finally',
- '''
- try:
- pass
- except:
- pass
- finally:
- pass
- '''),
- ('try_finally',
- '''
- try:
- pass
- finally:
- pass
- '''),
- ('unpacking_binop', '[*([1, 2, 3] + [3, 4, 5])]'),
- ('unpacking_call', '[*b()]'),
- ('unpacking_compare', '[*(x < y)]'),
- ('unpacking_constant', '[*3]'),
- ('unpacking_dict', '[*{1: 2, 3: 4}]'),
- ('unpacking_dict_comprehension', '[*{x:y for x,y in z}]'),
- ('unpacking_ifexpr', '[*([1, 2, 3] if x else y)]'),
- ('unpacking_list', '[*[1,2,3]]'),
- ('unpacking_list_comprehension', '[*[x for x in y]]'),
- ('unpacking_namedexpr', '[*(x:=[1, 2, 3])]'),
- ('unpacking_set', '[*{1,2,3}]'),
- ('unpacking_set_comprehension', '[*{x for x in y}]'),
- ('unpacking_string', '[*"myvalue"]'),
- ('unpacking_tuple', '[*(1,2,3)]'),
- ('unpacking_unaryop', '[*(not [1, 2, 3])]'),
- ('unpacking_yield', '[*(yield 42)]'),
- ('unpacking_yieldfrom', '[*(yield from x)]'),
- ('tuple', '(1, 2, 3)'),
- ('vararg',
- '''
- def f(*a):
- pass
- '''),
- ('vararg_kwonly_args',
- '''
- def f(*a, b):
- pass
- '''),
- ('while',
- '''
- while a:
- pass
- '''),
- ('while_else',
- '''
- while a:
- pass
- else:
- pass
- '''),
- ('with',
- '''
- with a:
- pass
- '''),
- ('with_as',
- '''
- with a as b:
- pass
- '''),
- ('with_as_paren',
- '''
- with a as (b):
- pass
- '''),
- ('with_as_empty', 'with a as (): pass'),
- ('with_list_recursive',
- '''
- with a as [x, [y, z]]:
- pass
- '''),
- ('with_tuple_recursive',
- '''
- with a as ((x, y), z):
- pass
- '''),
- ('with_tuple_target',
- '''
- with a as (x, y):
- pass
- '''),
- ('with_list_target',
- '''
- with a as [x, y]:
- pass
- '''),
- ('yield', 'yield'),
- ('yield_expr', 'yield a'),
- ('yield_from', 'yield from a'),
-]
-
-FAIL_TEST_CASES = [
- ("annotation_multiple_targets", "(a, b): int = 42"),
- ("annotation_nested_tuple", "((a, b)): int"),
- ("annotation_list", "[a]: int"),
- ("annotation_lambda", "lambda: int = 42"),
- ("annotation_tuple", "(a,): int"),
- ("annotation_tuple_without_paren", "a,: int"),
- ("assignment_keyword", "a = if"),
- ("augmented_assignment_list", "[a, b] += 1"),
- ("augmented_assignment_tuple", "a, b += 1"),
- ("augmented_assignment_tuple_paren", "(a, b) += (1, 2)"),
- ("comprehension_lambda", "(a for a in lambda: b)"),
- ("comprehension_else", "(a for a in b if c else d"),
- ("del_call", "del a()"),
- ("del_call_genexp", "del a(i for i in b)"),
- ("del_subscript_call", "del a[b]()"),
- ("del_attribute_call", "del a.b()"),
- ("del_mixed_call", "del a[0].b().c.d()"),
- ("for_star_targets_call", "for a() in b: pass"),
- ("for_star_targets_subscript_call", "for a[b]() in c: pass"),
- ("for_star_targets_attribute_call", "for a.b() in c: pass"),
- ("for_star_targets_mixed_call", "for a[0].b().c.d() in e: pass"),
- ("for_star_targets_in", "for a, in in b: pass"),
- ("f-string_assignment", "f'{x = 42}'"),
- ("f-string_empty", "f'{}'"),
- ("f-string_function_def", "f'{def f(): pass}'"),
- ("f-string_lambda", "f'{lambda x: 42}'"),
- ("f-string_singe_brace", "f'{'"),
- ("f-string_single_closing_brace", "f'}'"),
- ("from_import_invalid", "from import import a"),
- ("from_import_trailing_comma", "from a import b,"),
- ("import_non_ascii_syntax_error", "import ä £"),
- # This test case checks error paths involving tokens with uninitialized
- # values of col_offset and end_col_offset.
- ("invalid indentation",
- """
- def f():
- a
- a
- """),
- ("not_terminated_string", "a = 'example"),
- ("try_except_attribute_target",
- """
- try:
- pass
- except Exception as a.b:
- pass
- """),
- ("try_except_subscript_target",
- """
- try:
- pass
- except Exception as a[0]:
- pass
- """),
-]
-
-FAIL_SPECIALIZED_MESSAGE_CASES = [
- ("f(x, y, z=1, **b, *a", "iterable argument unpacking follows keyword argument unpacking"),
- ("f(x, y=1, *z, **a, b", "positional argument follows keyword argument unpacking"),
- ("f(x, y, z=1, a=2, b", "positional argument follows keyword argument"),
- ("True = 1", "cannot assign to True"),
- ("a() = 1", "cannot assign to function call"),
- ("(a, b): int", "only single target (not tuple) can be annotated"),
- ("[a, b]: int", "only single target (not list) can be annotated"),
- ("a(): int", "illegal target for annotation"),
- ("1 += 1", "'literal' is an illegal expression for augmented assignment"),
- ("pass\n pass", "unexpected indent"),
- ("def f():\npass", "expected an indented block"),
- ("def f(*): pass", "named arguments must follow bare *"),
- ("def f(*,): pass", "named arguments must follow bare *"),
- ("def f(*, **a): pass", "named arguments must follow bare *"),
- ("lambda *: pass", "named arguments must follow bare *"),
- ("lambda *,: pass", "named arguments must follow bare *"),
- ("lambda *, **a: pass", "named arguments must follow bare *"),
- ("f(g()=2", "expression cannot contain assignment, perhaps you meant \"==\"?"),
- ("f(a, b, *c, d.e=2", "expression cannot contain assignment, perhaps you meant \"==\"?"),
- ("f(*a, **b, c=0, d[1]=3)", "expression cannot contain assignment, perhaps you meant \"==\"?"),
-]
-
-GOOD_BUT_FAIL_TEST_CASES = [
- ('string_concatenation_format', 'f"{hello} world" f"again {and_again}"'),
- ('string_concatenation_multiple',
- '''
- f"hello" f"{world} again" f"and_again"
- '''),
- ('f-string_multiline_comp',
- """
- f'''
- {(i for i in a
- if b)}
- '''
- """),
-]
-
-FSTRINGS_TRACEBACKS = {
- 'multiline_fstrings_same_line_with_brace': (
- """
- f'''
- {a$b}
- '''
- """,
- '(a$b)',
- ),
- 'multiline_fstring_brace_on_next_line': (
- """
- f'''
- {a$b
- }'''
- """,
- '(a$b',
- ),
- 'multiline_fstring_brace_on_previous_line': (
- """
- f'''
- {
- a$b}'''
- """,
- 'a$b)',
- ),
-}
-
-EXPRESSIONS_TEST_CASES = [
- ("expression_add", "1+1"),
- ("expression_add_2", "a+b"),
- ("expression_call", "f(a, b=2, **kw)"),
- ("expression_tuple", "1, 2, 3"),
- ("expression_tuple_one_value", "1,")
-]
-
-
-def cleanup_source(source: Any) -> str:
- if isinstance(source, str):
- result = dedent(source)
- elif not isinstance(source, (list, tuple)):
- result = "\n".join(source)
- else:
- raise TypeError(f"Invalid type for test source: {source}")
- return result
-
-
-def prepare_test_cases(
- test_cases: Iterable[Tuple[str, Union[str, Iterable[str]]]]
-) -> Tuple[Iterable[str], Iterable[str]]:
-
- test_ids, _test_sources = zip(*test_cases)
- test_sources = list(_test_sources)
- for index, source in enumerate(test_sources):
- result = cleanup_source(source)
- test_sources[index] = result
- return test_ids, test_sources
-
-
-TEST_IDS, TEST_SOURCES = prepare_test_cases(TEST_CASES)
-
-GOOD_BUT_FAIL_TEST_IDS, GOOD_BUT_FAIL_SOURCES = prepare_test_cases(
- GOOD_BUT_FAIL_TEST_CASES
-)
-
-FAIL_TEST_IDS, FAIL_SOURCES = prepare_test_cases(FAIL_TEST_CASES)
-
-EXPRESSIONS_TEST_IDS, EXPRESSIONS_TEST_SOURCES = prepare_test_cases(
- EXPRESSIONS_TEST_CASES
-)
-
-
-class ASTGenerationTest(unittest.TestCase):
- def test_correct_ast_generation_on_source_files(self) -> None:
- self.maxDiff = None
- for source in TEST_SOURCES:
- actual_ast = peg_parser.parse_string(source)
- expected_ast = peg_parser.parse_string(source, oldparser=True)
- self.assertEqual(
- ast.dump(actual_ast, include_attributes=True),
- ast.dump(expected_ast, include_attributes=True),
- f"Wrong AST generation for source: {source}",
- )
-
- def test_incorrect_ast_generation_on_source_files(self) -> None:
- for source in FAIL_SOURCES:
- with self.assertRaises(SyntaxError, msg=f"Parsing {source} did not raise an exception"):
- peg_parser.parse_string(source)
-
- def test_incorrect_ast_generation_with_specialized_errors(self) -> None:
- for source, error_text in FAIL_SPECIALIZED_MESSAGE_CASES:
- exc = IndentationError if "indent" in error_text else SyntaxError
- with self.assertRaises(exc) as se:
- peg_parser.parse_string(source)
- self.assertTrue(
- error_text in se.exception.msg,
- f"Actual error message does not match expexted for {source}"
- )
-
- @unittest.expectedFailure
- def test_correct_but_known_to_fail_ast_generation_on_source_files(self) -> None:
- for source in GOOD_BUT_FAIL_SOURCES:
- actual_ast = peg_parser.parse_string(source)
- expected_ast = peg_parser.parse_string(source, oldparser=True)
- self.assertEqual(
- ast.dump(actual_ast, include_attributes=True),
- ast.dump(expected_ast, include_attributes=True),
- f"Wrong AST generation for source: {source}",
- )
-
- def test_correct_ast_generation_without_pos_info(self) -> None:
- for source in GOOD_BUT_FAIL_SOURCES:
- actual_ast = peg_parser.parse_string(source)
- expected_ast = peg_parser.parse_string(source, oldparser=True)
- self.assertEqual(
- ast.dump(actual_ast),
- ast.dump(expected_ast),
- f"Wrong AST generation for source: {source}",
- )
-
- def test_fstring_parse_error_tracebacks(self) -> None:
- for source, error_text in FSTRINGS_TRACEBACKS.values():
- with self.assertRaises(SyntaxError) as se:
- peg_parser.parse_string(dedent(source))
- self.assertEqual(error_text, se.exception.text)
-
- def test_correct_ast_generatrion_eval(self) -> None:
- for source in EXPRESSIONS_TEST_SOURCES:
- actual_ast = peg_parser.parse_string(source, mode='eval')
- expected_ast = peg_parser.parse_string(source, mode='eval', oldparser=True)
- self.assertEqual(
- ast.dump(actual_ast, include_attributes=True),
- ast.dump(expected_ast, include_attributes=True),
- f"Wrong AST generation for source: {source}",
- )
-
- def test_tokenizer_errors_are_propagated(self) -> None:
- n=201
- with self.assertRaisesRegex(SyntaxError, "too many nested parentheses"):
- peg_parser.parse_string(n*'(' + ')'*n)
diff --git a/Lib/test/test_string_literals.py b/Lib/test/test_string_literals.py
index 9565ee2..7231970 100644
--- a/Lib/test/test_string_literals.py
+++ b/Lib/test/test_string_literals.py
@@ -33,7 +33,6 @@
import tempfile
import unittest
import warnings
-from test.support import use_old_parser
TEMPLATE = r"""# coding: %s
@@ -168,8 +167,7 @@
eval("b'''\n\\z'''")
self.assertEqual(len(w), 1)
self.assertEqual(w[0].filename, '<string>')
- if use_old_parser():
- self.assertEqual(w[0].lineno, 1)
+ self.assertEqual(w[0].lineno, 1)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('error', category=DeprecationWarning)
@@ -178,8 +176,7 @@
exc = cm.exception
self.assertEqual(w, [])
self.assertEqual(exc.filename, '<string>')
- if use_old_parser():
- self.assertEqual(exc.lineno, 1)
+ self.assertEqual(exc.lineno, 1)
def test_eval_bytes_raw(self):
self.assertEqual(eval(""" br'x' """), b'x')
diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py
index 0c207ec..6ea9a55 100644
--- a/Lib/test/test_syntax.py
+++ b/Lib/test/test_syntax.py
@@ -733,7 +733,6 @@
def test_assign_call(self):
self._check_error("f() = 1", "assign")
- @unittest.skipIf(support.use_old_parser(), "The old parser cannot generate these error messages")
def test_assign_del(self):
self._check_error("del (,)", "invalid syntax")
self._check_error("del 1", "delete literal")
diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py
index f9a5f2f..925a6bc 100644
--- a/Lib/test/test_traceback.py
+++ b/Lib/test/test_traceback.py
@@ -655,7 +655,6 @@
self.assertIn('inner_raise() # Marker', blocks[2])
self.check_zero_div(blocks[2])
- @unittest.skipIf(support.use_old_parser(), "Pegen is arguably better here, so no need to fix this")
def test_syntax_error_offset_at_eol(self):
# See #10186.
def e():