Lib/test/test_parser.py

import copy
import warnings
with warnings.catch_warnings():
    warnings.filterwarnings('ignore', 'The parser module is deprecated',
                            DeprecationWarning)
    import parser
import pickle
import unittest
import operator
import struct
from test import support
from test.support.script_helper import assert_python_failure
from test.support.script_helper import assert_python_ok

#
#  First, we test that we can generate trees from valid source fragments,
#  and that these valid trees are indeed allowed by the tree-loading side
#  of the parser module.
#

class RoundtripLegalSyntaxTestCase(unittest.TestCase):

    def roundtrip(self, f, s):
        st1 = f(s)
        t = st1.totuple()
        try:
            st2 = parser.sequence2st(t)
        except parser.ParserError as why:
            self.fail("could not roundtrip %r: %s" % (s, why))

        self.assertEqual(t, st2.totuple(),
                         "could not re-generate syntax tree")

    def check_expr(self, s):
        self.roundtrip(parser.expr, s)

    def test_flags_passed(self):
        # The unicode literals flags has to be passed from the parser to AST
        # generation.
        suite = parser.suite("from __future__ import unicode_literals; x = ''")
        code = suite.compile()
        scope = {}
        exec(code, {}, scope)
        self.assertIsInstance(scope["x"], str)

    def check_suite(self, s):
        self.roundtrip(parser.suite, s)

    def test_yield_statement(self):
        self.check_suite("def f(): yield 1")
        self.check_suite("def f(): yield")
        self.check_suite("def f(): x += yield")
        self.check_suite("def f(): x = yield 1")
        self.check_suite("def f(): x = y = yield 1")
        self.check_suite("def f(): x = yield")
        self.check_suite("def f(): x = y = yield")
        self.check_suite("def f(): 1 + (yield)*2")
        self.check_suite("def f(): (yield 1)*2")
        self.check_suite("def f(): return; yield 1")
        self.check_suite("def f(): yield 1; return")
        self.check_suite("def f(): yield from 1")
        self.check_suite("def f(): x = yield from 1")
        self.check_suite("def f(): f((yield from 1))")
        self.check_suite("def f(): yield 1; return 1")
        self.check_suite("def f():\n"
                         "    for x in range(30):\n"
                         "        yield x\n")
        self.check_suite("def f():\n"
                         "    if (yield):\n"
                         "        yield x\n")

    def test_await_statement(self):
        self.check_suite("async def f():\n await smth()")
        self.check_suite("async def f():\n foo = await smth()")
        self.check_suite("async def f():\n foo, bar = await smth()")
        self.check_suite("async def f():\n (await smth())")
        self.check_suite("async def f():\n foo((await smth()))")
        self.check_suite("async def f():\n await foo(); return 42")

    def test_async_with_statement(self):
        self.check_suite("async def f():\n async with 1: pass")
        self.check_suite("async def f():\n async with a as b, c as d: pass")

    def test_async_for_statement(self):
        self.check_suite("async def f():\n async for i in (): pass")
        self.check_suite("async def f():\n async for i, b in (): pass")

    def test_nonlocal_statement(self):
        self.check_suite("def f():\n"
                         "    x = 0\n"
                         "    def g():\n"
                         "        nonlocal x\n")
        self.check_suite("def f():\n"
                         "    x = y = 0\n"
                         "    def g():\n"
                         "        nonlocal x, y\n")

    def test_expressions(self):
        self.check_expr("foo(1)")
        self.check_expr("[1, 2, 3]")
        self.check_expr("[x**3 for x in range(20)]")
        self.check_expr("[x**3 for x in range(20) if x % 3]")
        self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]")
        self.check_expr("list(x**3 for x in range(20))")
        self.check_expr("list(x**3 for x in range(20) if x % 3)")
        self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)")
        self.check_expr("foo(*args)")
        self.check_expr("foo(*args, **kw)")
        self.check_expr("foo(**kw)")
        self.check_expr("foo(key=value)")
        self.check_expr("foo(key=value, *args)")
        self.check_expr("foo(key=value, *args, **kw)")
        self.check_expr("foo(key=value, **kw)")
        self.check_expr("foo(a, b, c, *args)")
        self.check_expr("foo(a, b, c, *args, **kw)")
        self.check_expr("foo(a, b, c, **kw)")
        self.check_expr("foo(a, *args, keyword=23)")
        self.check_expr("foo + bar")
        self.check_expr("foo - bar")
        self.check_expr("foo * bar")
        self.check_expr("foo / bar")
        self.check_expr("foo // bar")
        self.check_expr("(foo := 1)")
        self.check_expr("lambda: 0")
        self.check_expr("lambda x: 0")
        self.check_expr("lambda *y: 0")
        self.check_expr("lambda *y, **z: 0")
        self.check_expr("lambda **z: 0")
        self.check_expr("lambda x, y: 0")
        self.check_expr("lambda foo=bar: 0")
        self.check_expr("lambda foo=bar, spaz=nifty+spit: 0")
        self.check_expr("lambda foo=bar, **z: 0")
        self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0")
        self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0")
        self.check_expr("lambda x, *y, **z: 0")
        self.check_expr("(x for x in range(10))")
        self.check_expr("foo(x for x in range(10))")
        self.check_expr("...")
        self.check_expr("a[...]")

    def test_simple_expression(self):
        # expr_stmt
        self.check_suite("a")

    def test_simple_assignments(self):
        self.check_suite("a = b")
        self.check_suite("a = b = c = d = e")

    def test_var_annot(self):
        self.check_suite("x: int = 5")
        self.check_suite("y: List[T] = []; z: [list] = fun()")
        self.check_suite("x: tuple = (1, 2)")
        self.check_suite("d[f()]: int = 42")
        self.check_suite("f(d[x]): str = 'abc'")
        self.check_suite("x.y.z.w: complex = 42j")
        self.check_suite("x: int")
        self.check_suite("def f():\n"
                         "    x: str\n"
                         "    y: int = 5\n")
        self.check_suite("class C:\n"
                         "    x: str\n"
                         "    y: int = 5\n")
        self.check_suite("class C:\n"
                         "    def __init__(self, x: int) -> None:\n"
                         "        self.x: int = x\n")
        # double check for nonsense
        with self.assertRaises(SyntaxError):
            exec("2+2: int", {}, {})
        with self.assertRaises(SyntaxError):
            exec("[]: int = 5", {}, {})
        with self.assertRaises(SyntaxError):
            exec("x, *y, z: int = range(5)", {}, {})
        with self.assertRaises(SyntaxError):
            exec("x: int = 1, y = 2", {}, {})
        with self.assertRaises(SyntaxError):
            exec("u = v: int", {}, {})
        with self.assertRaises(SyntaxError):
            exec("False: int", {}, {})
        with self.assertRaises(SyntaxError):
            exec("x.False: int", {}, {})
        with self.assertRaises(SyntaxError):
            exec("x.y,: int", {}, {})
        with self.assertRaises(SyntaxError):
            exec("[0]: int", {}, {})
        with self.assertRaises(SyntaxError):
            exec("f(): int", {}, {})

    def test_simple_augmented_assignments(self):
        self.check_suite("a += b")
        self.check_suite("a -= b")
        self.check_suite("a *= b")
        self.check_suite("a /= b")
        self.check_suite("a //= b")
        self.check_suite("a %= b")
        self.check_suite("a &= b")
        self.check_suite("a |= b")
        self.check_suite("a ^= b")
        self.check_suite("a <<= b")
        self.check_suite("a >>= b")
        self.check_suite("a **= b")

    def test_function_defs(self):
        self.check_suite("def f(): pass")
        self.check_suite("def f(*args): pass")
        self.check_suite("def f(*args, **kw): pass")
        self.check_suite("def f(**kw): pass")
        self.check_suite("def f(foo=bar): pass")
        self.check_suite("def f(foo=bar, *args): pass")
        self.check_suite("def f(foo=bar, *args, **kw): pass")
        self.check_suite("def f(foo=bar, **kw): pass")

        self.check_suite("def f(a, b): pass")
        self.check_suite("def f(a, b, *args): pass")
        self.check_suite("def f(a, b, *args, **kw): pass")
        self.check_suite("def f(a, b, **kw): pass")
        self.check_suite("def f(a, b, foo=bar): pass")
        self.check_suite("def f(a, b, foo=bar, *args): pass")
        self.check_suite("def f(a, b, foo=bar, *args, **kw): pass")
        self.check_suite("def f(a, b, foo=bar, **kw): pass")

        self.check_suite("@staticmethod\n"
                         "def f(): pass")
        self.check_suite("@staticmethod\n"
                         "@funcattrs(x, y)\n"
                         "def f(): pass")
        self.check_suite("@funcattrs()\n"
                         "def f(): pass")

        self.check_suite("@False or x\n"
                         "def f(): pass")
        self.check_suite("@d := x\n"
                         "def f(): pass")
        self.check_suite("@lambda f: x(f)\n"
                         "def f(): pass")
        self.check_suite("@[..., x, ...][1]\n"
                         "def f(): pass")
        self.check_suite("@x(x)(x)\n"
                         "def f(): pass")
        self.check_suite("@(x, x)\n"
                         "def f(): pass")
        self.check_suite("@...\n"
                         "def f(): pass")
        self.check_suite("@None\n"
                         "def f(): pass")
        self.check_suite("@w @(x @y) @(z)\n"
                         "def f(): pass")
        self.check_suite("@w[x].y.z\n"
                         "def f(): pass")

        # keyword-only arguments
        self.check_suite("def f(*, a): pass")
        self.check_suite("def f(*, a = 5): pass")
        self.check_suite("def f(*, a = 5, b): pass")
        self.check_suite("def f(*, a, b = 5): pass")
        self.check_suite("def f(*, a, b = 5, **kwds): pass")
        self.check_suite("def f(*args, a): pass")
        self.check_suite("def f(*args, a = 5): pass")
        self.check_suite("def f(*args, a = 5, b): pass")
        self.check_suite("def f(*args, a, b = 5): pass")
        self.check_suite("def f(*args, a, b = 5, **kwds): pass")

        # positional-only arguments
        self.check_suite("def f(a, /): pass")
        self.check_suite("def f(a, /,): pass")
        self.check_suite("def f(a, b, /): pass")
        self.check_suite("def f(a, b, /, c): pass")
        self.check_suite("def f(a, b, /, c = 6): pass")
        self.check_suite("def f(a, b, /, c, *, d): pass")
        self.check_suite("def f(a, b, /, c = 1, *, d): pass")
        self.check_suite("def f(a, b, /, c, *, d = 1): pass")
        self.check_suite("def f(a, b=1, /, c=2, *, d = 3): pass")
        self.check_suite("def f(a=0, b=1, /, c=2, *, d = 3): pass")

        # function annotations
        self.check_suite("def f(a: int): pass")
        self.check_suite("def f(a: int = 5): pass")
        self.check_suite("def f(*args: list): pass")
        self.check_suite("def f(**kwds: dict): pass")
        self.check_suite("def f(*, a: int): pass")
        self.check_suite("def f(*, a: int = 5): pass")
        self.check_suite("def f() -> int: pass")

    def test_class_defs(self):
        self.check_suite("class foo():pass")
        self.check_suite("class foo(object):pass")
        self.check_suite("@class_decorator\n"
                         "class foo():pass")
        self.check_suite("@class_decorator(arg)\n"
                         "class foo():pass")
        self.check_suite("@decorator1\n"
                         "@decorator2\n"
                         "class foo():pass")

        self.check_suite("@False or x\n"
                         "class C: pass")
        self.check_suite("@d := x\n"
                         "class C: pass")
        self.check_suite("@lambda f: x(f)\n"
                         "class C: pass")
        self.check_suite("@[..., x, ...][1]\n"
                         "class C: pass")
        self.check_suite("@x(x)(x)\n"
                         "class C: pass")
        self.check_suite("@(x, x)\n"
                         "class C: pass")
        self.check_suite("@...\n"
                         "class C: pass")
        self.check_suite("@None\n"
                         "class C: pass")
        self.check_suite("@w @(x @y) @(z)\n"
                         "class C: pass")
        self.check_suite("@w[x].y.z\n"
                         "class C: pass")

    def test_import_from_statement(self):
        self.check_suite("from sys.path import *")
        self.check_suite("from sys.path import dirname")
        self.check_suite("from sys.path import (dirname)")
        self.check_suite("from sys.path import (dirname,)")
        self.check_suite("from sys.path import dirname as my_dirname")
        self.check_suite("from sys.path import (dirname as my_dirname)")
        self.check_suite("from sys.path import (dirname as my_dirname,)")
        self.check_suite("from sys.path import dirname, basename")
        self.check_suite("from sys.path import (dirname, basename)")
        self.check_suite("from sys.path import (dirname, basename,)")
        self.check_suite(
            "from sys.path import dirname as my_dirname, basename")
        self.check_suite(
            "from sys.path import (dirname as my_dirname, basename)")
        self.check_suite(
            "from sys.path import (dirname as my_dirname, basename,)")
        self.check_suite(
            "from sys.path import dirname, basename as my_basename")
        self.check_suite(
            "from sys.path import (dirname, basename as my_basename)")
        self.check_suite(
            "from sys.path import (dirname, basename as my_basename,)")
        self.check_suite("from .bogus import x")

    def test_basic_import_statement(self):
        self.check_suite("import sys")
        self.check_suite("import sys as system")
        self.check_suite("import sys, math")
        self.check_suite("import sys as system, math")
        self.check_suite("import sys, math as my_math")

    def test_relative_imports(self):
        self.check_suite("from . import name")
        self.check_suite("from .. import name")
        # check all the way up to '....', since '...' is tokenized
        # differently from '.' (it's an ellipsis token).
        self.check_suite("from ... import name")
        self.check_suite("from .... import name")
        self.check_suite("from .pkg import name")
        self.check_suite("from ..pkg import name")
        self.check_suite("from ...pkg import name")
        self.check_suite("from ....pkg import name")

    def test_pep263(self):
        self.check_suite("# -*- coding: iso-8859-1 -*-\n"
                         "pass\n")

    def test_assert(self):
        self.check_suite("assert alo < ahi and blo < bhi\n")

    def test_with(self):
        self.check_suite("with open('x'): pass\n")
        self.check_suite("with open('x') as f: pass\n")
        self.check_suite("with open('x') as f, open('y') as g: pass\n")

    def test_try_stmt(self):
        self.check_suite("try: pass\nexcept: pass\n")
        self.check_suite("try: pass\nfinally: pass\n")
        self.check_suite("try: pass\nexcept A: pass\nfinally: pass\n")
        self.check_suite("try: pass\nexcept A: pass\nexcept: pass\n"
                         "finally: pass\n")
        self.check_suite("try: pass\nexcept: pass\nelse: pass\n")
        self.check_suite("try: pass\nexcept: pass\nelse: pass\n"
                         "finally: pass\n")

    def test_if_stmt(self):
        self.check_suite("if True:\n  pass\nelse:\n  pass\n")
        self.check_suite("if True:\n  pass\nelif True:\n  pass\nelse:\n  pass\n")

    def test_position(self):
        # An absolutely minimal test of position information.  Better
        # tests would be a big project.
        code = "def f(x):\n    return x + 1"
        st = parser.suite(code)

        def walk(tree):
            node_type = tree[0]
            next = tree[1]
            if isinstance(next, (tuple, list)):
                for elt in tree[1:]:
                    for x in walk(elt):
                        yield x
            else:
                yield tree

        expected = [
            (1, 'def', 1, 0),
            (1, 'f', 1, 4),
            (7, '(', 1, 5),
            (1, 'x', 1, 6),
            (8, ')', 1, 7),
            (11, ':', 1, 8),
            (4, '', 1, 9),
            (5, '', 2, -1),
            (1, 'return', 2, 4),
            (1, 'x', 2, 11),
            (14, '+', 2, 13),
            (2, '1', 2, 15),
            (4, '', 2, 16),
            (6, '', 2, -1),
            (4, '', 2, -1),
            (0, '', 2, -1),
        ]

        self.assertEqual(list(walk(st.totuple(line_info=True, col_info=True))),
                         expected)
        self.assertEqual(list(walk(st.totuple())),
                         [(t, n) for t, n, l, c in expected])
        self.assertEqual(list(walk(st.totuple(line_info=True))),
                         [(t, n, l) for t, n, l, c in expected])
        self.assertEqual(list(walk(st.totuple(col_info=True))),
                         [(t, n, c) for t, n, l, c in expected])
        self.assertEqual(list(walk(st.tolist(line_info=True, col_info=True))),
                         [list(x) for x in expected])
        self.assertEqual(list(walk(parser.st2tuple(st, line_info=True,
                                                   col_info=True))),
                         expected)
        self.assertEqual(list(walk(parser.st2list(st, line_info=True,
                                                  col_info=True))),
                         [list(x) for x in expected])

    def test_extended_unpacking(self):
        self.check_suite("*a = y")
        self.check_suite("x, *b, = m")
        self.check_suite("[*a, *b] = y")
        self.check_suite("for [*x, b] in x: pass")

    def test_raise_statement(self):
        self.check_suite("raise\n")
        self.check_suite("raise e\n")
        self.check_suite("try:\n"
                         "    suite\n"
                         "except Exception as e:\n"
                         "    raise ValueError from e\n")

    def test_list_displays(self):
        self.check_expr('[]')
        self.check_expr('[*{2}, 3, *[4]]')

    def test_set_displays(self):
        self.check_expr('{*{2}, 3, *[4]}')
        self.check_expr('{2}')
        self.check_expr('{2,}')
        self.check_expr('{2, 3}')
        self.check_expr('{2, 3,}')

    def test_dict_displays(self):
        self.check_expr('{}')
        self.check_expr('{a:b}')
        self.check_expr('{a:b,}')
        self.check_expr('{a:b, c:d}')
        self.check_expr('{a:b, c:d,}')
        self.check_expr('{**{}}')
        self.check_expr('{**{}, 3:4, **{5:6, 7:8}}')

    def test_argument_unpacking(self):
        self.check_expr("f(*a, **b)")
        self.check_expr('f(a, *b, *c, *d)')
        self.check_expr('f(**a, **b)')
        self.check_expr('f(2, *a, *b, **b, **c, **d)')
        self.check_expr("f(*b, *() or () and (), **{} and {}, **() or {})")

    def test_set_comprehensions(self):
        self.check_expr('{x for x in seq}')
        self.check_expr('{f(x) for x in seq}')
        self.check_expr('{f(x) for x in seq if condition(x)}')

    def test_dict_comprehensions(self):
        self.check_expr('{x:x for x in seq}')
        self.check_expr('{x**2:x[3] for x in seq if condition(x)}')
        self.check_expr('{x:x for x in seq1 for y in seq2 if condition(x, y)}')

    def test_named_expressions(self):
        self.check_suite("(a := 1)")
        self.check_suite("(a := a)")
        self.check_suite("if (match := pattern.search(data)) is None: pass")
        self.check_suite("while match := pattern.search(f.read()): pass")
        self.check_suite("[y := f(x), y**2, y**3]")
        self.check_suite("filtered_data = [y for x in data if (y := f(x)) is None]")
        self.check_suite("(y := f(x))")
        self.check_suite("y0 = (y1 := f(x))")
        self.check_suite("foo(x=(y := f(x)))")
        self.check_suite("def foo(answer=(p := 42)): pass")
        self.check_suite("def foo(answer: (p := 42) = 5): pass")
        self.check_suite("lambda: (x := 1)")
        self.check_suite("(x := lambda: 1)")
        self.check_suite("(x := lambda: (y := 1))")  # not in PEP
        self.check_suite("lambda line: (m := re.match(pattern, line)) and m.group(1)")
        self.check_suite("x = (y := 0)")
        self.check_suite("(z:=(y:=(x:=0)))")
        self.check_suite("(info := (name, phone, *rest))")
        self.check_suite("(x:=1,2)")
        self.check_suite("(total := total + tax)")
        self.check_suite("len(lines := f.readlines())")
        self.check_suite("foo(x := 3, cat='vector')")
        self.check_suite("foo(cat=(category := 'vector'))")
        self.check_suite("if any(len(longline := l) >= 100 for l in lines): print(longline)")
        self.check_suite(
            "if env_base := os.environ.get('PYTHONUSERBASE', None): return env_base"
        )
        self.check_suite(
            "if self._is_special and (ans := self._check_nans(context=context)): return ans"
        )
        self.check_suite("foo(b := 2, a=1)")
        self.check_suite("foo(b := 2, a=1)")
        self.check_suite("foo((b := 2), a=1)")
        self.check_suite("foo(c=(b := 2), a=1)")
        self.check_suite("{(x := C(i)).q: x for i in y}")


#
#  Second, we take *invalid* trees and make sure we get ParserError
#  rejections for them.
#

class IllegalSyntaxTestCase(unittest.TestCase):

    def check_bad_tree(self, tree, label):
        try:
            parser.sequence2st(tree)
        except parser.ParserError:
            pass
        else:
            self.fail("did not detect invalid tree for %r" % label)

    def test_junk(self):
        # not even remotely valid:
        self.check_bad_tree((1, 2, 3), "<junk>")

    def test_illegal_terminal(self):
        tree = \
            (257,
             (269,
              (270,
               (271,
                (277,
                 (1,))),
               (4, ''))),
             (4, ''),
             (0, ''))
        self.check_bad_tree(tree, "too small items in terminal node")
        tree = \
            (257,
             (269,
              (270,
               (271,
                (277,
                 (1, b'pass'))),
               (4, ''))),
             (4, ''),
             (0, ''))
        self.check_bad_tree(tree, "non-string second item in terminal node")
        tree = \
            (257,
             (269,
              (270,
               (271,
                (277,
                 (1, 'pass', '0', 0))),
               (4, ''))),
             (4, ''),
             (0, ''))
        self.check_bad_tree(tree, "non-integer third item in terminal node")
        tree = \
            (257,
             (269,
              (270,
               (271,
                (277,
                 (1, 'pass', 0, 0))),
               (4, ''))),
             (4, ''),
             (0, ''))
        self.check_bad_tree(tree, "too many items in terminal node")

    def test_illegal_yield_1(self):
        # Illegal yield statement: def f(): return 1; yield 1
        tree = \
        (257,
         (264,
          (285,
           (259,
            (1, 'def'),
            (1, 'f'),
            (260, (7, '('), (8, ')')),
            (11, ':'),
            (291,
             (4, ''),
             (5, ''),
             (264,
              (265,
               (266,
                (272,
                 (275,
                  (1, 'return'),
                  (313,
                   (292,
                    (293,
                     (294,
                      (295,
                       (297,
                        (298,
                         (299,
                          (300,
                           (301,
                            (302, (303, (304, (305, (2, '1')))))))))))))))))),
               (264,
                (265,
                 (266,
                  (272,
                   (276,
                    (1, 'yield'),
                    (313,
                     (292,
                      (293,
                       (294,
                        (295,
                         (297,
                          (298,
                           (299,
                            (300,
                             (301,
                              (302,
                               (303, (304, (305, (2, '1')))))))))))))))))),
                 (4, ''))),
               (6, ''))))),
           (4, ''),
           (0, ''))))
        self.check_bad_tree(tree, "def f():\n  return 1\n  yield 1")

    def test_illegal_yield_2(self):
        # Illegal return in generator: def f(): return 1; yield 1
        tree = \
        (257,
         (264,
          (265,
           (266,
            (278,
             (1, 'from'),
             (281, (1, '__future__')),
             (1, 'import'),
             (279, (1, 'generators')))),
           (4, ''))),
         (264,
          (285,
           (259,
            (1, 'def'),
            (1, 'f'),
            (260, (7, '('), (8, ')')),
            (11, ':'),
            (291,
             (4, ''),
             (5, ''),
             (264,
              (265,
               (266,
                (272,
                 (275,
                  (1, 'return'),
                  (313,
                   (292,
                    (293,
                     (294,
                      (295,
                       (297,
                        (298,
                         (299,
                          (300,
                           (301,
                            (302, (303, (304, (305, (2, '1')))))))))))))))))),
               (264,
                (265,
                 (266,
                  (272,
                   (276,
                    (1, 'yield'),
                    (313,
                     (292,
                      (293,
                       (294,
                        (295,
                         (297,
                          (298,
                           (299,
                            (300,
                             (301,
                              (302,
                               (303, (304, (305, (2, '1')))))))))))))))))),
                 (4, ''))),
               (6, ''))))),
           (4, ''),
           (0, ''))))
        self.check_bad_tree(tree, "def f():\n  return 1\n  yield 1")

    def test_a_comma_comma_c(self):
        # Illegal input: a,,c
        tree = \
        (258,
         (311,
          (290,
           (291,
            (292,
             (293,
              (295,
               (296,
                (297,
                 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))),
          (12, ','),
          (12, ','),
          (290,
           (291,
            (292,
             (293,
              (295,
               (296,
                (297,
                 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))),
         (4, ''),
         (0, ''))
        self.check_bad_tree(tree, "a,,c")

    def test_illegal_operator(self):
        # Illegal input: a $= b
        tree = \
        (257,
         (264,
          (265,
           (266,
            (267,
             (312,
              (291,
               (292,
                (293,
                 (294,
                  (296,
                   (297,
                    (298,
                     (299,
                      (300, (301, (302, (303, (304, (1, 'a'))))))))))))))),
             (268, (37, '$=')),
             (312,
              (291,
               (292,
                (293,
                 (294,
                  (296,
                   (297,
                    (298,
                     (299,
                      (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))),
           (4, ''))),
         (0, ''))
        self.check_bad_tree(tree, "a $= b")

    def test_malformed_global(self):
        #doesn't have global keyword in ast
        tree = (257,
                (264,
                 (265,
                  (266,
                   (282, (1, 'foo'))), (4, ''))),
                (4, ''),
                (0, ''))
        self.check_bad_tree(tree, "malformed global ast")

    def test_missing_import_source(self):
        # from import fred
        tree = \
            (257,
             (268,
              (269,
               (270,
                (282,
                 (284, (1, 'from'), (1, 'import'),
                  (287, (285, (1, 'fred')))))),
               (4, ''))),
             (4, ''), (0, ''))
        self.check_bad_tree(tree, "from import fred")

    def test_illegal_encoding(self):
        # Illegal encoding declaration
        tree = \
            (341,
             (257, (0, '')))
        self.check_bad_tree(tree, "missed encoding")
        tree = \
            (341,
             (257, (0, '')),
              b'iso-8859-1')
        self.check_bad_tree(tree, "non-string encoding")
        tree = \
            (341,
             (257, (0, '')),
              '\udcff')
        with self.assertRaises(UnicodeEncodeError):
            parser.sequence2st(tree)

    def test_invalid_node_id(self):
        tree = (257, (269, (-7, '')))
        self.check_bad_tree(tree, "negative node id")
        tree = (257, (269, (99, '')))
        self.check_bad_tree(tree, "invalid token id")
        tree = (257, (269, (9999, (0, ''))))
        self.check_bad_tree(tree, "invalid symbol id")

    def test_ParserError_message(self):
        try:
            parser.sequence2st((257,(269,(257,(0,'')))))
        except parser.ParserError as why:
            self.assertIn("compound_stmt", str(why))  # Expected
            self.assertIn("file_input", str(why))     # Got



class CompileTestCase(unittest.TestCase):

    # These tests are very minimal. :-(

    def test_compile_expr(self):
        st = parser.expr('2 + 3')
        code = parser.compilest(st)
        self.assertEqual(eval(code), 5)

    def test_compile_suite(self):
        st = parser.suite('x = 2; y = x + 3')
        code = parser.compilest(st)
        globs = {}
        exec(code, globs)
        self.assertEqual(globs['y'], 5)

    def test_compile_error(self):
        st = parser.suite('1 = 3 + 4')
        self.assertRaises(SyntaxError, parser.compilest, st)

    def test_compile_badunicode(self):
        st = parser.suite('a = "\\U12345678"')
        self.assertRaises(SyntaxError, parser.compilest, st)
        st = parser.suite('a = "\\u1"')
        self.assertRaises(SyntaxError, parser.compilest, st)

    def test_issue_9011(self):
        # Issue 9011: compilation of an unary minus expression changed
        # the meaning of the ST, so that a second compilation produced
        # incorrect results.
        st = parser.expr('-3')
        code1 = parser.compilest(st)
        self.assertEqual(eval(code1), -3)
        code2 = parser.compilest(st)
        self.assertEqual(eval(code2), -3)

    def test_compile_filename(self):
        st = parser.expr('a + 5')
        code = parser.compilest(st)
        self.assertEqual(code.co_filename, '<syntax-tree>')
        code = st.compile()
        self.assertEqual(code.co_filename, '<syntax-tree>')
        for filename in 'file.py', b'file.py':
            code = parser.compilest(st, filename)
            self.assertEqual(code.co_filename, 'file.py')
            code = st.compile(filename)
            self.assertEqual(code.co_filename, 'file.py')
        for filename in bytearray(b'file.py'), memoryview(b'file.py'):
            with self.assertWarns(DeprecationWarning):
                code = parser.compilest(st, filename)
            self.assertEqual(code.co_filename, 'file.py')
            with self.assertWarns(DeprecationWarning):
                code = st.compile(filename)
            self.assertEqual(code.co_filename, 'file.py')
        self.assertRaises(TypeError, parser.compilest, st, list(b'file.py'))
        self.assertRaises(TypeError, st.compile, list(b'file.py'))


class ParserStackLimitTestCase(unittest.TestCase):
    """try to push the parser to/over its limits.
    see http://bugs.python.org/issue1881 for a discussion
    """
    def _nested_expression(self, level):
        return "["*level+"]"*level

    def test_deeply_nested_list(self):
        # This has fluctuated between 99 levels in 2.x, down to 93 levels in
        # 3.7.X and back up to 99 in 3.8.X. Related to MAXSTACK size in Parser.h
        e = self._nested_expression(99)
        st = parser.expr(e)
        st.compile()

    def test_trigger_memory_error(self):
        e = self._nested_expression(100)
        rc, out, err = assert_python_failure('-c', e)
        # parsing the expression will result in an error message
        # followed by a MemoryError (see #11963)
        self.assertIn(b's_push: parser stack overflow', err)
        self.assertIn(b'MemoryError', err)

class STObjectTestCase(unittest.TestCase):
    """Test operations on ST objects themselves"""

    def test_comparisons(self):
        # ST objects should support order and equality comparisons
        st1 = parser.expr('2 + 3')
        st2 = parser.suite('x = 2; y = x + 3')
        st3 = parser.expr('list(x**3 for x in range(20))')
        st1_copy = parser.expr('2 + 3')
        st2_copy = parser.suite('x = 2; y = x + 3')
        st3_copy = parser.expr('list(x**3 for x in range(20))')

        # exercise fast path for object identity
        self.assertEqual(st1 == st1, True)
        self.assertEqual(st2 == st2, True)
        self.assertEqual(st3 == st3, True)
        # slow path equality
        self.assertEqual(st1, st1_copy)
        self.assertEqual(st2, st2_copy)
        self.assertEqual(st3, st3_copy)
        self.assertEqual(st1 == st2, False)
        self.assertEqual(st1 == st3, False)
        self.assertEqual(st2 == st3, False)
        self.assertEqual(st1 != st1, False)
        self.assertEqual(st2 != st2, False)
        self.assertEqual(st3 != st3, False)
        self.assertEqual(st1 != st1_copy, False)
        self.assertEqual(st2 != st2_copy, False)
        self.assertEqual(st3 != st3_copy, False)
        self.assertEqual(st2 != st1, True)
        self.assertEqual(st1 != st3, True)
        self.assertEqual(st3 != st2, True)
        # we don't particularly care what the ordering is;  just that
        # it's usable and self-consistent
        self.assertEqual(st1 < st2, not (st2 <= st1))
        self.assertEqual(st1 < st3, not (st3 <= st1))
        self.assertEqual(st2 < st3, not (st3 <= st2))
        self.assertEqual(st1 < st2, st2 > st1)
        self.assertEqual(st1 < st3, st3 > st1)
        self.assertEqual(st2 < st3, st3 > st2)
        self.assertEqual(st1 <= st2, st2 >= st1)
        self.assertEqual(st3 <= st1, st1 >= st3)
        self.assertEqual(st2 <= st3, st3 >= st2)
        # transitivity
        bottom = min(st1, st2, st3)
        top = max(st1, st2, st3)
        mid = sorted([st1, st2, st3])[1]
        self.assertTrue(bottom < mid)
        self.assertTrue(bottom < top)
        self.assertTrue(mid < top)
        self.assertTrue(bottom <= mid)
        self.assertTrue(bottom <= top)
        self.assertTrue(mid <= top)
        self.assertTrue(bottom <= bottom)
        self.assertTrue(mid <= mid)
        self.assertTrue(top <= top)
        # interaction with other types
        self.assertEqual(st1 == 1588.602459, False)
        self.assertEqual('spanish armada' != st2, True)
        self.assertRaises(TypeError, operator.ge, st3, None)
        self.assertRaises(TypeError, operator.le, False, st1)
        self.assertRaises(TypeError, operator.lt, st1, 1815)
        self.assertRaises(TypeError, operator.gt, b'waterloo', st2)

    def test_copy_pickle(self):
        sts = [
            parser.expr('2 + 3'),
            parser.suite('x = 2; y = x + 3'),
            parser.expr('list(x**3 for x in range(20))')
        ]
        for st in sts:
            st_copy = copy.copy(st)
            self.assertEqual(st_copy.totuple(), st.totuple())
            st_copy = copy.deepcopy(st)
            self.assertEqual(st_copy.totuple(), st.totuple())
            for proto in range(pickle.HIGHEST_PROTOCOL+1):
                st_copy = pickle.loads(pickle.dumps(st, proto))
                self.assertEqual(st_copy.totuple(), st.totuple())

    check_sizeof = support.check_sizeof

    @support.cpython_only
    def test_sizeof(self):
        def XXXROUNDUP(n):
            if n <= 1:
                return n
            if n <= 128:
                return (n + 3) & ~3
            return 1 << (n - 1).bit_length()

        basesize = support.calcobjsize('Piii')
        nodesize = struct.calcsize('hP3iP0h2i')
        def sizeofchildren(node):
            if node is None:
                return 0
            res = 0
            hasstr = len(node) > 1 and isinstance(node[-1], str)
            if hasstr:
                res += len(node[-1]) + 1
            children = node[1:-1] if hasstr else node[1:]
            if children:
                res += XXXROUNDUP(len(children)) * nodesize
                for child in children:
                    res += sizeofchildren(child)
            return res

        def check_st_sizeof(st):
            self.check_sizeof(st, basesize + nodesize +
                                  sizeofchildren(st.totuple()))

        check_st_sizeof(parser.expr('2 + 3'))
        check_st_sizeof(parser.expr('2 + 3 + 4'))
        check_st_sizeof(parser.suite('x = 2 + 3'))
        check_st_sizeof(parser.suite(''))
        check_st_sizeof(parser.suite('# -*- coding: utf-8 -*-'))
        check_st_sizeof(parser.expr('[' + '2,' * 1000 + ']'))


    # XXX tests for pickling and unpickling of ST objects should go here

class OtherParserCase(unittest.TestCase):

    def test_two_args_to_expr(self):
        # See bug #12264
        with self.assertRaises(TypeError):
            parser.expr("a", "b")


class TestDeprecation(unittest.TestCase):
    def test_deprecation_message(self):
        code = "def f():\n  import parser\n\nf()"
        rc, out, err = assert_python_ok('-c', code)
        self.assertIn(b'<string>:2: DeprecationWarning', err)


if __name__ == "__main__":
    unittest.main()