bpo-35766: Merge typed_ast back into CPython (GH-11645)
diff --git a/Lib/ast.py b/Lib/ast.py
index 6c1e978..470a74b 100644
--- a/Lib/ast.py
+++ b/Lib/ast.py
@@ -27,12 +27,16 @@
from _ast import *
-def parse(source, filename='<unknown>', mode='exec'):
+def parse(source, filename='<unknown>', mode='exec', *, type_comments=False):
"""
Parse the source into an AST node.
Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
+ Pass type_comments=True to get back type comments where the syntax allows.
"""
- return compile(source, filename, mode, PyCF_ONLY_AST)
+ flags = PyCF_ONLY_AST
+ if type_comments:
+ flags |= PyCF_TYPE_COMMENTS
+ return compile(source, filename, mode, flags)
def literal_eval(node_or_string):
diff --git a/Lib/symbol.py b/Lib/symbol.py
index b3fa089..36e0eec 100644
--- a/Lib/symbol.py
+++ b/Lib/symbol.py
@@ -100,6 +100,10 @@
encoding_decl = 341
yield_expr = 342
yield_arg = 343
+func_body_suite = 344
+func_type_input = 345
+func_type = 346
+typelist = 347
#--end constants--
sym_name = {}
diff --git a/Lib/test/test_asdl_parser.py b/Lib/test/test_asdl_parser.py
index 30e6466..9eaceec 100644
--- a/Lib/test/test_asdl_parser.py
+++ b/Lib/test/test_asdl_parser.py
@@ -117,7 +117,8 @@
v = CustomVisitor()
v.visit(self.types['mod'])
- self.assertEqual(v.names_with_seq, ['Module', 'Interactive', 'Suite'])
+ self.assertEqual(v.names_with_seq,
+ ['Module', 'Module', 'Interactive', 'FunctionType', 'Suite'])
if __name__ == '__main__':
diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index 09e425d..609c8b2 100644
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -455,7 +455,7 @@
def test_module(self):
body = [ast.Num(42)]
- x = ast.Module(body)
+ x = ast.Module(body, [])
self.assertEqual(x.body, body)
def test_nodeclasses(self):
@@ -524,13 +524,13 @@
def test_invalid_sum(self):
pos = dict(lineno=2, col_offset=3)
- m = ast.Module([ast.Expr(ast.expr(**pos), **pos)])
+ m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], [])
with self.assertRaises(TypeError) as cm:
compile(m, "<test>", "exec")
self.assertIn("but got <_ast.expr", str(cm.exception))
def test_invalid_identitifer(self):
- m = ast.Module([ast.Expr(ast.Name(42, ast.Load()))])
+ m = ast.Module([ast.Expr(ast.Name(42, ast.Load()))], [])
ast.fix_missing_locations(m)
with self.assertRaises(TypeError) as cm:
compile(m, "<test>", "exec")
@@ -575,11 +575,11 @@
self.assertEqual(ast.dump(node),
"Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), "
"args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], "
- "keywords=[]))])"
+ "keywords=[]))], type_ignores=[])"
)
self.assertEqual(ast.dump(node, annotate_fields=False),
"Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), "
- "Constant('and cheese')], []))])"
+ "Constant('and cheese')], []))], [])"
)
self.assertEqual(ast.dump(node, include_attributes=True),
"Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), "
@@ -588,7 +588,7 @@
"end_lineno=1, end_col_offset=9), Constant(value='and cheese', "
"lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], keywords=[], "
"lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), "
- "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])"
+ "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)], type_ignores=[])"
)
def test_copy_location(self):
@@ -617,7 +617,8 @@
"lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), "
"args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, "
"end_col_offset=0)], keywords=[], lineno=1, col_offset=0, end_lineno=1, "
- "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])"
+ "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)], "
+ "type_ignores=[])"
)
def test_increment_lineno(self):
@@ -760,7 +761,7 @@
names=[ast.alias(name='sleep')],
level=None,
lineno=None, col_offset=None)]
- mod = ast.Module(body)
+ mod = ast.Module(body, [])
with self.assertRaises(ValueError) as cm:
compile(mod, 'test', 'exec')
self.assertIn("invalid integer value: None", str(cm.exception))
@@ -770,7 +771,7 @@
names=[ast.alias(name='sleep')],
level=None,
lineno=0, col_offset=0)]
- mod = ast.Module(body)
+ mod = ast.Module(body, [])
code = compile(mod, 'test', 'exec')
ns = {}
exec(code, ns)
@@ -790,11 +791,11 @@
self.assertIn(msg, str(cm.exception))
def expr(self, node, msg=None, *, exc=ValueError):
- mod = ast.Module([ast.Expr(node)])
+ mod = ast.Module([ast.Expr(node)], [])
self.mod(mod, msg, exc=exc)
def stmt(self, stmt, msg=None):
- mod = ast.Module([stmt])
+ mod = ast.Module([stmt], [])
self.mod(mod, msg)
def test_module(self):
@@ -1603,61 +1604,61 @@
raise SystemExit
unittest.main()
-#### EVERYTHING BELOW IS GENERATED #####
+#### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast.py -g #####
exec_results = [
-('Module', [('Expr', (1, 0), ('Constant', (1, 0), None))]),
-('Module', [('Expr', (1, 0), ('Constant', (1, 0), 'module docstring'))]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (1, 9))], [], None)]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (1, 9), ('Constant', (1, 9), 'function docstring'))], [], None)]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, []), [('Pass', (1, 10))], [], None)]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, [('Constant', (1, 8), 0)]), [('Pass', (1, 12))], [], None)]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None), [], [], None, []), [('Pass', (1, 14))], [], None)]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None), []), [('Pass', (1, 17))], [], None)]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None), ('arg', (1, 9), 'b', None), ('arg', (1, 14), 'c', None), ('arg', (1, 22), 'd', None), ('arg', (1, 28), 'e', None)], ('arg', (1, 35), 'args', None), [('arg', (1, 41), 'f', None)], [('Constant', (1, 43), 42)], ('arg', (1, 49), 'kwargs', None), [('Constant', (1, 11), 1), ('Constant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Expr', (1, 58), ('Constant', (1, 58), 'doc for f()'))], [], None)]),
-('Module', [('ClassDef', (1, 0), 'C', [], [], [('Pass', (1, 8))], [])]),
-('Module', [('ClassDef', (1, 0), 'C', [], [], [('Expr', (1, 9), ('Constant', (1, 9), 'docstring for class C'))], [])]),
-('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], [('Pass', (1, 17))], [])]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Constant', (1, 15), 1))], [], None)]),
-('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
-('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Constant', (1, 4), 1))]),
-('Module', [('Assign', (1, 0), [('Tuple', (1, 0), [('Name', (1, 0), 'a', ('Store',)), ('Name', (1, 2), 'b', ('Store',))], ('Store',))], ('Name', (1, 6), 'c', ('Load',)))]),
-('Module', [('Assign', (1, 0), [('Tuple', (1, 0), [('Name', (1, 1), 'a', ('Store',)), ('Name', (1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 8), 'c', ('Load',)))]),
-('Module', [('Assign', (1, 0), [('List', (1, 0), [('Name', (1, 1), 'a', ('Store',)), ('Name', (1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 8), 'c', ('Load',)))]),
-('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Constant', (1, 5), 1))]),
-('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]),
-('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]),
-('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]),
-('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]),
-('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]),
-('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Constant', (1, 16), 'string')], []), None)]),
-('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])]),
-('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])]),
-('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]),
-('Module', [('Import', (1, 0), [('alias', 'sys', None)])]),
-('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]),
-('Module', [('Global', (1, 0), ['v'])]),
-('Module', [('Expr', (1, 0), ('Constant', (1, 0), 1))]),
-('Module', [('Pass', (1, 0))]),
-('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Break', (1, 11))], [])]),
-('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Continue', (1, 11))], [])]),
-('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [])]),
-('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 5), 'a', ('Store',)), ('Name', (1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 13), 'c', ('Load',)), [('Pass', (1, 16))], [])]),
-('Module', [('For', (1, 0), ('List', (1, 4), [('Name', (1, 5), 'a', ('Store',)), ('Name', (1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 13), 'c', ('Load',)), [('Pass', (1, 16))], [])]),
-('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 0), ('Tuple', (2, 4), [('Name', (3, 4), 'Aa', ('Load',)), ('Name', (5, 7), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4), [('Name', (8, 4), 'Aa', ('Store',)), ('Name', (10, 4), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10), 'Cc', ('Load',)), [], 0)]))]),
-('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Name', (1, 11), 'w', ('Store',)), ('Name', (1, 16), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22), 'm', ('Store',)), ('Name', (1, 27), 'p', ('Load',)), [('Name', (1, 32), 'g', ('Load',))], 0)]))]),
-('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [], 0)]))]),
-('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))], 0)]))]),
-('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [], 0)]))]),
-('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('Constant', (2, 1), 'async function')), ('Expr', (3, 1), ('Await', (3, 1), ('Call', (3, 7), ('Name', (3, 7), 'something', ('Load',)), [], [])))], [], None)]),
-('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 1), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Constant', (2, 19), 1))], [('Expr', (3, 7), ('Constant', (3, 7), 2))])], [], None)]),
-('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 1), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Constant', (2, 20), 1))])], [], None)]),
-('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Constant', (1, 10), 2)], [('Dict', (1, 3), [('Constant', (1, 4), 1)], [('Constant', (1, 6), 2)]), ('Constant', (1, 12), 3)]))]),
-('Module', [('Expr', (1, 0), ('Set', (1, 0), [('Starred', (1, 1), ('Set', (1, 2), [('Constant', (1, 3), 1), ('Constant', (1, 6), 2)]), ('Load',)), ('Constant', (1, 10), 3)]))]),
-('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('ListComp', (2, 1), ('Name', (2, 2), 'i', ('Load',)), [('comprehension', ('Name', (2, 14), 'b', ('Store',)), ('Name', (2, 19), 'c', ('Load',)), [], 1)]))], [], None)]),
-('Module', [('FunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None)]),
-('Module', [('AsyncFunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 15))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None)]),
-('Module', [('ClassDef', (3, 0), 'C', [], [], [('Pass', (3, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])])]),
-('Module', [('FunctionDef', (2, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (2, 9))], [('Call', (1, 1), ('Name', (1, 1), 'deco', ('Load',)), [('GeneratorExp', (1, 5), ('Name', (1, 6), 'a', ('Load',)), [('comprehension', ('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 17), 'b', ('Load',)), [], 0)])], [])], None)]),
+('Module', [('Expr', (1, 0), ('Constant', (1, 0), None))], []),
+('Module', [('Expr', (1, 0), ('Constant', (1, 0), 'module docstring'))], []),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (1, 9))], [], None, None)], []),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (1, 9), ('Constant', (1, 9), 'function docstring'))], [], None, None)], []),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10))], [], None, None)], []),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None, None)], None, [], [], None, [('Constant', (1, 8), 0)]), [('Pass', (1, 12))], [], None, None)], []),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None, None), [], [], None, []), [('Pass', (1, 14))], [], None, None)], []),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None, None), []), [('Pass', (1, 17))], [], None, None)], []),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None, None), ('arg', (1, 9), 'b', None, None), ('arg', (1, 14), 'c', None, None), ('arg', (1, 22), 'd', None, None), ('arg', (1, 28), 'e', None, None)], ('arg', (1, 35), 'args', None, None), [('arg', (1, 41), 'f', None, None)], [('Constant', (1, 43), 42)], ('arg', (1, 49), 'kwargs', None, None), [('Constant', (1, 11), 1), ('Constant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Expr', (1, 58), ('Constant', (1, 58), 'doc for f()'))], [], None, None)], []),
+('Module', [('ClassDef', (1, 0), 'C', [], [], [('Pass', (1, 8))], [])], []),
+('Module', [('ClassDef', (1, 0), 'C', [], [], [('Expr', (1, 9), ('Constant', (1, 9), 'docstring for class C'))], [])], []),
+('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], [('Pass', (1, 17))], [])], []),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Constant', (1, 15), 1))], [], None, None)], []),
+('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])], []),
+('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Constant', (1, 4), 1), None)], []),
+('Module', [('Assign', (1, 0), [('Tuple', (1, 0), [('Name', (1, 0), 'a', ('Store',)), ('Name', (1, 2), 'b', ('Store',))], ('Store',))], ('Name', (1, 6), 'c', ('Load',)), None)], []),
+('Module', [('Assign', (1, 0), [('Tuple', (1, 0), [('Name', (1, 1), 'a', ('Store',)), ('Name', (1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 8), 'c', ('Load',)), None)], []),
+('Module', [('Assign', (1, 0), [('List', (1, 0), [('Name', (1, 1), 'a', ('Store',)), ('Name', (1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 8), 'c', ('Load',)), None)], []),
+('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Constant', (1, 5), 1))], []),
+('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [], None)], []),
+('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])], []),
+('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])], []),
+('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))], None)], []),
+('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))], None)], []),
+('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Constant', (1, 16), 'string')], []), None)], []),
+('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])], []),
+('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])], []),
+('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)], []),
+('Module', [('Import', (1, 0), [('alias', 'sys', None)])], []),
+('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)], []),
+('Module', [('Global', (1, 0), ['v'])], []),
+('Module', [('Expr', (1, 0), ('Constant', (1, 0), 1))], []),
+('Module', [('Pass', (1, 0))], []),
+('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Break', (1, 11))], [], None)], []),
+('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Continue', (1, 11))], [], None)], []),
+('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [], None)], []),
+('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 5), 'a', ('Store',)), ('Name', (1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 13), 'c', ('Load',)), [('Pass', (1, 16))], [], None)], []),
+('Module', [('For', (1, 0), ('List', (1, 4), [('Name', (1, 5), 'a', ('Store',)), ('Name', (1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 13), 'c', ('Load',)), [('Pass', (1, 16))], [], None)], []),
+('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 0), ('Tuple', (2, 4), [('Name', (3, 4), 'Aa', ('Load',)), ('Name', (5, 7), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4), [('Name', (8, 4), 'Aa', ('Store',)), ('Name', (10, 4), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10), 'Cc', ('Load',)), [], 0)]))], []),
+('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Name', (1, 11), 'w', ('Store',)), ('Name', (1, 16), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22), 'm', ('Store',)), ('Name', (1, 27), 'p', ('Load',)), [('Name', (1, 32), 'g', ('Load',))], 0)]))], []),
+('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [], 0)]))], []),
+('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))], 0)]))], []),
+('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [], 0)]))], []),
+('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('Constant', (2, 1), 'async function')), ('Expr', (3, 1), ('Await', (3, 1), ('Call', (3, 7), ('Name', (3, 7), 'something', ('Load',)), [], [])))], [], None, None)], []),
+('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 1), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Constant', (2, 19), 1))], [('Expr', (3, 7), ('Constant', (3, 7), 2))], None)], [], None, None)], []),
+('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 1), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Constant', (2, 20), 1))], None)], [], None, None)], []),
+('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Constant', (1, 10), 2)], [('Dict', (1, 3), [('Constant', (1, 4), 1)], [('Constant', (1, 6), 2)]), ('Constant', (1, 12), 3)]))], []),
+('Module', [('Expr', (1, 0), ('Set', (1, 0), [('Starred', (1, 1), ('Set', (1, 2), [('Constant', (1, 3), 1), ('Constant', (1, 6), 2)]), ('Load',)), ('Constant', (1, 10), 3)]))], []),
+('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('ListComp', (2, 1), ('Name', (2, 2), 'i', ('Load',)), [('comprehension', ('Name', (2, 14), 'b', ('Store',)), ('Name', (2, 19), 'c', ('Load',)), [], 1)]))], [], None, None)], []),
+('Module', [('FunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None, None)], []),
+('Module', [('AsyncFunctionDef', (3, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (3, 15))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])], None, None)], []),
+('Module', [('ClassDef', (3, 0), 'C', [], [], [('Pass', (3, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 0), ('Name', (2, 1), 'deco2', ('Load',)), [], [])])], []),
+('Module', [('FunctionDef', (2, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (2, 9))], [('Call', (1, 1), ('Name', (1, 1), 'deco', ('Load',)), [('GeneratorExp', (1, 5), ('Name', (1, 6), 'a', ('Load',)), [('comprehension', ('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 17), 'b', ('Load',)), [], 0)])], [])], None, None)], []),
]
single_results = [
('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Constant', (1, 0), 1), ('Add',), ('Constant', (1, 2), 2)))]),
diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py
new file mode 100644
index 0000000..3065ddc
--- /dev/null
+++ b/Lib/test/test_type_comments.py
@@ -0,0 +1,295 @@
+import ast
+import unittest
+
+
+funcdef = """\
+def foo():
+ # type: () -> int
+ pass
+
+def bar(): # type: () -> None
+ pass
+"""
+
+asyncdef = """\
+async def foo():
+ # type: () -> int
+ return await bar()
+
+async def bar(): # type: () -> int
+ return await bar()
+"""
+
+redundantdef = """\
+def foo(): # type: () -> int
+ # type: () -> str
+ return ''
+"""
+
+nonasciidef = """\
+def foo():
+ # type: () -> àçčéñt
+ pass
+"""
+
+forstmt = """\
+for a in []: # type: int
+ pass
+"""
+
+withstmt = """\
+with context() as a: # type: int
+ pass
+"""
+
+vardecl = """\
+a = 0 # type: int
+"""
+
+ignores = """\
+def foo():
+ pass # type: ignore
+
+def bar():
+ x = 1 # type: ignore
+"""
+
+# Test for long-form type-comments in arguments. A test function
+# named 'fabvk' would have two positional args, a and b, plus a
+# var-arg *v, plus a kw-arg **k. It is verified in test_longargs()
+# that it has exactly these arguments, no more, no fewer.
+longargs = """\
+def fa(
+ a = 1, # type: A
+):
+ pass
+
+def fa(
+ a = 1 # type: A
+):
+ pass
+
+def fab(
+ a, # type: A
+ b, # type: B
+):
+ pass
+
+def fab(
+ a, # type: A
+ b # type: B
+):
+ pass
+
+def fv(
+ *v, # type: V
+):
+ pass
+
+def fv(
+ *v # type: V
+):
+ pass
+
+def fk(
+ **k, # type: K
+):
+ pass
+
+def fk(
+ **k # type: K
+):
+ pass
+
+def fvk(
+ *v, # type: V
+ **k, # type: K
+):
+ pass
+
+def fvk(
+ *v, # type: V
+ **k # type: K
+):
+ pass
+
+def fav(
+ a, # type: A
+ *v, # type: V
+):
+ pass
+
+def fav(
+ a, # type: A
+ *v # type: V
+):
+ pass
+
+def fak(
+ a, # type: A
+ **k, # type: K
+):
+ pass
+
+def fak(
+ a, # type: A
+ **k # type: K
+):
+ pass
+
+def favk(
+ a, # type: A
+ *v, # type: V
+ **k, # type: K
+):
+ pass
+
+def favk(
+ a, # type: A
+ *v, # type: V
+ **k # type: K
+):
+ pass
+"""
+
+
+class TypeCommentTests(unittest.TestCase):
+
+ def parse(self, source):
+ return ast.parse(source, type_comments=True)
+
+ def classic_parse(self, source):
+ return ast.parse(source)
+
+ def test_funcdef(self):
+ tree = self.parse(funcdef)
+ self.assertEqual(tree.body[0].type_comment, "() -> int")
+ self.assertEqual(tree.body[1].type_comment, "() -> None")
+ tree = self.classic_parse(funcdef)
+ self.assertEqual(tree.body[0].type_comment, None)
+ self.assertEqual(tree.body[1].type_comment, None)
+
+ def test_asyncdef(self):
+ tree = self.parse(asyncdef)
+ self.assertEqual(tree.body[0].type_comment, "() -> int")
+ self.assertEqual(tree.body[1].type_comment, "() -> int")
+ tree = self.classic_parse(asyncdef)
+ self.assertEqual(tree.body[0].type_comment, None)
+ self.assertEqual(tree.body[1].type_comment, None)
+
+ def test_redundantdef(self):
+ with self.assertRaisesRegex(SyntaxError, "^Cannot have two type comments on def"):
+ tree = self.parse(redundantdef)
+
+ def test_nonasciidef(self):
+ tree = self.parse(nonasciidef)
+ self.assertEqual(tree.body[0].type_comment, "() -> àçčéñt")
+
+ def test_forstmt(self):
+ tree = self.parse(forstmt)
+ self.assertEqual(tree.body[0].type_comment, "int")
+ tree = self.classic_parse(forstmt)
+ self.assertEqual(tree.body[0].type_comment, None)
+
+ def test_withstmt(self):
+ tree = self.parse(withstmt)
+ self.assertEqual(tree.body[0].type_comment, "int")
+ tree = self.classic_parse(withstmt)
+ self.assertEqual(tree.body[0].type_comment, None)
+
+ def test_vardecl(self):
+ tree = self.parse(vardecl)
+ self.assertEqual(tree.body[0].type_comment, "int")
+ tree = self.classic_parse(vardecl)
+ self.assertEqual(tree.body[0].type_comment, None)
+
+ def test_ignores(self):
+ tree = self.parse(ignores)
+ self.assertEqual([ti.lineno for ti in tree.type_ignores], [2, 5])
+ tree = self.classic_parse(ignores)
+ self.assertEqual(tree.type_ignores, [])
+
+ def test_longargs(self):
+ tree = self.parse(longargs)
+ for t in tree.body:
+ # The expected args are encoded in the function name
+ todo = set(t.name[1:])
+ self.assertEqual(len(t.args.args),
+ len(todo) - bool(t.args.vararg) - bool(t.args.kwarg))
+ self.assertTrue(t.name.startswith('f'), t.name)
+ for c in t.name[1:]:
+ todo.remove(c)
+ if c == 'v':
+ arg = t.args.vararg
+ elif c == 'k':
+ arg = t.args.kwarg
+ else:
+ assert 0 <= ord(c) - ord('a') < len(t.args.args)
+ arg = t.args.args[ord(c) - ord('a')]
+ self.assertEqual(arg.arg, c) # That's the argument name
+ self.assertEqual(arg.type_comment, arg.arg.upper())
+ assert not todo
+ tree = self.classic_parse(longargs)
+ for t in tree.body:
+ for arg in t.args.args + [t.args.vararg, t.args.kwarg]:
+ if arg is not None:
+ self.assertIsNone(arg.type_comment, "%s(%s:%r)" %
+ (t.name, arg.arg, arg.type_comment))
+
+ def test_inappropriate_type_comments(self):
+ """Tests for inappropriately-placed type comments.
+
+ These should be silently ignored with type comments off,
+ but raise SyntaxError with type comments on.
+
+ This is not meant to be exhaustive.
+ """
+
+ def check_both_ways(source):
+ ast.parse(source, type_comments=False)
+ with self.assertRaises(SyntaxError):
+ ast.parse(source, type_comments=True)
+
+ check_both_ways("pass # type: int\n")
+ check_both_ways("foo() # type: int\n")
+ check_both_ways("x += 1 # type: int\n")
+ check_both_ways("while True: # type: int\n continue\n")
+ check_both_ways("while True:\n continue # type: int\n")
+ check_both_ways("try: # type: int\n pass\nfinally:\n pass\n")
+ check_both_ways("try:\n pass\nfinally: # type: int\n pass\n")
+
+ def test_func_type_input(self):
+
+ def parse_func_type_input(source):
+ return ast.parse(source, "<unknown>", "func_type")
+
+ # Some checks below will crash if the returned structure is wrong
+ tree = parse_func_type_input("() -> int")
+ self.assertEqual(tree.argtypes, [])
+ self.assertEqual(tree.returns.id, "int")
+
+ tree = parse_func_type_input("(int) -> List[str]")
+ self.assertEqual(len(tree.argtypes), 1)
+ arg = tree.argtypes[0]
+ self.assertEqual(arg.id, "int")
+ self.assertEqual(tree.returns.value.id, "List")
+ self.assertEqual(tree.returns.slice.value.id, "str")
+
+ tree = parse_func_type_input("(int, *str, **Any) -> float")
+ self.assertEqual(tree.argtypes[0].id, "int")
+ self.assertEqual(tree.argtypes[1].id, "str")
+ self.assertEqual(tree.argtypes[2].id, "Any")
+ self.assertEqual(tree.returns.id, "float")
+
+ with self.assertRaises(SyntaxError):
+ tree = parse_func_type_input("(int, *str, *Any) -> float")
+
+ with self.assertRaises(SyntaxError):
+ tree = parse_func_type_input("(int, **str, Any) -> float")
+
+ with self.assertRaises(SyntaxError):
+ tree = parse_func_type_input("(**int, **str) -> float")
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Lib/token.py b/Lib/token.py
index 7224eca..9bf80a5 100644
--- a/Lib/token.py
+++ b/Lib/token.py
@@ -58,12 +58,14 @@
ELLIPSIS = 52
COLONEQUAL = 53
OP = 54
+TYPE_IGNORE = 55
+TYPE_COMMENT = 56
# These aren't used by the C tokenizer but are needed for tokenize.py
-ERRORTOKEN = 55
-COMMENT = 56
-NL = 57
-ENCODING = 58
-N_TOKENS = 59
+ERRORTOKEN = 57
+COMMENT = 58
+NL = 59
+ENCODING = 60
+N_TOKENS = 61
# Special definitions for cooperation with parser
NT_OFFSET = 256