bpo-35766: Merge typed_ast back into CPython (GH-11645)
diff --git a/Grammar/Grammar b/Grammar/Grammar
index 8455c12..e65a688 100644
--- a/Grammar/Grammar
+++ b/Grammar/Grammar
@@ -7,7 +7,9 @@
# single_input is a single interactive statement;
# file_input is a module or sequence of commands read from an input file;
# eval_input is the input for the eval() functions.
+# func_type_input is a PEP 484 Python 2 function type comment
# NB: compound_stmt in single_input is followed by extra NEWLINE!
+# NB: due to the way TYPE_COMMENT is tokenized it will always be followed by a NEWLINE
single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
file_input: (NEWLINE | stmt)* ENDMARKER
eval_input: testlist NEWLINE* ENDMARKER
@@ -17,14 +19,14 @@
decorated: decorators (classdef | funcdef | async_funcdef)
async_funcdef: 'async' funcdef
-funcdef: 'def' NAME parameters ['->' test] ':' suite
+funcdef: 'def' NAME parameters ['->' test] ':' [TYPE_COMMENT] func_body_suite
parameters: '(' [typedargslist] ')'
-typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [
- '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
- | '**' tfpdef [',']]]
- | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
- | '**' tfpdef [','])
+typedargslist: (tfpdef ['=' test] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] [
+ '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]])
+ | '**' tfpdef [','] [TYPE_COMMENT]]])
+ | '*' [tfpdef] (',' [TYPE_COMMENT] tfpdef ['=' test])* (TYPE_COMMENT | [',' [TYPE_COMMENT] ['**' tfpdef [','] [TYPE_COMMENT]]])
+ | '**' tfpdef [','] [TYPE_COMMENT])
tfpdef: NAME [':' test]
varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
'*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
@@ -39,7 +41,7 @@
small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
- ('=' (yield_expr|testlist_star_expr))*)
+ [('=' (yield_expr|testlist_star_expr))+ [TYPE_COMMENT]] )
annassign: ':' test ['=' (yield_expr|testlist)]
testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
@@ -71,13 +73,13 @@
async_stmt: 'async' (funcdef | with_stmt | for_stmt)
if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite]
while_stmt: 'while' test ':' suite ['else' ':' suite]
-for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
+for_stmt: 'for' exprlist 'in' testlist ':' [TYPE_COMMENT] suite ['else' ':' suite]
try_stmt: ('try' ':' suite
((except_clause ':' suite)+
['else' ':' suite]
['finally' ':' suite] |
'finally' ':' suite))
-with_stmt: 'with' with_item (',' with_item)* ':' suite
+with_stmt: 'with' with_item (',' with_item)* ':' [TYPE_COMMENT] suite
with_item: test ['as' expr]
# NB compile.c makes sure that the default except clause is last
except_clause: 'except' [test ['as' NAME]]
@@ -150,3 +152,14 @@
yield_expr: 'yield' [yield_arg]
yield_arg: 'from' test | testlist_star_expr
+
+# the TYPE_COMMENT in suites is only parsed for funcdefs,
+# but can't go elsewhere due to ambiguity
+func_body_suite: simple_stmt | NEWLINE [TYPE_COMMENT NEWLINE] INDENT stmt+ DEDENT
+
+func_type_input: func_type NEWLINE* ENDMARKER
+func_type: '(' [typelist] ')' '->' test
+# typelist is a modified typedargslist (see above)
+typelist: (test (',' test)* [','
+ ['*' [test] (',' test)* [',' '**' test] | '**' test]]
+ | '*' [test] (',' test)* [',' '**' test] | '**' test)
diff --git a/Grammar/Tokens b/Grammar/Tokens
index f6f303b..1d45e05 100644
--- a/Grammar/Tokens
+++ b/Grammar/Tokens
@@ -55,6 +55,8 @@
COLONEQUAL ':='
OP
+TYPE_IGNORE
+TYPE_COMMENT
ERRORTOKEN
# These aren't used by the C tokenizer but are needed for tokenize.py