PEP 0492 -- Coroutines with async and await syntax. Issue #24017.
diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt
index ce95d26..c954669 100644
--- a/Lib/lib2to3/Grammar.txt
+++ b/Lib/lib2to3/Grammar.txt
@@ -33,7 +33,8 @@
decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
decorators: decorator+
-decorated: decorators (classdef | funcdef)
+decorated: decorators (classdef | funcdef | async_funcdef)
+async_funcdef: ASYNC funcdef
funcdef: 'def' NAME parameters ['->' test] ':' suite
parameters: '(' [typedargslist] ')'
typedargslist: ((tfpdef ['=' test] ',')*
@@ -82,7 +83,8 @@
exec_stmt: 'exec' expr ['in' test [',' test]]
assert_stmt: 'assert' test [',' test]
-compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+async_stmt: ASYNC (funcdef | with_stmt | for_stmt)
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
while_stmt: 'while' test ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
@@ -121,7 +123,7 @@
arith_expr: term (('+'|'-') term)*
term: factor (('*'|'@'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
-power: atom trailer* ['**' factor]
+power: [AWAIT] atom trailer* ['**' factor]
atom: ('(' [yield_expr|testlist_gexp] ')' |
'[' [listmaker] ']' |
'{' [dictsetmaker] '}' |
diff --git a/Lib/lib2to3/pgen2/token.py b/Lib/lib2to3/pgen2/token.py
index 7599396..1a67955 100755
--- a/Lib/lib2to3/pgen2/token.py
+++ b/Lib/lib2to3/pgen2/token.py
@@ -62,8 +62,10 @@
COMMENT = 53
NL = 54
RARROW = 55
-ERRORTOKEN = 56
-N_TOKENS = 57
+AWAIT = 56
+ASYNC = 57
+ERRORTOKEN = 58
+N_TOKENS = 59
NT_OFFSET = 256
#--end constants--
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index 3dd1ee9..690fec4 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -220,7 +220,7 @@
for tok in iterable:
toknum, tokval = tok[:2]
- if toknum in (NAME, NUMBER):
+ if toknum in (NAME, NUMBER, ASYNC, AWAIT):
tokval += ' '
if toknum == INDENT:
@@ -366,6 +366,10 @@
contline = None
indents = [0]
+ # 'stashed' and 'ctx' are used for async/await parsing
+ stashed = None
+ ctx = [('sync', 0)]
+
while 1: # loop over lines in stream
try:
line = readline()
@@ -406,6 +410,10 @@
pos = pos + 1
if pos == max: break
+ if stashed:
+ yield stashed
+ stashed = None
+
if line[pos] in '#\r\n': # skip comments or blank lines
if line[pos] == '#':
comment_token = line[pos:].rstrip('\r\n')
@@ -449,9 +457,15 @@
newline = NEWLINE
if parenlev > 0:
newline = NL
+ if stashed:
+ yield stashed
+ stashed = None
yield (newline, token, spos, epos, line)
elif initial == '#':
assert not token.endswith("\n")
+ if stashed:
+ yield stashed
+ stashed = None
yield (COMMENT, token, spos, epos, line)
elif token in triple_quoted:
endprog = endprogs[token]
@@ -459,6 +473,9 @@
if endmatch: # all on one line
pos = endmatch.end(0)
token = line[start:pos]
+ if stashed:
+ yield stashed
+ stashed = None
yield (STRING, token, spos, (lnum, pos), line)
else:
strstart = (lnum, start) # multiple lines
@@ -476,22 +493,64 @@
contline = line
break
else: # ordinary string
+ if stashed:
+ yield stashed
+ stashed = None
yield (STRING, token, spos, epos, line)
elif initial in namechars: # ordinary name
- yield (NAME, token, spos, epos, line)
+ if token in ('async', 'await'):
+ if ctx[-1][0] == 'async' and ctx[-1][1] < indents[-1]:
+ yield (ASYNC if token == 'async' else AWAIT,
+ token, spos, epos, line)
+ continue
+
+ tok = (NAME, token, spos, epos, line)
+ if token == 'async' and not stashed:
+ stashed = tok
+ continue
+
+ if token == 'def':
+ if (stashed
+ and stashed[0] == NAME
+ and stashed[1] == 'async'):
+
+ ctx.append(('async', indents[-1]))
+
+ yield (ASYNC, stashed[1],
+ stashed[2], stashed[3],
+ stashed[4])
+ stashed = None
+ else:
+ ctx.append(('sync', indents[-1]))
+
+ if stashed:
+ yield stashed
+ stashed = None
+
+ yield tok
elif initial == '\\': # continued stmt
# This yield is new; needed for better idempotency:
+ if stashed:
+ yield stashed
+ stashed = None
yield (NL, token, spos, (lnum, pos), line)
continued = 1
else:
if initial in '([{': parenlev = parenlev + 1
elif initial in ')]}': parenlev = parenlev - 1
+ if stashed:
+ yield stashed
+ stashed = None
yield (OP, token, spos, epos, line)
else:
yield (ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos+1), line)
pos = pos + 1
+ if stashed:
+ yield stashed
+ stashed = None
+
for indent in indents[1:]: # pop remaining indent levels
yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index 5bb9d2b..107b5ab 100644
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -55,12 +55,42 @@
class TestYieldFrom(GrammarTest):
- def test_matrix_multiplication_operator(self):
+ def test_yield_from(self):
self.validate("yield from x")
self.validate("(yield from x) + y")
self.invalid_syntax("yield from")
+class TestAsyncAwait(GrammarTest):
+ def test_await_expr(self):
+ self.validate("""async def foo():
+ await x
+ """)
+
+ self.invalid_syntax("await x")
+ self.invalid_syntax("""def foo():
+ await x""")
+
+ def test_async_var(self):
+ self.validate("""async = 1""")
+ self.validate("""await = 1""")
+ self.validate("""def async(): pass""")
+
+ def test_async_with(self):
+ self.validate("""async def foo():
+ async for a in b: pass""")
+
+ self.invalid_syntax("""def foo():
+ async for a in b: pass""")
+
+ def test_async_for(self):
+ self.validate("""async def foo():
+ async with a: pass""")
+
+ self.invalid_syntax("""def foo():
+ async with a: pass""")
+
+
class TestRaiseChanges(GrammarTest):
def test_2x_style_1(self):
self.validate("raise")