make lib2to3 parse async generators everywhere (GH-6588)
(cherry picked from commit 149addd4960d634ce672ab5fc17e0e785a0cdcd0)
Co-authored-by: Zsolt Dollenstein <zsol.zsol@gmail.com>
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index 0e2685d..099dfa7 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -512,13 +512,14 @@ def generate_tokens(readline):
stashed = tok
continue
- if token == 'def':
+ if token in ('def', 'for'):
if (stashed
and stashed[0] == NAME
and stashed[1] == 'async'):
- async_def = True
- async_def_indent = indents[-1]
+ if token == 'def':
+ async_def = True
+ async_def_indent = indents[-1]
yield (ASYNC, stashed[1],
stashed[2], stashed[3],
diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index 1aba0e8..5eefb5a 100644
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -200,20 +200,27 @@ def test_async_var(self):
self.validate("""await = 1""")
self.validate("""def async(): pass""")
- def test_async_with(self):
+ def test_async_for(self):
self.validate("""async def foo():
async for a in b: pass""")
- self.invalid_syntax("""def foo():
- async for a in b: pass""")
-
- def test_async_for(self):
+ def test_async_with(self):
self.validate("""async def foo():
async with a: pass""")
self.invalid_syntax("""def foo():
async with a: pass""")
+ def test_async_generator(self):
+ self.validate(
+ """async def foo():
+ return (i * 2 async for i in arange(42))"""
+ )
+ self.validate(
+ """def foo():
+ return (i * 2 async for i in arange(42))"""
+ )
+
class TestRaiseChanges(GrammarTest):
def test_2x_style_1(self):