bpo-30406: Make async and await proper keywords (#1669)
Per PEP 492, 'async' and 'await' should become proper keywords in 3.7.
diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index 45afc5f..14560e4 100644
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -234,7 +234,7 @@
for tok in iterable:
toknum, tokval = tok[:2]
- if toknum in (NAME, NUMBER, ASYNC, AWAIT):
+ if toknum in (NAME, NUMBER):
tokval += ' '
if toknum == INDENT:
@@ -380,12 +380,6 @@
contline = None
indents = [0]
- # 'stashed' and 'async_*' are used for async/await parsing
- stashed = None
- async_def = False
- async_def_indent = 0
- async_def_nl = False
-
while 1: # loop over lines in stream
try:
line = readline()
@@ -426,10 +420,6 @@
pos = pos + 1
if pos == max: break
- if stashed:
- yield stashed
- stashed = None
-
if line[pos] in '#\r\n': # skip comments or blank lines
if line[pos] == '#':
comment_token = line[pos:].rstrip('\r\n')
@@ -453,18 +443,8 @@
("<tokenize>", lnum, pos, line))
indents = indents[:-1]
- if async_def and async_def_indent >= indents[-1]:
- async_def = False
- async_def_nl = False
- async_def_indent = 0
-
yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
- if async_def and async_def_nl and async_def_indent >= indents[-1]:
- async_def = False
- async_def_nl = False
- async_def_indent = 0
-
else: # continued statement
if not line:
raise TokenError("EOF in multi-line statement", (lnum, 0))
@@ -484,18 +464,10 @@
newline = NEWLINE
if parenlev > 0:
newline = NL
- elif async_def:
- async_def_nl = True
- if stashed:
- yield stashed
- stashed = None
yield (newline, token, spos, epos, line)
elif initial == '#':
assert not token.endswith("\n")
- if stashed:
- yield stashed
- stashed = None
yield (COMMENT, token, spos, epos, line)
elif token in triple_quoted:
endprog = endprogs[token]
@@ -503,9 +475,6 @@
if endmatch: # all on one line
pos = endmatch.end(0)
token = line[start:pos]
- if stashed:
- yield stashed
- stashed = None
yield (STRING, token, spos, (lnum, pos), line)
else:
strstart = (lnum, start) # multiple lines
@@ -523,63 +492,22 @@
contline = line
break
else: # ordinary string
- if stashed:
- yield stashed
- stashed = None
yield (STRING, token, spos, epos, line)
elif initial in namechars: # ordinary name
- if token in ('async', 'await'):
- if async_def:
- yield (ASYNC if token == 'async' else AWAIT,
- token, spos, epos, line)
- continue
-
- tok = (NAME, token, spos, epos, line)
- if token == 'async' and not stashed:
- stashed = tok
- continue
-
- if token == 'def':
- if (stashed
- and stashed[0] == NAME
- and stashed[1] == 'async'):
-
- async_def = True
- async_def_indent = indents[-1]
-
- yield (ASYNC, stashed[1],
- stashed[2], stashed[3],
- stashed[4])
- stashed = None
-
- if stashed:
- yield stashed
- stashed = None
-
- yield tok
+ yield (NAME, token, spos, epos, line)
elif initial == '\\': # continued stmt
# This yield is new; needed for better idempotency:
- if stashed:
- yield stashed
- stashed = None
yield (NL, token, spos, (lnum, pos), line)
continued = 1
else:
if initial in '([{': parenlev = parenlev + 1
elif initial in ')]}': parenlev = parenlev - 1
- if stashed:
- yield stashed
- stashed = None
yield (OP, token, spos, epos, line)
else:
yield (ERRORTOKEN, line[pos],
(lnum, pos), (lnum, pos+1), line)
pos = pos + 1
- if stashed:
- yield stashed
- stashed = None
-
for indent in indents[1:]: # pop remaining indent levels
yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')