PEP 0492 -- Coroutines with async and await syntax. Issue #24017.
diff --git a/Parser/Python.asdl b/Parser/Python.asdl
index 5476c53..cd0832d 100644
--- a/Parser/Python.asdl
+++ b/Parser/Python.asdl
@@ -10,7 +10,10 @@
| Suite(stmt* body)
stmt = FunctionDef(identifier name, arguments args,
- stmt* body, expr* decorator_list, expr? returns)
+ stmt* body, expr* decorator_list, expr? returns)
+ | AsyncFunctionDef(identifier name, arguments args,
+ stmt* body, expr* decorator_list, expr? returns)
+
| ClassDef(identifier name,
expr* bases,
keyword* keywords,
@@ -24,9 +27,11 @@
-- use 'orelse' because else is a keyword in target languages
| For(expr target, expr iter, stmt* body, stmt* orelse)
+ | AsyncFor(expr target, expr iter, stmt* body, stmt* orelse)
| While(expr test, stmt* body, stmt* orelse)
| If(expr test, stmt* body, stmt* orelse)
| With(withitem* items, stmt* body)
+ | AsyncWith(withitem* items, stmt* body)
| Raise(expr? exc, expr? cause)
| Try(stmt* body, excepthandler* handlers, stmt* orelse, stmt* finalbody)
@@ -57,6 +62,7 @@
| DictComp(expr key, expr value, comprehension* generators)
| GeneratorExp(expr elt, comprehension* generators)
-- the grammar constrains where yield expressions can occur
+ | Await(expr value)
| Yield(expr? value)
| YieldFrom(expr value)
-- need sequences for compare to distinguish between
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index ac413a8..798758d 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -103,6 +103,8 @@
"ELLIPSIS",
/* This table must match the #defines in token.h! */
"OP",
+ "AWAIT",
+ "ASYNC",
"<ERRORTOKEN>",
"<N_TOKENS>"
};
@@ -124,6 +126,11 @@
tok->tabsize = TABSIZE;
tok->indent = 0;
tok->indstack[0] = 0;
+
+ tok->def = 0;
+ tok->defstack[0] = 0;
+ tok->deftypestack[0] = 0;
+
tok->atbol = 1;
tok->pendin = 0;
tok->prompt = tok->nextprompt = NULL;
@@ -1335,6 +1342,11 @@
int c;
int blankline, nonascii;
+ int tok_len;
+ struct tok_state ahead_tok;
+ char *ahead_tok_start = NULL, *ahead_top_end = NULL;
+ int ahead_tok_kind;
+
*p_start = *p_end = NULL;
nextline:
tok->start = NULL;
@@ -1422,6 +1434,11 @@
if (tok->pendin != 0) {
if (tok->pendin < 0) {
tok->pendin++;
+
+ while (tok->def && tok->defstack[tok->def] >= tok->indent) {
+ tok->def--;
+ }
+
return DEDENT;
}
else {
@@ -1481,6 +1498,57 @@
return ERRORTOKEN;
*p_start = tok->start;
*p_end = tok->cur;
+
+ tok_len = tok->cur - tok->start;
+ if (tok_len == 3 && memcmp(tok->start, "def", 3) == 0) {
+
+ if (tok->def + 1 >= MAXINDENT) {
+ tok->done = E_TOODEEP;
+ tok->cur = tok->inp;
+ return ERRORTOKEN;
+ }
+
+ if (tok->def && tok->deftypestack[tok->def] == 3) {
+ tok->deftypestack[tok->def] = 2;
+ }
+ else {
+ tok->def++;
+ tok->defstack[tok->def] = tok->indent;
+ tok->deftypestack[tok->def] = 1;
+ }
+ }
+ else if (tok_len == 5) {
+ if (memcmp(tok->start, "async", 5) == 0) {
+ memcpy(&ahead_tok, tok, sizeof(ahead_tok));
+
+ ahead_tok_kind = tok_get(&ahead_tok, &ahead_tok_start,
+ &ahead_top_end);
+
+ if (ahead_tok_kind == NAME &&
+ ahead_tok.cur - ahead_tok.start == 3 &&
+ memcmp(ahead_tok.start, "def", 3) == 0) {
+
+ tok->def++;
+ tok->defstack[tok->def] = tok->indent;
+ tok->deftypestack[tok->def] = 3;
+
+ return ASYNC;
+ }
+ else if (tok->def && tok->deftypestack[tok->def] == 2
+ && tok->defstack[tok->def] < tok->indent) {
+
+ return ASYNC;
+ }
+
+ }
+ else if (memcmp(tok->start, "await", 5) == 0
+ && tok->def && tok->deftypestack[tok->def] == 2
+ && tok->defstack[tok->def] < tok->indent) {
+
+ return AWAIT;
+ }
+ }
+
return NAME;
}
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 1ce6eeb..3bcdad6 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -65,6 +65,13 @@
const char* enc; /* Encoding for the current str. */
const char* str;
const char* input; /* Tokenizer's newline translated copy of the string. */
+
+ int defstack[MAXINDENT]; /* stack if funcs & indents where they
+ were defined */
+ int deftypestack[MAXINDENT]; /* stack of func types
+ (0 not func; 1: "def name";
+ 2: "async def name") */
+ int def; /* Length of stack of func types */
};
extern struct tok_state *PyTokenizer_FromString(const char *, int);