bpo-40334: Add support for feature_version in new PEG parser (GH-19827)
`ast.parse` and `compile` support a `feature_version` parameter that
tells the parser to parse the input string, as if it were written in
an older Python version.
The `feature_version` is propagated to the tokenizer, which uses it
to handle the three different stages of support for `async` and
`await`. Additionally, it disallows the following at parser level:
- The '@' operator in < 3.5
- Async functions in < 3.5
- Async comprehensions in < 3.6
- Underscores in numeric literals in < 3.6
- Await expression in < 3.5
- Variable annotations in < 3.6
- Async for-loops in < 3.5
- Async with-statements in < 3.5
- F-strings in < 3.6
Closes we-like-parsers/cpython#124.
diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c
index 5a2491c..40c09ff 100644
--- a/Parser/pegen/pegen.c
+++ b/Parser/pegen/pegen.c
@@ -933,11 +933,16 @@
}
char *num_raw = PyBytes_AsString(t->bytes);
-
if (num_raw == NULL) {
return NULL;
}
+ if (p->feature_version < 6 && strchr(num_raw, '_') != NULL) {
+ p->error_indicator = 1;
+ return RAISE_SYNTAX_ERROR("Underscores in numeric literals are only supported"
+ "in Python 3.6 and greater");
+ }
+
PyObject *c = parsenumber(num_raw);
if (c == NULL) {
@@ -1030,12 +1035,15 @@
if (flags->cf_flags & PyCF_TYPE_COMMENTS) {
parser_flags |= PyPARSE_TYPE_COMMENTS;
}
+ if (flags->cf_feature_version < 7) {
+ parser_flags |= PyPARSE_ASYNC_HACKS;
+ }
return parser_flags;
}
Parser *
_PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
- int *errcode, PyArena *arena)
+ int feature_version, int *errcode, PyArena *arena)
{
Parser *p = PyMem_Malloc(sizeof(Parser));
if (p == NULL) {
@@ -1077,6 +1085,7 @@
p->starting_lineno = 0;
p->starting_col_offset = 0;
p->flags = flags;
+ p->feature_version = feature_version;
return p;
}
@@ -1138,7 +1147,8 @@
mod_ty result = NULL;
int parser_flags = compute_parser_flags(flags);
- Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, errcode, arena);
+ Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, PY_MINOR_VERSION,
+ errcode, arena);
if (p == NULL) {
goto error;
}
@@ -1194,9 +1204,12 @@
mod_ty result = NULL;
int parser_flags = compute_parser_flags(flags);
+ int feature_version = flags ? flags->cf_feature_version : PY_MINOR_VERSION;
tok->type_comments = (parser_flags & PyPARSE_TYPE_COMMENTS) > 0;
+ tok->async_hacks = (parser_flags & PyPARSE_ASYNC_HACKS) > 0;
- Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, NULL, arena);
+ Parser *p = _PyPegen_Parser_New(tok, start_rule, parser_flags, feature_version,
+ NULL, arena);
if (p == NULL) {
goto error;
}