bpo-40334: PEP 617 implementation: New PEG parser for CPython (GH-19503)
Co-authored-by: Guido van Rossum <guido@python.org>
Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>
diff --git a/Python/ast_opt.c b/Python/ast_opt.c
index 1766321..ff786d6 100644
--- a/Python/ast_opt.c
+++ b/Python/ast_opt.c
@@ -563,7 +563,8 @@
CALL(fold_tuple, expr_ty, node_);
break;
case Name_kind:
- if (_PyUnicode_EqualToASCIIString(node_->v.Name.id, "__debug__")) {
+ if (node_->v.Name.ctx == Load &&
+ _PyUnicode_EqualToASCIIString(node_->v.Name.id, "__debug__")) {
return make_const(node_, PyBool_FromLong(!state->optimize), ctx_);
}
break;
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 22ee596..1888335 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -816,7 +816,12 @@
if (str == NULL)
goto error;
+ int current_use_peg = PyInterpreterState_Get()->config.use_peg;
+ if (flags & PyCF_TYPE_COMMENTS || feature_version >= 0) {
+ PyInterpreterState_Get()->config.use_peg = 0;
+ }
result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize);
+ PyInterpreterState_Get()->config.use_peg = current_use_peg;
Py_XDECREF(source_copy);
goto finally;
diff --git a/Python/compile.c b/Python/compile.c
index 54e6516..3c21fba 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -2153,6 +2153,55 @@
}
static int
+forbidden_name(struct compiler *c, identifier name, expr_context_ty ctx)
+{
+
+ if (ctx == Store && _PyUnicode_EqualToASCIIString(name, "__debug__")) {
+ compiler_error(c, "cannot assign to __debug__");
+ return 1;
+ }
+ return 0;
+}
+
+static int
+compiler_check_debug_one_arg(struct compiler *c, arg_ty arg)
+{
+ if (arg != NULL) {
+ if (forbidden_name(c, arg->arg, Store))
+ return 0;
+ }
+ return 1;
+}
+
+static int
+compiler_check_debug_args_seq(struct compiler *c, asdl_seq *args)
+{
+ if (args != NULL) {
+ for (int i = 0, n = asdl_seq_LEN(args); i < n; i++) {
+ if (!compiler_check_debug_one_arg(c, asdl_seq_GET(args, i)))
+ return 0;
+ }
+ }
+ return 1;
+}
+
+static int
+compiler_check_debug_args(struct compiler *c, arguments_ty args)
+{
+ if (!compiler_check_debug_args_seq(c, args->posonlyargs))
+ return 0;
+ if (!compiler_check_debug_args_seq(c, args->args))
+ return 0;
+ if (!compiler_check_debug_one_arg(c, args->vararg))
+ return 0;
+ if (!compiler_check_debug_args_seq(c, args->kwonlyargs))
+ return 0;
+ if (!compiler_check_debug_one_arg(c, args->kwarg))
+ return 0;
+ return 1;
+}
+
+static int
compiler_function(struct compiler *c, stmt_ty s, int is_async)
{
PyCodeObject *co;
@@ -2189,6 +2238,9 @@
scope_type = COMPILER_SCOPE_FUNCTION;
}
+ if (!compiler_check_debug_args(c, args))
+ return 0;
+
if (!compiler_decorators(c, decos))
return 0;
@@ -2596,6 +2648,9 @@
arguments_ty args = e->v.Lambda.args;
assert(e->kind == Lambda_kind);
+ if (!compiler_check_debug_args(c, args))
+ return 0;
+
if (!name) {
name = PyUnicode_InternFromString("<lambda>");
if (!name)
@@ -3505,6 +3560,9 @@
!_PyUnicode_EqualToASCIIString(name, "True") &&
!_PyUnicode_EqualToASCIIString(name, "False"));
+ if (forbidden_name(c, name, ctx))
+ return 0;
+
mangled = _Py_Mangle(c->u->u_private, name);
if (!mangled)
return 0;
@@ -4056,6 +4114,9 @@
if (key->arg == NULL) {
continue;
}
+ if (forbidden_name(c, key->arg, Store)) {
+ return -1;
+ }
for (Py_ssize_t j = i + 1; j < nkeywords; j++) {
keyword_ty other = ((keyword_ty)asdl_seq_GET(keywords, j));
if (other->arg && !PyUnicode_Compare(key->arg, other->arg)) {
@@ -5013,6 +5074,8 @@
ADDOP_NAME(c, LOAD_ATTR, e->v.Attribute.attr, names);
break;
case Store:
+ if (forbidden_name(c, e->v.Attribute.attr, e->v.Attribute.ctx))
+ return 0;
ADDOP_NAME(c, STORE_ATTR, e->v.Attribute.attr, names);
break;
case Del:
@@ -5183,6 +5246,8 @@
}
switch (targ->kind) {
case Name_kind:
+ if (forbidden_name(c, targ->v.Name.id, Store))
+ return 0;
/* If we have a simple name in a module or class, store annotation. */
if (s->v.AnnAssign.simple &&
(c->u->u_scope_type == COMPILER_SCOPE_MODULE ||
@@ -5200,6 +5265,8 @@
}
break;
case Attribute_kind:
+ if (forbidden_name(c, targ->v.Attribute.attr, Store))
+ return 0;
if (!s->v.AnnAssign.value &&
!check_ann_expr(c, targ->v.Attribute.value)) {
return 0;
diff --git a/Python/importlib.h b/Python/importlib.h
index 4bd8b62..59e0272 100644
--- a/Python/importlib.h
+++ b/Python/importlib.h
@@ -1594,50 +1594,51 @@
0,218,1,120,90,5,119,104,101,114,101,90,9,102,114,111,
109,95,110,97,109,101,90,3,101,120,99,114,10,0,0,0,
114,10,0,0,0,114,11,0,0,0,114,215,0,0,0,9,
- 4,0,0,115,44,0,0,0,0,10,8,1,10,1,4,1,
- 12,2,4,1,28,2,8,1,14,1,10,1,2,255,8,2,
- 10,1,14,1,2,1,14,1,14,4,10,1,16,255,2,2,
- 12,1,26,1,114,215,0,0,0,99,1,0,0,0,0,0,
- 0,0,0,0,0,0,3,0,0,0,6,0,0,0,67,0,
- 0,0,115,146,0,0,0,124,0,160,0,100,1,161,1,125,
- 1,124,0,160,0,100,2,161,1,125,2,124,1,100,3,117,
- 1,114,82,124,2,100,3,117,1,114,78,124,1,124,2,106,
- 1,107,3,114,78,116,2,106,3,100,4,124,1,155,2,100,
- 5,124,2,106,1,155,2,100,6,157,5,116,4,100,7,100,
- 8,141,3,1,0,124,1,83,0,124,2,100,3,117,1,114,
- 96,124,2,106,1,83,0,116,2,106,3,100,9,116,4,100,
- 7,100,8,141,3,1,0,124,0,100,10,25,0,125,1,100,
- 11,124,0,118,1,114,142,124,1,160,5,100,12,161,1,100,
- 13,25,0,125,1,124,1,83,0,41,14,122,167,67,97,108,
- 99,117,108,97,116,101,32,119,104,97,116,32,95,95,112,97,
- 99,107,97,103,101,95,95,32,115,104,111,117,108,100,32,98,
- 101,46,10,10,32,32,32,32,95,95,112,97,99,107,97,103,
- 101,95,95,32,105,115,32,110,111,116,32,103,117,97,114,97,
- 110,116,101,101,100,32,116,111,32,98,101,32,100,101,102,105,
- 110,101,100,32,111,114,32,99,111,117,108,100,32,98,101,32,
- 115,101,116,32,116,111,32,78,111,110,101,10,32,32,32,32,
- 116,111,32,114,101,112,114,101,115,101,110,116,32,116,104,97,
- 116,32,105,116,115,32,112,114,111,112,101,114,32,118,97,108,
- 117,101,32,105,115,32,117,110,107,110,111,119,110,46,10,10,
- 32,32,32,32,114,146,0,0,0,114,106,0,0,0,78,122,
- 32,95,95,112,97,99,107,97,103,101,95,95,32,33,61,32,
- 95,95,115,112,101,99,95,95,46,112,97,114,101,110,116,32,
- 40,122,4,32,33,61,32,250,1,41,233,3,0,0,0,41,
- 1,90,10,115,116,97,99,107,108,101,118,101,108,122,89,99,
- 97,110,39,116,32,114,101,115,111,108,118,101,32,112,97,99,
- 107,97,103,101,32,102,114,111,109,32,95,95,115,112,101,99,
- 95,95,32,111,114,32,95,95,112,97,99,107,97,103,101,95,
- 95,44,32,102,97,108,108,105,110,103,32,98,97,99,107,32,
- 111,110,32,95,95,110,97,109,101,95,95,32,97,110,100,32,
- 95,95,112,97,116,104,95,95,114,1,0,0,0,114,142,0,
- 0,0,114,129,0,0,0,114,22,0,0,0,41,6,114,35,
- 0,0,0,114,131,0,0,0,114,193,0,0,0,114,194,0,
- 0,0,114,195,0,0,0,114,130,0,0,0,41,3,218,7,
- 103,108,111,98,97,108,115,114,187,0,0,0,114,96,0,0,
- 0,114,10,0,0,0,114,10,0,0,0,114,11,0,0,0,
- 218,17,95,99,97,108,99,95,95,95,112,97,99,107,97,103,
- 101,95,95,46,4,0,0,115,34,0,0,0,0,7,10,1,
- 10,1,8,1,18,1,22,2,4,254,6,3,4,1,8,1,
+ 4,0,0,115,52,0,0,0,0,10,8,1,10,1,4,1,
+ 12,2,4,1,4,1,2,255,4,1,8,255,10,2,8,1,
+ 14,1,10,1,2,255,8,2,10,1,14,1,2,1,14,1,
+ 14,4,10,1,16,255,2,2,12,1,26,1,114,215,0,0,
+ 0,99,1,0,0,0,0,0,0,0,0,0,0,0,3,0,
+ 0,0,6,0,0,0,67,0,0,0,115,146,0,0,0,124,
+ 0,160,0,100,1,161,1,125,1,124,0,160,0,100,2,161,
+ 1,125,2,124,1,100,3,117,1,114,82,124,2,100,3,117,
+ 1,114,78,124,1,124,2,106,1,107,3,114,78,116,2,106,
+ 3,100,4,124,1,155,2,100,5,124,2,106,1,155,2,100,
+ 6,157,5,116,4,100,7,100,8,141,3,1,0,124,1,83,
+ 0,124,2,100,3,117,1,114,96,124,2,106,1,83,0,116,
+ 2,106,3,100,9,116,4,100,7,100,8,141,3,1,0,124,
+ 0,100,10,25,0,125,1,100,11,124,0,118,1,114,142,124,
+ 1,160,5,100,12,161,1,100,13,25,0,125,1,124,1,83,
+ 0,41,14,122,167,67,97,108,99,117,108,97,116,101,32,119,
+ 104,97,116,32,95,95,112,97,99,107,97,103,101,95,95,32,
+ 115,104,111,117,108,100,32,98,101,46,10,10,32,32,32,32,
+ 95,95,112,97,99,107,97,103,101,95,95,32,105,115,32,110,
+ 111,116,32,103,117,97,114,97,110,116,101,101,100,32,116,111,
+ 32,98,101,32,100,101,102,105,110,101,100,32,111,114,32,99,
+ 111,117,108,100,32,98,101,32,115,101,116,32,116,111,32,78,
+ 111,110,101,10,32,32,32,32,116,111,32,114,101,112,114,101,
+ 115,101,110,116,32,116,104,97,116,32,105,116,115,32,112,114,
+ 111,112,101,114,32,118,97,108,117,101,32,105,115,32,117,110,
+ 107,110,111,119,110,46,10,10,32,32,32,32,114,146,0,0,
+ 0,114,106,0,0,0,78,122,32,95,95,112,97,99,107,97,
+ 103,101,95,95,32,33,61,32,95,95,115,112,101,99,95,95,
+ 46,112,97,114,101,110,116,32,40,122,4,32,33,61,32,250,
+ 1,41,233,3,0,0,0,41,1,90,10,115,116,97,99,107,
+ 108,101,118,101,108,122,89,99,97,110,39,116,32,114,101,115,
+ 111,108,118,101,32,112,97,99,107,97,103,101,32,102,114,111,
+ 109,32,95,95,115,112,101,99,95,95,32,111,114,32,95,95,
+ 112,97,99,107,97,103,101,95,95,44,32,102,97,108,108,105,
+ 110,103,32,98,97,99,107,32,111,110,32,95,95,110,97,109,
+ 101,95,95,32,97,110,100,32,95,95,112,97,116,104,95,95,
+ 114,1,0,0,0,114,142,0,0,0,114,129,0,0,0,114,
+ 22,0,0,0,41,6,114,35,0,0,0,114,131,0,0,0,
+ 114,193,0,0,0,114,194,0,0,0,114,195,0,0,0,114,
+ 130,0,0,0,41,3,218,7,103,108,111,98,97,108,115,114,
+ 187,0,0,0,114,96,0,0,0,114,10,0,0,0,114,10,
+ 0,0,0,114,11,0,0,0,218,17,95,99,97,108,99,95,
+ 95,95,112,97,99,107,97,103,101,95,95,46,4,0,0,115,
+ 42,0,0,0,0,7,10,1,10,1,8,1,18,1,6,1,
+ 2,255,4,1,4,255,6,2,4,254,6,3,4,1,8,1,
6,2,6,2,4,254,6,3,8,1,8,1,14,1,114,221,
0,0,0,114,10,0,0,0,99,5,0,0,0,0,0,0,
0,0,0,0,0,9,0,0,0,5,0,0,0,67,0,0,
diff --git a/Python/importlib_external.h b/Python/importlib_external.h
index 9618f9f..dd23742 100644
--- a/Python/importlib_external.h
+++ b/Python/importlib_external.h
@@ -481,10 +481,11 @@
108,101,118,101,108,90,13,98,97,115,101,95,102,105,108,101,
110,97,109,101,114,5,0,0,0,114,5,0,0,0,114,8,
0,0,0,218,17,115,111,117,114,99,101,95,102,114,111,109,
- 95,99,97,99,104,101,116,1,0,0,115,52,0,0,0,0,
+ 95,99,97,99,104,101,116,1,0,0,115,68,0,0,0,0,
9,12,1,8,1,10,1,12,1,4,1,10,1,12,1,14,
- 1,16,1,4,1,4,1,12,1,8,1,18,2,10,1,8,
- 1,16,1,10,1,16,1,10,1,14,2,16,1,10,1,16,
+ 1,16,1,4,1,4,1,12,1,8,1,2,1,2,255,4,
+ 1,2,255,8,2,10,1,8,1,16,1,10,1,16,1,10,
+ 1,4,1,2,255,8,2,16,1,10,1,4,1,2,255,10,
2,14,1,114,102,0,0,0,99,1,0,0,0,0,0,0,
0,0,0,0,0,5,0,0,0,9,0,0,0,67,0,0,
0,115,124,0,0,0,116,0,124,0,131,1,100,1,107,2,
diff --git a/Python/initconfig.c b/Python/initconfig.c
index c313d91..7662d61 100644
--- a/Python/initconfig.c
+++ b/Python/initconfig.c
@@ -68,6 +68,7 @@
-X opt : set implementation-specific option. The following options are available:\n\
\n\
-X faulthandler: enable faulthandler\n\
+ -X oldparser: enable the traditional LL(1) parser; also PYTHONOLDPARSER\n\
-X showrefcount: output the total reference count and number of used\n\
memory blocks when the program finishes or after each statement in the\n\
interactive interpreter. This only works on debug builds\n\
@@ -634,6 +635,7 @@
#ifdef MS_WINDOWS
config->legacy_windows_stdio = -1;
#endif
+ config->use_peg = 1;
}
@@ -791,6 +793,7 @@
COPY_ATTR(isolated);
COPY_ATTR(use_environment);
COPY_ATTR(dev_mode);
+ COPY_ATTR(use_peg);
COPY_ATTR(install_signal_handlers);
COPY_ATTR(use_hash_seed);
COPY_ATTR(hash_seed);
@@ -894,6 +897,7 @@
SET_ITEM_INT(isolated);
SET_ITEM_INT(use_environment);
SET_ITEM_INT(dev_mode);
+ SET_ITEM_INT(use_peg);
SET_ITEM_INT(install_signal_handlers);
SET_ITEM_INT(use_hash_seed);
SET_ITEM_UINT(hash_seed);
@@ -1428,6 +1432,11 @@
config->import_time = 1;
}
+ if (config_get_env(config, "PYTHONOLDPARSER")
+ || config_get_xoption(config, L"oldparser")) {
+ config->use_peg = 0;
+ }
+
PyStatus status;
if (config->tracemalloc < 0) {
status = config_init_tracemalloc(config);
@@ -2507,6 +2516,7 @@
assert(config->isolated >= 0);
assert(config->use_environment >= 0);
assert(config->dev_mode >= 0);
+ assert(config->use_peg >= 0);
assert(config->install_signal_handlers >= 0);
assert(config->use_hash_seed >= 0);
assert(config->faulthandler >= 0);
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index 0a25ebc..6199f0c 100644
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -29,6 +29,8 @@
#include "ast.h" // PyAST_FromNodeObject()
#include "marshal.h" // PyMarshal_ReadLongFromFile()
+#include <pegen_interface.h> // PyPegen_ASTFrom*
+
#ifdef MS_WINDOWS
# include "malloc.h" // alloca()
#endif
@@ -183,6 +185,7 @@
PyArena *arena;
const char *ps1 = "", *ps2 = "", *enc = NULL;
int errcode = 0;
+ int use_peg = _PyInterpreterState_GET()->config.use_peg;
_Py_IDENTIFIER(encoding);
_Py_IDENTIFIER(__main__);
@@ -235,9 +238,17 @@
Py_XDECREF(oenc);
return -1;
}
- mod = PyParser_ASTFromFileObject(fp, filename, enc,
- Py_single_input, ps1, ps2,
- flags, &errcode, arena);
+
+ if (use_peg) {
+ mod = PyPegen_ASTFromFileObject(fp, filename, Py_single_input,
+ enc, ps1, ps2, &errcode, arena);
+ }
+ else {
+ mod = PyParser_ASTFromFileObject(fp, filename, enc,
+ Py_single_input, ps1, ps2,
+ flags, &errcode, arena);
+ }
+
Py_XDECREF(v);
Py_XDECREF(w);
Py_XDECREF(oenc);
@@ -1019,6 +1030,7 @@
mod_ty mod;
PyArena *arena;
PyObject *filename;
+ int use_peg = _PyInterpreterState_GET()->config.use_peg;
filename = _PyUnicode_FromId(&PyId_string); /* borrowed */
if (filename == NULL)
@@ -1028,7 +1040,13 @@
if (arena == NULL)
return NULL;
- mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
+ if (use_peg) {
+ mod = PyPegen_ASTFromStringObject(str, filename, start, flags, arena);
+ }
+ else {
+ mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
+ }
+
if (mod != NULL)
ret = run_mod(mod, filename, globals, locals, flags, arena);
PyArena_Free(arena);
@@ -1043,6 +1061,7 @@
mod_ty mod;
PyArena *arena = NULL;
PyObject *filename;
+ int use_peg = _PyInterpreterState_GET()->config.use_peg;
filename = PyUnicode_DecodeFSDefault(filename_str);
if (filename == NULL)
@@ -1052,8 +1071,15 @@
if (arena == NULL)
goto exit;
- mod = PyParser_ASTFromFileObject(fp, filename, NULL, start, 0, 0,
- flags, NULL, arena);
+ if (use_peg) {
+ mod = PyPegen_ASTFromFileObject(fp, filename, start, NULL, NULL, NULL,
+ NULL, arena);
+ }
+ else {
+ mod = PyParser_ASTFromFileObject(fp, filename, NULL, start, 0, 0,
+ flags, NULL, arena);
+ }
+
if (closeit)
fclose(fp);
if (mod == NULL) {
@@ -1196,11 +1222,17 @@
{
PyCodeObject *co;
mod_ty mod;
+ int use_peg = _PyInterpreterState_GET()->config.use_peg;
PyArena *arena = PyArena_New();
if (arena == NULL)
return NULL;
- mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
+ if (use_peg) {
+ mod = PyPegen_ASTFromStringObject(str, filename, start, flags, arena);
+ }
+ else {
+ mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
+ }
if (mod == NULL) {
PyArena_Free(arena);
return NULL;
@@ -1297,13 +1329,19 @@
{
struct symtable *st;
mod_ty mod;
+ int use_peg = _PyInterpreterState_GET()->config.use_peg;
PyArena *arena;
arena = PyArena_New();
if (arena == NULL)
return NULL;
- mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
+ if (use_peg) {
+ mod = PyPegen_ASTFromStringObject(str, filename, start, flags, arena);
+ }
+ else {
+ mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
+ }
if (mod == NULL) {
PyArena_Free(arena);
return NULL;
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 92ea5e7..cf3ddff 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -2427,6 +2427,7 @@
{"inspect", "-i"},
{"interactive", "-i"},
{"optimize", "-O or -OO"},
+ {"use_peg", "-p old or -p new"},
{"dont_write_bytecode", "-B"},
{"no_user_site", "-s"},
{"no_site", "-S"},
@@ -2447,7 +2448,7 @@
"sys.flags", /* name */
flags__doc__, /* doc */
flags_fields, /* fields */
- 15
+ 16
};
static PyObject*
@@ -2470,6 +2471,7 @@
SetFlag(config->inspect);
SetFlag(config->interactive);
SetFlag(config->optimization_level);
+ SetFlag(config->use_peg);
SetFlag(!config->write_bytecode);
SetFlag(!config->user_site_directory);
SetFlag(!config->site_import);