Add ast.Constant
Issue #26146: Add a new kind of AST node: ast.Constant. It can be used by
external AST optimizers, but the compiler does not emit directly such node.
An optimizer can replace the following AST nodes with ast.Constant:
* ast.NameConstant: None, False, True
* ast.Num: int, float, complex
* ast.Str: str
* ast.Bytes: bytes
* ast.Tuple if items are constants too: tuple
* frozenset
Update code to accept ast.Constant instead of ast.Num and/or ast.Str:
* compiler
* docstrings
* ast.literal_eval()
* Tools/parser/unparse.py
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index 07d9b3e..4dde11f 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -306,6 +306,10 @@
"value",
};
static PyTypeObject *Ellipsis_type;
+static PyTypeObject *Constant_type;
+static char *Constant_fields[]={
+ "value",
+};
static PyTypeObject *Attribute_type;
_Py_IDENTIFIER(attr);
_Py_IDENTIFIER(ctx);
@@ -709,6 +713,7 @@
return (PyObject*)o;
}
#define ast2obj_singleton ast2obj_object
+#define ast2obj_constant ast2obj_object
#define ast2obj_identifier ast2obj_object
#define ast2obj_string ast2obj_object
#define ast2obj_bytes ast2obj_object
@@ -746,6 +751,26 @@
return 0;
}
+static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena)
+{
+ if (obj == Py_None || obj == Py_True || obj == Py_False) {
+ /* don't increment the reference counter, Constant uses a borrowed
+ * reference, not a strong reference */
+ *out = obj;
+ return 0;
+ }
+
+ if (obj) {
+ if (PyArena_AddPyObject(arena, obj) < 0) {
+ *out = NULL;
+ return -1;
+ }
+ Py_INCREF(obj);
+ }
+ *out = obj;
+ return 0;
+}
+
static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena)
{
if (!PyUnicode_CheckExact(obj) && obj != Py_None) {
@@ -941,6 +966,8 @@
if (!NameConstant_type) return 0;
Ellipsis_type = make_type("Ellipsis", expr_type, NULL, 0);
if (!Ellipsis_type) return 0;
+ Constant_type = make_type("Constant", expr_type, Constant_fields, 1);
+ if (!Constant_type) return 0;
Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3);
if (!Attribute_type) return 0;
Subscript_type = make_type("Subscript", expr_type, Subscript_fields, 3);
@@ -2167,6 +2194,25 @@
}
expr_ty
+Constant(constant value, int lineno, int col_offset, PyArena *arena)
+{
+ expr_ty p;
+ if (!value) {
+ PyErr_SetString(PyExc_ValueError,
+ "field value is required for Constant");
+ return NULL;
+ }
+ p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
+ if (!p)
+ return NULL;
+ p->kind = Constant_kind;
+ p->v.Constant.value = value;
+ p->lineno = lineno;
+ p->col_offset = col_offset;
+ return p;
+}
+
+expr_ty
Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int
col_offset, PyArena *arena)
{
@@ -3267,6 +3313,15 @@
result = PyType_GenericNew(Ellipsis_type, NULL, NULL);
if (!result) goto failed;
break;
+ case Constant_kind:
+ result = PyType_GenericNew(Constant_type, NULL, NULL);
+ if (!result) goto failed;
+ value = ast2obj_constant(o->v.Constant.value);
+ if (!value) goto failed;
+ if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+ goto failed;
+ Py_DECREF(value);
+ break;
case Attribute_kind:
result = PyType_GenericNew(Attribute_type, NULL, NULL);
if (!result) goto failed;
@@ -6240,6 +6295,28 @@
if (*out == NULL) goto failed;
return 0;
}
+ isinstance = PyObject_IsInstance(obj, (PyObject*)Constant_type);
+ if (isinstance == -1) {
+ return 1;
+ }
+ if (isinstance) {
+ constant value;
+
+ if (_PyObject_HasAttrId(obj, &PyId_value)) {
+ int res;
+ tmp = _PyObject_GetAttrId(obj, &PyId_value);
+ if (tmp == NULL) goto failed;
+ res = obj2ast_constant(tmp, &value, arena);
+ if (res != 0) goto failed;
+ Py_CLEAR(tmp);
+ } else {
+ PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from Constant");
+ return 1;
+ }
+ *out = Constant(value, lineno, col_offset, arena);
+ if (*out == NULL) goto failed;
+ return 0;
+ }
isinstance = PyObject_IsInstance(obj, (PyObject*)Attribute_type);
if (isinstance == -1) {
return 1;
@@ -7517,6 +7594,8 @@
0) return NULL;
if (PyDict_SetItemString(d, "Ellipsis", (PyObject*)Ellipsis_type) < 0)
return NULL;
+ if (PyDict_SetItemString(d, "Constant", (PyObject*)Constant_type) < 0)
+ return NULL;
if (PyDict_SetItemString(d, "Attribute", (PyObject*)Attribute_type) < 0)
return NULL;
if (PyDict_SetItemString(d, "Subscript", (PyObject*)Subscript_type) < 0)
diff --git a/Python/ast.c b/Python/ast.c
index a5d8dba..5422e9c 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -132,6 +132,50 @@
}
static int
+validate_constant(PyObject *value)
+{
+ if (value == Py_None || value == Py_Ellipsis)
+ return 1;
+
+ if (PyLong_CheckExact(value)
+ || PyFloat_CheckExact(value)
+ || PyComplex_CheckExact(value)
+ || PyBool_Check(value)
+ || PyUnicode_CheckExact(value)
+ || PyBytes_CheckExact(value))
+ return 1;
+
+ if (PyTuple_CheckExact(value) || PyFrozenSet_CheckExact(value)) {
+ PyObject *it;
+
+ it = PyObject_GetIter(value);
+ if (it == NULL)
+ return 0;
+
+ while (1) {
+ PyObject *item = PyIter_Next(it);
+ if (item == NULL) {
+ if (PyErr_Occurred()) {
+ Py_DECREF(it);
+ return 0;
+ }
+ break;
+ }
+
+ if (!validate_constant(item)) {
+ Py_DECREF(it);
+ return 0;
+ }
+ }
+
+ Py_DECREF(it);
+ return 1;
+ }
+
+ return 0;
+}
+
+static int
validate_expr(expr_ty exp, expr_context_ty ctx)
{
int check_ctx = 1;
@@ -240,6 +284,12 @@
return validate_expr(exp->v.Call.func, Load) &&
validate_exprs(exp->v.Call.args, Load, 0) &&
validate_keywords(exp->v.Call.keywords);
+ case Constant_kind:
+ if (!validate_constant(exp->v.Constant.value)) {
+ PyErr_SetString(PyExc_TypeError, "invalid type in Constant");
+ return 0;
+ }
+ return 1;
case Num_kind: {
PyObject *n = exp->v.Num.n;
if (!PyLong_CheckExact(n) && !PyFloat_CheckExact(n) &&
diff --git a/Python/compile.c b/Python/compile.c
index a710e82..ccb05cf 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -1314,7 +1314,11 @@
{
if (s->kind != Expr_kind)
return 0;
- return s->v.Expr.value->kind == Str_kind;
+ if (s->v.Expr.value->kind == Str_kind)
+ return 1;
+ if (s->v.Expr.value->kind == Constant_kind)
+ return PyUnicode_CheckExact(s->v.Expr.value->v.Constant.value);
+ return 0;
}
/* Compile a sequence of statements, checking for a docstring. */
@@ -1688,8 +1692,12 @@
st = (stmt_ty)asdl_seq_GET(body, 0);
docstring = compiler_isdocstring(st);
- if (docstring && c->c_optimize < 2)
- first_const = st->v.Expr.value->v.Str.s;
+ if (docstring && c->c_optimize < 2) {
+ if (st->v.Expr.value->kind == Constant_kind)
+ first_const = st->v.Expr.value->v.Constant.value;
+ else
+ first_const = st->v.Expr.value->v.Str.s;
+ }
if (compiler_add_o(c, c->u->u_consts, first_const) < 0) {
compiler_exit_scope(c);
return 0;
@@ -2600,6 +2608,36 @@
}
static int
+compiler_visit_stmt_expr(struct compiler *c, expr_ty value)
+{
+ if (c->c_interactive && c->c_nestlevel <= 1) {
+ VISIT(c, expr, value);
+ ADDOP(c, PRINT_EXPR);
+ return 1;
+ }
+
+ if (value->kind == Str_kind || value->kind == Num_kind) {
+ /* ignore strings and numbers */
+ return 1;
+ }
+
+ if (value->kind == Constant_kind) {
+ PyObject *cst = value->v.Constant.value;
+ if (PyUnicode_CheckExact(cst)
+ || PyLong_CheckExact(cst)
+ || PyFloat_CheckExact(cst)
+ || PyComplex_CheckExact(cst)) {
+ /* ignore strings and numbers */
+ return 1;
+ }
+ }
+
+ VISIT(c, expr, value);
+ ADDOP(c, POP_TOP);
+ return 1;
+}
+
+static int
compiler_visit_stmt(struct compiler *c, stmt_ty s)
{
Py_ssize_t i, n;
@@ -2669,16 +2707,7 @@
case Nonlocal_kind:
break;
case Expr_kind:
- if (c->c_interactive && c->c_nestlevel <= 1) {
- VISIT(c, expr, s->v.Expr.value);
- ADDOP(c, PRINT_EXPR);
- }
- else if (s->v.Expr.value->kind != Str_kind &&
- s->v.Expr.value->kind != Num_kind) {
- VISIT(c, expr, s->v.Expr.value);
- ADDOP(c, POP_TOP);
- }
- break;
+ return compiler_visit_stmt_expr(c, s->v.Expr.value);
case Pass_kind:
break;
case Break_kind:
@@ -3625,6 +3654,8 @@
switch (e->kind) {
case Ellipsis_kind:
return 1;
+ case Constant_kind:
+ return PyObject_IsTrue(e->v.Constant.value);
case Num_kind:
return PyObject_IsTrue(e->v.Num.n);
case Str_kind:
@@ -3912,6 +3943,9 @@
return compiler_compare(c, e);
case Call_kind:
return compiler_call(c, e);
+ case Constant_kind:
+ ADDOP_O(c, LOAD_CONST, e->v.Constant.value, consts);
+ break;
case Num_kind:
ADDOP_O(c, LOAD_CONST, e->v.Num.n, consts);
break;
diff --git a/Python/future.c b/Python/future.c
index 163f87f..75f2107 100644
--- a/Python/future.c
+++ b/Python/future.c
@@ -79,7 +79,10 @@
i = 0;
first = (stmt_ty)asdl_seq_GET(mod->v.Module.body, i);
- if (first->kind == Expr_kind && first->v.Expr.value->kind == Str_kind)
+ if (first->kind == Expr_kind
+ && (first->v.Expr.value->kind == Str_kind
+ || (first->v.Expr.value->kind == Constant_kind
+ && PyUnicode_CheckExact(first->v.Expr.value->v.Constant.value))))
i++;
diff --git a/Python/symtable.c b/Python/symtable.c
index 9f5149f..558bb06 100644
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -1455,6 +1455,7 @@
case JoinedStr_kind:
VISIT_SEQ(st, expr, e->v.JoinedStr.values);
break;
+ case Constant_kind:
case Num_kind:
case Str_kind:
case Bytes_kind: