Patch #1810 by Thomas Lee, reviewed by myself:
allow compiling Python AST objects into code objects
in compile().
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 17d9dfd..4b6b7cf 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -73,12 +73,12 @@
A sum is simple if its types have no fields, e.g.
unaryop = Invert | Not | UAdd | USub
"""
-
for t in sum.types:
if t.fields:
return False
return True
+
class EmitVisitor(asdl.VisitorBase):
"""Visit that emits lines"""
@@ -96,6 +96,7 @@
line = (" " * TABSIZE * depth) + line + "\n"
self.file.write(line)
+
class TypeDefVisitor(EmitVisitor):
def visitModule(self, mod):
for dfn in mod.dfns:
@@ -133,6 +134,7 @@
self.emit(s, depth)
self.emit("", depth)
+
class StructVisitor(EmitVisitor):
"""Visitor to generate typdefs for AST."""
@@ -202,6 +204,7 @@
self.emit("};", depth)
self.emit("", depth)
+
class PrototypeVisitor(EmitVisitor):
"""Generate function prototypes for the .h file"""
@@ -271,6 +274,7 @@
self.emit_function(name, get_c_type(name),
self.get_args(prod.fields), [], union=0)
+
class FunctionVisitor(PrototypeVisitor):
"""Visitor to generate constructor functions for AST."""
@@ -325,6 +329,7 @@
emit("p->%s = %s;" % (argname, argname), 1)
assert not attrs
+
class PickleVisitor(EmitVisitor):
def visitModule(self, mod):
@@ -346,6 +351,181 @@
def visitField(self, sum):
pass
+
+class Obj2ModPrototypeVisitor(PickleVisitor):
+ def visitProduct(self, prod, name):
+ code = "static int obj2ast_%s(PyObject* obj, %s* out, PyArena* arena);"
+ self.emit(code % (name, get_c_type(name)), 0)
+
+ visitSum = visitProduct
+
+
+class Obj2ModVisitor(PickleVisitor):
+ def funcHeader(self, name):
+ ctype = get_c_type(name)
+ self.emit("int", 0)
+ self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)
+ self.emit("{", 0)
+ self.emit("PyObject* tmp = NULL;", 1)
+ self.emit("", 0)
+
+ def sumTrailer(self, name):
+ self.emit("", 0)
+ self.emit("tmp = PyObject_Repr(obj);", 1)
+ # there's really nothing more we can do if this fails ...
+ self.emit("if (tmp == NULL) goto failed;", 1)
+ error = "expected some sort of %s, but got %%.400s" % name
+ format = "PyErr_Format(PyExc_TypeError, \"%s\", PyString_AS_STRING(tmp));"
+ self.emit(format % error, 1, reflow=False)
+ self.emit("failed:", 0)
+ self.emit("Py_XDECREF(tmp);", 1)
+ self.emit("return 1;", 1)
+ self.emit("}", 0)
+ self.emit("", 0)
+
+ def simpleSum(self, sum, name):
+ self.funcHeader(name)
+ for t in sum.types:
+ self.emit("if (PyObject_IsInstance(obj, (PyObject*)%s_type)) {" % t.name, 1)
+ self.emit("*out = %s;" % t.name, 2)
+ self.emit("return 0;", 2)
+ self.emit("}", 1)
+ self.sumTrailer(name)
+
+ def buildArgs(self, fields):
+ return ", ".join(fields + ["arena"])
+
+ def complexSum(self, sum, name):
+ self.funcHeader(name)
+ for a in sum.attributes:
+ self.visitAttributeDeclaration(a, name, sum=sum)
+ self.emit("", 0)
+ # XXX: should we only do this for 'expr'?
+ self.emit("if (obj == Py_None) {", 1)
+ self.emit("*out = NULL;", 2)
+ self.emit("return 0;", 2)
+ self.emit("}", 1)
+ for a in sum.attributes:
+ self.visitField(a, name, sum=sum, depth=1)
+ for t in sum.types:
+ self.emit("if (PyObject_IsInstance(obj, (PyObject*)%s_type)) {" % t.name, 1)
+ for f in t.fields:
+ self.visitFieldDeclaration(f, t.name, sum=sum, depth=2)
+ self.emit("", 0)
+ for f in t.fields:
+ self.visitField(f, t.name, sum=sum, depth=2)
+ args = [f.name.value for f in t.fields] + [a.name.value for a in sum.attributes]
+ self.emit("*out = %s(%s);" % (t.name, self.buildArgs(args)), 2)
+ self.emit("if (*out == NULL) goto failed;", 2)
+ self.emit("return 0;", 2)
+ self.emit("}", 1)
+ self.sumTrailer(name)
+
+ def visitAttributeDeclaration(self, a, name, sum=sum):
+ ctype = get_c_type(a.type)
+ self.emit("%s %s;" % (ctype, a.name), 1)
+
+ def visitSum(self, sum, name):
+ if is_simple(sum):
+ self.simpleSum(sum, name)
+ else:
+ self.complexSum(sum, name)
+
+ def visitProduct(self, prod, name):
+ ctype = get_c_type(name)
+ self.emit("int", 0)
+ self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)
+ self.emit("{", 0)
+ self.emit("PyObject* tmp = NULL;", 1)
+ for f in prod.fields:
+ self.visitFieldDeclaration(f, name, prod=prod, depth=1)
+ self.emit("", 0)
+ for f in prod.fields:
+ self.visitField(f, name, prod=prod, depth=1)
+ args = [f.name.value for f in prod.fields]
+ self.emit("*out = %s(%s);" % (name, self.buildArgs(args)), 1)
+ self.emit("return 0;", 1)
+ self.emit("failed:", 0)
+ self.emit("Py_XDECREF(tmp);", 1)
+ self.emit("return 1;", 1)
+ self.emit("}", 0)
+ self.emit("", 0)
+
+ def visitFieldDeclaration(self, field, name, sum=None, prod=None, depth=0):
+ ctype = get_c_type(field.type)
+ if field.seq:
+ if self.isSimpleType(field):
+ self.emit("asdl_int_seq* %s;" % field.name, depth)
+ else:
+ self.emit("asdl_seq* %s;" % field.name, depth)
+ else:
+ ctype = get_c_type(field.type)
+ self.emit("%s %s;" % (ctype, field.name), depth)
+
+ def isSimpleSum(self, field):
+ # XXX can the members of this list be determined automatically?
+ return field.type.value in ('expr_context', 'boolop', 'operator',
+ 'unaryop', 'cmpop')
+
+ def isNumeric(self, field):
+ return get_c_type(field.type) in ("int", "bool")
+
+ def isSimpleType(self, field):
+ return self.isSimpleSum(field) or self.isNumeric(field)
+
+ def visitField(self, field, name, sum=None, prod=None, depth=0):
+ ctype = get_c_type(field.type)
+ self.emit("if (PyObject_HasAttrString(obj, \"%s\")) {" % field.name, depth)
+ self.emit("int res;", depth+1)
+ if field.seq:
+ self.emit("Py_ssize_t len;", depth+1)
+ self.emit("Py_ssize_t i;", depth+1)
+ self.emit("tmp = PyObject_GetAttrString(obj, \"%s\");" % field.name, depth+1)
+ self.emit("if (tmp == NULL) goto failed;", depth+1)
+ if field.seq:
+ self.emit("if (!PyList_Check(tmp)) {", depth+1)
+ self.emit("PyErr_Format(PyExc_TypeError, \"%s field \\\"%s\\\" must "
+ "be a list, not a %%.200s\", tmp->ob_type->tp_name);" %
+ (name, field.name),
+ depth+2, reflow=False)
+ self.emit("goto failed;", depth+2)
+ self.emit("}", depth+1)
+ self.emit("len = PyList_GET_SIZE(tmp);", depth+1)
+ if self.isSimpleType(field):
+ self.emit("%s = asdl_int_seq_new(len, arena);" % field.name, depth+1)
+ else:
+ self.emit("%s = asdl_seq_new(len, arena);" % field.name, depth+1)
+ self.emit("if (%s == NULL) goto failed;" % field.name, depth+1)
+ self.emit("for (i = 0; i < len; i++) {", depth+1)
+ self.emit("%s value;" % ctype, depth+2)
+ self.emit("res = obj2ast_%s(PyList_GET_ITEM(tmp, i), &value, arena);" %
+ field.type, depth+2, reflow=False)
+ self.emit("if (res != 0) goto failed;", depth+2)
+ self.emit("asdl_seq_SET(%s, i, value);" % field.name, depth+2)
+ self.emit("}", depth+1)
+ else:
+ self.emit("res = obj2ast_%s(tmp, &%s, arena);" %
+ (field.type, field.name), depth+1)
+ self.emit("if (res != 0) goto failed;", depth+1)
+
+ self.emit("Py_XDECREF(tmp);", depth+1)
+ self.emit("tmp = NULL;", depth+1)
+ self.emit("} else {", depth)
+ if not field.opt:
+ message = "required field \\\"%s\\\" missing from %s" % (field.name, name)
+ format = "PyErr_SetString(PyExc_TypeError, \"%s\");"
+ self.emit(format % message, depth+1, reflow=False)
+ self.emit("return 1;", depth+1)
+ else:
+ if self.isNumeric(field):
+ self.emit("%s = 0;" % field.name, depth+1)
+ elif not self.isSimpleType(field):
+ self.emit("%s = NULL;" % field.name, depth+1)
+ else:
+ raise TypeError("could not determine the default value for %s" % field.name)
+ self.emit("}", depth)
+
+
class MarshalPrototypeVisitor(PickleVisitor):
def prototype(self, sum, name):
@@ -355,6 +535,7 @@
visitProduct = visitSum = prototype
+
class PyTypesDeclareVisitor(PickleVisitor):
def visitProduct(self, prod, name):
@@ -440,6 +621,8 @@
return result;
}
+/* Conversion AST -> Python */
+
static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
{
int i, n = asdl_seq_LEN(seq);
@@ -476,6 +659,57 @@
{
return PyInt_FromLong(b);
}
+
+/* Conversion Python -> AST */
+
+static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena)
+{
+ if (obj == Py_None)
+ obj = NULL;
+ if (obj)
+ PyArena_AddPyObject(arena, obj);
+ Py_XINCREF(obj);
+ *out = obj;
+ return 0;
+}
+
+#define obj2ast_identifier obj2ast_object
+#define obj2ast_string obj2ast_object
+
+static int obj2ast_int(PyObject* obj, int* out, PyArena* arena)
+{
+ int i;
+ if (!PyInt_Check(obj) && !PyLong_Check(obj)) {
+ PyObject *s = PyObject_Repr(obj);
+ if (s == NULL) return 1;
+ PyErr_Format(PyExc_ValueError, "invalid integer value: %.400s",
+ PyString_AS_STRING(s));
+ Py_DECREF(s);
+ return 1;
+ }
+
+ i = (int)PyLong_AsLong(obj);
+ if (i == -1 && PyErr_Occurred())
+ return 1;
+ *out = i;
+ return 0;
+}
+
+static int obj2ast_bool(PyObject* obj, bool* out, PyArena* arena)
+{
+ if (!PyBool_Check(obj)) {
+ PyObject *s = PyObject_Repr(obj);
+ if (s == NULL) return 1;
+ PyErr_Format(PyExc_ValueError, "invalid boolean value: %.400s",
+ PyString_AS_STRING(s));
+ Py_DECREF(s);
+ return 1;
+ }
+
+ *out = (obj == Py_True);
+ return 0;
+}
+
""", 0, reflow=False)
self.emit("static int init_types(void)",0)
@@ -523,6 +757,7 @@
(cons.name, cons.name), 1)
self.emit("if (!%s_singleton) return 0;" % cons.name, 1)
+
def parse_version(mod):
return mod.version.value[12:-3]
@@ -562,6 +797,7 @@
def addObj(self, name):
self.emit('if (PyDict_SetItemString(d, "%s", (PyObject*)%s_type) < 0) return;' % (name, name), 1)
+
_SPECIALIZED_SEQUENCES = ('stmt', 'expr')
def find_sequence(fields, doing_specialization):
@@ -587,6 +823,7 @@
def visit(self, object):
self.emit(self.CODE, 0, reflow=False)
+
class ObjVisitor(PickleVisitor):
def func_begin(self, name):
@@ -637,8 +874,12 @@
self.emit("case %s:" % t.name, 2)
self.emit("Py_INCREF(%s_singleton);" % t.name, 3)
self.emit("return %s_singleton;" % t.name, 3)
+ self.emit("default:" % name, 2)
+ self.emit('/* should never happen, but just in case ... */', 3)
+ code = "PyErr_Format(PyExc_SystemError, \"unknown %s found\");" % name
+ self.emit(code, 3, reflow=False)
+ self.emit("return NULL;", 3)
self.emit("}", 1)
- self.emit("return NULL; /* cannot happen */", 1)
self.emit("}", 0)
def visitProduct(self, prod, name):
@@ -712,6 +953,27 @@
init_types();
return ast2obj_mod(t);
}
+
+mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena)
+{
+ mod_ty res;
+ init_types();
+ if (!PyObject_IsInstance(ast, mod_type)) {
+ PyErr_SetString(PyExc_TypeError, "expected either Module, Interactive "
+ "or Expression node");
+ return NULL;
+ }
+ if (obj2ast_mod(ast, &res, arena) != 0)
+ return NULL;
+ else
+ return res;
+}
+
+int PyAST_Check(PyObject* obj)
+{
+ init_types();
+ return PyObject_IsInstance(obj, (PyObject*)AST_type);
+}
"""
class ChainOfVisitors:
@@ -754,6 +1016,8 @@
)
c.visit(mod)
print >>f, "PyObject* PyAST_mod2obj(mod_ty t);"
+ print >>f, "mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena);"
+ print >>f, "int PyAST_Check(PyObject* obj);"
f.close()
if SRC_DIR:
@@ -768,8 +1032,10 @@
v = ChainOfVisitors(
PyTypesDeclareVisitor(f),
PyTypesVisitor(f),
+ Obj2ModPrototypeVisitor(f),
FunctionVisitor(f),
ObjVisitor(f),
+ Obj2ModVisitor(f),
ASTModuleVisitor(f),
PartingShots(f),
)