blob: e81506cc9a62d9773e76523ed9a9c65abefcb60b [file] [log] [blame]
Benjamin Petersonf51d36a2011-12-29 12:07:21 -06001#! /usr/bin/env python
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00002"""Generate C code from an ASDL description."""
3
Benjamin Peterson5a3f49b2011-08-11 14:42:28 -05004import os, sys
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00005
6import asdl
7
Victor Stinnerce72e1c2013-07-27 00:00:36 +02008TABSIZE = 4
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00009MAX_COL = 80
10
11def get_c_type(name):
12 """Return a string for the C name of the type.
13
Eli Bendersky5e3d3382014-05-09 17:58:22 -070014 This function special cases the default types provided by asdl.
Jeremy Hylton3e0055f2005-10-20 19:59:25 +000015 """
Jeremy Hylton3e0055f2005-10-20 19:59:25 +000016 if name in asdl.builtin_types:
17 return name
18 else:
19 return "%s_ty" % name
20
21def reflow_lines(s, depth):
22 """Reflow the line s indented depth tabs.
23
24 Return a sequence of lines where no line extends beyond MAX_COL
25 when properly indented. The first line is properly indented based
26 exclusively on depth * TABSIZE. All following lines -- these are
27 the reflowed lines generated by this function -- start at the same
28 column as the first character beyond the opening { in the first
29 line.
30 """
31 size = MAX_COL - depth * TABSIZE
32 if len(s) < size:
33 return [s]
34
35 lines = []
36 cur = s
37 padding = ""
38 while len(cur) > size:
39 i = cur.rfind(' ', 0, size)
40 # XXX this should be fixed for real
41 if i == -1 and 'GeneratorExp' in cur:
42 i = size + 3
Brett Cannonf0365512006-08-25 04:06:31 +000043 assert i != -1, "Impossible line %d to reflow: %r" % (size, s)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +000044 lines.append(padding + cur[:i])
45 if len(lines) == 1:
46 # find new size based on brace
47 j = cur.find('{', 0, i)
48 if j >= 0:
49 j += 2 # account for the brace and the space after it
50 size -= j
51 padding = " " * j
52 else:
53 j = cur.find('(', 0, i)
54 if j >= 0:
55 j += 1 # account for the paren (no space after it)
56 size -= j
57 padding = " " * j
58 cur = cur[i+1:]
59 else:
60 lines.append(padding + cur)
61 return lines
62
63def is_simple(sum):
64 """Return True if a sum is a simple.
65
66 A sum is simple if its types have no fields, e.g.
67 unaryop = Invert | Not | UAdd | USub
68 """
Jeremy Hylton3e0055f2005-10-20 19:59:25 +000069 for t in sum.types:
70 if t.fields:
71 return False
72 return True
73
Martin v. Löwis618dc5e2008-03-30 20:03:44 +000074
Jeremy Hylton3e0055f2005-10-20 19:59:25 +000075class EmitVisitor(asdl.VisitorBase):
76 """Visit that emits lines"""
77
78 def __init__(self, file):
79 self.file = file
Martin v. Löwis1ee1b6f2011-10-10 18:11:30 +020080 self.identifiers = set()
Dino Viehlandac46eb42019-09-11 10:16:34 -070081 self.singletons = set()
82 self.types = set()
Jeremy Hylton3e0055f2005-10-20 19:59:25 +000083 super(EmitVisitor, self).__init__()
84
Martin v. Löwis1ee1b6f2011-10-10 18:11:30 +020085 def emit_identifier(self, name):
Dino Viehlandac46eb42019-09-11 10:16:34 -070086 self.identifiers.add(str(name))
87
88 def emit_singleton(self, name):
89 self.singletons.add(str(name))
90
91 def emit_type(self, name):
92 self.types.add(str(name))
Martin v. Löwis1ee1b6f2011-10-10 18:11:30 +020093
Benjamin Peterson87c8d872009-06-11 22:54:11 +000094 def emit(self, s, depth, reflow=True):
Jeremy Hylton3e0055f2005-10-20 19:59:25 +000095 # XXX reflow long lines?
96 if reflow:
97 lines = reflow_lines(s, depth)
98 else:
99 lines = [s]
100 for line in lines:
Serhiy Storchaka5d84cb32017-09-15 06:28:22 +0300101 if line:
102 line = (" " * TABSIZE * depth) + line
103 self.file.write(line + "\n")
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000104
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000105
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000106class TypeDefVisitor(EmitVisitor):
107 def visitModule(self, mod):
108 for dfn in mod.dfns:
109 self.visit(dfn)
110
111 def visitType(self, type, depth=0):
112 self.visit(type.value, type.name, depth)
113
114 def visitSum(self, sum, name, depth):
115 if is_simple(sum):
116 self.simple_sum(sum, name, depth)
117 else:
118 self.sum_with_constructors(sum, name, depth)
119
120 def simple_sum(self, sum, name, depth):
121 enum = []
122 for i in range(len(sum.types)):
123 type = sum.types[i]
124 enum.append("%s=%d" % (type.name, i + 1))
125 enums = ", ".join(enum)
126 ctype = get_c_type(name)
127 s = "typedef enum _%s { %s } %s;" % (name, enums, ctype)
128 self.emit(s, depth)
129 self.emit("", depth)
130
131 def sum_with_constructors(self, sum, name, depth):
132 ctype = get_c_type(name)
133 s = "typedef struct _%(name)s *%(ctype)s;" % locals()
134 self.emit(s, depth)
135 self.emit("", depth)
136
137 def visitProduct(self, product, name, depth):
138 ctype = get_c_type(name)
139 s = "typedef struct _%(name)s *%(ctype)s;" % locals()
140 self.emit(s, depth)
141 self.emit("", depth)
142
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000143
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000144class StructVisitor(EmitVisitor):
Eli Bendersky5e3d3382014-05-09 17:58:22 -0700145 """Visitor to generate typedefs for AST."""
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000146
147 def visitModule(self, mod):
148 for dfn in mod.dfns:
149 self.visit(dfn)
150
151 def visitType(self, type, depth=0):
152 self.visit(type.value, type.name, depth)
153
154 def visitSum(self, sum, name, depth):
155 if not is_simple(sum):
156 self.sum_with_constructors(sum, name, depth)
157
158 def sum_with_constructors(self, sum, name, depth):
159 def emit(s, depth=depth):
160 self.emit(s % sys._getframe(1).f_locals, depth)
161 enum = []
162 for i in range(len(sum.types)):
163 type = sum.types[i]
164 enum.append("%s_kind=%d" % (type.name, i + 1))
165
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000166 emit("enum _%(name)s_kind {" + ", ".join(enum) + "};")
167
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000168 emit("struct _%(name)s {")
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000169 emit("enum _%(name)s_kind kind;", depth + 1)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000170 emit("union {", depth + 1)
171 for t in sum.types:
172 self.visit(t, depth + 2)
173 emit("} v;", depth + 1)
174 for field in sum.attributes:
175 # rudimentary attribute handling
176 type = str(field.type)
177 assert type in asdl.builtin_types, type
178 emit("%s %s;" % (type, field.name), depth + 1);
179 emit("};")
180 emit("")
181
182 def visitConstructor(self, cons, depth):
183 if cons.fields:
184 self.emit("struct {", depth)
185 for f in cons.fields:
186 self.visit(f, depth + 1)
187 self.emit("} %s;" % cons.name, depth)
188 self.emit("", depth)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000189
190 def visitField(self, field, depth):
191 # XXX need to lookup field.type, because it might be something
192 # like a builtin...
193 ctype = get_c_type(field.type)
194 name = field.name
195 if field.seq:
Eli Bendersky5e3d3382014-05-09 17:58:22 -0700196 if field.type == 'cmpop':
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000197 self.emit("asdl_int_seq *%(name)s;" % locals(), depth)
198 else:
199 self.emit("asdl_seq *%(name)s;" % locals(), depth)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000200 else:
201 self.emit("%(ctype)s %(name)s;" % locals(), depth)
202
203 def visitProduct(self, product, name, depth):
204 self.emit("struct _%(name)s {" % locals(), depth)
205 for f in product.fields:
206 self.visit(f, depth + 1)
Benjamin Petersoncda75be2013-03-18 10:48:58 -0700207 for field in product.attributes:
208 # rudimentary attribute handling
209 type = str(field.type)
210 assert type in asdl.builtin_types, type
211 self.emit("%s %s;" % (type, field.name), depth + 1);
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000212 self.emit("};", depth)
213 self.emit("", depth)
214
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000215
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000216class PrototypeVisitor(EmitVisitor):
217 """Generate function prototypes for the .h file"""
218
219 def visitModule(self, mod):
220 for dfn in mod.dfns:
221 self.visit(dfn)
222
223 def visitType(self, type):
224 self.visit(type.value, type.name)
225
226 def visitSum(self, sum, name):
227 if is_simple(sum):
228 pass # XXX
229 else:
230 for t in sum.types:
231 self.visit(t, name, sum.attributes)
232
233 def get_args(self, fields):
234 """Return list of C argument into, one for each field.
235
236 Argument info is 3-tuple of a C type, variable name, and flag
237 that is true if type can be NULL.
238 """
239 args = []
240 unnamed = {}
241 for f in fields:
242 if f.name is None:
243 name = f.type
244 c = unnamed[name] = unnamed.get(name, 0) + 1
245 if c > 1:
246 name = "name%d" % (c - 1)
247 else:
248 name = f.name
249 # XXX should extend get_c_type() to handle this
250 if f.seq:
Eli Bendersky5e3d3382014-05-09 17:58:22 -0700251 if f.type == 'cmpop':
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000252 ctype = "asdl_int_seq *"
253 else:
254 ctype = "asdl_seq *"
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000255 else:
256 ctype = get_c_type(f.type)
257 args.append((ctype, name, f.opt or f.seq))
258 return args
259
260 def visitConstructor(self, cons, type, attrs):
261 args = self.get_args(cons.fields)
262 attrs = self.get_args(attrs)
263 ctype = get_c_type(type)
264 self.emit_function(cons.name, ctype, args, attrs)
265
Benjamin Peterson87c8d872009-06-11 22:54:11 +0000266 def emit_function(self, name, ctype, args, attrs, union=True):
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000267 args = args + attrs
268 if args:
269 argstr = ", ".join(["%s %s" % (atype, aname)
270 for atype, aname, opt in args])
Neal Norwitzadb69fc2005-12-17 20:54:49 +0000271 argstr += ", PyArena *arena"
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000272 else:
Neal Norwitzadb69fc2005-12-17 20:54:49 +0000273 argstr = "PyArena *arena"
Thomas Woutersb2137042007-02-01 18:02:27 +0000274 margs = "a0"
275 for i in range(1, len(args)+1):
276 margs += ", a%d" % i
277 self.emit("#define %s(%s) _Py_%s(%s)" % (name, margs, name, margs), 0,
Benjamin Peterson87c8d872009-06-11 22:54:11 +0000278 reflow=False)
279 self.emit("%s _Py_%s(%s);" % (ctype, name, argstr), False)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000280
281 def visitProduct(self, prod, name):
282 self.emit_function(name, get_c_type(name),
Victor Stinnerc106c682015-11-06 17:01:48 +0100283 self.get_args(prod.fields),
284 self.get_args(prod.attributes),
285 union=False)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000286
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000287
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000288class FunctionVisitor(PrototypeVisitor):
289 """Visitor to generate constructor functions for AST."""
290
Benjamin Peterson87c8d872009-06-11 22:54:11 +0000291 def emit_function(self, name, ctype, args, attrs, union=True):
292 def emit(s, depth=0, reflow=True):
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000293 self.emit(s, depth, reflow)
294 argstr = ", ".join(["%s %s" % (atype, aname)
295 for atype, aname, opt in args + attrs])
Neal Norwitzadb69fc2005-12-17 20:54:49 +0000296 if argstr:
297 argstr += ", PyArena *arena"
298 else:
299 argstr = "PyArena *arena"
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000300 self.emit("%s" % ctype, 0)
301 emit("%s(%s)" % (name, argstr))
302 emit("{")
303 emit("%s p;" % ctype, 1)
304 for argtype, argname, opt in args:
Neal Norwitz3591bbe2007-02-26 19:04:49 +0000305 if not opt and argtype != "int":
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000306 emit("if (!%s) {" % argname, 1)
307 emit("PyErr_SetString(PyExc_ValueError,", 2)
308 msg = "field %s is required for %s" % (argname, name)
309 emit(' "%s");' % msg,
Benjamin Peterson87c8d872009-06-11 22:54:11 +0000310 2, reflow=False)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000311 emit('return NULL;', 2)
312 emit('}', 1)
313
Neal Norwitzadb69fc2005-12-17 20:54:49 +0000314 emit("p = (%s)PyArena_Malloc(arena, sizeof(*p));" % ctype, 1);
Thomas Woutersa44f3a32007-02-26 18:20:15 +0000315 emit("if (!p)", 1)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000316 emit("return NULL;", 2)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000317 if union:
318 self.emit_body_union(name, args, attrs)
319 else:
320 self.emit_body_struct(name, args, attrs)
321 emit("return p;", 1)
322 emit("}")
323 emit("")
324
325 def emit_body_union(self, name, args, attrs):
Benjamin Peterson87c8d872009-06-11 22:54:11 +0000326 def emit(s, depth=0, reflow=True):
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000327 self.emit(s, depth, reflow)
328 emit("p->kind = %s_kind;" % name, 1)
329 for argtype, argname, opt in args:
330 emit("p->v.%s.%s = %s;" % (name, argname, argname), 1)
331 for argtype, argname, opt in attrs:
332 emit("p->%s = %s;" % (argname, argname), 1)
333
334 def emit_body_struct(self, name, args, attrs):
Benjamin Peterson87c8d872009-06-11 22:54:11 +0000335 def emit(s, depth=0, reflow=True):
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000336 self.emit(s, depth, reflow)
337 for argtype, argname, opt in args:
338 emit("p->%s = %s;" % (argname, argname), 1)
Victor Stinnerc106c682015-11-06 17:01:48 +0100339 for argtype, argname, opt in attrs:
340 emit("p->%s = %s;" % (argname, argname), 1)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000341
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000342
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000343class PickleVisitor(EmitVisitor):
344
345 def visitModule(self, mod):
346 for dfn in mod.dfns:
347 self.visit(dfn)
348
349 def visitType(self, type):
350 self.visit(type.value, type.name)
351
352 def visitSum(self, sum, name):
353 pass
354
355 def visitProduct(self, sum, name):
356 pass
357
358 def visitConstructor(self, cons, name):
359 pass
360
361 def visitField(self, sum):
362 pass
363
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000364
365class Obj2ModPrototypeVisitor(PickleVisitor):
366 def visitProduct(self, prod, name):
367 code = "static int obj2ast_%s(PyObject* obj, %s* out, PyArena* arena);"
368 self.emit(code % (name, get_c_type(name)), 0)
369
370 visitSum = visitProduct
371
372
373class Obj2ModVisitor(PickleVisitor):
374 def funcHeader(self, name):
375 ctype = get_c_type(name)
376 self.emit("int", 0)
377 self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)
378 self.emit("{", 0)
Benjamin Peterson0e9e98e2010-11-20 02:01:45 +0000379 self.emit("int isinstance;", 1)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000380 self.emit("", 0)
381
Benjamin Peterson5b066812010-11-20 01:38:49 +0000382 def sumTrailer(self, name, add_label=False):
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000383 self.emit("", 0)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000384 # there's really nothing more we can do if this fails ...
Benjamin Peterson5b066812010-11-20 01:38:49 +0000385 error = "expected some sort of %s, but got %%R" % name
386 format = "PyErr_Format(PyExc_TypeError, \"%s\", obj);"
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000387 self.emit(format % error, 1, reflow=False)
Benjamin Peterson5b066812010-11-20 01:38:49 +0000388 if add_label:
389 self.emit("failed:", 1)
Benjamin Peterson0a4dae52010-11-21 15:12:34 +0000390 self.emit("Py_XDECREF(tmp);", 1)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000391 self.emit("return 1;", 1)
392 self.emit("}", 0)
393 self.emit("", 0)
394
395 def simpleSum(self, sum, name):
396 self.funcHeader(name)
397 for t in sum.types:
Benjamin Peterson97dd9872009-12-13 01:23:39 +0000398 line = ("isinstance = PyObject_IsInstance(obj, "
Dino Viehlandac46eb42019-09-11 10:16:34 -0700399 "astmodulestate_global->%s_type);")
Benjamin Peterson97dd9872009-12-13 01:23:39 +0000400 self.emit(line % (t.name,), 1)
401 self.emit("if (isinstance == -1) {", 1)
402 self.emit("return 1;", 2)
403 self.emit("}", 1)
404 self.emit("if (isinstance) {", 1)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000405 self.emit("*out = %s;" % t.name, 2)
406 self.emit("return 0;", 2)
407 self.emit("}", 1)
408 self.sumTrailer(name)
409
410 def buildArgs(self, fields):
411 return ", ".join(fields + ["arena"])
412
413 def complexSum(self, sum, name):
414 self.funcHeader(name)
Benjamin Petersond8f65972010-11-20 04:31:07 +0000415 self.emit("PyObject *tmp = NULL;", 1)
Dino Viehlandac46eb42019-09-11 10:16:34 -0700416 self.emit("PyObject *tp;", 1)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000417 for a in sum.attributes:
418 self.visitAttributeDeclaration(a, name, sum=sum)
419 self.emit("", 0)
420 # XXX: should we only do this for 'expr'?
421 self.emit("if (obj == Py_None) {", 1)
422 self.emit("*out = NULL;", 2)
423 self.emit("return 0;", 2)
424 self.emit("}", 1)
425 for a in sum.attributes:
426 self.visitField(a, name, sum=sum, depth=1)
427 for t in sum.types:
Dino Viehlandac46eb42019-09-11 10:16:34 -0700428 self.emit("tp = astmodulestate_global->%s_type;" % (t.name,), 1)
429 self.emit("isinstance = PyObject_IsInstance(obj, tp);", 1)
Benjamin Peterson97dd9872009-12-13 01:23:39 +0000430 self.emit("if (isinstance == -1) {", 1)
431 self.emit("return 1;", 2)
432 self.emit("}", 1)
433 self.emit("if (isinstance) {", 1)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000434 for f in t.fields:
435 self.visitFieldDeclaration(f, t.name, sum=sum, depth=2)
436 self.emit("", 0)
437 for f in t.fields:
438 self.visitField(f, t.name, sum=sum, depth=2)
Eli Bendersky5e3d3382014-05-09 17:58:22 -0700439 args = [f.name for f in t.fields] + [a.name for a in sum.attributes]
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000440 self.emit("*out = %s(%s);" % (t.name, self.buildArgs(args)), 2)
441 self.emit("if (*out == NULL) goto failed;", 2)
442 self.emit("return 0;", 2)
443 self.emit("}", 1)
Benjamin Peterson5b066812010-11-20 01:38:49 +0000444 self.sumTrailer(name, True)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000445
446 def visitAttributeDeclaration(self, a, name, sum=sum):
447 ctype = get_c_type(a.type)
448 self.emit("%s %s;" % (ctype, a.name), 1)
449
450 def visitSum(self, sum, name):
451 if is_simple(sum):
452 self.simpleSum(sum, name)
453 else:
454 self.complexSum(sum, name)
455
456 def visitProduct(self, prod, name):
457 ctype = get_c_type(name)
458 self.emit("int", 0)
459 self.emit("obj2ast_%s(PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0)
460 self.emit("{", 0)
461 self.emit("PyObject* tmp = NULL;", 1)
462 for f in prod.fields:
463 self.visitFieldDeclaration(f, name, prod=prod, depth=1)
Victor Stinnerc106c682015-11-06 17:01:48 +0100464 for a in prod.attributes:
465 self.visitFieldDeclaration(a, name, prod=prod, depth=1)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000466 self.emit("", 0)
467 for f in prod.fields:
468 self.visitField(f, name, prod=prod, depth=1)
Victor Stinnerc106c682015-11-06 17:01:48 +0100469 for a in prod.attributes:
470 self.visitField(a, name, prod=prod, depth=1)
Eli Bendersky5e3d3382014-05-09 17:58:22 -0700471 args = [f.name for f in prod.fields]
Victor Stinnerc106c682015-11-06 17:01:48 +0100472 args.extend([a.name for a in prod.attributes])
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000473 self.emit("*out = %s(%s);" % (name, self.buildArgs(args)), 1)
474 self.emit("return 0;", 1)
475 self.emit("failed:", 0)
476 self.emit("Py_XDECREF(tmp);", 1)
477 self.emit("return 1;", 1)
478 self.emit("}", 0)
479 self.emit("", 0)
480
481 def visitFieldDeclaration(self, field, name, sum=None, prod=None, depth=0):
482 ctype = get_c_type(field.type)
483 if field.seq:
484 if self.isSimpleType(field):
485 self.emit("asdl_int_seq* %s;" % field.name, depth)
486 else:
487 self.emit("asdl_seq* %s;" % field.name, depth)
488 else:
489 ctype = get_c_type(field.type)
490 self.emit("%s %s;" % (ctype, field.name), depth)
491
492 def isSimpleSum(self, field):
493 # XXX can the members of this list be determined automatically?
Eli Bendersky5e3d3382014-05-09 17:58:22 -0700494 return field.type in ('expr_context', 'boolop', 'operator',
495 'unaryop', 'cmpop')
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000496
497 def isNumeric(self, field):
498 return get_c_type(field.type) in ("int", "bool")
499
500 def isSimpleType(self, field):
501 return self.isSimpleSum(field) or self.isNumeric(field)
502
503 def visitField(self, field, name, sum=None, prod=None, depth=0):
504 ctype = get_c_type(field.type)
Dino Viehlandac46eb42019-09-11 10:16:34 -0700505 line = "if (_PyObject_LookupAttr(obj, astmodulestate_global->%s, &tmp) < 0) {"
506 self.emit(line % field.name, depth)
Serhiy Storchakaf320be72018-01-25 10:49:40 +0200507 self.emit("return 1;", depth+1)
508 self.emit("}", depth)
Serhiy Storchakabba22392017-11-11 16:41:32 +0200509 if not field.opt:
Serhiy Storchakaf320be72018-01-25 10:49:40 +0200510 self.emit("if (tmp == NULL) {", depth)
511 message = "required field \\\"%s\\\" missing from %s" % (field.name, name)
512 format = "PyErr_SetString(PyExc_TypeError, \"%s\");"
513 self.emit(format % message, depth+1, reflow=False)
514 self.emit("return 1;", depth+1)
Benjamin Petersoncda75be2013-03-18 10:48:58 -0700515 else:
Serhiy Storchakaf320be72018-01-25 10:49:40 +0200516 self.emit("if (tmp == NULL || tmp == Py_None) {", depth)
517 self.emit("Py_CLEAR(tmp);", depth+1)
518 if self.isNumeric(field):
519 self.emit("%s = 0;" % field.name, depth+1)
520 elif not self.isSimpleType(field):
521 self.emit("%s = NULL;" % field.name, depth+1)
522 else:
523 raise TypeError("could not determine the default value for %s" % field.name)
524 self.emit("}", depth)
525 self.emit("else {", depth)
526
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000527 self.emit("int res;", depth+1)
528 if field.seq:
529 self.emit("Py_ssize_t len;", depth+1)
530 self.emit("Py_ssize_t i;", depth+1)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000531 self.emit("if (!PyList_Check(tmp)) {", depth+1)
532 self.emit("PyErr_Format(PyExc_TypeError, \"%s field \\\"%s\\\" must "
Dino Viehlandac46eb42019-09-11 10:16:34 -0700533 "be a list, not a %%.200s\", _PyType_Name(Py_TYPE(tmp)));" %
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000534 (name, field.name),
535 depth+2, reflow=False)
536 self.emit("goto failed;", depth+2)
537 self.emit("}", depth+1)
538 self.emit("len = PyList_GET_SIZE(tmp);", depth+1)
539 if self.isSimpleType(field):
Antoine Pitroud01d396e2013-10-12 22:52:43 +0200540 self.emit("%s = _Py_asdl_int_seq_new(len, arena);" % field.name, depth+1)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000541 else:
Antoine Pitroud01d396e2013-10-12 22:52:43 +0200542 self.emit("%s = _Py_asdl_seq_new(len, arena);" % field.name, depth+1)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000543 self.emit("if (%s == NULL) goto failed;" % field.name, depth+1)
544 self.emit("for (i = 0; i < len; i++) {", depth+1)
Yuan Chao Chou2af565b2017-08-04 10:53:12 -0700545 self.emit("%s val;" % ctype, depth+2)
Serhiy Storchaka43c97312019-09-10 13:02:30 +0300546 self.emit("PyObject *tmp2 = PyList_GET_ITEM(tmp, i);", depth+2)
547 self.emit("Py_INCREF(tmp2);", depth+2)
548 self.emit("res = obj2ast_%s(tmp2, &val, arena);" %
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000549 field.type, depth+2, reflow=False)
Serhiy Storchaka43c97312019-09-10 13:02:30 +0300550 self.emit("Py_DECREF(tmp2);", depth+2)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000551 self.emit("if (res != 0) goto failed;", depth+2)
Serhiy Storchakacf380602016-10-07 21:51:28 +0300552 self.emit("if (len != PyList_GET_SIZE(tmp)) {", depth+2)
553 self.emit("PyErr_SetString(PyExc_RuntimeError, \"%s field \\\"%s\\\" "
554 "changed size during iteration\");" %
555 (name, field.name),
556 depth+3, reflow=False)
557 self.emit("goto failed;", depth+3)
558 self.emit("}", depth+2)
Yuan Chao Chou2af565b2017-08-04 10:53:12 -0700559 self.emit("asdl_seq_SET(%s, i, val);" % field.name, depth+2)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000560 self.emit("}", depth+1)
561 else:
562 self.emit("res = obj2ast_%s(tmp, &%s, arena);" %
563 (field.type, field.name), depth+1)
564 self.emit("if (res != 0) goto failed;", depth+1)
565
Victor Stinner1acc1292013-07-27 00:03:47 +0200566 self.emit("Py_CLEAR(tmp);", depth+1)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000567 self.emit("}", depth)
568
569
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000570class MarshalPrototypeVisitor(PickleVisitor):
571
572 def prototype(self, sum, name):
573 ctype = get_c_type(name)
Neal Norwitz6576bd82005-11-13 18:41:28 +0000574 self.emit("static int marshal_write_%s(PyObject **, int *, %s);"
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000575 % (name, ctype), 0)
576
577 visitProduct = visitSum = prototype
578
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000579
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000580class PyTypesDeclareVisitor(PickleVisitor):
Jeremy Hylton3e0055f2005-10-20 19:59:25 +0000581
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000582 def visitProduct(self, prod, name):
Dino Viehlandac46eb42019-09-11 10:16:34 -0700583 self.emit_type("%s_type" % name)
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000584 self.emit("static PyObject* ast2obj_%s(void*);" % name, 0)
Benjamin Petersoncda75be2013-03-18 10:48:58 -0700585 if prod.attributes:
586 for a in prod.attributes:
587 self.emit_identifier(a.name)
Serhiy Storchaka43c97312019-09-10 13:02:30 +0300588 self.emit("static const char * const %s_attributes[] = {" % name, 0)
Benjamin Petersoncda75be2013-03-18 10:48:58 -0700589 for a in prod.attributes:
590 self.emit('"%s",' % a.name, 1)
591 self.emit("};", 0)
Martin v. Löwis8d0701d2006-02-26 23:40:20 +0000592 if prod.fields:
Martin v. Löwis1ee1b6f2011-10-10 18:11:30 +0200593 for f in prod.fields:
594 self.emit_identifier(f.name)
Serhiy Storchaka43c97312019-09-10 13:02:30 +0300595 self.emit("static const char * const %s_fields[]={" % name,0)
Martin v. Löwis8d0701d2006-02-26 23:40:20 +0000596 for f in prod.fields:
597 self.emit('"%s",' % f.name, 1)
598 self.emit("};", 0)
Tim Peters710ab3b2006-02-28 18:30:36 +0000599
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000600 def visitSum(self, sum, name):
Dino Viehlandac46eb42019-09-11 10:16:34 -0700601 self.emit_type("%s_type" % name)
Martin v. Löwis577b5b92006-02-27 15:23:19 +0000602 if sum.attributes:
Martin v. Löwis1ee1b6f2011-10-10 18:11:30 +0200603 for a in sum.attributes:
604 self.emit_identifier(a.name)
Serhiy Storchaka43c97312019-09-10 13:02:30 +0300605 self.emit("static const char * const %s_attributes[] = {" % name, 0)
Martin v. Löwis577b5b92006-02-27 15:23:19 +0000606 for a in sum.attributes:
607 self.emit('"%s",' % a.name, 1)
608 self.emit("};", 0)
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000609 ptype = "void*"
610 if is_simple(sum):
611 ptype = get_c_type(name)
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000612 for t in sum.types:
Dino Viehlandac46eb42019-09-11 10:16:34 -0700613 self.emit_singleton("%s_singleton" % t.name)
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000614 self.emit("static PyObject* ast2obj_%s(%s);" % (name, ptype), 0)
615 for t in sum.types:
616 self.visitConstructor(t, name)
Tim Peters710ab3b2006-02-28 18:30:36 +0000617
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000618 def visitConstructor(self, cons, name):
Martin v. Löwis8d0701d2006-02-26 23:40:20 +0000619 if cons.fields:
Martin v. Löwis1ee1b6f2011-10-10 18:11:30 +0200620 for t in cons.fields:
621 self.emit_identifier(t.name)
Serhiy Storchaka43c97312019-09-10 13:02:30 +0300622 self.emit("static const char * const %s_fields[]={" % cons.name, 0)
Martin v. Löwis8d0701d2006-02-26 23:40:20 +0000623 for t in cons.fields:
624 self.emit('"%s",' % t.name, 1)
625 self.emit("};",0)
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000626
627class PyTypesVisitor(PickleVisitor):
628
629 def visitModule(self, mod):
630 self.emit("""
INADA Naokifc489082017-01-25 22:33:43 +0900631
Benjamin Peterson7e0dbfb2012-03-12 09:46:44 -0700632typedef struct {
Victor Stinner45e50de2012-03-13 01:17:31 +0100633 PyObject_HEAD
Benjamin Peterson7e0dbfb2012-03-12 09:46:44 -0700634 PyObject *dict;
635} AST_object;
636
Benjamin Peterson1767e022012-03-14 21:50:29 -0500637static void
638ast_dealloc(AST_object *self)
639{
INADA Naokia6296d32017-08-24 14:55:17 +0900640 /* bpo-31095: UnTrack is needed before calling any callbacks */
Eddie Elizondo0247e802019-09-14 09:38:17 -0400641 PyTypeObject *tp = Py_TYPE(self);
INADA Naokia6296d32017-08-24 14:55:17 +0900642 PyObject_GC_UnTrack(self);
Benjamin Peterson1767e022012-03-14 21:50:29 -0500643 Py_CLEAR(self->dict);
Eddie Elizondo0247e802019-09-14 09:38:17 -0400644 freefunc free_func = PyType_GetSlot(tp, Py_tp_free);
645 assert(free_func != NULL);
646 free_func(self);
647 Py_DECREF(tp);
Benjamin Peterson1767e022012-03-14 21:50:29 -0500648}
649
Neal Norwitz207c9f32008-03-31 04:42:11 +0000650static int
Benjamin Peterson81071762012-07-08 11:03:46 -0700651ast_traverse(AST_object *self, visitproc visit, void *arg)
652{
653 Py_VISIT(self->dict);
654 return 0;
655}
656
Serhiy Storchakaa5c42282018-05-31 07:34:34 +0300657static int
Benjamin Peterson81071762012-07-08 11:03:46 -0700658ast_clear(AST_object *self)
659{
660 Py_CLEAR(self->dict);
Serhiy Storchakaa5c42282018-05-31 07:34:34 +0300661 return 0;
Benjamin Peterson81071762012-07-08 11:03:46 -0700662}
663
664static int
Neal Norwitz207c9f32008-03-31 04:42:11 +0000665ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
666{
667 Py_ssize_t i, numfields = 0;
668 int res = -1;
669 PyObject *key, *value, *fields;
Dino Viehlandac46eb42019-09-11 10:16:34 -0700670 if (_PyObject_LookupAttr((PyObject*)Py_TYPE(self), astmodulestate_global->_fields, &fields) < 0) {
Serhiy Storchakaf320be72018-01-25 10:49:40 +0200671 goto cleanup;
672 }
Neal Norwitz207c9f32008-03-31 04:42:11 +0000673 if (fields) {
674 numfields = PySequence_Size(fields);
675 if (numfields == -1)
676 goto cleanup;
677 }
INADA Naoki4c78c522017-02-24 02:48:17 +0900678
Neal Norwitz207c9f32008-03-31 04:42:11 +0000679 res = 0; /* if no error occurs, this stays 0 to the end */
INADA Naoki4c78c522017-02-24 02:48:17 +0900680 if (numfields < PyTuple_GET_SIZE(args)) {
681 PyErr_Format(PyExc_TypeError, "%.400s constructor takes at most "
682 "%zd positional argument%s",
Dino Viehlandac46eb42019-09-11 10:16:34 -0700683 _PyType_Name(Py_TYPE(self)),
INADA Naoki4c78c522017-02-24 02:48:17 +0900684 numfields, numfields == 1 ? "" : "s");
685 res = -1;
686 goto cleanup;
687 }
688 for (i = 0; i < PyTuple_GET_SIZE(args); i++) {
689 /* cannot be reached when fields is NULL */
690 PyObject *name = PySequence_GetItem(fields, i);
691 if (!name) {
Neal Norwitz207c9f32008-03-31 04:42:11 +0000692 res = -1;
693 goto cleanup;
694 }
INADA Naoki4c78c522017-02-24 02:48:17 +0900695 res = PyObject_SetAttr(self, name, PyTuple_GET_ITEM(args, i));
696 Py_DECREF(name);
697 if (res < 0)
698 goto cleanup;
Neal Norwitz207c9f32008-03-31 04:42:11 +0000699 }
700 if (kw) {
701 i = 0; /* needed by PyDict_Next */
702 while (PyDict_Next(kw, &i, &key, &value)) {
703 res = PyObject_SetAttr(self, key, value);
704 if (res < 0)
705 goto cleanup;
706 }
707 }
708 cleanup:
709 Py_XDECREF(fields);
710 return res;
711}
712
Neal Norwitzee9b10a2008-03-31 05:29:39 +0000713/* Pickling support */
714static PyObject *
715ast_type_reduce(PyObject *self, PyObject *unused)
716{
Serhiy Storchakaf320be72018-01-25 10:49:40 +0200717 PyObject *dict;
Dino Viehlandac46eb42019-09-11 10:16:34 -0700718 if (_PyObject_LookupAttr(self, astmodulestate_global->__dict__, &dict) < 0) {
Serhiy Storchakaf320be72018-01-25 10:49:40 +0200719 return NULL;
Neal Norwitzee9b10a2008-03-31 05:29:39 +0000720 }
721 if (dict) {
Serhiy Storchakaf320be72018-01-25 10:49:40 +0200722 return Py_BuildValue("O()N", Py_TYPE(self), dict);
Neal Norwitzee9b10a2008-03-31 05:29:39 +0000723 }
724 return Py_BuildValue("O()", Py_TYPE(self));
725}
726
Eddie Elizondo3368f3c2019-09-19 09:29:05 -0700727static PyMemberDef ast_type_members[] = {
728 {"__dictoffset__", T_PYSSIZET, offsetof(AST_object, dict), READONLY},
729 {NULL} /* Sentinel */
730};
731
Neal Norwitzee9b10a2008-03-31 05:29:39 +0000732static PyMethodDef ast_type_methods[] = {
733 {"__reduce__", ast_type_reduce, METH_NOARGS, NULL},
734 {NULL}
735};
736
Benjamin Peterson7e0dbfb2012-03-12 09:46:44 -0700737static PyGetSetDef ast_type_getsets[] = {
738 {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict},
739 {NULL}
740};
741
Dino Viehlandac46eb42019-09-11 10:16:34 -0700742static PyType_Slot AST_type_slots[] = {
743 {Py_tp_dealloc, ast_dealloc},
744 {Py_tp_getattro, PyObject_GenericGetAttr},
745 {Py_tp_setattro, PyObject_GenericSetAttr},
746 {Py_tp_traverse, ast_traverse},
747 {Py_tp_clear, ast_clear},
Eddie Elizondo3368f3c2019-09-19 09:29:05 -0700748 {Py_tp_members, ast_type_members},
Dino Viehlandac46eb42019-09-11 10:16:34 -0700749 {Py_tp_methods, ast_type_methods},
750 {Py_tp_getset, ast_type_getsets},
751 {Py_tp_init, ast_type_init},
752 {Py_tp_alloc, PyType_GenericAlloc},
753 {Py_tp_new, PyType_GenericNew},
Dino Viehlandac46eb42019-09-11 10:16:34 -0700754 {Py_tp_free, PyObject_GC_Del},
755 {0, 0},
756};
757
758static PyType_Spec AST_type_spec = {
Neal Norwitzee9b10a2008-03-31 05:29:39 +0000759 "_ast.AST",
Benjamin Peterson7e0dbfb2012-03-12 09:46:44 -0700760 sizeof(AST_object),
Neal Norwitz207c9f32008-03-31 04:42:11 +0000761 0,
Dino Viehlandac46eb42019-09-11 10:16:34 -0700762 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
763 AST_type_slots
Neal Norwitz207c9f32008-03-31 04:42:11 +0000764};
765
Dino Viehlandac46eb42019-09-11 10:16:34 -0700766static PyObject *
767make_type(const char *type, PyObject* base, const char* const* fields, int num_fields)
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000768{
769 PyObject *fnames, *result;
770 int i;
Neal Norwitzee9b10a2008-03-31 05:29:39 +0000771 fnames = PyTuple_New(num_fields);
772 if (!fnames) return NULL;
773 for (i = 0; i < num_fields; i++) {
Serhiy Storchaka43c97312019-09-10 13:02:30 +0300774 PyObject *field = PyUnicode_InternFromString(fields[i]);
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000775 if (!field) {
776 Py_DECREF(fnames);
777 return NULL;
778 }
779 PyTuple_SET_ITEM(fnames, i, field);
780 }
INADA Naokifc489082017-01-25 22:33:43 +0900781 result = PyObject_CallFunction((PyObject*)&PyType_Type, "s(O){OOOO}",
782 type, base,
Dino Viehlandac46eb42019-09-11 10:16:34 -0700783 astmodulestate_global->_fields, fnames,
784 astmodulestate_global->__module__,
785 astmodulestate_global->_ast);
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000786 Py_DECREF(fnames);
Dino Viehlandac46eb42019-09-11 10:16:34 -0700787 return result;
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000788}
789
Serhiy Storchaka43c97312019-09-10 13:02:30 +0300790static int
Dino Viehlandac46eb42019-09-11 10:16:34 -0700791add_attributes(PyObject *type, const char * const *attrs, int num_fields)
Martin v. Löwis577b5b92006-02-27 15:23:19 +0000792{
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000793 int i, result;
Neal Norwitz207c9f32008-03-31 04:42:11 +0000794 PyObject *s, *l = PyTuple_New(num_fields);
Benjamin Peterson3e5cd1d2010-06-27 21:45:24 +0000795 if (!l)
796 return 0;
797 for (i = 0; i < num_fields; i++) {
Serhiy Storchaka43c97312019-09-10 13:02:30 +0300798 s = PyUnicode_InternFromString(attrs[i]);
Martin v. Löwis577b5b92006-02-27 15:23:19 +0000799 if (!s) {
800 Py_DECREF(l);
801 return 0;
802 }
Neal Norwitz207c9f32008-03-31 04:42:11 +0000803 PyTuple_SET_ITEM(l, i, s);
Martin v. Löwis577b5b92006-02-27 15:23:19 +0000804 }
Dino Viehlandac46eb42019-09-11 10:16:34 -0700805 result = PyObject_SetAttr(type, astmodulestate_global->_attributes, l) >= 0;
Thomas Wouters49fd7fa2006-04-21 10:40:58 +0000806 Py_DECREF(l);
807 return result;
Martin v. Löwis577b5b92006-02-27 15:23:19 +0000808}
809
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000810/* Conversion AST -> Python */
811
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000812static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
813{
Benjamin Peterson77fa9372012-05-15 10:10:27 -0700814 Py_ssize_t i, n = asdl_seq_LEN(seq);
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000815 PyObject *result = PyList_New(n);
816 PyObject *value;
817 if (!result)
818 return NULL;
819 for (i = 0; i < n; i++) {
820 value = func(asdl_seq_GET(seq, i));
821 if (!value) {
822 Py_DECREF(result);
823 return NULL;
824 }
825 PyList_SET_ITEM(result, i, value);
826 }
827 return result;
828}
829
830static PyObject* ast2obj_object(void *o)
831{
832 if (!o)
833 o = Py_None;
834 Py_INCREF((PyObject*)o);
835 return (PyObject*)o;
836}
Benjamin Peterson442f2092012-12-06 17:41:04 -0500837#define ast2obj_singleton ast2obj_object
Victor Stinnerf2c1aa12016-01-26 00:40:57 +0100838#define ast2obj_constant ast2obj_object
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000839#define ast2obj_identifier ast2obj_object
840#define ast2obj_string ast2obj_object
Benjamin Petersone2498412011-08-09 16:08:39 -0500841#define ast2obj_bytes ast2obj_object
Martin v. Löwis577b5b92006-02-27 15:23:19 +0000842
Thomas Woutersa44f3a32007-02-26 18:20:15 +0000843static PyObject* ast2obj_int(long b)
Martin v. Löwis577b5b92006-02-27 15:23:19 +0000844{
Christian Heimes2b1c5922007-12-02 22:43:00 +0000845 return PyLong_FromLong(b);
Martin v. Löwis577b5b92006-02-27 15:23:19 +0000846}
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000847
848/* Conversion Python -> AST */
849
850static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena)
851{
852 if (obj == Py_None)
853 obj = NULL;
Christian Heimes70c94e72013-07-27 00:33:13 +0200854 if (obj) {
855 if (PyArena_AddPyObject(arena, obj) < 0) {
856 *out = NULL;
857 return -1;
858 }
859 Py_INCREF(obj);
860 }
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000861 *out = obj;
862 return 0;
863}
864
Victor Stinnerf2c1aa12016-01-26 00:40:57 +0100865static int obj2ast_constant(PyObject* obj, PyObject** out, PyArena* arena)
866{
Serhiy Storchaka3f228112018-09-27 17:42:37 +0300867 if (PyArena_AddPyObject(arena, obj) < 0) {
868 *out = NULL;
869 return -1;
Victor Stinnerf2c1aa12016-01-26 00:40:57 +0100870 }
Serhiy Storchaka3f228112018-09-27 17:42:37 +0300871 Py_INCREF(obj);
Victor Stinnerf2c1aa12016-01-26 00:40:57 +0100872 *out = obj;
873 return 0;
874}
875
Benjamin Peterson180e6352011-07-22 11:09:07 -0500876static int obj2ast_identifier(PyObject* obj, PyObject** out, PyArena* arena)
Benjamin Peterson2193d2b2011-07-22 10:50:23 -0500877{
Benjamin Peterson180e6352011-07-22 11:09:07 -0500878 if (!PyUnicode_CheckExact(obj) && obj != Py_None) {
879 PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str");
Benjamin Peterson2193d2b2011-07-22 10:50:23 -0500880 return 1;
881 }
882 return obj2ast_object(obj, out, arena);
883}
884
Guido van Rossumdcfcd142019-01-31 03:40:27 -0800885static int obj2ast_string(PyObject* obj, PyObject** out, PyArena* arena)
886{
887 if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) {
888 PyErr_SetString(PyExc_TypeError, "AST string must be of type str");
889 return 1;
890 }
891 return obj2ast_object(obj, out, arena);
892}
893
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000894static int obj2ast_int(PyObject* obj, int* out, PyArena* arena)
895{
896 int i;
897 if (!PyLong_Check(obj)) {
Amaury Forgeot d'Arc58e87612011-11-22 21:51:55 +0100898 PyErr_Format(PyExc_ValueError, "invalid integer value: %R", obj);
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000899 return 1;
900 }
901
Serhiy Storchaka481d3af2015-09-06 23:29:04 +0300902 i = _PyLong_AsInt(obj);
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000903 if (i == -1 && PyErr_Occurred())
904 return 1;
905 *out = i;
906 return 0;
907}
908
Benjamin Peterson1a6e0d02008-10-25 15:49:17 +0000909static int add_ast_fields(void)
Benjamin Peterson206e3072008-10-19 14:07:49 +0000910{
Dino Viehlandac46eb42019-09-11 10:16:34 -0700911 PyObject *empty_tuple;
Benjamin Peterson206e3072008-10-19 14:07:49 +0000912 empty_tuple = PyTuple_New(0);
913 if (!empty_tuple ||
Dino Viehlandac46eb42019-09-11 10:16:34 -0700914 PyObject_SetAttrString(astmodulestate_global->AST_type, "_fields", empty_tuple) < 0 ||
915 PyObject_SetAttrString(astmodulestate_global->AST_type, "_attributes", empty_tuple) < 0) {
Benjamin Peterson206e3072008-10-19 14:07:49 +0000916 Py_XDECREF(empty_tuple);
917 return -1;
918 }
919 Py_DECREF(empty_tuple);
920 return 0;
921}
922
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000923""", 0, reflow=False)
924
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000925 self.emit("static int init_types(void)",0)
926 self.emit("{", 0)
Dino Viehlandac46eb42019-09-11 10:16:34 -0700927 self.emit("PyObject *m;", 1)
928 self.emit("if (PyState_FindModule(&_astmodule) == NULL) {", 1)
929 self.emit("m = PyModule_Create(&_astmodule);", 2)
930 self.emit("if (!m) return 0;", 2)
931 self.emit("PyState_AddModule(m, &_astmodule);", 2)
932 self.emit("}", 1)
933 self.emit("astmodulestate *state = astmodulestate_global;", 1)
934 self.emit("if (state->initialized) return 1;", 1)
935 self.emit("if (init_identifiers() < 0) return 0;", 1)
936 self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1)
937 self.emit("if (!state->AST_type) return 0;", 1)
Benjamin Peterson206e3072008-10-19 14:07:49 +0000938 self.emit("if (add_ast_fields() < 0) return 0;", 1)
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000939 for dfn in mod.dfns:
940 self.visit(dfn)
Dino Viehlandac46eb42019-09-11 10:16:34 -0700941 self.emit("state->initialized = 1;", 1)
Martin v. Löwis577b5b92006-02-27 15:23:19 +0000942 self.emit("return 1;", 1);
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000943 self.emit("}", 0)
944
945 def visitProduct(self, prod, name):
Martin v. Löwis8d0701d2006-02-26 23:40:20 +0000946 if prod.fields:
Eli Bendersky5e3d3382014-05-09 17:58:22 -0700947 fields = name+"_fields"
Martin v. Löwis8d0701d2006-02-26 23:40:20 +0000948 else:
949 fields = "NULL"
Dino Viehlandac46eb42019-09-11 10:16:34 -0700950 self.emit('state->%s_type = make_type("%s", state->AST_type, %s, %d);' %
Martin v. Löwis8d0701d2006-02-26 23:40:20 +0000951 (name, name, fields, len(prod.fields)), 1)
Dino Viehlandac46eb42019-09-11 10:16:34 -0700952 self.emit("if (!state->%s_type) return 0;" % name, 1)
953 self.emit_type("AST_type")
954 self.emit_type("%s_type" % name)
Benjamin Petersoncda75be2013-03-18 10:48:58 -0700955 if prod.attributes:
Dino Viehlandac46eb42019-09-11 10:16:34 -0700956 self.emit("if (!add_attributes(state->%s_type, %s_attributes, %d)) return 0;" %
Benjamin Petersoncda75be2013-03-18 10:48:58 -0700957 (name, name, len(prod.attributes)), 1)
958 else:
Dino Viehlandac46eb42019-09-11 10:16:34 -0700959 self.emit("if (!add_attributes(state->%s_type, NULL, 0)) return 0;" % name, 1)
Tim Peters710ab3b2006-02-28 18:30:36 +0000960
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000961 def visitSum(self, sum, name):
Dino Viehlandac46eb42019-09-11 10:16:34 -0700962 self.emit('state->%s_type = make_type("%s", state->AST_type, NULL, 0);' %
Neal Norwitz207c9f32008-03-31 04:42:11 +0000963 (name, name), 1)
Dino Viehlandac46eb42019-09-11 10:16:34 -0700964 self.emit_type("%s_type" % name)
965 self.emit("if (!state->%s_type) return 0;" % name, 1)
Martin v. Löwis577b5b92006-02-27 15:23:19 +0000966 if sum.attributes:
Dino Viehlandac46eb42019-09-11 10:16:34 -0700967 self.emit("if (!add_attributes(state->%s_type, %s_attributes, %d)) return 0;" %
Martin v. Löwis577b5b92006-02-27 15:23:19 +0000968 (name, name, len(sum.attributes)), 1)
969 else:
Dino Viehlandac46eb42019-09-11 10:16:34 -0700970 self.emit("if (!add_attributes(state->%s_type, NULL, 0)) return 0;" % name, 1)
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000971 simple = is_simple(sum)
972 for t in sum.types:
973 self.visitConstructor(t, name, simple)
Tim Peters710ab3b2006-02-28 18:30:36 +0000974
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000975 def visitConstructor(self, cons, name, simple):
Martin v. Löwis8d0701d2006-02-26 23:40:20 +0000976 if cons.fields:
Eli Bendersky5e3d3382014-05-09 17:58:22 -0700977 fields = cons.name+"_fields"
Martin v. Löwis8d0701d2006-02-26 23:40:20 +0000978 else:
979 fields = "NULL"
Dino Viehlandac46eb42019-09-11 10:16:34 -0700980 self.emit('state->%s_type = make_type("%s", state->%s_type, %s, %d);' %
Martin v. Löwis8d0701d2006-02-26 23:40:20 +0000981 (cons.name, cons.name, name, fields, len(cons.fields)), 1)
Dino Viehlandac46eb42019-09-11 10:16:34 -0700982 self.emit("if (!state->%s_type) return 0;" % cons.name, 1)
983 self.emit_type("%s_type" % cons.name)
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000984 if simple:
Dino Viehlandac46eb42019-09-11 10:16:34 -0700985 self.emit("state->%s_singleton = PyType_GenericNew((PyTypeObject *)"
986 "state->%s_type, NULL, NULL);" %
Martin v. Löwisbd260da2006-02-26 19:42:26 +0000987 (cons.name, cons.name), 1)
Dino Viehlandac46eb42019-09-11 10:16:34 -0700988 self.emit("if (!state->%s_singleton) return 0;" % cons.name, 1)
Tim Peters710ab3b2006-02-28 18:30:36 +0000989
Martin v. Löwis618dc5e2008-03-30 20:03:44 +0000990
Martin v. Löwis577b5b92006-02-27 15:23:19 +0000991class ASTModuleVisitor(PickleVisitor):
992
993 def visitModule(self, mod):
994 self.emit("PyMODINIT_FUNC", 0)
Martin v. Löwis1a214512008-06-11 05:26:20 +0000995 self.emit("PyInit__ast(void)", 0)
Martin v. Löwis577b5b92006-02-27 15:23:19 +0000996 self.emit("{", 0)
Dino Viehlandac46eb42019-09-11 10:16:34 -0700997 self.emit("PyObject *m;", 1)
Martin v. Löwis1a214512008-06-11 05:26:20 +0000998 self.emit("if (!init_types()) return NULL;", 1)
Dino Viehlandac46eb42019-09-11 10:16:34 -0700999 self.emit('m = PyState_FindModule(&_astmodule);', 1)
Martin v. Löwis1a214512008-06-11 05:26:20 +00001000 self.emit("if (!m) return NULL;", 1)
Brandt Bucherd2f96672020-02-06 06:45:46 -08001001 self.emit('if (PyModule_AddObject(m, "AST", astmodulestate_global->AST_type) < 0) {', 1)
1002 self.emit('goto error;', 2)
1003 self.emit('}', 1)
Dino Viehlandac46eb42019-09-11 10:16:34 -07001004 self.emit('Py_INCREF(astmodulestate(m)->AST_type);', 1)
Brandt Bucherd2f96672020-02-06 06:45:46 -08001005 self.emit('if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0) {', 1)
1006 self.emit("goto error;", 2)
1007 self.emit('}', 1)
1008 self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0) {', 1)
1009 self.emit("goto error;", 2)
1010 self.emit('}', 1)
1011 self.emit('if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0) {', 1)
1012 self.emit("goto error;", 2)
1013 self.emit('}', 1)
Martin v. Löwis577b5b92006-02-27 15:23:19 +00001014 for dfn in mod.dfns:
1015 self.visit(dfn)
Martin v. Löwis1a214512008-06-11 05:26:20 +00001016 self.emit("return m;", 1)
Brandt Bucherd2f96672020-02-06 06:45:46 -08001017 self.emit("error:", 0)
1018 self.emit("Py_DECREF(m);", 1)
1019 self.emit("return NULL;", 1)
Martin v. Löwis577b5b92006-02-27 15:23:19 +00001020 self.emit("}", 0)
1021
1022 def visitProduct(self, prod, name):
1023 self.addObj(name)
Tim Peters710ab3b2006-02-28 18:30:36 +00001024
Martin v. Löwis577b5b92006-02-27 15:23:19 +00001025 def visitSum(self, sum, name):
1026 self.addObj(name)
1027 for t in sum.types:
1028 self.visitConstructor(t, name)
Tim Peters710ab3b2006-02-28 18:30:36 +00001029
Martin v. Löwis577b5b92006-02-27 15:23:19 +00001030 def visitConstructor(self, cons, name):
1031 self.addObj(cons.name)
Tim Peters710ab3b2006-02-28 18:30:36 +00001032
Martin v. Löwis577b5b92006-02-27 15:23:19 +00001033 def addObj(self, name):
Dino Viehlandac46eb42019-09-11 10:16:34 -07001034 self.emit("if (PyModule_AddObject(m, \"%s\", "
Brandt Bucherd2f96672020-02-06 06:45:46 -08001035 "astmodulestate_global->%s_type) < 0) {" % (name, name), 1)
1036 self.emit("goto error;", 2)
1037 self.emit('}', 1)
Dino Viehlandac46eb42019-09-11 10:16:34 -07001038 self.emit("Py_INCREF(astmodulestate(m)->%s_type);" % name, 1)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001039
Martin v. Löwis618dc5e2008-03-30 20:03:44 +00001040
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001041_SPECIALIZED_SEQUENCES = ('stmt', 'expr')
1042
1043def find_sequence(fields, doing_specialization):
1044 """Return True if any field uses a sequence."""
1045 for f in fields:
1046 if f.seq:
1047 if not doing_specialization:
1048 return True
1049 if str(f.type) not in _SPECIALIZED_SEQUENCES:
1050 return True
1051 return False
1052
1053def has_sequence(types, doing_specialization):
1054 for t in types:
1055 if find_sequence(t.fields, doing_specialization):
1056 return True
1057 return False
1058
1059
1060class StaticVisitor(PickleVisitor):
Ezio Melotti7c4a7e62013-08-26 01:32:56 +03001061 CODE = '''Very simple, always emit this static code. Override CODE'''
Neal Norwitz7b5a6042005-11-13 19:14:20 +00001062
1063 def visit(self, object):
1064 self.emit(self.CODE, 0, reflow=False)
1065
Martin v. Löwis618dc5e2008-03-30 20:03:44 +00001066
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001067class ObjVisitor(PickleVisitor):
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001068
Martin v. Löwis577b5b92006-02-27 15:23:19 +00001069 def func_begin(self, name):
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001070 ctype = get_c_type(name)
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001071 self.emit("PyObject*", 0)
1072 self.emit("ast2obj_%s(void* _o)" % (name), 0)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001073 self.emit("{", 0)
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001074 self.emit("%s o = (%s)_o;" % (ctype, ctype), 1)
1075 self.emit("PyObject *result = NULL, *value = NULL;", 1)
Dino Viehlandac46eb42019-09-11 10:16:34 -07001076 self.emit("PyTypeObject *tp;", 1)
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001077 self.emit('if (!o) {', 1)
INADA Naokifc489082017-01-25 22:33:43 +09001078 self.emit("Py_RETURN_NONE;", 2)
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001079 self.emit("}", 1)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001080 self.emit('', 0)
1081
Martin v. Löwis577b5b92006-02-27 15:23:19 +00001082 def func_end(self):
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001083 self.emit("return result;", 1)
1084 self.emit("failed:", 0)
1085 self.emit("Py_XDECREF(value);", 1)
1086 self.emit("Py_XDECREF(result);", 1)
1087 self.emit("return NULL;", 1)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001088 self.emit("}", 0)
1089 self.emit("", 0)
1090
1091 def visitSum(self, sum, name):
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001092 if is_simple(sum):
1093 self.simpleSum(sum, name)
1094 return
Martin v. Löwis577b5b92006-02-27 15:23:19 +00001095 self.func_begin(name)
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001096 self.emit("switch (o->kind) {", 1)
1097 for i in range(len(sum.types)):
1098 t = sum.types[i]
1099 self.visitConstructor(t, i + 1, name)
1100 self.emit("}", 1)
Martin v. Löwis577b5b92006-02-27 15:23:19 +00001101 for a in sum.attributes:
1102 self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1)
1103 self.emit("if (!value) goto failed;", 1)
Dino Viehlandac46eb42019-09-11 10:16:34 -07001104 self.emit('if (PyObject_SetAttr(result, astmodulestate_global->%s, value) < 0)' % a.name, 1)
Martin v. Löwis03e5bc02006-03-02 00:31:27 +00001105 self.emit('goto failed;', 2)
1106 self.emit('Py_DECREF(value);', 1)
Martin v. Löwis577b5b92006-02-27 15:23:19 +00001107 self.func_end()
Tim Peters710ab3b2006-02-28 18:30:36 +00001108
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001109 def simpleSum(self, sum, name):
1110 self.emit("PyObject* ast2obj_%s(%s_ty o)" % (name, name), 0)
1111 self.emit("{", 0)
1112 self.emit("switch(o) {", 1)
1113 for t in sum.types:
1114 self.emit("case %s:" % t.name, 2)
Dino Viehlandac46eb42019-09-11 10:16:34 -07001115 self.emit("Py_INCREF(astmodulestate_global->%s_singleton);" % t.name, 3)
1116 self.emit("return astmodulestate_global->%s_singleton;" % t.name, 3)
Ezio Melotticb2916a2012-09-30 22:41:37 +03001117 self.emit("default:", 2)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +00001118 self.emit('/* should never happen, but just in case ... */', 3)
1119 code = "PyErr_Format(PyExc_SystemError, \"unknown %s found\");" % name
1120 self.emit(code, 3, reflow=False)
1121 self.emit("return NULL;", 3)
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001122 self.emit("}", 1)
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001123 self.emit("}", 0)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001124
1125 def visitProduct(self, prod, name):
Martin v. Löwis577b5b92006-02-27 15:23:19 +00001126 self.func_begin(name)
Dino Viehlandac46eb42019-09-11 10:16:34 -07001127 self.emit("tp = (PyTypeObject *)astmodulestate_global->%s_type;" % name, 1)
1128 self.emit("result = PyType_GenericNew(tp, NULL, NULL);", 1);
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001129 self.emit("if (!result) return NULL;", 1)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001130 for field in prod.fields:
1131 self.visitField(field, name, 1, True)
Benjamin Petersoncda75be2013-03-18 10:48:58 -07001132 for a in prod.attributes:
1133 self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1)
1134 self.emit("if (!value) goto failed;", 1)
Dino Viehlandac46eb42019-09-11 10:16:34 -07001135 self.emit("if (PyObject_SetAttr(result, astmodulestate_global->%s, value) < 0)" % a.name, 1)
Benjamin Petersoncda75be2013-03-18 10:48:58 -07001136 self.emit('goto failed;', 2)
1137 self.emit('Py_DECREF(value);', 1)
Martin v. Löwis577b5b92006-02-27 15:23:19 +00001138 self.func_end()
Tim Peters536cf992005-12-25 23:18:31 +00001139
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001140 def visitConstructor(self, cons, enum, name):
1141 self.emit("case %s_kind:" % cons.name, 1)
Dino Viehlandac46eb42019-09-11 10:16:34 -07001142 self.emit("tp = (PyTypeObject *)astmodulestate_global->%s_type;" % cons.name, 2)
1143 self.emit("result = PyType_GenericNew(tp, NULL, NULL);", 2);
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001144 self.emit("if (!result) goto failed;", 2)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001145 for f in cons.fields:
1146 self.visitField(f, cons.name, 2, False)
1147 self.emit("break;", 2)
1148
1149 def visitField(self, field, name, depth, product):
1150 def emit(s, d):
1151 self.emit(s, depth + d)
1152 if product:
1153 value = "o->%s" % field.name
1154 else:
1155 value = "o->v.%s.%s" % (name, field.name)
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001156 self.set(field, value, depth)
1157 emit("if (!value) goto failed;", 0)
Dino Viehlandac46eb42019-09-11 10:16:34 -07001158 emit("if (PyObject_SetAttr(result, astmodulestate_global->%s, value) == -1)" % field.name, 0)
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001159 emit("goto failed;", 1)
1160 emit("Py_DECREF(value);", 0)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001161
1162 def emitSeq(self, field, value, depth, emit):
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001163 emit("seq = %s;" % value, 0)
1164 emit("n = asdl_seq_LEN(seq);", 0)
1165 emit("value = PyList_New(n);", 0)
1166 emit("if (!value) goto failed;", 0)
1167 emit("for (i = 0; i < n; i++) {", 0)
1168 self.set("value", field, "asdl_seq_GET(seq, i)", depth + 1)
1169 emit("if (!value1) goto failed;", 1)
1170 emit("PyList_SET_ITEM(value, i, value1);", 1)
1171 emit("value1 = NULL;", 1)
1172 emit("}", 0)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001173
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001174 def set(self, field, value, depth):
1175 if field.seq:
Martin v. Löwisce1d5d22006-02-26 20:51:25 +00001176 # XXX should really check for is_simple, but that requires a symbol table
Eli Bendersky5e3d3382014-05-09 17:58:22 -07001177 if field.type == "cmpop":
Martin v. Löwisce1d5d22006-02-26 20:51:25 +00001178 # While the sequence elements are stored as void*,
1179 # ast2obj_cmpop expects an enum
1180 self.emit("{", depth)
Benjamin Peterson77fa9372012-05-15 10:10:27 -07001181 self.emit("Py_ssize_t i, n = asdl_seq_LEN(%s);" % value, depth+1)
Martin v. Löwisce1d5d22006-02-26 20:51:25 +00001182 self.emit("value = PyList_New(n);", depth+1)
1183 self.emit("if (!value) goto failed;", depth+1)
1184 self.emit("for(i = 0; i < n; i++)", depth+1)
1185 # This cannot fail, so no need for error handling
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001186 self.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(%s, i)));" % value,
1187 depth+2, reflow=False)
Martin v. Löwisce1d5d22006-02-26 20:51:25 +00001188 self.emit("}", depth)
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001189 else:
Martin v. Löwisce1d5d22006-02-26 20:51:25 +00001190 self.emit("value = ast2obj_list(%s, ast2obj_%s);" % (value, field.type), depth)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001191 else:
1192 ctype = get_c_type(field.type)
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001193 self.emit("value = ast2obj_%s(%s);" % (field.type, value), depth, reflow=False)
Tim Peters536cf992005-12-25 23:18:31 +00001194
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001195
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001196class PartingShots(StaticVisitor):
1197
1198 CODE = """
1199PyObject* PyAST_mod2obj(mod_ty t)
1200{
Victor Stinnerbdf630c2013-07-17 00:17:15 +02001201 if (!init_types())
1202 return NULL;
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001203 return ast2obj_mod(t);
1204}
Martin v. Löwis618dc5e2008-03-30 20:03:44 +00001205
Neal Norwitzdb4115f2008-03-31 04:20:05 +00001206/* mode is 0 for "exec", 1 for "eval" and 2 for "single" input */
1207mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
Martin v. Löwis618dc5e2008-03-30 20:03:44 +00001208{
Benjamin Petersonc2f665e2014-02-10 22:19:02 -05001209 PyObject *req_type[3];
Serhiy Storchaka43c97312019-09-10 13:02:30 +03001210 const char * const req_name[] = {"Module", "Expression", "Interactive"};
Benjamin Peterson97dd9872009-12-13 01:23:39 +00001211 int isinstance;
Benjamin Petersonc2f665e2014-02-10 22:19:02 -05001212
Steve Dowerb82e17e2019-05-23 08:45:22 -07001213 if (PySys_Audit("compile", "OO", ast, Py_None) < 0) {
1214 return NULL;
1215 }
1216
Dino Viehlandac46eb42019-09-11 10:16:34 -07001217 req_type[0] = astmodulestate_global->Module_type;
1218 req_type[1] = astmodulestate_global->Expression_type;
1219 req_type[2] = astmodulestate_global->Interactive_type;
Benjamin Petersonc2f665e2014-02-10 22:19:02 -05001220
Guido van Rossum3a32e3b2019-02-01 11:37:34 -08001221 assert(0 <= mode && mode <= 2);
Neal Norwitzdb4115f2008-03-31 04:20:05 +00001222
Victor Stinnerbdf630c2013-07-17 00:17:15 +02001223 if (!init_types())
1224 return NULL;
Neal Norwitzdb4115f2008-03-31 04:20:05 +00001225
Benjamin Peterson97dd9872009-12-13 01:23:39 +00001226 isinstance = PyObject_IsInstance(ast, req_type[mode]);
1227 if (isinstance == -1)
1228 return NULL;
1229 if (!isinstance) {
Neal Norwitzdb4115f2008-03-31 04:20:05 +00001230 PyErr_Format(PyExc_TypeError, "expected %s node, got %.400s",
Dino Viehlandac46eb42019-09-11 10:16:34 -07001231 req_name[mode], _PyType_Name(Py_TYPE(ast)));
Martin v. Löwis618dc5e2008-03-30 20:03:44 +00001232 return NULL;
1233 }
Dong-hee Naa05fcd32019-10-10 16:41:26 +09001234
1235 mod_ty res = NULL;
Martin v. Löwis618dc5e2008-03-30 20:03:44 +00001236 if (obj2ast_mod(ast, &res, arena) != 0)
1237 return NULL;
1238 else
1239 return res;
1240}
1241
1242int PyAST_Check(PyObject* obj)
1243{
Victor Stinnerbdf630c2013-07-17 00:17:15 +02001244 if (!init_types())
1245 return -1;
Dino Viehlandac46eb42019-09-11 10:16:34 -07001246 return PyObject_IsInstance(obj, astmodulestate_global->AST_type);
Martin v. Löwis618dc5e2008-03-30 20:03:44 +00001247}
Martin v. Löwisbd260da2006-02-26 19:42:26 +00001248"""
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001249
1250class ChainOfVisitors:
1251 def __init__(self, *visitors):
1252 self.visitors = visitors
1253
1254 def visit(self, object):
1255 for v in self.visitors:
1256 v.visit(object)
Neal Norwitz7b5a6042005-11-13 19:14:20 +00001257 v.emit("", 0)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001258
Dino Viehlandac46eb42019-09-11 10:16:34 -07001259
1260def generate_module_def(f, mod):
1261 # Gather all the data needed for ModuleSpec
1262 visitor_list = set()
1263 with open(os.devnull, "w") as devnull:
1264 visitor = PyTypesDeclareVisitor(devnull)
1265 visitor.visit(mod)
1266 visitor_list.add(visitor)
1267 visitor = PyTypesVisitor(devnull)
1268 visitor.visit(mod)
1269 visitor_list.add(visitor)
1270
1271 state_strings = set(["__dict__", "_attributes", "_fields", "__module__", "_ast"])
1272 module_state = set(["__dict__", "_attributes", "_fields", "__module__", "_ast"])
1273 for visitor in visitor_list:
1274 for identifier in visitor.identifiers:
1275 module_state.add(identifier)
1276 state_strings.add(identifier)
1277 for singleton in visitor.singletons:
1278 module_state.add(singleton)
1279 for tp in visitor.types:
1280 module_state.add(tp)
1281 state_strings = sorted(state_strings)
1282 module_state = sorted(module_state)
1283 f.write('typedef struct {\n')
1284 f.write(' int initialized;\n')
1285 for s in module_state:
1286 f.write(' PyObject *' + s + ';\n')
1287 f.write('} astmodulestate;\n\n')
1288 f.write("""
1289#define astmodulestate(o) ((astmodulestate *)PyModule_GetState(o))
1290
1291static int astmodule_clear(PyObject *module)
1292{
1293""")
1294 for s in module_state:
1295 f.write(" Py_CLEAR(astmodulestate(module)->" + s + ');\n')
1296 f.write("""
1297 return 0;
1298}
1299
1300static int astmodule_traverse(PyObject *module, visitproc visit, void* arg)
1301{
1302""")
1303 for s in module_state:
1304 f.write(" Py_VISIT(astmodulestate(module)->" + s + ');\n')
1305 f.write("""
1306 return 0;
1307}
1308
1309static void astmodule_free(void* module) {
1310 astmodule_clear((PyObject*)module);
1311}
1312
1313static struct PyModuleDef _astmodule = {
1314 PyModuleDef_HEAD_INIT,
1315 "_ast",
1316 NULL,
1317 sizeof(astmodulestate),
1318 NULL,
1319 NULL,
1320 astmodule_traverse,
1321 astmodule_clear,
1322 astmodule_free,
1323};
1324
1325#define astmodulestate_global ((astmodulestate *)PyModule_GetState(PyState_FindModule(&_astmodule)))
1326
1327""")
1328 f.write('static int init_identifiers(void)\n')
1329 f.write('{\n')
1330 f.write(' astmodulestate *state = astmodulestate_global;\n')
1331 for identifier in state_strings:
1332 f.write(' if ((state->' + identifier)
1333 f.write(' = PyUnicode_InternFromString("')
1334 f.write(identifier + '")) == NULL) return 0;\n')
1335 f.write(' return 1;\n')
1336 f.write('};\n\n')
1337
1338
Guido van Rossum805365e2007-05-07 22:24:25 +00001339common_msg = "/* File automatically generated by %s. */\n\n"
Thomas Wouterscf297e42007-02-23 15:07:44 +00001340
Eli Bendersky5e3d3382014-05-09 17:58:22 -07001341def main(srcfile, dump_module=False):
Neal Norwitz897ff812005-12-11 21:18:22 +00001342 argv0 = sys.argv[0]
Armin Rigo6b1793f2005-12-14 18:05:14 +00001343 components = argv0.split(os.sep)
Steve Dowera9d0a6a2019-12-17 14:14:13 -08001344 # Always join with '/' so different OS does not keep changing the file
1345 argv0 = '/'.join(components[-2:])
Thomas Wouterscf297e42007-02-23 15:07:44 +00001346 auto_gen_msg = common_msg % argv0
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001347 mod = asdl.parse(srcfile)
Eli Bendersky5e3d3382014-05-09 17:58:22 -07001348 if dump_module:
1349 print('Parsed Module:')
1350 print(mod)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001351 if not asdl.check(mod):
1352 sys.exit(1)
Antoine Pitroub091bec2017-09-20 23:57:56 +02001353 if H_FILE:
1354 with open(H_FILE, "w") as f:
1355 f.write(auto_gen_msg)
Victor Stinner5f2df882018-11-12 00:56:19 +01001356 f.write('#ifndef Py_PYTHON_AST_H\n')
1357 f.write('#define Py_PYTHON_AST_H\n')
1358 f.write('#ifdef __cplusplus\n')
1359 f.write('extern "C" {\n')
1360 f.write('#endif\n')
1361 f.write('\n')
Zackery Spytz421a72a2019-09-12 03:27:14 -06001362 f.write('#ifndef Py_LIMITED_API\n')
Victor Stinner5f2df882018-11-12 00:56:19 +01001363 f.write('#include "asdl.h"\n')
1364 f.write('\n')
Victor Stinner3bb183d2018-11-22 18:38:38 +01001365 f.write('#undef Yield /* undefine macro conflicting with <winbase.h> */\n')
Victor Stinner5f2df882018-11-12 00:56:19 +01001366 f.write('\n')
Antoine Pitroub091bec2017-09-20 23:57:56 +02001367 c = ChainOfVisitors(TypeDefVisitor(f),
Ivan Levkivskyi9932a222019-01-22 11:18:22 +00001368 StructVisitor(f))
1369
Antoine Pitroub091bec2017-09-20 23:57:56 +02001370 c.visit(mod)
Ivan Levkivskyi9932a222019-01-22 11:18:22 +00001371 f.write("// Note: these macros affect function definitions, not only call sites.\n")
1372 PrototypeVisitor(f).visit(mod)
1373 f.write("\n")
Antoine Pitroub091bec2017-09-20 23:57:56 +02001374 f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
1375 f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
1376 f.write("int PyAST_Check(PyObject* obj);\n")
Zackery Spytz421a72a2019-09-12 03:27:14 -06001377 f.write("#endif /* !Py_LIMITED_API */\n")
Victor Stinner5f2df882018-11-12 00:56:19 +01001378 f.write('\n')
1379 f.write('#ifdef __cplusplus\n')
1380 f.write('}\n')
1381 f.write('#endif\n')
1382 f.write('#endif /* !Py_PYTHON_AST_H */\n')
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001383
Antoine Pitroub091bec2017-09-20 23:57:56 +02001384 if C_FILE:
1385 with open(C_FILE, "w") as f:
1386 f.write(auto_gen_msg)
1387 f.write('#include <stddef.h>\n')
1388 f.write('\n')
1389 f.write('#include "Python.h"\n')
1390 f.write('#include "%s-ast.h"\n' % mod.name)
Eddie Elizondo3368f3c2019-09-19 09:29:05 -07001391 f.write('#include "structmember.h"\n')
Antoine Pitroub091bec2017-09-20 23:57:56 +02001392 f.write('\n')
Dino Viehlandac46eb42019-09-11 10:16:34 -07001393
1394 generate_module_def(f, mod)
1395
Antoine Pitroub091bec2017-09-20 23:57:56 +02001396 v = ChainOfVisitors(
1397 PyTypesDeclareVisitor(f),
1398 PyTypesVisitor(f),
1399 Obj2ModPrototypeVisitor(f),
1400 FunctionVisitor(f),
1401 ObjVisitor(f),
1402 Obj2ModVisitor(f),
1403 ASTModuleVisitor(f),
1404 PartingShots(f),
1405 )
1406 v.visit(mod)
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001407
1408if __name__ == "__main__":
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001409 import getopt
1410
Antoine Pitroub091bec2017-09-20 23:57:56 +02001411 H_FILE = ''
1412 C_FILE = ''
Eli Bendersky5e3d3382014-05-09 17:58:22 -07001413 dump_module = False
1414 opts, args = getopt.getopt(sys.argv[1:], "dh:c:")
Jeremy Hylton3e0055f2005-10-20 19:59:25 +00001415 for o, v in opts:
1416 if o == '-h':
Antoine Pitroub091bec2017-09-20 23:57:56 +02001417 H_FILE = v
Emmanuel Ariased5e29c2019-03-21 01:39:17 -03001418 elif o == '-c':
Antoine Pitroub091bec2017-09-20 23:57:56 +02001419 C_FILE = v
Emmanuel Ariased5e29c2019-03-21 01:39:17 -03001420 elif o == '-d':
Eli Bendersky5e3d3382014-05-09 17:58:22 -07001421 dump_module = True
Antoine Pitroub091bec2017-09-20 23:57:56 +02001422 if H_FILE and C_FILE:
Eli Bendersky5e3d3382014-05-09 17:58:22 -07001423 print('Must specify exactly one output file')
Thomas Wouters49fd7fa2006-04-21 10:40:58 +00001424 sys.exit(1)
Eli Bendersky5e3d3382014-05-09 17:58:22 -07001425 elif len(args) != 1:
1426 print('Must specify single input file')
1427 sys.exit(1)
1428 main(args[0], dump_module)