compiler.transformer: correct lineno attribute when possible
SF patch #1015989
The basic idea of this patch is to compute lineno attributes for all AST nodes. The actual
implementation lead to a lot of restructing and code cleanup.
The generated AST nodes now have an optional lineno argument to constructor. Remove the
top-level asList(), since it didn't seem to serve any purpose. Add an __iter__ to ast nodes.
Use isinstance() instead of explicit type tests.
Change transformer to use the new lineno attribute, which replaces three lines of code with one.
Use universal newlines so that we can get rid of special-case code for line endings. Use
lookup_node() in a few more frequently called, but simple com_xxx methods(). Change string
exception to class exception.
diff --git a/Lib/compiler/ast.py b/Lib/compiler/ast.py
index 6f585b7..cfc2a51 100644
--- a/Lib/compiler/ast.py
+++ b/Lib/compiler/ast.py
@@ -1,15 +1,14 @@
"""Python abstract syntax node definitions
-This file is automatically generated.
+This file is automatically generated by Tools/compiler/astgen.py
"""
-from types import TupleType, ListType
from consts import CO_VARARGS, CO_VARKEYWORDS
def flatten(list):
l = []
for elt in list:
t = type(elt)
- if t is TupleType or t is ListType:
+ if t is tuple or t is list:
for elt2 in flatten(elt):
l.append(elt2)
else:
@@ -19,29 +18,17 @@
def flatten_nodes(list):
return [n for n in flatten(list) if isinstance(n, Node)]
-def asList(nodearg):
- l = []
- for item in nodearg:
- if hasattr(item, "asList"):
- l.append(item.asList())
- else:
- t = type(item)
- if t is TupleType or t is ListType:
- l.append(tuple(asList(item)))
- else:
- l.append(item)
- return l
-
nodes = {}
-class Node: # an abstract base class
- lineno = None # provide a lineno for nodes that don't have one
- def getType(self):
- pass # implemented by subclass
+class Node:
+ """Abstract base class for ast nodes."""
def getChildren(self):
pass # implemented by subclasses
- def asList(self):
- return tuple(asList(self.getChildren()))
+ def __iter__(self):
+ for n in self.getChildren():
+ yield n
+ def asList(self): # for backwards compatibility
+ return self.getChildren()
def getChildNodes(self):
pass # implemented by subclasses
@@ -64,10 +51,10 @@
return "Expression(%s)" % (repr(self.node))
class Add(Node):
- nodes["add"] = "Add"
- def __init__(self, (left, right)):
+ def __init__(self, (left, right), lineno=None):
self.left = left
self.right = right
+ self.lineno = lineno
def getChildren(self):
return self.left, self.right
@@ -79,14 +66,12 @@
return "Add((%s, %s))" % (repr(self.left), repr(self.right))
class And(Node):
- nodes["and"] = "And"
- def __init__(self, nodes):
+ def __init__(self, nodes, lineno=None):
self.nodes = nodes
+ self.lineno = lineno
def getChildren(self):
- children = []
- children.extend(flatten(self.nodes))
- return tuple(children)
+ return tuple(flatten(self.nodes))
def getChildNodes(self):
nodelist = []
@@ -97,11 +82,11 @@
return "And(%s)" % (repr(self.nodes),)
class AssAttr(Node):
- nodes["assattr"] = "AssAttr"
- def __init__(self, expr, attrname, flags):
+ def __init__(self, expr, attrname, flags, lineno=None):
self.expr = expr
self.attrname = attrname
self.flags = flags
+ self.lineno = lineno
def getChildren(self):
return self.expr, self.attrname, self.flags
@@ -113,14 +98,12 @@
return "AssAttr(%s, %s, %s)" % (repr(self.expr), repr(self.attrname), repr(self.flags))
class AssList(Node):
- nodes["asslist"] = "AssList"
- def __init__(self, nodes):
+ def __init__(self, nodes, lineno=None):
self.nodes = nodes
+ self.lineno = lineno
def getChildren(self):
- children = []
- children.extend(flatten(self.nodes))
- return tuple(children)
+ return tuple(flatten(self.nodes))
def getChildNodes(self):
nodelist = []
@@ -131,10 +114,10 @@
return "AssList(%s)" % (repr(self.nodes),)
class AssName(Node):
- nodes["assname"] = "AssName"
- def __init__(self, name, flags):
+ def __init__(self, name, flags, lineno=None):
self.name = name
self.flags = flags
+ self.lineno = lineno
def getChildren(self):
return self.name, self.flags
@@ -146,14 +129,12 @@
return "AssName(%s, %s)" % (repr(self.name), repr(self.flags))
class AssTuple(Node):
- nodes["asstuple"] = "AssTuple"
- def __init__(self, nodes):
+ def __init__(self, nodes, lineno=None):
self.nodes = nodes
+ self.lineno = lineno
def getChildren(self):
- children = []
- children.extend(flatten(self.nodes))
- return tuple(children)
+ return tuple(flatten(self.nodes))
def getChildNodes(self):
nodelist = []
@@ -164,10 +145,10 @@
return "AssTuple(%s)" % (repr(self.nodes),)
class Assert(Node):
- nodes["assert"] = "Assert"
- def __init__(self, test, fail):
+ def __init__(self, test, fail, lineno=None):
self.test = test
self.fail = fail
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -178,17 +159,18 @@
def getChildNodes(self):
nodelist = []
nodelist.append(self.test)
- if self.fail is not None: nodelist.append(self.fail)
+ if self.fail is not None:
+ nodelist.append(self.fail)
return tuple(nodelist)
def __repr__(self):
return "Assert(%s, %s)" % (repr(self.test), repr(self.fail))
class Assign(Node):
- nodes["assign"] = "Assign"
- def __init__(self, nodes, expr):
+ def __init__(self, nodes, expr, lineno=None):
self.nodes = nodes
self.expr = expr
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -206,11 +188,11 @@
return "Assign(%s, %s)" % (repr(self.nodes), repr(self.expr))
class AugAssign(Node):
- nodes["augassign"] = "AugAssign"
- def __init__(self, node, op, expr):
+ def __init__(self, node, op, expr, lineno=None):
self.node = node
self.op = op
self.expr = expr
+ self.lineno = lineno
def getChildren(self):
return self.node, self.op, self.expr
@@ -222,9 +204,9 @@
return "AugAssign(%s, %s, %s)" % (repr(self.node), repr(self.op), repr(self.expr))
class Backquote(Node):
- nodes["backquote"] = "Backquote"
- def __init__(self, expr):
+ def __init__(self, expr, lineno=None):
self.expr = expr
+ self.lineno = lineno
def getChildren(self):
return self.expr,
@@ -236,14 +218,12 @@
return "Backquote(%s)" % (repr(self.expr),)
class Bitand(Node):
- nodes["bitand"] = "Bitand"
- def __init__(self, nodes):
+ def __init__(self, nodes, lineno=None):
self.nodes = nodes
+ self.lineno = lineno
def getChildren(self):
- children = []
- children.extend(flatten(self.nodes))
- return tuple(children)
+ return tuple(flatten(self.nodes))
def getChildNodes(self):
nodelist = []
@@ -254,14 +234,12 @@
return "Bitand(%s)" % (repr(self.nodes),)
class Bitor(Node):
- nodes["bitor"] = "Bitor"
- def __init__(self, nodes):
+ def __init__(self, nodes, lineno=None):
self.nodes = nodes
+ self.lineno = lineno
def getChildren(self):
- children = []
- children.extend(flatten(self.nodes))
- return tuple(children)
+ return tuple(flatten(self.nodes))
def getChildNodes(self):
nodelist = []
@@ -272,14 +250,12 @@
return "Bitor(%s)" % (repr(self.nodes),)
class Bitxor(Node):
- nodes["bitxor"] = "Bitxor"
- def __init__(self, nodes):
+ def __init__(self, nodes, lineno=None):
self.nodes = nodes
+ self.lineno = lineno
def getChildren(self):
- children = []
- children.extend(flatten(self.nodes))
- return tuple(children)
+ return tuple(flatten(self.nodes))
def getChildNodes(self):
nodelist = []
@@ -290,9 +266,8 @@
return "Bitxor(%s)" % (repr(self.nodes),)
class Break(Node):
- nodes["break"] = "Break"
- def __init__(self, ):
- pass
+ def __init__(self, lineno=None):
+ self.lineno = lineno
def getChildren(self):
return ()
@@ -304,12 +279,12 @@
return "Break()"
class CallFunc(Node):
- nodes["callfunc"] = "CallFunc"
- def __init__(self, node, args, star_args = None, dstar_args = None):
+ def __init__(self, node, args, star_args = None, dstar_args = None, lineno=None):
self.node = node
self.args = args
self.star_args = star_args
self.dstar_args = dstar_args
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -323,20 +298,22 @@
nodelist = []
nodelist.append(self.node)
nodelist.extend(flatten_nodes(self.args))
- if self.star_args is not None: nodelist.append(self.star_args)
- if self.dstar_args is not None: nodelist.append(self.dstar_args)
+ if self.star_args is not None:
+ nodelist.append(self.star_args)
+ if self.dstar_args is not None:
+ nodelist.append(self.dstar_args)
return tuple(nodelist)
def __repr__(self):
return "CallFunc(%s, %s, %s, %s)" % (repr(self.node), repr(self.args), repr(self.star_args), repr(self.dstar_args))
class Class(Node):
- nodes["class"] = "Class"
- def __init__(self, name, bases, doc, code):
+ def __init__(self, name, bases, doc, code, lineno=None):
self.name = name
self.bases = bases
self.doc = doc
self.code = code
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -356,10 +333,10 @@
return "Class(%s, %s, %s, %s)" % (repr(self.name), repr(self.bases), repr(self.doc), repr(self.code))
class Compare(Node):
- nodes["compare"] = "Compare"
- def __init__(self, expr, ops):
+ def __init__(self, expr, ops, lineno=None):
self.expr = expr
self.ops = ops
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -377,9 +354,9 @@
return "Compare(%s, %s)" % (repr(self.expr), repr(self.ops))
class Const(Node):
- nodes["const"] = "Const"
- def __init__(self, value):
+ def __init__(self, value, lineno=None):
self.value = value
+ self.lineno = lineno
def getChildren(self):
return self.value,
@@ -391,9 +368,8 @@
return "Const(%s)" % (repr(self.value),)
class Continue(Node):
- nodes["continue"] = "Continue"
- def __init__(self, ):
- pass
+ def __init__(self, lineno=None):
+ self.lineno = lineno
def getChildren(self):
return ()
@@ -405,28 +381,28 @@
return "Continue()"
class Decorators(Node):
- nodes["decorators"] = "Decorators"
- def __init__(self, nodes):
+ def __init__(self, nodes, lineno=None):
self.nodes = nodes
+ self.lineno = lineno
def getChildren(self):
return tuple(flatten(self.nodes))
def getChildNodes(self):
- return flatten_nodes(self.nodes)
+ nodelist = []
+ nodelist.extend(flatten_nodes(self.nodes))
+ return tuple(nodelist)
def __repr__(self):
return "Decorators(%s)" % (repr(self.nodes),)
class Dict(Node):
- nodes["dict"] = "Dict"
- def __init__(self, items):
+ def __init__(self, items, lineno=None):
self.items = items
+ self.lineno = lineno
def getChildren(self):
- children = []
- children.extend(flatten(self.items))
- return tuple(children)
+ return tuple(flatten(self.items))
def getChildNodes(self):
nodelist = []
@@ -437,9 +413,9 @@
return "Dict(%s)" % (repr(self.items),)
class Discard(Node):
- nodes["discard"] = "Discard"
- def __init__(self, expr):
+ def __init__(self, expr, lineno=None):
self.expr = expr
+ self.lineno = lineno
def getChildren(self):
return self.expr,
@@ -451,10 +427,10 @@
return "Discard(%s)" % (repr(self.expr),)
class Div(Node):
- nodes["div"] = "Div"
- def __init__(self, (left, right)):
+ def __init__(self, (left, right), lineno=None):
self.left = left
self.right = right
+ self.lineno = lineno
def getChildren(self):
return self.left, self.right
@@ -466,9 +442,8 @@
return "Div((%s, %s))" % (repr(self.left), repr(self.right))
class Ellipsis(Node):
- nodes["ellipsis"] = "Ellipsis"
- def __init__(self, ):
- pass
+ def __init__(self, lineno=None):
+ self.lineno = lineno
def getChildren(self):
return ()
@@ -480,11 +455,11 @@
return "Ellipsis()"
class Exec(Node):
- nodes["exec"] = "Exec"
- def __init__(self, expr, locals, globals):
+ def __init__(self, expr, locals, globals, lineno=None):
self.expr = expr
self.locals = locals
self.globals = globals
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -496,18 +471,20 @@
def getChildNodes(self):
nodelist = []
nodelist.append(self.expr)
- if self.locals is not None: nodelist.append(self.locals)
- if self.globals is not None: nodelist.append(self.globals)
+ if self.locals is not None:
+ nodelist.append(self.locals)
+ if self.globals is not None:
+ nodelist.append(self.globals)
return tuple(nodelist)
def __repr__(self):
return "Exec(%s, %s, %s)" % (repr(self.expr), repr(self.locals), repr(self.globals))
class FloorDiv(Node):
- nodes["floordiv"] = "FloorDiv"
- def __init__(self, (left, right)):
+ def __init__(self, (left, right), lineno=None):
self.left = left
self.right = right
+ self.lineno = lineno
def getChildren(self):
return self.left, self.right
@@ -519,12 +496,12 @@
return "FloorDiv((%s, %s))" % (repr(self.left), repr(self.right))
class For(Node):
- nodes["for"] = "For"
- def __init__(self, assign, list, body, else_):
+ def __init__(self, assign, list, body, else_, lineno=None):
self.assign = assign
self.list = list
self.body = body
self.else_ = else_
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -539,17 +516,18 @@
nodelist.append(self.assign)
nodelist.append(self.list)
nodelist.append(self.body)
- if self.else_ is not None: nodelist.append(self.else_)
+ if self.else_ is not None:
+ nodelist.append(self.else_)
return tuple(nodelist)
def __repr__(self):
return "For(%s, %s, %s, %s)" % (repr(self.assign), repr(self.list), repr(self.body), repr(self.else_))
class From(Node):
- nodes["from"] = "From"
- def __init__(self, modname, names):
+ def __init__(self, modname, names, lineno=None):
self.modname = modname
self.names = names
+ self.lineno = lineno
def getChildren(self):
return self.modname, self.names
@@ -561,8 +539,7 @@
return "From(%s, %s)" % (repr(self.modname), repr(self.names))
class Function(Node):
- nodes["function"] = "Function"
- def __init__(self, decorators, name, argnames, defaults, flags, doc, code):
+ def __init__(self, decorators, name, argnames, defaults, flags, doc, code, lineno=None):
self.decorators = decorators
self.name = name
self.argnames = argnames
@@ -570,18 +547,18 @@
self.flags = flags
self.doc = doc
self.code = code
+ self.lineno = lineno
self.varargs = self.kwargs = None
if flags & CO_VARARGS:
self.varargs = 1
if flags & CO_VARKEYWORDS:
self.kwargs = 1
-
+
def getChildren(self):
children = []
- if self.decorators:
- children.append(flatten(self.decorators.nodes))
+ children.append(self.decorators)
children.append(self.name)
children.append(self.argnames)
children.extend(flatten(self.defaults))
@@ -592,8 +569,8 @@
def getChildNodes(self):
nodelist = []
- if self.decorators:
- nodelist.extend(flatten_nodes(self.decorators.nodes))
+ if self.decorators is not None:
+ nodelist.append(self.decorators)
nodelist.extend(flatten_nodes(self.defaults))
nodelist.append(self.code)
return tuple(nodelist)
@@ -602,12 +579,12 @@
return "Function(%s, %s, %s, %s, %s, %s, %s)" % (repr(self.decorators), repr(self.name), repr(self.argnames), repr(self.defaults), repr(self.flags), repr(self.doc), repr(self.code))
class GenExpr(Node):
- nodes["genexpr"] = "GenExpr"
- def __init__(self, code):
+ def __init__(self, code, lineno=None):
self.code = code
+ self.lineno = lineno
self.argnames = ['[outmost-iterable]']
self.varargs = self.kwargs = None
-
+
def getChildren(self):
@@ -620,11 +597,11 @@
return "GenExpr(%s)" % (repr(self.code),)
class GenExprFor(Node):
- nodes["genexprfor"] = "GenExprFor"
- def __init__(self, assign, iter, ifs):
+ def __init__(self, assign, iter, ifs, lineno=None):
self.assign = assign
self.iter = iter
self.ifs = ifs
+ self.lineno = lineno
self.is_outmost = False
@@ -646,9 +623,9 @@
return "GenExprFor(%s, %s, %s)" % (repr(self.assign), repr(self.iter), repr(self.ifs))
class GenExprIf(Node):
- nodes["genexprif"] = "GenExprIf"
- def __init__(self, test):
+ def __init__(self, test, lineno=None):
self.test = test
+ self.lineno = lineno
def getChildren(self):
return self.test,
@@ -660,10 +637,10 @@
return "GenExprIf(%s)" % (repr(self.test),)
class GenExprInner(Node):
- nodes["genexprinner"] = "GenExprInner"
- def __init__(self, expr, quals):
+ def __init__(self, expr, quals, lineno=None):
self.expr = expr
self.quals = quals
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -681,10 +658,10 @@
return "GenExprInner(%s, %s)" % (repr(self.expr), repr(self.quals))
class Getattr(Node):
- nodes["getattr"] = "Getattr"
- def __init__(self, expr, attrname):
+ def __init__(self, expr, attrname, lineno=None):
self.expr = expr
self.attrname = attrname
+ self.lineno = lineno
def getChildren(self):
return self.expr, self.attrname
@@ -696,9 +673,9 @@
return "Getattr(%s, %s)" % (repr(self.expr), repr(self.attrname))
class Global(Node):
- nodes["global"] = "Global"
- def __init__(self, names):
+ def __init__(self, names, lineno=None):
self.names = names
+ self.lineno = lineno
def getChildren(self):
return self.names,
@@ -710,10 +687,10 @@
return "Global(%s)" % (repr(self.names),)
class If(Node):
- nodes["if"] = "If"
- def __init__(self, tests, else_):
+ def __init__(self, tests, else_, lineno=None):
self.tests = tests
self.else_ = else_
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -724,16 +701,17 @@
def getChildNodes(self):
nodelist = []
nodelist.extend(flatten_nodes(self.tests))
- if self.else_ is not None: nodelist.append(self.else_)
+ if self.else_ is not None:
+ nodelist.append(self.else_)
return tuple(nodelist)
def __repr__(self):
return "If(%s, %s)" % (repr(self.tests), repr(self.else_))
class Import(Node):
- nodes["import"] = "Import"
- def __init__(self, names):
+ def __init__(self, names, lineno=None):
self.names = names
+ self.lineno = lineno
def getChildren(self):
return self.names,
@@ -745,9 +723,9 @@
return "Import(%s)" % (repr(self.names),)
class Invert(Node):
- nodes["invert"] = "Invert"
- def __init__(self, expr):
+ def __init__(self, expr, lineno=None):
self.expr = expr
+ self.lineno = lineno
def getChildren(self):
return self.expr,
@@ -759,10 +737,10 @@
return "Invert(%s)" % (repr(self.expr),)
class Keyword(Node):
- nodes["keyword"] = "Keyword"
- def __init__(self, name, expr):
+ def __init__(self, name, expr, lineno=None):
self.name = name
self.expr = expr
+ self.lineno = lineno
def getChildren(self):
return self.name, self.expr
@@ -774,18 +752,18 @@
return "Keyword(%s, %s)" % (repr(self.name), repr(self.expr))
class Lambda(Node):
- nodes["lambda"] = "Lambda"
- def __init__(self, argnames, defaults, flags, code):
+ def __init__(self, argnames, defaults, flags, code, lineno=None):
self.argnames = argnames
self.defaults = defaults
self.flags = flags
self.code = code
+ self.lineno = lineno
self.varargs = self.kwargs = None
if flags & CO_VARARGS:
self.varargs = 1
if flags & CO_VARKEYWORDS:
self.kwargs = 1
-
+
def getChildren(self):
@@ -806,10 +784,10 @@
return "Lambda(%s, %s, %s, %s)" % (repr(self.argnames), repr(self.defaults), repr(self.flags), repr(self.code))
class LeftShift(Node):
- nodes["leftshift"] = "LeftShift"
- def __init__(self, (left, right)):
+ def __init__(self, (left, right), lineno=None):
self.left = left
self.right = right
+ self.lineno = lineno
def getChildren(self):
return self.left, self.right
@@ -821,14 +799,12 @@
return "LeftShift((%s, %s))" % (repr(self.left), repr(self.right))
class List(Node):
- nodes["list"] = "List"
- def __init__(self, nodes):
+ def __init__(self, nodes, lineno=None):
self.nodes = nodes
+ self.lineno = lineno
def getChildren(self):
- children = []
- children.extend(flatten(self.nodes))
- return tuple(children)
+ return tuple(flatten(self.nodes))
def getChildNodes(self):
nodelist = []
@@ -839,10 +815,10 @@
return "List(%s)" % (repr(self.nodes),)
class ListComp(Node):
- nodes["listcomp"] = "ListComp"
- def __init__(self, expr, quals):
+ def __init__(self, expr, quals, lineno=None):
self.expr = expr
self.quals = quals
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -860,11 +836,11 @@
return "ListComp(%s, %s)" % (repr(self.expr), repr(self.quals))
class ListCompFor(Node):
- nodes["listcompfor"] = "ListCompFor"
- def __init__(self, assign, list, ifs):
+ def __init__(self, assign, list, ifs, lineno=None):
self.assign = assign
self.list = list
self.ifs = ifs
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -884,9 +860,9 @@
return "ListCompFor(%s, %s, %s)" % (repr(self.assign), repr(self.list), repr(self.ifs))
class ListCompIf(Node):
- nodes["listcompif"] = "ListCompIf"
- def __init__(self, test):
+ def __init__(self, test, lineno=None):
self.test = test
+ self.lineno = lineno
def getChildren(self):
return self.test,
@@ -898,10 +874,10 @@
return "ListCompIf(%s)" % (repr(self.test),)
class Mod(Node):
- nodes["mod"] = "Mod"
- def __init__(self, (left, right)):
+ def __init__(self, (left, right), lineno=None):
self.left = left
self.right = right
+ self.lineno = lineno
def getChildren(self):
return self.left, self.right
@@ -913,10 +889,10 @@
return "Mod((%s, %s))" % (repr(self.left), repr(self.right))
class Module(Node):
- nodes["module"] = "Module"
- def __init__(self, doc, node):
+ def __init__(self, doc, node, lineno=None):
self.doc = doc
self.node = node
+ self.lineno = lineno
def getChildren(self):
return self.doc, self.node
@@ -928,10 +904,10 @@
return "Module(%s, %s)" % (repr(self.doc), repr(self.node))
class Mul(Node):
- nodes["mul"] = "Mul"
- def __init__(self, (left, right)):
+ def __init__(self, (left, right), lineno=None):
self.left = left
self.right = right
+ self.lineno = lineno
def getChildren(self):
return self.left, self.right
@@ -943,9 +919,9 @@
return "Mul((%s, %s))" % (repr(self.left), repr(self.right))
class Name(Node):
- nodes["name"] = "Name"
- def __init__(self, name):
+ def __init__(self, name, lineno=None):
self.name = name
+ self.lineno = lineno
def getChildren(self):
return self.name,
@@ -957,9 +933,9 @@
return "Name(%s)" % (repr(self.name),)
class Not(Node):
- nodes["not"] = "Not"
- def __init__(self, expr):
+ def __init__(self, expr, lineno=None):
self.expr = expr
+ self.lineno = lineno
def getChildren(self):
return self.expr,
@@ -971,14 +947,12 @@
return "Not(%s)" % (repr(self.expr),)
class Or(Node):
- nodes["or"] = "Or"
- def __init__(self, nodes):
+ def __init__(self, nodes, lineno=None):
self.nodes = nodes
+ self.lineno = lineno
def getChildren(self):
- children = []
- children.extend(flatten(self.nodes))
- return tuple(children)
+ return tuple(flatten(self.nodes))
def getChildNodes(self):
nodelist = []
@@ -989,9 +963,8 @@
return "Or(%s)" % (repr(self.nodes),)
class Pass(Node):
- nodes["pass"] = "Pass"
- def __init__(self, ):
- pass
+ def __init__(self, lineno=None):
+ self.lineno = lineno
def getChildren(self):
return ()
@@ -1003,10 +976,10 @@
return "Pass()"
class Power(Node):
- nodes["power"] = "Power"
- def __init__(self, (left, right)):
+ def __init__(self, (left, right), lineno=None):
self.left = left
self.right = right
+ self.lineno = lineno
def getChildren(self):
return self.left, self.right
@@ -1018,10 +991,10 @@
return "Power((%s, %s))" % (repr(self.left), repr(self.right))
class Print(Node):
- nodes["print"] = "Print"
- def __init__(self, nodes, dest):
+ def __init__(self, nodes, dest, lineno=None):
self.nodes = nodes
self.dest = dest
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -1032,17 +1005,18 @@
def getChildNodes(self):
nodelist = []
nodelist.extend(flatten_nodes(self.nodes))
- if self.dest is not None: nodelist.append(self.dest)
+ if self.dest is not None:
+ nodelist.append(self.dest)
return tuple(nodelist)
def __repr__(self):
return "Print(%s, %s)" % (repr(self.nodes), repr(self.dest))
class Printnl(Node):
- nodes["printnl"] = "Printnl"
- def __init__(self, nodes, dest):
+ def __init__(self, nodes, dest, lineno=None):
self.nodes = nodes
self.dest = dest
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -1053,18 +1027,19 @@
def getChildNodes(self):
nodelist = []
nodelist.extend(flatten_nodes(self.nodes))
- if self.dest is not None: nodelist.append(self.dest)
+ if self.dest is not None:
+ nodelist.append(self.dest)
return tuple(nodelist)
def __repr__(self):
return "Printnl(%s, %s)" % (repr(self.nodes), repr(self.dest))
class Raise(Node):
- nodes["raise"] = "Raise"
- def __init__(self, expr1, expr2, expr3):
+ def __init__(self, expr1, expr2, expr3, lineno=None):
self.expr1 = expr1
self.expr2 = expr2
self.expr3 = expr3
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -1075,18 +1050,21 @@
def getChildNodes(self):
nodelist = []
- if self.expr1 is not None: nodelist.append(self.expr1)
- if self.expr2 is not None: nodelist.append(self.expr2)
- if self.expr3 is not None: nodelist.append(self.expr3)
+ if self.expr1 is not None:
+ nodelist.append(self.expr1)
+ if self.expr2 is not None:
+ nodelist.append(self.expr2)
+ if self.expr3 is not None:
+ nodelist.append(self.expr3)
return tuple(nodelist)
def __repr__(self):
return "Raise(%s, %s, %s)" % (repr(self.expr1), repr(self.expr2), repr(self.expr3))
class Return(Node):
- nodes["return"] = "Return"
- def __init__(self, value):
+ def __init__(self, value, lineno=None):
self.value = value
+ self.lineno = lineno
def getChildren(self):
return self.value,
@@ -1098,10 +1076,10 @@
return "Return(%s)" % (repr(self.value),)
class RightShift(Node):
- nodes["rightshift"] = "RightShift"
- def __init__(self, (left, right)):
+ def __init__(self, (left, right), lineno=None):
self.left = left
self.right = right
+ self.lineno = lineno
def getChildren(self):
return self.left, self.right
@@ -1113,12 +1091,12 @@
return "RightShift((%s, %s))" % (repr(self.left), repr(self.right))
class Slice(Node):
- nodes["slice"] = "Slice"
- def __init__(self, expr, flags, lower, upper):
+ def __init__(self, expr, flags, lower, upper, lineno=None):
self.expr = expr
self.flags = flags
self.lower = lower
self.upper = upper
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -1131,22 +1109,22 @@
def getChildNodes(self):
nodelist = []
nodelist.append(self.expr)
- if self.lower is not None: nodelist.append(self.lower)
- if self.upper is not None: nodelist.append(self.upper)
+ if self.lower is not None:
+ nodelist.append(self.lower)
+ if self.upper is not None:
+ nodelist.append(self.upper)
return tuple(nodelist)
def __repr__(self):
return "Slice(%s, %s, %s, %s)" % (repr(self.expr), repr(self.flags), repr(self.lower), repr(self.upper))
class Sliceobj(Node):
- nodes["sliceobj"] = "Sliceobj"
- def __init__(self, nodes):
+ def __init__(self, nodes, lineno=None):
self.nodes = nodes
+ self.lineno = lineno
def getChildren(self):
- children = []
- children.extend(flatten(self.nodes))
- return tuple(children)
+ return tuple(flatten(self.nodes))
def getChildNodes(self):
nodelist = []
@@ -1157,14 +1135,12 @@
return "Sliceobj(%s)" % (repr(self.nodes),)
class Stmt(Node):
- nodes["stmt"] = "Stmt"
- def __init__(self, nodes):
+ def __init__(self, nodes, lineno=None):
self.nodes = nodes
+ self.lineno = lineno
def getChildren(self):
- children = []
- children.extend(flatten(self.nodes))
- return tuple(children)
+ return tuple(flatten(self.nodes))
def getChildNodes(self):
nodelist = []
@@ -1175,10 +1151,10 @@
return "Stmt(%s)" % (repr(self.nodes),)
class Sub(Node):
- nodes["sub"] = "Sub"
- def __init__(self, (left, right)):
+ def __init__(self, (left, right), lineno=None):
self.left = left
self.right = right
+ self.lineno = lineno
def getChildren(self):
return self.left, self.right
@@ -1190,11 +1166,11 @@
return "Sub((%s, %s))" % (repr(self.left), repr(self.right))
class Subscript(Node):
- nodes["subscript"] = "Subscript"
- def __init__(self, expr, flags, subs):
+ def __init__(self, expr, flags, subs, lineno=None):
self.expr = expr
self.flags = flags
self.subs = subs
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -1213,11 +1189,11 @@
return "Subscript(%s, %s, %s)" % (repr(self.expr), repr(self.flags), repr(self.subs))
class TryExcept(Node):
- nodes["tryexcept"] = "TryExcept"
- def __init__(self, body, handlers, else_):
+ def __init__(self, body, handlers, else_, lineno=None):
self.body = body
self.handlers = handlers
self.else_ = else_
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -1230,17 +1206,18 @@
nodelist = []
nodelist.append(self.body)
nodelist.extend(flatten_nodes(self.handlers))
- if self.else_ is not None: nodelist.append(self.else_)
+ if self.else_ is not None:
+ nodelist.append(self.else_)
return tuple(nodelist)
def __repr__(self):
return "TryExcept(%s, %s, %s)" % (repr(self.body), repr(self.handlers), repr(self.else_))
class TryFinally(Node):
- nodes["tryfinally"] = "TryFinally"
- def __init__(self, body, final):
+ def __init__(self, body, final, lineno=None):
self.body = body
self.final = final
+ self.lineno = lineno
def getChildren(self):
return self.body, self.final
@@ -1252,14 +1229,12 @@
return "TryFinally(%s, %s)" % (repr(self.body), repr(self.final))
class Tuple(Node):
- nodes["tuple"] = "Tuple"
- def __init__(self, nodes):
+ def __init__(self, nodes, lineno=None):
self.nodes = nodes
+ self.lineno = lineno
def getChildren(self):
- children = []
- children.extend(flatten(self.nodes))
- return tuple(children)
+ return tuple(flatten(self.nodes))
def getChildNodes(self):
nodelist = []
@@ -1270,9 +1245,9 @@
return "Tuple(%s)" % (repr(self.nodes),)
class UnaryAdd(Node):
- nodes["unaryadd"] = "UnaryAdd"
- def __init__(self, expr):
+ def __init__(self, expr, lineno=None):
self.expr = expr
+ self.lineno = lineno
def getChildren(self):
return self.expr,
@@ -1284,9 +1259,9 @@
return "UnaryAdd(%s)" % (repr(self.expr),)
class UnarySub(Node):
- nodes["unarysub"] = "UnarySub"
- def __init__(self, expr):
+ def __init__(self, expr, lineno=None):
self.expr = expr
+ self.lineno = lineno
def getChildren(self):
return self.expr,
@@ -1298,11 +1273,11 @@
return "UnarySub(%s)" % (repr(self.expr),)
class While(Node):
- nodes["while"] = "While"
- def __init__(self, test, body, else_):
+ def __init__(self, test, body, else_, lineno=None):
self.test = test
self.body = body
self.else_ = else_
+ self.lineno = lineno
def getChildren(self):
children = []
@@ -1315,16 +1290,17 @@
nodelist = []
nodelist.append(self.test)
nodelist.append(self.body)
- if self.else_ is not None: nodelist.append(self.else_)
+ if self.else_ is not None:
+ nodelist.append(self.else_)
return tuple(nodelist)
def __repr__(self):
return "While(%s, %s, %s)" % (repr(self.test), repr(self.body), repr(self.else_))
class Yield(Node):
- nodes["yield"] = "Yield"
- def __init__(self, value):
+ def __init__(self, value, lineno=None):
self.value = value
+ self.lineno = lineno
def getChildren(self):
return self.value,
@@ -1335,6 +1311,6 @@
def __repr__(self):
return "Yield(%s)" % (repr(self.value),)
-klasses = globals()
-for k in nodes.keys():
- nodes[k] = klasses[nodes[k]]
+for name, obj in globals().items():
+ if isinstance(obj, type) and issubclass(obj, Node):
+ nodes[name.lower()] = obj
diff --git a/Lib/compiler/transformer.py b/Lib/compiler/transformer.py
index 9fb18c2..9d2f378 100644
--- a/Lib/compiler/transformer.py
+++ b/Lib/compiler/transformer.py
@@ -14,7 +14,10 @@
#
# Modifications and improvements for Python 2.0 by Jeremy Hylton and
# Mark Hammond
-
+#
+# Some fixes to try to have correct line number on almost all nodes
+# (except Module, Discard and Stmt) added by Sylvain Thenault
+#
# Portions of this file are:
# Copyright (C) 1997-1998 Greg Stein. All Rights Reserved.
#
@@ -22,21 +25,20 @@
# http://www.opensource.org/licenses/bsd-license.html
# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
-from ast import *
+from compiler.ast import *
import parser
-# Care must be taken to use only symbols and tokens defined in Python
-# 1.5.2 for code branches executed in 1.5.2
import symbol
import token
import sys
-error = 'walker.error'
+class WalkerError(StandardError):
+ pass
from consts import CO_VARARGS, CO_VARKEYWORDS
from consts import OP_ASSIGN, OP_DELETE, OP_APPLY
def parseFile(path):
- f = open(path)
+ f = open(path, "U")
# XXX The parser API tolerates files without a trailing newline,
# but not strings without a trailing newline. Always add an extra
# newline to the file contents, since we're going through the string
@@ -68,6 +70,16 @@
l.append(item)
return l
+def extractLineNo(ast):
+ if not isinstance(ast[1], tuple):
+ # get a terminal node
+ return ast[2]
+ for child in ast[1:]:
+ if isinstance(child, tuple):
+ lineno = extractLineNo(child)
+ if lineno is not None:
+ return lineno
+
def Node(*args):
kind = args[0]
if nodes.has_key(kind):
@@ -77,7 +89,7 @@
print nodes[kind], len(args), args
raise
else:
- raise error, "Can't find appropriate Node type: %s" % str(args)
+ raise WalkerEror, "Can't find appropriate Node type: %s" % str(args)
#return apply(ast.Node, args)
class Transformer:
@@ -108,17 +120,14 @@
def transform(self, tree):
"""Transform an AST into a modified parse tree."""
- if type(tree) != type(()) and type(tree) != type([]):
+ if not (isinstance(tree, tuple) or isinstance(tree, list)):
tree = parser.ast2tuple(tree, line_info=1)
return self.compile_node(tree)
def parsesuite(self, text):
"""Return a modified parse tree for the given suite text."""
- # Hack for handling non-native line endings on non-DOS like OSs.
- # this can go now we have universal newlines?
- text = text.replace('\x0d', '')
return self.transform(parser.suite(text))
-
+
def parseexpr(self, text):
"""Return a modified parse tree for the given expression text."""
return self.transform(parser.expr(text))
@@ -156,7 +165,7 @@
if n == symbol.classdef:
return self.classdef(node[1:])
- raise error, ('unexpected node type', n)
+ raise WalkerEror, ('unexpected node type', n)
def single_input(self, node):
### do we want to do anything about being "interactive" ?
@@ -254,9 +263,8 @@
assert isinstance(code, Stmt)
assert isinstance(code.nodes[0], Discard)
del code.nodes[0]
- n = Function(decorators, name, names, defaults, flags, doc, code)
- n.lineno = lineno
- return n
+ return Function(decorators, name, names, defaults, flags, doc, code,
+ lineno=lineno)
def lambdef(self, nodelist):
# lambdef: 'lambda' [varargslist] ':' test
@@ -269,9 +277,7 @@
# code for lambda
code = self.com_node(nodelist[-1])
- n = Lambda(names, defaults, flags, code)
- n.lineno = nodelist[1][2]
- return n
+ return Lambda(names, defaults, flags, code, lineno=nodelist[1][2])
def classdef(self, nodelist):
# classdef: 'class' NAME ['(' testlist ')'] ':' suite
@@ -291,9 +297,7 @@
assert isinstance(code.nodes[0], Discard)
del code.nodes[0]
- n = Class(name, bases, doc, code)
- n.lineno = nodelist[1][2]
- return n
+ return Class(name, bases, doc, code, lineno=nodelist[1][2])
def stmt(self, nodelist):
return self.com_stmt(nodelist[0])
@@ -310,31 +314,31 @@
return Stmt(stmts)
def parameters(self, nodelist):
- raise error
+ raise WalkerEror
def varargslist(self, nodelist):
- raise error
+ raise WalkerEror
def fpdef(self, nodelist):
- raise error
+ raise WalkerEror
def fplist(self, nodelist):
- raise error
+ raise WalkerEror
def dotted_name(self, nodelist):
- raise error
+ raise WalkerEror
def comp_op(self, nodelist):
- raise error
+ raise WalkerEror
def trailer(self, nodelist):
- raise error
+ raise WalkerEror
def sliceop(self, nodelist):
- raise error
+ raise WalkerEror
def argument(self, nodelist):
- raise error
+ raise WalkerEror
# --------------------------------------------------------------
#
@@ -346,21 +350,17 @@
en = nodelist[-1]
exprNode = self.lookup_node(en)(en[1:])
if len(nodelist) == 1:
- n = Discard(exprNode)
- n.lineno = exprNode.lineno
- return n
+ return Discard(exprNode, lineno=exprNode.lineno)
if nodelist[1][0] == token.EQUAL:
nodesl = []
for i in range(0, len(nodelist) - 2, 2):
nodesl.append(self.com_assign(nodelist[i], OP_ASSIGN))
- n = Assign(nodesl, exprNode)
- n.lineno = nodelist[1][2]
+ return Assign(nodesl, exprNode, lineno=nodelist[1][2])
else:
lval = self.com_augassign(nodelist[0])
op = self.com_augassign_op(nodelist[1])
- n = AugAssign(lval, op[1], exprNode)
- n.lineno = op[2]
- return n
+ return AugAssign(lval, op[1], exprNode, lineno=op[2])
+ raise WalkerError, "can't get here"
def print_stmt(self, nodelist):
# print ([ test (',' test)* [','] ] | '>>' test [ (',' test)+ [','] ])
@@ -379,45 +379,29 @@
for i in range(start, len(nodelist), 2):
items.append(self.com_node(nodelist[i]))
if nodelist[-1][0] == token.COMMA:
- n = Print(items, dest)
- n.lineno = nodelist[0][2]
- return n
- n = Printnl(items, dest)
- n.lineno = nodelist[0][2]
- return n
+ return Print(items, dest, lineno=nodelist[0][2])
+ return Printnl(items, dest, lineno=nodelist[0][2])
def del_stmt(self, nodelist):
return self.com_assign(nodelist[1], OP_DELETE)
def pass_stmt(self, nodelist):
- n = Pass()
- n.lineno = nodelist[0][2]
- return n
+ return Pass(lineno=nodelist[0][2])
def break_stmt(self, nodelist):
- n = Break()
- n.lineno = nodelist[0][2]
- return n
+ return Break(lineno=nodelist[0][2])
def continue_stmt(self, nodelist):
- n = Continue()
- n.lineno = nodelist[0][2]
- return n
+ return Continue(lineno=nodelist[0][2])
def return_stmt(self, nodelist):
# return: [testlist]
if len(nodelist) < 2:
- n = Return(Const(None))
- n.lineno = nodelist[0][2]
- return n
- n = Return(self.com_node(nodelist[1]))
- n.lineno = nodelist[0][2]
- return n
+ return Return(Const(None), lineno=nodelist[0][2])
+ return Return(self.com_node(nodelist[1]), lineno=nodelist[0][2])
def yield_stmt(self, nodelist):
- n = Yield(self.com_node(nodelist[1]))
- n.lineno = nodelist[0][2]
- return n
+ return Yield(self.com_node(nodelist[1]), lineno=nodelist[0][2])
def raise_stmt(self, nodelist):
# raise: [test [',' test [',' test]]]
@@ -433,9 +417,7 @@
expr1 = self.com_node(nodelist[1])
else:
expr1 = None
- n = Raise(expr1, expr2, expr3)
- n.lineno = nodelist[0][2]
- return n
+ return Raise(expr1, expr2, expr3, lineno=nodelist[0][2])
def import_stmt(self, nodelist):
# import_stmt: import_name | import_from
@@ -444,9 +426,8 @@
def import_name(self, nodelist):
# import_name: 'import' dotted_as_names
- n = Import(self.com_dotted_as_names(nodelist[1]))
- n.lineno = nodelist[0][2]
- return n
+ return Import(self.com_dotted_as_names(nodelist[1]),
+ lineno=nodelist[0][2])
def import_from(self, nodelist):
# import_from: 'from' dotted_name 'import' ('*' |
@@ -456,21 +437,19 @@
assert nodelist[2][1] == 'import'
fromname = self.com_dotted_name(nodelist[1])
if nodelist[3][0] == token.STAR:
- n = From(fromname, [('*', None)])
+ # TODO(jhylton): where is the lineno?
+ return From(fromname, [('*', None)])
else:
node = nodelist[3 + (nodelist[3][0] == token.LPAR)]
- n = From(fromname, self.com_import_as_names(node))
- n.lineno = nodelist[0][2]
- return n
+ return From(fromname, self.com_import_as_names(node),
+ lineno=nodelist[0][2])
def global_stmt(self, nodelist):
# global: NAME (',' NAME)*
names = []
for i in range(1, len(nodelist), 2):
names.append(nodelist[i][1])
- n = Global(names)
- n.lineno = nodelist[0][2]
- return n
+ return Global(names, lineno=nodelist[0][2])
def exec_stmt(self, nodelist):
# exec_stmt: 'exec' expr ['in' expr [',' expr]]
@@ -484,9 +463,7 @@
else:
expr2 = expr3 = None
- n = Exec(expr1, expr2, expr3)
- n.lineno = nodelist[0][2]
- return n
+ return Exec(expr1, expr2, expr3, lineno=nodelist[0][2])
def assert_stmt(self, nodelist):
# 'assert': test, [',' test]
@@ -495,9 +472,7 @@
expr2 = self.com_node(nodelist[3])
else:
expr2 = None
- n = Assert(expr1, expr2)
- n.lineno = nodelist[0][2]
- return n
+ return Assert(expr1, expr2, lineno=nodelist[0][2])
def if_stmt(self, nodelist):
# if: test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
@@ -512,9 +487,7 @@
## elseNode.lineno = nodelist[-1][1][2]
else:
elseNode = None
- n = If(tests, elseNode)
- n.lineno = nodelist[0][2]
- return n
+ return If(tests, elseNode, lineno=nodelist[0][2])
def while_stmt(self, nodelist):
# 'while' test ':' suite ['else' ':' suite]
@@ -527,9 +500,7 @@
else:
elseNode = None
- n = While(testNode, bodyNode, elseNode)
- n.lineno = nodelist[0][2]
- return n
+ return While(testNode, bodyNode, elseNode, lineno=nodelist[0][2])
def for_stmt(self, nodelist):
# 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite]
@@ -543,9 +514,8 @@
else:
elseNode = None
- n = For(assignNode, listNode, bodyNode, elseNode)
- n.lineno = nodelist[0][2]
- return n
+ return For(assignNode, listNode, bodyNode, elseNode,
+ lineno=nodelist[0][2])
def try_stmt(self, nodelist):
# 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite]
@@ -601,9 +571,7 @@
# 'not' not_test | comparison
result = self.com_node(nodelist[-1])
if len(nodelist) == 2:
- n = Not(result)
- n.lineno = nodelist[0][2]
- return n
+ return Not(result, lineno=nodelist[0][2])
return result
def comparison(self, nodelist):
@@ -637,9 +605,7 @@
# the two have very different semantics and results (note that the
# latter form is always true)
- n = Compare(node, results)
- n.lineno = lineno
- return n
+ return Compare(node, results, lineno=lineno)
def expr(self, nodelist):
# xor_expr ('|' xor_expr)*
@@ -659,11 +625,9 @@
for i in range(2, len(nodelist), 2):
right = self.com_node(nodelist[i])
if nodelist[i-1][0] == token.LEFTSHIFT:
- node = LeftShift([node, right])
- node.lineno = nodelist[1][2]
+ node = LeftShift([node, right], lineno=nodelist[1][2])
elif nodelist[i-1][0] == token.RIGHTSHIFT:
- node = RightShift([node, right])
- node.lineno = nodelist[1][2]
+ node = RightShift([node, right], lineno=nodelist[1][2])
else:
raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
return node
@@ -673,11 +637,9 @@
for i in range(2, len(nodelist), 2):
right = self.com_node(nodelist[i])
if nodelist[i-1][0] == token.PLUS:
- node = Add([node, right])
- node.lineno = nodelist[1][2]
+ node = Add([node, right], lineno=nodelist[1][2])
elif nodelist[i-1][0] == token.MINUS:
- node = Sub([node, right])
- node.lineno = nodelist[1][2]
+ node = Sub([node, right], lineno=nodelist[1][2])
else:
raise ValueError, "unexpected token: %s" % nodelist[i-1][0]
return node
@@ -703,17 +665,14 @@
def factor(self, nodelist):
elt = nodelist[0]
t = elt[0]
- node = self.com_node(nodelist[-1])
+ node = self.lookup_node(nodelist[-1])(nodelist[-1][1:])
# need to handle (unary op)constant here...
if t == token.PLUS:
- node = UnaryAdd(node)
- node.lineno = elt[2]
+ return UnaryAdd(node, lineno=elt[2])
elif t == token.MINUS:
- node = UnarySub(node)
- node.lineno = elt[2]
+ return UnarySub(node, lineno=elt[2])
elif t == token.TILDE:
- node = Invert(node)
- node.lineno = elt[2]
+ node = Invert(node, lineno=elt[2])
return node
def power(self, nodelist):
@@ -722,31 +681,26 @@
for i in range(1, len(nodelist)):
elt = nodelist[i]
if elt[0] == token.DOUBLESTAR:
- n = Power([node, self.com_node(nodelist[i+1])])
- n.lineno = elt[2]
- return n
+ return Power([node, self.com_node(nodelist[i+1])],
+ lineno=elt[2])
node = self.com_apply_trailer(node, elt)
return node
def atom(self, nodelist):
- n = self._atom_dispatch[nodelist[0][0]](nodelist)
+ return self._atom_dispatch[nodelist[0][0]](nodelist)
n.lineno = nodelist[0][2]
return n
def atom_lpar(self, nodelist):
if nodelist[1][0] == token.RPAR:
- n = Tuple(())
- n.lineno = nodelist[0][2]
- return n
+ return Tuple(())
return self.com_node(nodelist[1])
def atom_lsqb(self, nodelist):
if nodelist[1][0] == token.RSQB:
- n = List(())
- n.lineno = nodelist[0][2]
- return n
+ return List(())
return self.com_list_constructor(nodelist[1])
def atom_lbrace(self, nodelist):
@@ -755,16 +709,12 @@
return self.com_dictmaker(nodelist[1])
def atom_backquote(self, nodelist):
- n = Backquote(self.com_node(nodelist[1]))
- n.lineno = nodelist[0][2]
- return n
+ return Backquote(self.com_node(nodelist[1]))
def atom_number(self, nodelist):
### need to verify this matches compile.c
k = eval(nodelist[0][1])
- n = Const(k)
- n.lineno = nodelist[0][2]
- return n
+ return Const(k, lineno=nodelist[0][2])
def decode_literal(self, lit):
if self.encoding:
@@ -781,15 +731,10 @@
k = ''
for node in nodelist:
k += self.decode_literal(node[1])
- n = Const(k)
- n.lineno = nodelist[0][2]
- return n
+ return Const(k, lineno=nodelist[0][2])
def atom_name(self, nodelist):
- ### any processing to do?
- n = Name(nodelist[0][1])
- n.lineno = nodelist[0][2]
- return n
+ return Name(nodelist[0][1], lineno=nodelist[0][2])
# --------------------------------------------------------------
#
@@ -807,6 +752,8 @@
def lookup_node(self, node):
return self._dispatch[node[0]]
+ _callers = {}
+
def com_node(self, node):
# Note: compile.c has handling in com_node for del_stmt, pass_stmt,
# break_stmt, stmt, small_stmt, flow_stmt, simple_stmt,
@@ -865,6 +812,7 @@
i = i + 2
elif len(defaults):
# Treat "(a=1, b)" as "(a=1, b=None)"
+ print nodelist[i]
defaults.append(Const(None))
i = i + 1
@@ -938,10 +886,9 @@
def com_try_finally(self, nodelist):
# try_fin_stmt: "try" ":" suite "finally" ":" suite
- n = TryFinally(self.com_node(nodelist[2]),
- self.com_node(nodelist[5]))
- n.lineno = nodelist[0][2]
- return n
+ return TryFinally(self.com_node(nodelist[2]),
+ self.com_node(nodelist[5]),
+ lineno=nodelist[0][2])
def com_try_except(self, nodelist):
# try_except: 'try' ':' suite (except_clause ':' suite)* ['else' suite]
@@ -965,9 +912,8 @@
if node[0] == token.NAME:
elseNode = self.com_node(nodelist[i+2])
- n = TryExcept(self.com_node(nodelist[2]), clauses, elseNode)
- n.lineno = nodelist[0][2]
- return n
+ return TryExcept(self.com_node(nodelist[2]), clauses, elseNode,
+ lineno=nodelist[0][2])
def com_augassign_op(self, node):
assert node[0] == symbol.augassign
@@ -1031,7 +977,7 @@
assigns = []
for i in range(1, len(node), 2):
assigns.append(self.com_assign(node[i], assigning))
- return AssTuple(assigns)
+ return AssTuple(assigns, lineno=extractLineNo(node))
def com_assign_list(self, node, assigning):
assigns = []
@@ -1041,12 +987,10 @@
raise SyntaxError, "can't assign to list comprehension"
assert node[i + 1][0] == token.COMMA, node[i + 1]
assigns.append(self.com_assign(node[i], assigning))
- return AssList(assigns)
+ return AssList(assigns, lineno=extractLineNo(node))
def com_assign_name(self, node, assigning):
- n = AssName(node[1], assigning)
- n.lineno = node[2]
- return n
+ return AssName(node[1], assigning, lineno=node[2])
def com_assign_trailer(self, primary, node, assigning):
t = node[1][0]
@@ -1059,7 +1003,7 @@
raise SyntaxError, "unknown trailer type: %s" % t
def com_assign_attr(self, primary, node, assigning):
- return AssAttr(primary, node[1], assigning)
+ return AssAttr(primary, node[1], assigning, lineno=node[-1])
def com_binary(self, constructor, nodelist):
"Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])."
@@ -1071,7 +1015,7 @@
for i in range(0, l, 2):
n = nodelist[i]
items.append(self.lookup_node(n)(n[1:]))
- return constructor(items)
+ return constructor(items, lineno=extractLineNo(nodelist))
def com_stmt(self, node):
result = self.lookup_node(node)(node[1:])
@@ -1081,7 +1025,7 @@
return Stmt([result])
def com_append_stmt(self, stmts, node):
- result = self.com_node(node)
+ result = self.lookup_node(node)(node[1:])
assert result is not None
if isinstance(result, Stmt):
stmts.extend(result.nodes)
@@ -1100,7 +1044,7 @@
elif nodelist[i][0] == token.COMMA:
continue
values.append(self.com_node(nodelist[i]))
- return List(values)
+ return List(values, lineno=values[0].lineno)
def com_list_comprehension(self, expr, node):
# list_iter: list_for | list_if
@@ -1125,8 +1069,7 @@
node = self.com_list_iter(node[5])
elif t == 'if':
test = self.com_node(node[2])
- newif = ListCompIf(test)
- newif.lineno = node[1][2]
+ newif = ListCompIf(test, lineno=node[1][2])
newfor.ifs.append(newif)
if len(node) == 3:
node = None
@@ -1136,9 +1079,7 @@
raise SyntaxError, \
("unexpected list comprehension element: %s %d"
% (node, lineno))
- n = ListComp(expr, fors)
- n.lineno = lineno
- return n
+ return ListComp(expr, fors, lineno=lineno)
def com_list_iter(self, node):
assert node[0] == symbol.list_iter
@@ -1163,8 +1104,8 @@
if t == 'for':
assignNode = self.com_assign(node[2], OP_ASSIGN)
genNode = self.com_node(node[4])
- newfor = GenExprFor(assignNode, genNode, [])
- newfor.lineno = node[1][2]
+ newfor = GenExprFor(assignNode, genNode, [],
+ lineno=node[1][2])
fors.append(newfor)
if (len(node)) == 5:
node = None
@@ -1172,8 +1113,7 @@
node = self.com_gen_iter(node[5])
elif t == 'if':
test = self.com_node(node[2])
- newif = GenExprIf(test)
- newif.lineno = node[1][2]
+ newif = GenExprIf(test, lineno=node[1][2])
newfor.ifs.append(newif)
if len(node) == 3:
node = None
@@ -1184,9 +1124,7 @@
("unexpected generator expression element: %s %d"
% (node, lineno))
fors[0].is_outmost = True
- n = GenExpr(GenExprInner(expr, fors))
- n.lineno = lineno
- return n
+ return GenExpr(GenExprInner(expr, fors), lineno=lineno)
def com_gen_iter(self, node):
assert node[0] == symbol.gen_iter
@@ -1214,13 +1152,11 @@
def com_select_member(self, primaryNode, nodelist):
if nodelist[0] != token.NAME:
raise SyntaxError, "member must be a name"
- n = Getattr(primaryNode, nodelist[1])
- n.lineno = nodelist[2]
- return n
+ return Getattr(primaryNode, nodelist[1], lineno=nodelist[2])
def com_call_function(self, primaryNode, nodelist):
if nodelist[0] == token.RPAR:
- return CallFunc(primaryNode, [])
+ return CallFunc(primaryNode, [], lineno=extractLineNo(nodelist))
args = []
kw = 0
len_nodelist = len(nodelist)
@@ -1253,8 +1189,8 @@
dstar_node = self.com_node(ch)
else:
raise SyntaxError, 'unknown node type: %s' % tok
-
- return CallFunc(primaryNode, args, star_node, dstar_node)
+ return CallFunc(primaryNode, args, star_node, dstar_node,
+ lineno=extractLineNo(nodelist))
def com_argument(self, nodelist, kw):
if len(nodelist) == 3 and nodelist[2][0] == symbol.gen_for:
@@ -1270,8 +1206,7 @@
n = n[1]
if n[0] != token.NAME:
raise SyntaxError, "keyword can't be an expression (%s)"%n[0]
- node = Keyword(n[1], result)
- node.lineno = n[2]
+ node = Keyword(n[1], result, lineno=n[2])
return 1, node
def com_subscriptlist(self, primary, nodelist, assigning):
@@ -1291,8 +1226,8 @@
subscripts = []
for i in range(1, len(nodelist), 2):
subscripts.append(self.com_subscript(nodelist[i]))
-
- return Subscript(primary, assigning, subscripts)
+ return Subscript(primary, assigning, subscripts,
+ lineno=extractLineNo(nodelist))
def com_subscript(self, node):
# slice_item: expression | proper_slice | ellipsis
@@ -1338,8 +1273,7 @@
items.append(Const(None))
else:
items.append(self.com_node(ch[2]))
-
- return Sliceobj(items)
+ return Sliceobj(items, lineno=extractLineNo(node))
def com_slice(self, primary, node, assigning):
# short_slice: [lower_bound] ":" [upper_bound]
@@ -1352,7 +1286,8 @@
elif len(node) == 4:
lower = self.com_node(node[1])
upper = self.com_node(node[3])
- return Slice(primary, assigning, lower, upper)
+ return Slice(primary, assigning, lower, upper,
+ lineno=extractLineNo(node))
def get_docstring(self, node, n=None):
if n is None:
diff --git a/Lib/test/test_compiler.py b/Lib/test/test_compiler.py
index 63cbc00..1d77318 100644
--- a/Lib/test/test_compiler.py
+++ b/Lib/test/test_compiler.py
@@ -33,6 +33,65 @@
else:
compiler.compile(buf, basename, "exec")
+ def testLineNo(self):
+ # Test that all nodes except Module have a correct lineno attribute.
+ filename = __file__
+ if filename.endswith(".pyc") or filename.endswith(".pyo"):
+ filename = filename[:-1]
+ tree = compiler.parseFile(filename)
+ self.check_lineno(tree)
+
+ def check_lineno(self, node):
+ try:
+ self._check_lineno(node)
+ except AssertionError:
+ print node.__class__, node.lineno
+ raise
+
+ def _check_lineno(self, node):
+ if not node.__class__ in NOLINENO:
+ self.assert_(isinstance(node.lineno, int),
+ "lineno=%s on %s" % (node.lineno, node.__class__))
+ self.assert_(node.lineno > 0,
+ "lineno=%s on %s" % (node.lineno, node.__class__))
+ for child in node.getChildNodes():
+ self.check_lineno(child)
+
+NOLINENO = (compiler.ast.Module, compiler.ast.Stmt, compiler.ast.Discard)
+
+###############################################################################
+# code below is just used to trigger some possible errors, for the benefit of
+# testLineNo
+###############################################################################
+
+class Toto:
+ """docstring"""
+ pass
+
+a, b = 2, 3
+[c, d] = 5, 6
+l = [(x, y) for x, y in zip(range(5), range(5,10))]
+l[0]
+l[3:4]
+if l:
+ pass
+else:
+ a, b = b, a
+
+try:
+ print yo
+except:
+ yo = 3
+else:
+ yo += 3
+
+try:
+ a += b
+finally:
+ b = 0
+
+###############################################################################
+
def test_main():
global TEST_ALL
TEST_ALL = test.test_support.is_resource_enabled("compiler")
diff --git a/Lib/test/test_transformer.py b/Lib/test/test_transformer.py
index 6a8c98a..909cda5 100644
--- a/Lib/test/test_transformer.py
+++ b/Lib/test/test_transformer.py
@@ -29,9 +29,7 @@
assert vals['b'] == 2
def test_main():
- test_support.run_unittest(
- Tests
- )
+ test_support.run_unittest(Tests)
if __name__ == "__main__":
test_main()
diff --git a/Misc/ACKS b/Misc/ACKS
index dfdf005..1dc50a3 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -563,6 +563,7 @@
Ken Stox
Daniel Stutzbach
Paul Swartz
+Thenault Sylvain
Geoff Talvola
William Tanksley
Christian Tanzer
diff --git a/Tools/compiler/astgen.py b/Tools/compiler/astgen.py
index 4fe4bbe..0156567 100644
--- a/Tools/compiler/astgen.py
+++ b/Tools/compiler/astgen.py
@@ -94,7 +94,6 @@
def gen_source(self):
buf = StringIO()
print >> buf, "class %s(Node):" % self.name
- print >> buf, ' nodes["%s"] = "%s"' % (self.name.lower(), self.name)
self._gen_init(buf)
print >> buf
self._gen_getChildren(buf)
@@ -106,12 +105,14 @@
return buf.read()
def _gen_init(self, buf):
- print >> buf, " def __init__(self, %s):" % self.args
+ if self.args:
+ print >> buf, " def __init__(self, %s, lineno=None):" % self.args
+ else:
+ print >> buf, " def __init__(self, lineno=None):"
if self.argnames:
for name in self.argnames:
print >> buf, " self.%s = %s" % (name, name)
- else:
- print >> buf, " pass"
+ print >> buf, " self.lineno = lineno"
if self.init:
print >> buf, "".join([" " + line for line in self.init])
@@ -128,15 +129,18 @@
else:
print >> buf, " return %s" % clist
else:
- print >> buf, " children = []"
- template = " children.%s(%sself.%s%s)"
- for name in self.argnames:
- if self.argprops[name] == P_NESTED:
- print >> buf, template % ("extend", "flatten(",
- name, ")")
- else:
- print >> buf, template % ("append", "", name, "")
- print >> buf, " return tuple(children)"
+ if len(self.argnames) == 1:
+ print >> buf, " return tuple(flatten(self.%s))" % self.argnames[0]
+ else:
+ print >> buf, " children = []"
+ template = " children.%s(%sself.%s%s)"
+ for name in self.argnames:
+ if self.argprops[name] == P_NESTED:
+ print >> buf, template % ("extend", "flatten(",
+ name, ")")
+ else:
+ print >> buf, template % ("append", "", name, "")
+ print >> buf, " return tuple(children)"
def _gen_getChildNodes(self, buf):
print >> buf, " def getChildNodes(self):"
@@ -158,7 +162,7 @@
template = " nodelist.%s(%sself.%s%s)"
for name in self.argnames:
if self.argprops[name] == P_NONE:
- tmp = (" if self.%s is not None:"
+ tmp = (" if self.%s is not None:\n"
" nodelist.append(self.%s)")
print >> buf, tmp % (name, name)
elif self.argprops[name] == P_NESTED:
@@ -226,16 +230,15 @@
### PROLOGUE
"""Python abstract syntax node definitions
-This file is automatically generated.
+This file is automatically generated by Tools/compiler/astgen.py
"""
-from types import TupleType, ListType
from consts import CO_VARARGS, CO_VARKEYWORDS
def flatten(list):
l = []
for elt in list:
t = type(elt)
- if t is TupleType or t is ListType:
+ if t is tuple or t is list:
for elt2 in flatten(elt):
l.append(elt2)
else:
@@ -245,29 +248,17 @@
def flatten_nodes(list):
return [n for n in flatten(list) if isinstance(n, Node)]
-def asList(nodearg):
- l = []
- for item in nodearg:
- if hasattr(item, "asList"):
- l.append(item.asList())
- else:
- t = type(item)
- if t is TupleType or t is ListType:
- l.append(tuple(asList(item)))
- else:
- l.append(item)
- return l
-
nodes = {}
-class Node: # an abstract base class
- lineno = None # provide a lineno for nodes that don't have one
- def getType(self):
- pass # implemented by subclass
+class Node:
+ """Abstract base class for ast nodes."""
def getChildren(self):
pass # implemented by subclasses
- def asList(self):
- return tuple(asList(self.getChildren()))
+ def __iter__(self):
+ for n in self.getChildren():
+ yield n
+ def asList(self): # for backwards compatibility
+ return self.getChildren()
def getChildNodes(self):
pass # implemented by subclasses
@@ -290,6 +281,6 @@
return "Expression(%s)" % (repr(self.node))
### EPILOGUE
-klasses = globals()
-for k in nodes.keys():
- nodes[k] = klasses[nodes[k]]
+for name, obj in globals().items():
+ if isinstance(obj, type) and issubclass(obj, Node):
+ nodes[name.lower()] = obj