Add lookup_name() to optimize use of stack frames
The use of com_node() introduces a lot of extra stack frames, enough
to cause a stack overflow compiling test.test_parser with the standard
interpreter recursionlimit. The com_node() is a convenience function
that hides the dispatch details, but comes at a very high cost. It is
more efficient to dispatch directly in the callers. In these cases,
use lookup_node() and call the dispatched node directly.
Also handle yield_stmt in a way that will work with Python 2.1
(suggested by Shane Hathaway)
diff --git a/Lib/compiler/transformer.py b/Lib/compiler/transformer.py
index ebb3481..7f6a1b4 100644
--- a/Lib/compiler/transformer.py
+++ b/Lib/compiler/transformer.py
@@ -274,7 +274,8 @@
def expr_stmt(self, nodelist):
# augassign testlist | testlist ('=' testlist)*
- exprNode = self.com_node(nodelist[-1])
+ en = nodelist[-1]
+ exprNode = self.lookup_node(en)(en[1:])
if len(nodelist) == 1:
n = Discard(exprNode)
n.lineno = exprNode.lineno
@@ -696,6 +697,17 @@
# INTERNAL PARSING UTILITIES
#
+ # The use of com_node() introduces a lot of extra stack frames,
+ # enough to cause a stack overflow compiling test.test_parser with
+ # the standard interpreter recursionlimit. The com_node() is a
+ # convenience function that hides the dispatch details, but comes
+ # at a very high cost. It is more efficient to dispatch directly
+ # in the callers. In these cases, use lookup_node() and call the
+ # dispatched node directly.
+
+ def lookup_node(self, node):
+ return self._dispatch[node[0]]
+
def com_node(self, node):
# Note: compile.c has handling in com_node for del_stmt, pass_stmt,
# break_stmt, stmt, small_stmt, flow_stmt, simple_stmt,
@@ -938,14 +950,16 @@
"Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])."
l = len(nodelist)
if l == 1:
- return self.com_node(nodelist[0])
+ n = nodelist[0]
+ return self.lookup_node(n)(n[1:])
items = []
for i in range(0, l, 2):
- items.append(self.com_node(nodelist[i]))
+ n = nodelist[i]
+ items.append(self.lookup_node(n)(n[1:]))
return constructor(items)
def com_stmt(self, node):
- result = self.com_node(node)
+ result = self.lookup_node(node)(node[1:])
assert result is not None
if isinstance(result, Stmt):
return result
@@ -1254,7 +1268,6 @@
symbol.continue_stmt,
symbol.return_stmt,
symbol.raise_stmt,
- symbol.yield_stmt,
symbol.import_stmt,
symbol.global_stmt,
symbol.exec_stmt,
@@ -1281,6 +1294,9 @@
symbol.atom,
]
+if hasattr(symbol, 'yield_stmt'):
+ _legal_node_types.append(symbol.yield_stmt)
+
_assign_types = [
symbol.test,
symbol.and_test,
diff --git a/Tools/compiler/compiler/transformer.py b/Tools/compiler/compiler/transformer.py
index ebb3481..7f6a1b4 100644
--- a/Tools/compiler/compiler/transformer.py
+++ b/Tools/compiler/compiler/transformer.py
@@ -274,7 +274,8 @@
def expr_stmt(self, nodelist):
# augassign testlist | testlist ('=' testlist)*
- exprNode = self.com_node(nodelist[-1])
+ en = nodelist[-1]
+ exprNode = self.lookup_node(en)(en[1:])
if len(nodelist) == 1:
n = Discard(exprNode)
n.lineno = exprNode.lineno
@@ -696,6 +697,17 @@
# INTERNAL PARSING UTILITIES
#
+ # The use of com_node() introduces a lot of extra stack frames,
+ # enough to cause a stack overflow compiling test.test_parser with
+ # the standard interpreter recursionlimit. The com_node() is a
+ # convenience function that hides the dispatch details, but comes
+ # at a very high cost. It is more efficient to dispatch directly
+ # in the callers. In these cases, use lookup_node() and call the
+ # dispatched node directly.
+
+ def lookup_node(self, node):
+ return self._dispatch[node[0]]
+
def com_node(self, node):
# Note: compile.c has handling in com_node for del_stmt, pass_stmt,
# break_stmt, stmt, small_stmt, flow_stmt, simple_stmt,
@@ -938,14 +950,16 @@
"Compile 'NODE (OP NODE)*' into (type, [ node1, ..., nodeN ])."
l = len(nodelist)
if l == 1:
- return self.com_node(nodelist[0])
+ n = nodelist[0]
+ return self.lookup_node(n)(n[1:])
items = []
for i in range(0, l, 2):
- items.append(self.com_node(nodelist[i]))
+ n = nodelist[i]
+ items.append(self.lookup_node(n)(n[1:]))
return constructor(items)
def com_stmt(self, node):
- result = self.com_node(node)
+ result = self.lookup_node(node)(node[1:])
assert result is not None
if isinstance(result, Stmt):
return result
@@ -1254,7 +1268,6 @@
symbol.continue_stmt,
symbol.return_stmt,
symbol.raise_stmt,
- symbol.yield_stmt,
symbol.import_stmt,
symbol.global_stmt,
symbol.exec_stmt,
@@ -1281,6 +1294,9 @@
symbol.atom,
]
+if hasattr(symbol, 'yield_stmt'):
+ _legal_node_types.append(symbol.yield_stmt)
+
_assign_types = [
symbol.test,
symbol.and_test,