patches from Mark Hammond
Attached is a set of diffs for the .py compiler that adds support
for the new extended call syntax.
compiler/ast.py:
CallFunc node gets 2 new children to support extended call syntax -
"star_args" (for "*args") and "dstar_args" (for "**args")
compiler/pyassem.py
It appear that self.lnotab is supposed to be responsible for
tracking line numbers, but self.firstlineno was still hanging
around. Removed self.firstlineno completely. NOTE - I didnt
actually test that the generated code has the correct line numbers!!
Stack depth tracking appeared a little broken - the checks never
made it beyond the "self.patterns" check - thus, the custom methods
were never called! Fixed this.
(XXX Jeremy notes: I think this code is still broken because it
doesn't track stack effects across block bounaries.)
Added support for the new extended call syntax opcodes for depth
calculations.
compiler/pycodegen.py
Added support for the new extended call syntax opcodes.
compiler/transformer.py
Added support for the new extended call syntax.
diff --git a/Lib/compiler/ast.py b/Lib/compiler/ast.py
index 5686d8b..a3e51b7 100644
--- a/Lib/compiler/ast.py
+++ b/Lib/compiler/ast.py
@@ -445,13 +445,15 @@
class CallFunc(Node):
nodes['call_func'] = 'CallFunc'
- def __init__(self, node, args):
+ def __init__(self, node, args, star_args = None, dstar_args = None):
self.node = node
self.args = args
- self._children = ('call_func', node, args)
+ self.star_args = star_args
+ self.dstar_args = dstar_args
+ self._children = ('call_func', node, args, star_args, dstar_args)
def __repr__(self):
- return "CallFunc(%s,%s)" % self._children[1:]
+ return "CallFunc(%s,%s,*%s, **%s)" % self._children[1:]
class Keyword(Node):
nodes['keyword'] = 'Keyword'
diff --git a/Lib/compiler/pyassem.py b/Lib/compiler/pyassem.py
index 3272419..6e07987 100644
--- a/Lib/compiler/pyassem.py
+++ b/Lib/compiler/pyassem.py
@@ -149,7 +149,6 @@
self.flags = CO_OPTIMIZED | CO_NEWLOCALS
else:
self.flags = 0
- self.firstlineno = None
self.consts = []
self.names = []
self.varnames = list(args) or []
@@ -314,8 +313,6 @@
oparg = t[1]
if opname == "SET_LINENO":
lnotab.nextLine(oparg)
- if self.firstlineno is None:
- self.firstlineno = oparg
hi, lo = twobyte(oparg)
try:
lnotab.addCode(self.opnum[opname], lo, hi)
@@ -342,7 +339,7 @@
return new.code(argcount, nlocals, self.stacksize, self.flags,
self.lnotab.getCode(), self.getConsts(),
tuple(self.names), tuple(self.varnames),
- self.filename, self.name, self.firstlineno,
+ self.filename, self.name, self.lnotab.firstline,
self.lnotab.getTable())
def getConsts(self):
@@ -464,14 +461,16 @@
if depth > maxDepth:
maxDepth = depth
# now check patterns
- for pat, delta in self.patterns:
+ for pat, pat_delta in self.patterns:
if opname[:len(pat)] == pat:
+ delta = pat_delta
depth = depth + delta
break
# if we still haven't found a match
if delta == 0:
- meth = getattr(self, opname)
- depth = depth + meth(i[1])
+ meth = getattr(self, opname, None)
+ if meth is not None:
+ depth = depth + meth(i[1])
if depth < 0:
depth = 0
return maxDepth
@@ -527,6 +526,12 @@
def CALL_FUNCTION(self, argc):
hi, lo = divmod(argc, 256)
return lo + hi * 2
+ def CALL_FUNCTION_VAR(self, argc):
+ return self.CALL_FUNCTION(argc)+1
+ def CALL_FUNCTION_KW(self, argc):
+ return self.CALL_FUNCTION(argc)+1
+ def CALL_FUNCTION_VAR_KW(self, argc):
+ return self.CALL_FUNCTION(argc)+2
def MAKE_FUNCTION(self, argc):
return -argc
def BUILD_SLICE(self, argc):
diff --git a/Lib/compiler/pycodegen.py b/Lib/compiler/pycodegen.py
index 2e98d4e..a697b55 100644
--- a/Lib/compiler/pycodegen.py
+++ b/Lib/compiler/pycodegen.py
@@ -9,6 +9,14 @@
from compiler import pyassem, misc
from compiler.pyassem import CO_VARARGS, CO_VARKEYWORDS, TupleArg
+callfunc_opcode_info = {
+ # (Have *args, Have **args) : opcode
+ (0,0) : "CALL_FUNCTION",
+ (1,0) : "CALL_FUNCTION_VAR",
+ (0,1) : "CALL_FUNCTION_KW",
+ (1,1) : "CALL_FUNCTION_VAR_KW",
+}
+
def compile(filename):
f = open(filename)
buf = f.read()
@@ -478,7 +486,14 @@
kw = kw + 1
else:
pos = pos + 1
- self.emit('CALL_FUNCTION', kw << 8 | pos)
+ if node.star_args is not None:
+ self.visit(node.star_args)
+ if node.dstar_args is not None:
+ self.visit(node.dstar_args)
+ have_star = node.star_args is not None
+ have_dstar = node.dstar_args is not None
+ opcode = callfunc_opcode_info[have_star, have_dstar]
+ self.emit(opcode, kw << 8 | pos)
def visitPrint(self, node):
self.emit('SET_LINENO', node.lineno)
diff --git a/Lib/compiler/transformer.py b/Lib/compiler/transformer.py
index 1531ec3..d23fdb8 100644
--- a/Lib/compiler/transformer.py
+++ b/Lib/compiler/transformer.py
@@ -984,10 +984,32 @@
return Node('call_func', primaryNode, [ ])
args = [ ]
kw = 0
- for i in range(1, len(nodelist), 2):
- kw, result = self.com_argument(nodelist[i], kw)
+ len_nodelist = len(nodelist)
+ for i in range(1, len_nodelist, 2):
+ node = nodelist[i]
+ if node[0] == token.STAR or node[0] == token.DOUBLESTAR:
+ break
+ kw, result = self.com_argument(node, kw)
args.append(result)
- return Node('call_func', primaryNode, args)
+ else:
+ i = i + 1 # No broken by star arg, so skip the last one we processed.
+ star_node = dstar_node = None
+ while i < len_nodelist:
+ tok = nodelist[i]
+ ch = nodelist[i+1]
+ i = i + 3
+ if tok[0]==token.STAR:
+ if star_node is not None:
+ raise SyntaxError, 'already have the varargs indentifier'
+ star_node = self.com_node(ch)
+ elif tok[0]==token.DOUBLESTAR:
+ if dstar_node is not None:
+ raise SyntaxError, 'already have the kwargs indentifier'
+ dstar_node = self.com_node(ch)
+ else:
+ raise SyntaxError, 'unknown node type: %s' % tok
+
+ return Node('call_func', primaryNode, args, star_node, dstar_node)
def com_argument(self, nodelist, kw):
if len(nodelist) == 2:
diff --git a/Tools/compiler/compiler/ast.py b/Tools/compiler/compiler/ast.py
index 5686d8b..a3e51b7 100644
--- a/Tools/compiler/compiler/ast.py
+++ b/Tools/compiler/compiler/ast.py
@@ -445,13 +445,15 @@
class CallFunc(Node):
nodes['call_func'] = 'CallFunc'
- def __init__(self, node, args):
+ def __init__(self, node, args, star_args = None, dstar_args = None):
self.node = node
self.args = args
- self._children = ('call_func', node, args)
+ self.star_args = star_args
+ self.dstar_args = dstar_args
+ self._children = ('call_func', node, args, star_args, dstar_args)
def __repr__(self):
- return "CallFunc(%s,%s)" % self._children[1:]
+ return "CallFunc(%s,%s,*%s, **%s)" % self._children[1:]
class Keyword(Node):
nodes['keyword'] = 'Keyword'
diff --git a/Tools/compiler/compiler/pyassem.py b/Tools/compiler/compiler/pyassem.py
index 3272419..6e07987 100644
--- a/Tools/compiler/compiler/pyassem.py
+++ b/Tools/compiler/compiler/pyassem.py
@@ -149,7 +149,6 @@
self.flags = CO_OPTIMIZED | CO_NEWLOCALS
else:
self.flags = 0
- self.firstlineno = None
self.consts = []
self.names = []
self.varnames = list(args) or []
@@ -314,8 +313,6 @@
oparg = t[1]
if opname == "SET_LINENO":
lnotab.nextLine(oparg)
- if self.firstlineno is None:
- self.firstlineno = oparg
hi, lo = twobyte(oparg)
try:
lnotab.addCode(self.opnum[opname], lo, hi)
@@ -342,7 +339,7 @@
return new.code(argcount, nlocals, self.stacksize, self.flags,
self.lnotab.getCode(), self.getConsts(),
tuple(self.names), tuple(self.varnames),
- self.filename, self.name, self.firstlineno,
+ self.filename, self.name, self.lnotab.firstline,
self.lnotab.getTable())
def getConsts(self):
@@ -464,14 +461,16 @@
if depth > maxDepth:
maxDepth = depth
# now check patterns
- for pat, delta in self.patterns:
+ for pat, pat_delta in self.patterns:
if opname[:len(pat)] == pat:
+ delta = pat_delta
depth = depth + delta
break
# if we still haven't found a match
if delta == 0:
- meth = getattr(self, opname)
- depth = depth + meth(i[1])
+ meth = getattr(self, opname, None)
+ if meth is not None:
+ depth = depth + meth(i[1])
if depth < 0:
depth = 0
return maxDepth
@@ -527,6 +526,12 @@
def CALL_FUNCTION(self, argc):
hi, lo = divmod(argc, 256)
return lo + hi * 2
+ def CALL_FUNCTION_VAR(self, argc):
+ return self.CALL_FUNCTION(argc)+1
+ def CALL_FUNCTION_KW(self, argc):
+ return self.CALL_FUNCTION(argc)+1
+ def CALL_FUNCTION_VAR_KW(self, argc):
+ return self.CALL_FUNCTION(argc)+2
def MAKE_FUNCTION(self, argc):
return -argc
def BUILD_SLICE(self, argc):
diff --git a/Tools/compiler/compiler/pycodegen.py b/Tools/compiler/compiler/pycodegen.py
index 2e98d4e..a697b55 100644
--- a/Tools/compiler/compiler/pycodegen.py
+++ b/Tools/compiler/compiler/pycodegen.py
@@ -9,6 +9,14 @@
from compiler import pyassem, misc
from compiler.pyassem import CO_VARARGS, CO_VARKEYWORDS, TupleArg
+callfunc_opcode_info = {
+ # (Have *args, Have **args) : opcode
+ (0,0) : "CALL_FUNCTION",
+ (1,0) : "CALL_FUNCTION_VAR",
+ (0,1) : "CALL_FUNCTION_KW",
+ (1,1) : "CALL_FUNCTION_VAR_KW",
+}
+
def compile(filename):
f = open(filename)
buf = f.read()
@@ -478,7 +486,14 @@
kw = kw + 1
else:
pos = pos + 1
- self.emit('CALL_FUNCTION', kw << 8 | pos)
+ if node.star_args is not None:
+ self.visit(node.star_args)
+ if node.dstar_args is not None:
+ self.visit(node.dstar_args)
+ have_star = node.star_args is not None
+ have_dstar = node.dstar_args is not None
+ opcode = callfunc_opcode_info[have_star, have_dstar]
+ self.emit(opcode, kw << 8 | pos)
def visitPrint(self, node):
self.emit('SET_LINENO', node.lineno)
diff --git a/Tools/compiler/compiler/transformer.py b/Tools/compiler/compiler/transformer.py
index 1531ec3..d23fdb8 100644
--- a/Tools/compiler/compiler/transformer.py
+++ b/Tools/compiler/compiler/transformer.py
@@ -984,10 +984,32 @@
return Node('call_func', primaryNode, [ ])
args = [ ]
kw = 0
- for i in range(1, len(nodelist), 2):
- kw, result = self.com_argument(nodelist[i], kw)
+ len_nodelist = len(nodelist)
+ for i in range(1, len_nodelist, 2):
+ node = nodelist[i]
+ if node[0] == token.STAR or node[0] == token.DOUBLESTAR:
+ break
+ kw, result = self.com_argument(node, kw)
args.append(result)
- return Node('call_func', primaryNode, args)
+ else:
+ i = i + 1 # No broken by star arg, so skip the last one we processed.
+ star_node = dstar_node = None
+ while i < len_nodelist:
+ tok = nodelist[i]
+ ch = nodelist[i+1]
+ i = i + 3
+ if tok[0]==token.STAR:
+ if star_node is not None:
+ raise SyntaxError, 'already have the varargs indentifier'
+ star_node = self.com_node(ch)
+ elif tok[0]==token.DOUBLESTAR:
+ if dstar_node is not None:
+ raise SyntaxError, 'already have the kwargs indentifier'
+ dstar_node = self.com_node(ch)
+ else:
+ raise SyntaxError, 'unknown node type: %s' % tok
+
+ return Node('call_func', primaryNode, args, star_node, dstar_node)
def com_argument(self, nodelist, kw):
if len(nodelist) == 2: