SF patch [ 597919 ] compiler package and SET_LINENO
A variety of changes from Michael Hudson to get the compiler working
with 2.3. The primary change is the handling of SET_LINENO:
# The set_lineno() function and the explicit emit() calls for
# SET_LINENO below are only used to generate the line number table.
# As of Python 2.3, the interpreter does not have a SET_LINENO
# instruction. pyassem treats SET_LINENO opcodes as a special case.
A few other small changes:
- Remove unused code from pycodegen and pyassem.
- Fix error handling in parsermodule. When PyParser_SimplerParseString()
fails, it sets an exception with detailed info. The parsermodule
was clobbering that exception and replacing it was a generic
"could not parse string" exception. Keep the original exception.
diff --git a/Lib/compiler/transformer.py b/Lib/compiler/transformer.py
index 382ea41..d1001bd 100644
--- a/Lib/compiler/transformer.py
+++ b/Lib/compiler/transformer.py
@@ -37,7 +37,11 @@
def parseFile(path):
f = open(path)
- src = f.read()
+ # XXX The parser API tolerates files without a trailing newline,
+ # but not strings without a trailing newline. Always add an extra
+ # newline to the file contents, since we're going through the string
+ # version of the API.
+ src = f.read() + "\n"
f.close()
return parse(src)
@@ -100,6 +104,7 @@
token.STRING: self.atom_string,
token.NAME: self.atom_name,
}
+ self.encoding = None
def transform(self, tree):
"""Transform an AST into a modified parse tree."""
@@ -110,6 +115,7 @@
def parsesuite(self, text):
"""Return a modified parse tree for the given suite text."""
# Hack for handling non-native line endings on non-DOS like OSs.
+ # this can go now we have universal newlines?
text = text.replace('\x0d', '')
return self.transform(parser.suite(text))
@@ -131,6 +137,12 @@
def compile_node(self, node):
### emit a line-number node?
n = node[0]
+
+ if n == symbol.encoding_decl:
+ self.encoding = node[2]
+ node = node[1]
+ n = node[0]
+
if n == symbol.single_input:
return self.single_input(node[1:])
if n == symbol.file_input:
@@ -519,6 +531,7 @@
return self.com_binary(Tuple, nodelist)
testlist_safe = testlist # XXX
+ testlist1 = testlist
exprlist = testlist
def test(self, nodelist):
@@ -637,11 +650,14 @@
def factor(self, nodelist):
elt = nodelist[0]
t = elt[0]
+ print "source", nodelist[-1]
node = self.com_node(nodelist[-1])
+ # need to handle (unary op)constant here...
if t == token.PLUS:
node = UnaryAdd(node)
node.lineno = elt[2]
elif t == token.MINUS:
+ print node
node = UnarySub(node)
node.lineno = elt[2]
elif t == token.TILDE:
@@ -699,11 +715,21 @@
n.lineno = nodelist[0][2]
return n
+ def decode_literal(self, lit):
+ if self.encoding:
+ # this is particularly fragile & a bit of a
+ # hack... changes in compile.c:parsestr and
+ # tokenizer.c must be reflected here.
+ if self.encoding not in ['utf-8', 'iso-8859-1']:
+ lit = unicode(lit, 'utf-8').encode(self.encoding)
+ return eval("# coding: %s\n%s" % (self.encoding, lit))
+ else:
+ return eval(lit)
+
def atom_string(self, nodelist):
- ### need to verify this matches compile.c
k = ''
for node in nodelist:
- k = k + eval(node[1])
+ k += self.decode_literal(node[1])
n = Const(k)
n.lineno = nodelist[0][2]
return n