Update Demo/parser directory; backport unparse fixes from py3k.
diff --git a/Demo/parser/unparse.py b/Demo/parser/unparse.py
index 98aeb32..1b1ff29 100644
--- a/Demo/parser/unparse.py
+++ b/Demo/parser/unparse.py
@@ -1,15 +1,19 @@
"Usage: unparse.py <path to source file>"
import sys
-import _ast
+import ast
import cStringIO
import os
+# Large float and imaginary literals get turned into infinities in the AST.
+# We unparse those infinities to INFSTR.
+INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)
+
def interleave(inter, f, seq):
"""Call f on each item in seq, calling inter() in between.
"""
seq = iter(seq)
try:
- f(seq.next())
+ f(next(seq))
except StopIteration:
pass
else:
@@ -20,15 +24,16 @@
class Unparser:
"""Methods in this class recursively traverse an AST and
output source code for the abstract syntax; original formatting
- is disregarged. """
+ is disregarded. """
def __init__(self, tree, file = sys.stdout):
"""Unparser(tree, file=sys.stdout) -> None.
Print the source for tree to file."""
self.f = file
+ self.future_imports = []
self._indent = 0
self.dispatch(tree)
- print >>self.f,""
+ self.f.write("")
self.f.flush()
def fill(self, text = ""):
@@ -79,11 +84,16 @@
interleave(lambda: self.write(", "), self.dispatch, t.names)
def _ImportFrom(self, t):
+ # A from __future__ import may affect unparsing, so record it.
+ if t.module and t.module == '__future__':
+ self.future_imports.extend(n.name for n in t.names)
+
self.fill("from ")
- self.write(t.module)
+ self.write("." * t.level)
+ if t.module:
+ self.write(t.module)
self.write(" import ")
interleave(lambda: self.write(", "), self.dispatch, t.names)
- # XXX(jpe) what is level for?
def _Assign(self, t):
self.fill()
@@ -186,10 +196,14 @@
self.leave()
def _TryFinally(self, t):
- self.fill("try")
- self.enter()
- self.dispatch(t.body)
- self.leave()
+ if len(t.body) == 1 and isinstance(t.body[0], ast.TryExcept):
+ # try-except-finally
+ self.dispatch(t.body)
+ else:
+ self.fill("try")
+ self.enter()
+ self.dispatch(t.body)
+ self.leave()
self.fill("finally")
self.enter()
@@ -202,7 +216,7 @@
self.write(" ")
self.dispatch(t.type)
if t.name:
- self.write(", ")
+ self.write(" as ")
self.dispatch(t.name)
self.enter()
self.dispatch(t.body)
@@ -210,6 +224,9 @@
def _ClassDef(self, t):
self.write("\n")
+ for deco in t.decorator_list:
+ self.fill("@")
+ self.dispatch(deco)
self.fill("class "+t.name)
if t.bases:
self.write("(")
@@ -251,9 +268,18 @@
self.fill("if ")
self.dispatch(t.test)
self.enter()
- # XXX elif?
self.dispatch(t.body)
self.leave()
+ # collapse nested ifs into equivalent elifs.
+ while (t.orelse and len(t.orelse) == 1 and
+ isinstance(t.orelse[0], ast.If)):
+ t = t.orelse[0]
+ self.fill("elif ")
+ self.dispatch(t.test)
+ self.enter()
+ self.dispatch(t.body)
+ self.leave()
+ # final else
if t.orelse:
self.fill("else")
self.enter()
@@ -284,7 +310,17 @@
# expr
def _Str(self, tree):
- self.write(repr(tree.s))
+ # if from __future__ import unicode_literals is in effect,
+ # then we want to output string literals using a 'b' prefix
+ # and unicode literals with no prefix.
+ if "unicode_literals" not in self.future_imports:
+ self.write(repr(tree.s))
+ elif isinstance(tree.s, str):
+ self.write("b" + repr(tree.s))
+ elif isinstance(tree.s, unicode):
+ self.write(repr(tree.s).lstrip("u"))
+ else:
+ assert False, "shouldn't get here"
def _Name(self, t):
self.write(t.id)
@@ -295,16 +331,14 @@
self.write("`")
def _Num(self, t):
- # There are no negative numeric literals in Python; however,
- # some optimizations produce a negative Num in the AST. Add
- # parentheses to avoid turning (-1)**2 into -1**2.
- strnum = repr(t.n)
- if strnum.startswith("-"):
+ repr_n = repr(t.n)
+ # Parenthesize negative numbers, to avoid turning (-1)**2 into -1**2.
+ if repr_n.startswith("-"):
self.write("(")
- self.write(strnum)
+ # Substitute overflowing decimal literal for AST infinities.
+ self.write(repr_n.replace("inf", INFSTR))
+ if repr_n.startswith("-"):
self.write(")")
- else:
- self.write(repr(t.n))
def _List(self, t):
self.write("[")
@@ -325,6 +359,22 @@
self.dispatch(gen)
self.write(")")
+ def _SetComp(self, t):
+ self.write("{")
+ self.dispatch(t.elt)
+ for gen in t.generators:
+ self.dispatch(gen)
+ self.write("}")
+
+ def _DictComp(self, t):
+ self.write("{")
+ self.dispatch(t.key)
+ self.write(": ")
+ self.dispatch(t.value)
+ for gen in t.generators:
+ self.dispatch(gen)
+ self.write("}")
+
def _comprehension(self, t):
self.write(" for ")
self.dispatch(t.target)
@@ -343,13 +393,20 @@
self.dispatch(t.orelse)
self.write(")")
+ def _Set(self, t):
+ assert(t.elts) # should be at least one element
+ self.write("{")
+ interleave(lambda: self.write(", "), self.dispatch, t.elts)
+ self.write("}")
+
def _Dict(self, t):
self.write("{")
- def writem((k, v)):
+ def write_pair(pair):
+ (k, v) = pair
self.dispatch(k)
self.write(": ")
self.dispatch(v)
- interleave(lambda: self.write(", "), writem, zip(t.keys, t.values))
+ interleave(lambda: self.write(", "), write_pair, zip(t.keys, t.values))
self.write("}")
def _Tuple(self, t):
@@ -367,7 +424,17 @@
self.write("(")
self.write(self.unop[t.op.__class__.__name__])
self.write(" ")
- self.dispatch(t.operand)
+ # If we're applying unary minus to a number, parenthesize the number.
+ # This is necessary: -2147483648 is different from -(2147483648) on
+ # a 32-bit machine (the first is an int, the second a long), and
+ # -7j is different from -(7j). (The first has real part 0.0, the second
+ # has real part -0.0.)
+ if isinstance(t.op, ast.USub) and isinstance(t.operand, ast.Num):
+ self.write("(")
+ self.dispatch(t.operand)
+ self.write(")")
+ else:
+ self.dispatch(t.operand)
self.write(")")
binop = { "Add":"+", "Sub":"-", "Mult":"*", "Div":"/", "Mod":"%",
@@ -390,7 +457,7 @@
self.dispatch(e)
self.write(")")
- boolops = {_ast.And: 'and', _ast.Or: 'or'}
+ boolops = {ast.And: 'and', ast.Or: 'or'}
def _BoolOp(self, t):
self.write("(")
s = " %s " % self.boolops[t.op.__class__]
@@ -399,6 +466,11 @@
def _Attribute(self,t):
self.dispatch(t.value)
+ # Special case: 3.__abs__() is a syntax error, so if t.value
+ # is an integer literal then we need to either parenthesize
+ # it or add an extra space to get 3 .__abs__().
+ if isinstance(t.value, ast.Num) and isinstance(t.value.n, int):
+ self.write(" ")
self.write(".")
self.write(t.attr)
@@ -455,21 +527,24 @@
# others
def _arguments(self, t):
first = True
- nonDef = len(t.args)-len(t.defaults)
- for a in t.args[0:nonDef]:
- if first:first = False
- else: self.write(", ")
- self.dispatch(a)
- for a,d in zip(t.args[nonDef:], t.defaults):
+ # normal arguments
+ defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults
+ for a,d in zip(t.args, defaults):
if first:first = False
else: self.write(", ")
self.dispatch(a),
- self.write("=")
- self.dispatch(d)
+ if d:
+ self.write("=")
+ self.dispatch(d)
+
+ # varargs
if t.vararg:
if first:first = False
else: self.write(", ")
- self.write("*"+t.vararg)
+ self.write("*")
+ self.write(t.vararg)
+
+ # kwargs
if t.kwarg:
if first:first = False
else: self.write(", ")
@@ -481,10 +556,12 @@
self.dispatch(t.value)
def _Lambda(self, t):
+ self.write("(")
self.write("lambda ")
self.dispatch(t.args)
self.write(": ")
self.dispatch(t.body)
+ self.write(")")
def _alias(self, t):
self.write(t.name)
@@ -492,8 +569,9 @@
self.write(" as "+t.asname)
def roundtrip(filename, output=sys.stdout):
- source = open(filename).read()
- tree = compile(source, filename, "exec", _ast.PyCF_ONLY_AST)
+ with open(filename, "r") as pyfile:
+ source = pyfile.read()
+ tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST)
Unparser(tree, output)
@@ -502,7 +580,7 @@
try:
names = [n for n in os.listdir(a) if n.endswith('.py')]
except OSError:
- print >> sys.stderr, "Directory not readable: %s" % a
+ sys.stderr.write("Directory not readable: %s" % a)
else:
for n in names:
fullname = os.path.join(a, n)
@@ -511,7 +589,7 @@
print 'Testing %s' % fullname
try:
roundtrip(fullname, output)
- except Exception, e:
+ except Exception as e:
print ' Failed to compile, exception is %s' % repr(e)
elif os.path.isdir(fullname):
testdir(fullname)