Merged revisions 73771,73811,73840,73842,73848-73849,73861,73957-73960,73964-73969,73972-73974,73977,73981,73984,74065,74113 via svnmerge from
svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3
........
r73771 | benjamin.peterson | 2009-07-02 10:56:55 -0500 (Thu, 02 Jul 2009) | 1 line
force the imports fixer to be run after the import one #6400
........
r73811 | benjamin.peterson | 2009-07-03 09:03:14 -0500 (Fri, 03 Jul 2009) | 1 line
check for sep, not pathsep when looking for a subpackage #6408
........
r73840 | benjamin.peterson | 2009-07-04 09:52:28 -0500 (Sat, 04 Jul 2009) | 1 line
don't print diffs by default; it's annoying
........
r73842 | benjamin.peterson | 2009-07-04 09:58:46 -0500 (Sat, 04 Jul 2009) | 1 line
complain when not showing diffs or writing
........
r73848 | alexandre.vassalotti | 2009-07-04 23:38:19 -0500 (Sat, 04 Jul 2009) | 2 lines
Fix test_refactor_stdin to handle print_output() method with 4 arguments.
........
r73849 | alexandre.vassalotti | 2009-07-04 23:43:18 -0500 (Sat, 04 Jul 2009) | 5 lines
Issue 2370: Add fixer for the removal of operator.isCallable() and
operator.sequenceIncludes().
Patch contributed by Jeff Balogh (and updated by me).
........
r73861 | benjamin.peterson | 2009-07-05 09:15:53 -0500 (Sun, 05 Jul 2009) | 1 line
cleanup and use unicode where appropiate
........
r73957 | benjamin.peterson | 2009-07-11 15:49:56 -0500 (Sat, 11 Jul 2009) | 1 line
fix calls to str() with unicode()
........
r73958 | benjamin.peterson | 2009-07-11 15:51:51 -0500 (Sat, 11 Jul 2009) | 1 line
more str() -> unicode()
........
r73959 | benjamin.peterson | 2009-07-11 16:40:08 -0500 (Sat, 11 Jul 2009) | 1 line
add tests for refactor_dir()
........
r73960 | benjamin.peterson | 2009-07-11 16:44:32 -0500 (Sat, 11 Jul 2009) | 1 line
don't parse files just because they end with 'py' (no dot)
........
r73964 | benjamin.peterson | 2009-07-11 17:30:15 -0500 (Sat, 11 Jul 2009) | 1 line
simplify
........
r73965 | benjamin.peterson | 2009-07-11 17:31:30 -0500 (Sat, 11 Jul 2009) | 1 line
remove usage of get_prefix()
........
r73966 | benjamin.peterson | 2009-07-11 17:33:35 -0500 (Sat, 11 Jul 2009) | 1 line
revert unintended change in 73965
........
r73967 | benjamin.peterson | 2009-07-11 17:34:44 -0500 (Sat, 11 Jul 2009) | 1 line
avoid expensive checks and assume the node did change
........
r73968 | benjamin.peterson | 2009-07-11 20:46:46 -0500 (Sat, 11 Jul 2009) | 1 line
use a regular dict for the heads to avoid adding lists in the loop
........
r73969 | benjamin.peterson | 2009-07-11 20:50:43 -0500 (Sat, 11 Jul 2009) | 1 line
prefix headnode functions with '_'
........
r73972 | benjamin.peterson | 2009-07-11 21:25:45 -0500 (Sat, 11 Jul 2009) | 1 line
try to make the head node dict as sparse as possible
........
r73973 | benjamin.peterson | 2009-07-11 21:59:49 -0500 (Sat, 11 Jul 2009) | 1 line
a better idea; add an option to *not* print diffs
........
r73974 | benjamin.peterson | 2009-07-11 22:00:29 -0500 (Sat, 11 Jul 2009) | 1 line
add space
........
r73977 | benjamin.peterson | 2009-07-12 10:16:07 -0500 (Sun, 12 Jul 2009) | 1 line
update get_headnode_dict tests for recent changes
........
r73981 | benjamin.peterson | 2009-07-12 12:06:39 -0500 (Sun, 12 Jul 2009) | 4 lines
detect when "from __future__ import print_function" is given
Deprecate the 'print_function' option and the -p flag
........
r73984 | benjamin.peterson | 2009-07-12 16:16:37 -0500 (Sun, 12 Jul 2009) | 1 line
add tests for Call; thanks Joe Amenta
........
r74065 | benjamin.peterson | 2009-07-17 12:52:49 -0500 (Fri, 17 Jul 2009) | 1 line
pathname2url and url2pathname are in urllib.request not urllib.parse #6496
........
r74113 | benjamin.peterson | 2009-07-20 08:56:57 -0500 (Mon, 20 Jul 2009) | 1 line
fix deprecation warnings in tests
........
diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py
index d1cdb5d..97a540d 100644
--- a/Lib/lib2to3/refactor.py
+++ b/Lib/lib2to3/refactor.py
@@ -14,14 +14,15 @@
# Python imports
import os
import sys
-import difflib
import logging
import operator
-from collections import defaultdict
+import collections
+import StringIO
+import warnings
from itertools import chain
# Local imports
-from .pgen2 import driver, tokenize
+from .pgen2 import driver, tokenize, token
from . import pytree, pygram
@@ -37,7 +38,12 @@
fix_names.append(name[:-3])
return fix_names
-def get_head_types(pat):
+
+class _EveryNode(Exception):
+ pass
+
+
+def _get_head_types(pat):
""" Accepts a pytree Pattern Node and returns a set
of the pattern types which will match first. """
@@ -45,34 +51,50 @@
# NodePatters must either have no type and no content
# or a type and content -- so they don't get any farther
# Always return leafs
+ if pat.type is None:
+ raise _EveryNode
return set([pat.type])
if isinstance(pat, pytree.NegatedPattern):
if pat.content:
- return get_head_types(pat.content)
- return set([None]) # Negated Patterns don't have a type
+ return _get_head_types(pat.content)
+ raise _EveryNode # Negated Patterns don't have a type
if isinstance(pat, pytree.WildcardPattern):
# Recurse on each node in content
r = set()
for p in pat.content:
for x in p:
- r.update(get_head_types(x))
+ r.update(_get_head_types(x))
return r
raise Exception("Oh no! I don't understand pattern %s" %(pat))
-def get_headnode_dict(fixer_list):
+
+def _get_headnode_dict(fixer_list):
""" Accepts a list of fixers and returns a dictionary
of head node type --> fixer list. """
- head_nodes = defaultdict(list)
+ head_nodes = collections.defaultdict(list)
+ every = []
for fixer in fixer_list:
- if not fixer.pattern:
- head_nodes[None].append(fixer)
- continue
- for t in get_head_types(fixer.pattern):
- head_nodes[t].append(fixer)
- return head_nodes
+ if fixer.pattern:
+ try:
+ heads = _get_head_types(fixer.pattern)
+ except _EveryNode:
+ every.append(fixer)
+ else:
+ for node_type in heads:
+ head_nodes[node_type].append(fixer)
+ else:
+ if fixer._accept_type is not None:
+ head_nodes[fixer._accept_type].append(fixer)
+ else:
+ every.append(fixer)
+ for node_type in chain(pygram.python_grammar.symbol2number.itervalues(),
+ pygram.python_grammar.tokens):
+ head_nodes[node_type].extend(every)
+ return dict(head_nodes)
+
def get_fixers_from_package(pkg_name):
"""
@@ -101,13 +123,56 @@
_to_system_newlines = _identity
+def _detect_future_print(source):
+ have_docstring = False
+ gen = tokenize.generate_tokens(StringIO.StringIO(source).readline)
+ def advance():
+ tok = next(gen)
+ return tok[0], tok[1]
+ ignore = frozenset((token.NEWLINE, tokenize.NL, token.COMMENT))
+ try:
+ while True:
+ tp, value = advance()
+ if tp in ignore:
+ continue
+ elif tp == token.STRING:
+ if have_docstring:
+ break
+ have_docstring = True
+ elif tp == token.NAME:
+ if value == u"from":
+ tp, value = advance()
+ if tp != token.NAME and value != u"__future__":
+ break
+ tp, value = advance()
+ if tp != token.NAME and value != u"import":
+ break
+ tp, value = advance()
+ if tp == token.OP and value == u"(":
+ tp, value = advance()
+ while tp == token.NAME:
+ if value == u"print_function":
+ return True
+ tp, value = advance()
+ if tp != token.OP and value != u",":
+ break
+ tp, value = advance()
+ else:
+ break
+ else:
+ break
+ except StopIteration:
+ pass
+ return False
+
+
class FixerError(Exception):
"""A fixer could not be loaded."""
class RefactoringTool(object):
- _default_options = {"print_function": False}
+ _default_options = {}
CLASS_PREFIX = "Fix" # The prefix for fixer classes
FILE_PREFIX = "fix_" # The prefix for modules with a fixer within
@@ -124,20 +189,21 @@
self.explicit = explicit or []
self.options = self._default_options.copy()
if options is not None:
+ if "print_function" in options:
+ warnings.warn("the 'print_function' option is deprecated",
+ DeprecationWarning)
self.options.update(options)
self.errors = []
self.logger = logging.getLogger("RefactoringTool")
self.fixer_log = []
self.wrote = False
- if self.options["print_function"]:
- del pygram.python_grammar.keywords["print"]
self.driver = driver.Driver(pygram.python_grammar,
convert=pytree.convert,
logger=self.logger)
self.pre_order, self.post_order = self.get_fixers()
- self.pre_order_heads = get_headnode_dict(self.pre_order)
- self.post_order_heads = get_headnode_dict(self.post_order)
+ self.pre_order_heads = _get_headnode_dict(self.pre_order)
+ self.post_order_heads = _get_headnode_dict(self.post_order)
self.files = [] # List of files that were or should be modified
@@ -196,8 +262,9 @@
msg = msg % args
self.logger.debug(msg)
- def print_output(self, lines):
- """Called with lines of output to give to the user."""
+ def print_output(self, old_text, new_text, filename, equal):
+ """Called with the old version, new version, and filename of a
+ refactored file."""
pass
def refactor(self, items, write=False, doctests_only=False):
@@ -220,7 +287,8 @@
dirnames.sort()
filenames.sort()
for name in filenames:
- if not name.startswith(".") and name.endswith("py"):
+ if not name.startswith(".") and \
+ os.path.splitext(name)[1].endswith("py"):
fullname = os.path.join(dirpath, name)
self.refactor_file(fullname, write, doctests_only)
# Modify dirnames in-place to remove subdirs with leading dots
@@ -276,12 +344,16 @@
An AST corresponding to the refactored input stream; None if
there were errors during the parse.
"""
+ if _detect_future_print(data):
+ self.driver.grammar = pygram.python_grammar_no_print_statement
try:
tree = self.driver.parse_string(data)
except Exception, err:
self.log_error("Can't parse %s: %s: %s",
name, err.__class__.__name__, err)
return
+ finally:
+ self.driver.grammar = pygram.python_grammar
self.log_debug("Refactoring %s", name)
self.refactor_tree(tree, name)
return tree
@@ -298,7 +370,7 @@
else:
tree = self.refactor_string(input, "<stdin>")
if tree and tree.was_changed:
- self.processed_file(str(tree), "<stdin>", input)
+ self.processed_file(unicode(tree), "<stdin>", input)
else:
self.log_debug("No changes in stdin")
@@ -338,12 +410,11 @@
if not fixers:
return
for node in traversal:
- for fixer in fixers[node.type] + fixers[None]:
+ for fixer in fixers[node.type]:
results = fixer.match(node)
if results:
new = fixer.transform(node, results)
- if new is not None and (new != node or
- str(new) != str(node)):
+ if new is not None:
node.replace(new)
node = new
@@ -357,10 +428,11 @@
old_text = self._read_python_source(filename)[0]
if old_text is None:
return
- if old_text == new_text:
+ equal = old_text == new_text
+ self.print_output(old_text, new_text, filename, equal)
+ if equal:
self.log_debug("No changes to %s", filename)
return
- self.print_output(diff_texts(old_text, new_text, filename))
if write:
self.write_file(new_text, filename, old_text, encoding)
else:
@@ -451,7 +523,7 @@
filename, lineno, err.__class__.__name__, err)
return block
if self.refactor_tree(tree, filename):
- new = str(tree).splitlines(True)
+ new = unicode(tree).splitlines(True)
# Undo the adjustment of the line numbers in wrap_toks() below.
clipped, new = new[:lineno-1], new[lineno-1:]
assert clipped == [u"\n"] * (lineno-1), clipped
@@ -582,12 +654,3 @@
else:
return super(MultiprocessRefactoringTool, self).refactor_file(
*args, **kwargs)
-
-
-def diff_texts(a, b, filename):
- """Return a unified diff of two strings."""
- a = a.splitlines()
- b = b.splitlines()
- return difflib.unified_diff(a, b, filename, filename,
- "(original)", "(refactored)",
- lineterm="")