Merged revisions 61596-61597 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/trunk
........
r61596 | martin.v.loewis | 2008-03-18 23:43:46 -0500 (Di, 18 Mär 2008) | 2 lines
Import lib2to3.
........
r61597 | martin.v.loewis | 2008-03-18 23:58:04 -0500 (Di, 18 Mär 2008) | 3 lines
Initialized merge tracking via "svnmerge" with revisions "1-61595" from
svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3
........
diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py
new file mode 100755
index 0000000..f7a3b15
--- /dev/null
+++ b/Lib/lib2to3/refactor.py
@@ -0,0 +1,526 @@
+#!/usr/bin/env python2.5
+# Copyright 2006 Google, Inc. All Rights Reserved.
+# Licensed to PSF under a Contributor Agreement.
+
+"""Refactoring framework.
+
+Used as a main program, this can refactor any number of files and/or
+recursively descend down directories. Imported as a module, this
+provides infrastructure to write your own refactoring tool.
+"""
+
+__author__ = "Guido van Rossum <guido@python.org>"
+
+
+# Python imports
+import os
+import sys
+import difflib
+import optparse
+import logging
+
+# Local imports
+from .pgen2 import driver
+from .pgen2 import tokenize
+
+from . import pytree
+from . import patcomp
+from . import fixes
+from . import pygram
+
+if sys.version_info < (2, 4):
+ hdlr = logging.StreamHandler()
+ fmt = logging.Formatter('%(name)s: %(message)s')
+ hdlr.setFormatter(fmt)
+ logging.root.addHandler(hdlr)
+else:
+ logging.basicConfig(format='%(name)s: %(message)s', level=logging.INFO)
+
+
+def main(args=None):
+ """Main program.
+
+ Call without arguments to use sys.argv[1:] as the arguments; or
+ call with a list of arguments (excluding sys.argv[0]).
+
+ Returns a suggested exit status (0, 1, 2).
+ """
+ # Set up option parser
+ parser = optparse.OptionParser(usage="refactor.py [options] file|dir ...")
+ parser.add_option("-d", "--doctests_only", action="store_true",
+ help="Fix up doctests only")
+ parser.add_option("-f", "--fix", action="append", default=[],
+ help="Each FIX specifies a transformation; default all")
+ parser.add_option("-l", "--list-fixes", action="store_true",
+ help="List available transformations (fixes/fix_*.py)")
+ parser.add_option("-p", "--print-function", action="store_true",
+ help="Modify the grammar so that print() is a function")
+ parser.add_option("-v", "--verbose", action="store_true",
+ help="More verbose logging")
+ parser.add_option("-w", "--write", action="store_true",
+ help="Write back modified files")
+
+ # Parse command line arguments
+ options, args = parser.parse_args(args)
+ if options.list_fixes:
+ print "Available transformations for the -f/--fix option:"
+ for fixname in get_all_fix_names():
+ print fixname
+ if not args:
+ return 0
+ if not args:
+ print >>sys.stderr, "At least one file or directory argument required."
+ print >>sys.stderr, "Use --help to show usage."
+ return 2
+
+ # Initialize the refactoring tool
+ rt = RefactoringTool(options)
+
+ # Refactor all files and directories passed as arguments
+ if not rt.errors:
+ rt.refactor_args(args)
+ rt.summarize()
+
+ # Return error status (0 if rt.errors is zero)
+ return int(bool(rt.errors))
+
+
+def get_all_fix_names():
+ """Return a sorted list of all available fix names."""
+ fix_names = []
+ names = os.listdir(os.path.dirname(fixes.__file__))
+ names.sort()
+ for name in names:
+ if name.startswith("fix_") and name.endswith(".py"):
+ fix_names.append(name[4:-3])
+ fix_names.sort()
+ return fix_names
+
+
+class RefactoringTool(object):
+
+ def __init__(self, options):
+ """Initializer.
+
+ The argument is an optparse.Values instance.
+ """
+ self.options = options
+ self.errors = []
+ self.logger = logging.getLogger("RefactoringTool")
+ self.fixer_log = []
+ if self.options.print_function:
+ del pygram.python_grammar.keywords["print"]
+ self.driver = driver.Driver(pygram.python_grammar,
+ convert=pytree.convert,
+ logger=self.logger)
+ self.pre_order, self.post_order = self.get_fixers()
+ self.files = [] # List of files that were or should be modified
+
+ def get_fixers(self):
+ """Inspects the options to load the requested patterns and handlers.
+
+ Returns:
+ (pre_order, post_order), where pre_order is the list of fixers that
+ want a pre-order AST traversal, and post_order is the list that want
+ post-order traversal.
+ """
+ pre_order_fixers = []
+ post_order_fixers = []
+ fix_names = self.options.fix
+ if not fix_names or "all" in fix_names:
+ fix_names = get_all_fix_names()
+ for fix_name in fix_names:
+ try:
+ mod = __import__("lib2to3.fixes.fix_" + fix_name, {}, {}, ["*"])
+ except ImportError:
+ self.log_error("Can't find transformation %s", fix_name)
+ continue
+ parts = fix_name.split("_")
+ class_name = "Fix" + "".join([p.title() for p in parts])
+ try:
+ fix_class = getattr(mod, class_name)
+ except AttributeError:
+ self.log_error("Can't find fixes.fix_%s.%s",
+ fix_name, class_name)
+ continue
+ try:
+ fixer = fix_class(self.options, self.fixer_log)
+ except Exception, err:
+ self.log_error("Can't instantiate fixes.fix_%s.%s()",
+ fix_name, class_name, exc_info=True)
+ continue
+ if fixer.explicit and fix_name not in self.options.fix:
+ self.log_message("Skipping implicit fixer: %s", fix_name)
+ continue
+
+ if self.options.verbose:
+ self.log_message("Adding transformation: %s", fix_name)
+ if fixer.order == "pre":
+ pre_order_fixers.append(fixer)
+ elif fixer.order == "post":
+ post_order_fixers.append(fixer)
+ else:
+ raise ValueError("Illegal fixer order: %r" % fixer.order)
+ return (pre_order_fixers, post_order_fixers)
+
+ def log_error(self, msg, *args, **kwds):
+ """Increments error count and log a message."""
+ self.errors.append((msg, args, kwds))
+ self.logger.error(msg, *args, **kwds)
+
+ def log_message(self, msg, *args):
+ """Hook to log a message."""
+ if args:
+ msg = msg % args
+ self.logger.info(msg)
+
+ def refactor_args(self, args):
+ """Refactors files and directories from an argument list."""
+ for arg in args:
+ if arg == "-":
+ self.refactor_stdin()
+ elif os.path.isdir(arg):
+ self.refactor_dir(arg)
+ else:
+ self.refactor_file(arg)
+
+ def refactor_dir(self, arg):
+ """Descends down a directory and refactor every Python file found.
+
+ Python files are assumed to have a .py extension.
+
+ Files and subdirectories starting with '.' are skipped.
+ """
+ for dirpath, dirnames, filenames in os.walk(arg):
+ if self.options.verbose:
+ self.log_message("Descending into %s", dirpath)
+ dirnames.sort()
+ filenames.sort()
+ for name in filenames:
+ if not name.startswith(".") and name.endswith("py"):
+ fullname = os.path.join(dirpath, name)
+ self.refactor_file(fullname)
+ # Modify dirnames in-place to remove subdirs with leading dots
+ dirnames[:] = [dn for dn in dirnames if not dn.startswith(".")]
+
+ def refactor_file(self, filename):
+ """Refactors a file."""
+ try:
+ f = open(filename)
+ except IOError, err:
+ self.log_error("Can't open %s: %s", filename, err)
+ return
+ try:
+ input = f.read() + "\n" # Silence certain parse errors
+ finally:
+ f.close()
+ if self.options.doctests_only:
+ if self.options.verbose:
+ self.log_message("Refactoring doctests in %s", filename)
+ output = self.refactor_docstring(input, filename)
+ if output != input:
+ self.write_file(output, filename, input)
+ elif self.options.verbose:
+ self.log_message("No doctest changes in %s", filename)
+ else:
+ tree = self.refactor_string(input, filename)
+ if tree and tree.was_changed:
+ # The [:-1] is to take off the \n we added earlier
+ self.write_file(str(tree)[:-1], filename)
+ elif self.options.verbose:
+ self.log_message("No changes in %s", filename)
+
+ def refactor_string(self, data, name):
+ """Refactor a given input string.
+
+ Args:
+ data: a string holding the code to be refactored.
+ name: a human-readable name for use in error/log messages.
+
+ Returns:
+ An AST corresponding to the refactored input stream; None if
+ there were errors during the parse.
+ """
+ try:
+ tree = self.driver.parse_string(data,1)
+ except Exception, err:
+ self.log_error("Can't parse %s: %s: %s",
+ name, err.__class__.__name__, err)
+ return
+ if self.options.verbose:
+ self.log_message("Refactoring %s", name)
+ self.refactor_tree(tree, name)
+ return tree
+
+ def refactor_stdin(self):
+ if self.options.write:
+ self.log_error("Can't write changes back to stdin")
+ return
+ input = sys.stdin.read()
+ if self.options.doctests_only:
+ if self.options.verbose:
+ self.log_message("Refactoring doctests in stdin")
+ output = self.refactor_docstring(input, "<stdin>")
+ if output != input:
+ self.write_file(output, "<stdin>", input)
+ elif self.options.verbose:
+ self.log_message("No doctest changes in stdin")
+ else:
+ tree = self.refactor_string(input, "<stdin>")
+ if tree and tree.was_changed:
+ self.write_file(str(tree), "<stdin>", input)
+ elif self.options.verbose:
+ self.log_message("No changes in stdin")
+
+ def refactor_tree(self, tree, name):
+ """Refactors a parse tree (modifying the tree in place).
+
+ Args:
+ tree: a pytree.Node instance representing the root of the tree
+ to be refactored.
+ name: a human-readable name for this tree.
+
+ Returns:
+ True if the tree was modified, False otherwise.
+ """
+ all_fixers = self.pre_order + self.post_order
+ for fixer in all_fixers:
+ fixer.start_tree(tree, name)
+
+ self.traverse_by(self.pre_order, tree.pre_order())
+ self.traverse_by(self.post_order, tree.post_order())
+
+ for fixer in all_fixers:
+ fixer.finish_tree(tree, name)
+ return tree.was_changed
+
+ def traverse_by(self, fixers, traversal):
+ """Traverse an AST, applying a set of fixers to each node.
+
+ This is a helper method for refactor_tree().
+
+ Args:
+ fixers: a list of fixer instances.
+ traversal: a generator that yields AST nodes.
+
+ Returns:
+ None
+ """
+ if not fixers:
+ return
+ for node in traversal:
+ for fixer in fixers:
+ results = fixer.match(node)
+ if results:
+ new = fixer.transform(node, results)
+ if new is not None and (new != node or
+ str(new) != str(node)):
+ node.replace(new)
+ node = new
+
+ def write_file(self, new_text, filename, old_text=None):
+ """Writes a string to a file.
+
+ If there are no changes, this is a no-op.
+
+ Otherwise, it first shows a unified diff between the old text
+ and the new text, and then rewrites the file; the latter is
+ only done if the write option is set.
+ """
+ self.files.append(filename)
+ if old_text is None:
+ try:
+ f = open(filename, "r")
+ except IOError, err:
+ self.log_error("Can't read %s: %s", filename, err)
+ return
+ try:
+ old_text = f.read()
+ finally:
+ f.close()
+ if old_text == new_text:
+ if self.options.verbose:
+ self.log_message("No changes to %s", filename)
+ return
+ diff_texts(old_text, new_text, filename)
+ if not self.options.write:
+ if self.options.verbose:
+ self.log_message("Not writing changes to %s", filename)
+ return
+ backup = filename + ".bak"
+ if os.path.lexists(backup):
+ try:
+ os.remove(backup)
+ except os.error, err:
+ self.log_message("Can't remove backup %s", backup)
+ try:
+ os.rename(filename, backup)
+ except os.error, err:
+ self.log_message("Can't rename %s to %s", filename, backup)
+ try:
+ f = open(filename, "w")
+ except os.error, err:
+ self.log_error("Can't create %s: %s", filename, err)
+ return
+ try:
+ try:
+ f.write(new_text)
+ except os.error, err:
+ self.log_error("Can't write %s: %s", filename, err)
+ finally:
+ f.close()
+ if self.options.verbose:
+ self.log_message("Wrote changes to %s", filename)
+
+ PS1 = ">>> "
+ PS2 = "... "
+
+ def refactor_docstring(self, input, filename):
+ """Refactors a docstring, looking for doctests.
+
+ This returns a modified version of the input string. It looks
+ for doctests, which start with a ">>>" prompt, and may be
+ continued with "..." prompts, as long as the "..." is indented
+ the same as the ">>>".
+
+ (Unfortunately we can't use the doctest module's parser,
+ since, like most parsers, it is not geared towards preserving
+ the original source.)
+ """
+ result = []
+ block = None
+ block_lineno = None
+ indent = None
+ lineno = 0
+ for line in input.splitlines(True):
+ lineno += 1
+ if line.lstrip().startswith(self.PS1):
+ if block is not None:
+ result.extend(self.refactor_doctest(block, block_lineno,
+ indent, filename))
+ block_lineno = lineno
+ block = [line]
+ i = line.find(self.PS1)
+ indent = line[:i]
+ elif (indent is not None and
+ (line.startswith(indent + self.PS2) or
+ line == indent + self.PS2.rstrip() + "\n")):
+ block.append(line)
+ else:
+ if block is not None:
+ result.extend(self.refactor_doctest(block, block_lineno,
+ indent, filename))
+ block = None
+ indent = None
+ result.append(line)
+ if block is not None:
+ result.extend(self.refactor_doctest(block, block_lineno,
+ indent, filename))
+ return "".join(result)
+
+ def refactor_doctest(self, block, lineno, indent, filename):
+ """Refactors one doctest.
+
+ A doctest is given as a block of lines, the first of which starts
+ with ">>>" (possibly indented), while the remaining lines start
+ with "..." (identically indented).
+
+ """
+ try:
+ tree = self.parse_block(block, lineno, indent)
+ except Exception, err:
+ if self.options.verbose:
+ for line in block:
+ self.log_message("Source: %s", line.rstrip("\n"))
+ self.log_error("Can't parse docstring in %s line %s: %s: %s",
+ filename, lineno, err.__class__.__name__, err)
+ return block
+ if self.refactor_tree(tree, filename):
+ new = str(tree).splitlines(True)
+ # Undo the adjustment of the line numbers in wrap_toks() below.
+ clipped, new = new[:lineno-1], new[lineno-1:]
+ assert clipped == ["\n"] * (lineno-1), clipped
+ if not new[-1].endswith("\n"):
+ new[-1] += "\n"
+ block = [indent + self.PS1 + new.pop(0)]
+ if new:
+ block += [indent + self.PS2 + line for line in new]
+ return block
+
+ def summarize(self):
+ if self.options.write:
+ were = "were"
+ else:
+ were = "need to be"
+ if not self.files:
+ self.log_message("No files %s modified.", were)
+ else:
+ self.log_message("Files that %s modified:", were)
+ for file in self.files:
+ self.log_message(file)
+ if self.fixer_log:
+ self.log_message("Warnings/messages while refactoring:")
+ for message in self.fixer_log:
+ self.log_message(message)
+ if self.errors:
+ if len(self.errors) == 1:
+ self.log_message("There was 1 error:")
+ else:
+ self.log_message("There were %d errors:", len(self.errors))
+ for msg, args, kwds in self.errors:
+ self.log_message(msg, *args, **kwds)
+
+ def parse_block(self, block, lineno, indent):
+ """Parses a block into a tree.
+
+ This is necessary to get correct line number / offset information
+ in the parser diagnostics and embedded into the parse tree.
+ """
+ return self.driver.parse_tokens(self.wrap_toks(block, lineno, indent))
+
+ def wrap_toks(self, block, lineno, indent):
+ """Wraps a tokenize stream to systematically modify start/end."""
+ tokens = tokenize.generate_tokens(self.gen_lines(block, indent).next)
+ for type, value, (line0, col0), (line1, col1), line_text in tokens:
+ line0 += lineno - 1
+ line1 += lineno - 1
+ # Don't bother updating the columns; this is too complicated
+ # since line_text would also have to be updated and it would
+ # still break for tokens spanning lines. Let the user guess
+ # that the column numbers for doctests are relative to the
+ # end of the prompt string (PS1 or PS2).
+ yield type, value, (line0, col0), (line1, col1), line_text
+
+
+ def gen_lines(self, block, indent):
+ """Generates lines as expected by tokenize from a list of lines.
+
+ This strips the first len(indent + self.PS1) characters off each line.
+ """
+ prefix1 = indent + self.PS1
+ prefix2 = indent + self.PS2
+ prefix = prefix1
+ for line in block:
+ if line.startswith(prefix):
+ yield line[len(prefix):]
+ elif line == prefix.rstrip() + "\n":
+ yield "\n"
+ else:
+ raise AssertionError("line=%r, prefix=%r" % (line, prefix))
+ prefix = prefix2
+ while True:
+ yield ""
+
+
+def diff_texts(a, b, filename):
+ """Prints a unified diff of two strings."""
+ a = a.splitlines()
+ b = b.splitlines()
+ for line in difflib.unified_diff(a, b, filename, filename,
+ "(original)", "(refactored)",
+ lineterm=""):
+ print line
+
+
+if __name__ == "__main__":
+ sys.exit(main())