Thomas Heller fixed modulefinder and added a test. Thanks!
diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py
index 2a31dbf..9c90d8d 100644
--- a/Lib/modulefinder.py
+++ b/Lib/modulefinder.py
@@ -1,13 +1,14 @@
"""Find modules used by a script, using introspection."""
-
# This module should be kept compatible with Python 2.2, see PEP 291.
+from __future__ import generators
import dis
import imp
import marshal
import os
import sys
import new
+import struct
if hasattr(sys.__stdout__, "newlines"):
READ_MODE = "U" # universal line endings
@@ -15,11 +16,12 @@
# remain compatible with Python < 2.3
READ_MODE = "r"
-LOAD_CONST = dis.opname.index('LOAD_CONST')
-IMPORT_NAME = dis.opname.index('IMPORT_NAME')
-STORE_NAME = dis.opname.index('STORE_NAME')
-STORE_GLOBAL = dis.opname.index('STORE_GLOBAL')
+LOAD_CONST = chr(dis.opname.index('LOAD_CONST'))
+IMPORT_NAME = chr(dis.opname.index('IMPORT_NAME'))
+STORE_NAME = chr(dis.opname.index('STORE_NAME'))
+STORE_GLOBAL = chr(dis.opname.index('STORE_GLOBAL'))
STORE_OPS = [STORE_NAME, STORE_GLOBAL]
+HAVE_ARGUMENT = chr(dis.HAVE_ARGUMENT)
# Modulefinder does a good job at simulating Python's, but it can not
# handle __path__ modifications packages make at runtime. Therefore there
@@ -118,9 +120,9 @@
stuff = (ext, "r", imp.PY_SOURCE)
self.load_module(name, fp, pathname, stuff)
- def import_hook(self, name, caller=None, fromlist=None):
- self.msg(3, "import_hook", name, caller, fromlist)
- parent = self.determine_parent(caller)
+ def import_hook(self, name, caller=None, fromlist=None, level=-1):
+ self.msg(3, "import_hook", name, caller, fromlist, level)
+ parent = self.determine_parent(caller, level=level)
q, tail = self.find_head_package(parent, name)
m = self.load_tail(q, tail)
if not fromlist:
@@ -129,12 +131,26 @@
self.ensure_fromlist(m, fromlist)
return None
- def determine_parent(self, caller):
- self.msgin(4, "determine_parent", caller)
- if not caller:
+ def determine_parent(self, caller, level=-1):
+ self.msgin(4, "determine_parent", caller, level)
+ if not caller or level == 0:
self.msgout(4, "determine_parent -> None")
return None
pname = caller.__name__
+ if level >= 1: # relative import
+ if caller.__path__:
+ level -= 1
+ if level == 0:
+ parent = self.modules[pname]
+ assert parent is caller
+ self.msgout(4, "determine_parent ->", parent)
+ return parent
+ if pname.count(".") < level:
+ raise ImportError, "relative importpath too deep"
+ pname = ".".join(pname.split(".")[:-level])
+ parent = self.modules[pname]
+ self.msgout(4, "determine_parent ->", parent)
+ return parent
if caller.__path__:
parent = self.modules[pname]
assert caller is parent
@@ -294,13 +310,13 @@
self.badmodules[name] = {}
self.badmodules[name][caller.__name__] = 1
- def _safe_import_hook(self, name, caller, fromlist):
+ def _safe_import_hook(self, name, caller, fromlist, level=-1):
# wrapper for self.import_hook() that won't raise ImportError
if name in self.badmodules:
self._add_badmodule(name, caller)
return
try:
- self.import_hook(name, caller)
+ self.import_hook(name, caller, level=level)
except ImportError, msg:
self.msg(2, "ImportError:", str(msg))
self._add_badmodule(name, caller)
@@ -311,38 +327,83 @@
self._add_badmodule(sub, caller)
continue
try:
- self.import_hook(name, caller, [sub])
+ self.import_hook(name, caller, [sub], level=level)
except ImportError, msg:
self.msg(2, "ImportError:", str(msg))
fullname = name + "." + sub
self._add_badmodule(fullname, caller)
+ def scan_opcodes(self, co,
+ unpack = struct.unpack):
+ # Scan the code, and yield 'interesting' opcode combinations
+ # Version for Python 2.4 and older
+ code = co.co_code
+ names = co.co_names
+ consts = co.co_consts
+ while code:
+ c = code[0]
+ if c in STORE_OPS:
+ oparg, = unpack('<H', code[1:3])
+ yield "store", (names[oparg],)
+ code = code[3:]
+ continue
+ if c == LOAD_CONST and code[3] == IMPORT_NAME:
+ oparg_1, oparg_2 = unpack('<xHxH', code[:6])
+ yield "import", (consts[oparg_1], names[oparg_2])
+ code = code[6:]
+ continue
+ if c >= HAVE_ARGUMENT:
+ code = code[3:]
+ else:
+ code = code[1:]
+
+ def scan_opcodes_25(self, co,
+ unpack = struct.unpack):
+ # Scan the code, and yield 'interesting' opcode combinations
+ # Python 2.5 version (has absolute and relative imports)
+ code = co.co_code
+ names = co.co_names
+ consts = co.co_consts
+ LOAD_LOAD_AND_IMPORT = LOAD_CONST + LOAD_CONST + IMPORT_NAME
+ while code:
+ c = code[0]
+ if c in STORE_OPS:
+ oparg, = unpack('<H', code[1:3])
+ yield "store", (names[oparg],)
+ code = code[3:]
+ continue
+ if code[:9:3] == LOAD_LOAD_AND_IMPORT:
+ oparg_1, oparg_2, oparg_3 = unpack('<xHxHxH', code[:9])
+ level = consts[oparg_1]
+ if level == 0: # absolute import
+ yield "absolute_import", (consts[oparg_2], names[oparg_3])
+ else: # relative import
+ yield "relative_import", (level, consts[oparg_2], names[oparg_3])
+ code = code[9:]
+ continue
+ if c >= HAVE_ARGUMENT:
+ code = code[3:]
+ else:
+ code = code[1:]
+
def scan_code(self, co, m):
code = co.co_code
- n = len(code)
- i = 0
- fromlist = None
- while i < n:
- c = code[i]
- i = i+1
- op = ord(c)
- if op >= dis.HAVE_ARGUMENT:
- oparg = ord(code[i]) + ord(code[i+1])*256
- i = i+2
- if op == LOAD_CONST:
- # An IMPORT_NAME is always preceded by a LOAD_CONST, it's
- # a tuple of "from" names, or None for a regular import.
- # The tuple may contain "*" for "from <mod> import *"
- fromlist = co.co_consts[oparg]
- elif op == IMPORT_NAME:
- assert fromlist is None or type(fromlist) is tuple
- name = co.co_names[oparg]
+ if sys.version_info >= (2, 5):
+ scanner = self.scan_opcodes_25
+ else:
+ scanner = self.scan_opcodes
+ for what, args in scanner(co):
+ if what == "store":
+ name, = args
+ m.globalnames[name] = 1
+ elif what == "absolute_import":
+ fromlist, name = args
have_star = 0
if fromlist is not None:
if "*" in fromlist:
have_star = 1
fromlist = [f for f in fromlist if f != "*"]
- self._safe_import_hook(name, m, fromlist)
+ self._safe_import_hook(name, m, fromlist, level=0)
if have_star:
# We've encountered an "import *". If it is a Python module,
# the code has already been parsed and we can suck out the
@@ -362,10 +423,17 @@
m.starimports[name] = 1
else:
m.starimports[name] = 1
- elif op in STORE_OPS:
- # keep track of all global names that are assigned to
- name = co.co_names[oparg]
- m.globalnames[name] = 1
+ elif what == "relative_import":
+ level, fromlist, name = args
+ if name:
+ self._safe_import_hook(name, m, fromlist, level=level)
+ else:
+ parent = self.determine_parent(m, level=level)
+ self._safe_import_hook(parent.__name__, None, fromlist, level=0)
+ else:
+ # We don't expect anything else from the generator.
+ raise RuntimeError(what)
+
for c in co.co_consts:
if isinstance(c, type(co)):
self.scan_code(c, m)
diff --git a/Lib/test/test_modulefinder.py b/Lib/test/test_modulefinder.py
new file mode 100644
index 0000000..0b8e451
--- /dev/null
+++ b/Lib/test/test_modulefinder.py
@@ -0,0 +1,259 @@
+import __future__
+import sys, os
+import unittest
+import distutils.dir_util
+import tempfile
+
+from test import test_support
+
+try: set
+except NameError: from sets import Set as set
+
+import modulefinder
+
+# Note: To test modulefinder with Python 2.2, sets.py and
+# modulefinder.py must be available - they are not in the standard
+# library.
+
+TEST_DIR = tempfile.mkdtemp()
+TEST_PATH = [TEST_DIR, os.path.dirname(__future__.__file__)]
+
+# Each test description is a list of 5 items:
+#
+# 1. a module name that will be imported by modulefinder
+# 2. a list of module names that modulefinder is required to find
+# 3. a list of module names that modulefinder should complain
+# about because they are not found
+# 4. a list of module names that modulefinder should complain
+# about because they MAY be not found
+# 5. a string specifying packages to create; the format is obvious imo.
+#
+# Each package will be created in TEST_DIR, and TEST_DIR will be
+# removed after the tests again.
+# Modulefinder searches in a path that contains TEST_DIR, plus
+# the standard Lib directory.
+
+maybe_test = [
+ "a.module",
+ ["a", "a.module", "sys",
+ "b"],
+ ["c"], ["b.something"],
+ """\
+a/__init__.py
+a/module.py
+ from b import something
+ from c import something
+b/__init__.py
+ from sys import *
+"""]
+
+maybe_test_new = [
+ "a.module",
+ ["a", "a.module", "sys",
+ "b"],
+ ["c"], ["b.something"],
+ """\
+a/__init__.py
+a/module.py
+ from b import something
+ from c import something
+b/__init__.py
+ from sys import *
+"""]
+
+package_test = [
+ "a.module",
+ ["a", "a.b", "a.c", "a.module", "mymodule", "sys"],
+ ["blahblah", "c"], [],
+ """\
+mymodule.py
+a/__init__.py
+ import blahblah
+ from a import b
+ import c
+a/module.py
+ import sys
+ from a import b as x
+ from a.c import sillyname
+a/b.py
+a/c.py
+ from a.module import x
+ import mymodule as sillyname
+ from sys import version_info
+"""]
+
+absolute_import_test = [
+ "a.module",
+ ["a", "a.module",
+ "b", "b.x", "b.y", "b.z",
+ "sys", "exceptions"],
+ ["blahblah", "z"], [],
+ """\
+mymodule.py
+a/__init__.py
+a/module.py
+ import sys # sys
+ import blahblah # fails
+ import exceptions # exceptions
+ import b.x # b.x
+ from b import y # b.y
+ from b.z import * # b.z.*
+a/exceptions.py
+a/sys.py
+ import mymodule
+a/b/__init__.py
+a/b/x.py
+a/b/y.py
+a/b/z.py
+b/__init__.py
+ import z
+b/unused.py
+b/x.py
+b/y.py
+b/z.py
+"""]
+
+relative_import_test = [
+ "a.module",
+ ["a", "a.module",
+ "a.b", "a.b.y", "a.b.z",
+ "a.b.c", "a.b.c.moduleC",
+ "a.b.c.d", "a.b.c.e",
+ "a.b.x",
+ "exceptions"],
+ [], [],
+ """\
+mymodule.py
+a/__init__.py
+ from .b import y, z # a.b.y, a.b.z
+a/module.py
+ import exceptions # exceptions
+a/exceptions.py
+a/sys.py
+a/b/__init__.py
+ from ..b import x # a.b.x
+ #from a.b.c import moduleC
+ from .c import moduleC # a.b.moduleC
+a/b/x.py
+a/b/y.py
+a/b/z.py
+a/b/g.py
+a/b/c/__init__.py
+ from ..c import e # a.b.c.e
+a/b/c/moduleC.py
+ from ..c import d # a.b.c.d
+a/b/c/d.py
+a/b/c/e.py
+a/b/c/x.py
+"""]
+
+relative_import_test_2 = [
+ "a.module",
+ ["a", "a.module",
+ "a.sys",
+ "a.b", "a.b.y", "a.b.z",
+ "a.b.c", "a.b.c.d",
+ "a.b.c.e",
+ "a.b.c.moduleC",
+ "a.b.c.f",
+ "a.b.x",
+ "a.another"],
+ [], [],
+ """\
+mymodule.py
+a/__init__.py
+ from . import sys # a.sys
+a/another.py
+a/module.py
+ from .b import y, z # a.b.y, a.b.z
+a/exceptions.py
+a/sys.py
+a/b/__init__.py
+ from .c import moduleC # a.b.c.moduleC
+ from .c import d # a.b.c.d
+a/b/x.py
+a/b/y.py
+a/b/z.py
+a/b/c/__init__.py
+ from . import e # a.b.c.e
+a/b/c/moduleC.py
+ #
+ from . import f # a.b.c.f
+ from .. import x # a.b.x
+ from ... import another # a.another
+a/b/c/d.py
+a/b/c/e.py
+a/b/c/f.py
+"""]
+
+def open_file(path):
+ ##print "#", os.path.abspath(path)
+ dirname = os.path.dirname(path)
+ distutils.dir_util.mkpath(dirname)
+ return open(path, "w")
+
+def create_package(source):
+ ofi = None
+ for line in source.splitlines():
+ if line.startswith(" ") or line.startswith("\t"):
+ ofi.write(line.strip() + "\n")
+ else:
+ ofi = open_file(os.path.join(TEST_DIR, line.strip()))
+
+class ModuleFinderTest(unittest.TestCase):
+ def _do_test(self, info, report=False):
+ import_this, modules, missing, maybe_missing, source = info
+ create_package(source)
+ try:
+ mf = modulefinder.ModuleFinder(path=TEST_PATH)
+ mf.import_hook(import_this)
+ if report:
+ mf.report()
+## # This wouldn't work in general when executed several times:
+## opath = sys.path[:]
+## sys.path = TEST_PATH
+## try:
+## __import__(import_this)
+## except:
+## import traceback; traceback.print_exc()
+## sys.path = opath
+## return
+ modules = set(modules)
+ found = set(mf.modules.keys())
+ more = list(found - modules)
+ less = list(modules - found)
+ # check if we found what we expected, not more, not less
+ self.failUnlessEqual((more, less), ([], []))
+
+ # check for missing and maybe missing modules
+ bad, maybe = mf.any_missing_maybe()
+ self.failUnlessEqual(bad, missing)
+ self.failUnlessEqual(maybe, maybe_missing)
+ finally:
+ distutils.dir_util.remove_tree(TEST_DIR)
+
+ def test_package(self):
+ self._do_test(package_test)
+
+ def test_maybe(self):
+ self._do_test(maybe_test)
+
+ if getattr(__future__, "absolute_import", None):
+
+ def test_maybe_new(self):
+ self._do_test(maybe_test_new)
+
+ def test_absolute_imports(self):
+ self._do_test(absolute_import_test)
+
+ def test_relative_imports(self):
+ self._do_test(relative_import_test)
+
+ def test_relative_imports_2(self):
+ self._do_test(relative_import_test_2)
+
+def test_main():
+ test_support.run_unittest(ModuleFinderTest)
+
+if __name__ == "__main__":
+ unittest.main()