Issue #15343: Handle importlib.machinery.FileFinder instances in pkgutil.walk_packages (et al)
diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py
index 487c8cd..8407b6d 100644
--- a/Lib/pkgutil.py
+++ b/Lib/pkgutil.py
@@ -157,6 +157,49 @@
iter_importer_modules = simplegeneric(iter_importer_modules)
+# Implement a file walker for the normal importlib path hook
+def _iter_file_finder_modules(importer, prefix=''):
+ if importer.path is None or not os.path.isdir(importer.path):
+ return
+
+ yielded = {}
+ import inspect
+ try:
+ filenames = os.listdir(importer.path)
+ except OSError:
+ # ignore unreadable directories like import does
+ filenames = []
+ filenames.sort() # handle packages before same-named modules
+
+ for fn in filenames:
+ modname = inspect.getmodulename(fn)
+ if modname=='__init__' or modname in yielded:
+ continue
+
+ path = os.path.join(importer.path, fn)
+ ispkg = False
+
+ if not modname and os.path.isdir(path) and '.' not in fn:
+ modname = fn
+ try:
+ dircontents = os.listdir(path)
+ except OSError:
+ # ignore unreadable directories like import does
+ dircontents = []
+ for fn in dircontents:
+ subname = inspect.getmodulename(fn)
+ if subname=='__init__':
+ ispkg = True
+ break
+ else:
+ continue # not a package
+
+ if modname and '.' not in modname:
+ yielded[modname] = 1
+ yield prefix + modname, ispkg
+
+iter_importer_modules.register(
+ importlib.machinery.FileFinder, _iter_file_finder_modules)
class ImpImporter:
"""PEP 302 Importer that wraps Python's "classic" import algorithm
diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py
index 51f5dee..73c6869 100644
--- a/Lib/test/test_pkgutil.py
+++ b/Lib/test/test_pkgutil.py
@@ -9,7 +9,11 @@
import shutil
import zipfile
-
+# Note: pkgutil.walk_packages is currently tested in test_runpy. This is
+# a hack to get a major issue resolved for 3.3b2. Longer term, it should
+# be moved back here, perhaps by factoring out the helper code for
+# creating interesting package layouts to a separate module.
+# Issue #15348 declares this is indeed a dodgy hack ;)
class PkgutilTests(unittest.TestCase):
diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py
index c39e281..abb7dd9 100644
--- a/Lib/test/test_runpy.py
+++ b/Lib/test/test_runpy.py
@@ -13,6 +13,7 @@
from test.script_helper import (
make_pkg, make_script, make_zip_pkg, make_zip_script, temp_dir)
+
import runpy
from runpy import _run_code, _run_module_code, run_module, run_path
# Note: This module can't safely test _run_module_as_main as it
@@ -148,7 +149,7 @@
mod_package)
self.check_code_execution(create_ns, expected_ns)
-
+# TODO: Use self.addCleanup to get rid of a lot of try-finally blocks
class RunModuleTestCase(unittest.TestCase, CodeExecutionMixin):
"""Unit tests for runpy.run_module"""
@@ -413,6 +414,40 @@
finally:
self._del_pkg(pkg_dir, depth, mod_name)
+ def test_pkgutil_walk_packages(self):
+ # This is a dodgy hack to use the test_runpy infrastructure to test
+ # issue #15343. Issue #15348 declares this is indeed a dodgy hack ;)
+ import pkgutil
+ max_depth = 4
+ base_name = "__runpy_pkg__"
+ package_suffixes = ["uncle", "uncle.cousin"]
+ module_suffixes = ["uncle.cousin.nephew", base_name + ".sibling"]
+ expected_packages = set()
+ expected_modules = set()
+ for depth in range(1, max_depth):
+ pkg_name = ".".join([base_name] * depth)
+ expected_packages.add(pkg_name)
+ for name in package_suffixes:
+ expected_packages.add(pkg_name + "." + name)
+ for name in module_suffixes:
+ expected_modules.add(pkg_name + "." + name)
+ pkg_name = ".".join([base_name] * max_depth)
+ expected_packages.add(pkg_name)
+ expected_modules.add(pkg_name + ".runpy_test")
+ pkg_dir, mod_fname, mod_name = (
+ self._make_pkg("", max_depth))
+ self.addCleanup(self._del_pkg, pkg_dir, max_depth, mod_name)
+ for depth in range(2, max_depth+1):
+ self._add_relative_modules(pkg_dir, "", depth)
+ for finder, mod_name, ispkg in pkgutil.walk_packages([pkg_dir]):
+ self.assertIsInstance(finder,
+ importlib.machinery.FileFinder)
+ if ispkg:
+ expected_packages.remove(mod_name)
+ else:
+ expected_modules.remove(mod_name)
+ self.assertEqual(len(expected_packages), 0, expected_packages)
+ self.assertEqual(len(expected_modules), 0, expected_modules)
class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
"""Unit tests for runpy.run_path"""