bpo-39769: Fix compileall ddir for subpkgs. (GH-18676) (GH-18718)
Fix compileall.compile_dir() ddir= behavior on sub-packages.
Fixes compileall.compile_dir's ddir parameter and compileall command
line flag `-d` to no longer write the wrong pathname to the generated
pyc file for submodules beneath the root of the directory tree being
compiled. This fixes a regression introduced with Python 3.5.
Tests backported from GH 02673352b5db6ca4d3dc804965facbedfe66425d, the
implementation is different due to intervening code changes. But still
quiet simple.
Why was the bug ever introduced? The refactoring to add parallel
execution kept the ddir -> dfile computations but discarded the results
instead of sending them to compile_file(). This fixes that. Lack of tests
meant this went unnoticed.
diff --git a/Lib/compileall.py b/Lib/compileall.py
index 49306d9..bfac8ef 100644
--- a/Lib/compileall.py
+++ b/Lib/compileall.py
@@ -41,7 +41,7 @@
else:
dfile = None
if not os.path.isdir(fullname):
- yield fullname
+ yield fullname, ddir
elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
os.path.isdir(fullname) and not os.path.islink(fullname)):
yield from _walk_dir(fullname, ddir=dfile,
@@ -76,28 +76,33 @@
from concurrent.futures import ProcessPoolExecutor
except ImportError:
workers = 1
- files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
- ddir=ddir)
+ files_and_ddirs = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels,
+ ddir=ddir)
success = True
if workers != 1 and ProcessPoolExecutor is not None:
# If workers == 0, let ProcessPoolExecutor choose
workers = workers or None
with ProcessPoolExecutor(max_workers=workers) as executor:
- results = executor.map(partial(compile_file,
- ddir=ddir, force=force,
- rx=rx, quiet=quiet,
- legacy=legacy,
- optimize=optimize,
- invalidation_mode=invalidation_mode),
- files)
+ results = executor.map(
+ partial(_compile_file_tuple,
+ force=force, rx=rx, quiet=quiet,
+ legacy=legacy, optimize=optimize,
+ invalidation_mode=invalidation_mode,
+ ),
+ files_and_ddirs)
success = min(results, default=True)
else:
- for file in files:
- if not compile_file(file, ddir, force, rx, quiet,
+ for file, dfile in files_and_ddirs:
+ if not compile_file(file, dfile, force, rx, quiet,
legacy, optimize, invalidation_mode):
success = False
return success
+def _compile_file_tuple(file_and_dfile, **kwargs):
+ """Needs to be toplevel for ProcessPoolExecutor."""
+ file, dfile = file_and_dfile
+ return compile_file(file, dfile, **kwargs)
+
def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
legacy=False, optimize=-1,
invalidation_mode=None):