bpo-34022: Stop forcing of hash-based invalidation with SOURCE_DATE_EPOCH (GH-9607)
Unconditional forcing of ``CHECKED_HASH`` invalidation was introduced in
3.7.0 in bpo-29708. The change is bad, as it unconditionally overrides
*invalidation_mode*, even if it was passed as an explicit argument to
``py_compile.compile()`` or ``compileall``. An environment variable
should *never* override an explicit argument to a library function.
That change leads to multiple test failures if the ``SOURCE_DATE_EPOCH``
environment variable is set.
This changes ``py_compile.compile()`` to only look at
``SOURCE_DATE_EPOCH`` if no explicit *invalidation_mode* was specified.
I also made various relevant tests run with explicit control over the
value of ``SOURCE_DATE_EPOCH``.
While looking at this, I noticed that ``zipimport`` does not work
with hash-based .pycs _at all_, though I left the fixes for
subsequent commits.
(cherry picked from commit a6b3ec5b6d4f6387820fccc570eea08b9615620d)
Co-authored-by: Elvis Pranskevichus <elvis@magic.io>
diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
index 2995e08..2e25523 100644
--- a/Lib/test/test_compileall.py
+++ b/Lib/test/test_compileall.py
@@ -22,7 +22,11 @@
from test import support
from test.support import script_helper
-class CompileallTests(unittest.TestCase):
+from .test_py_compile import without_source_date_epoch
+from .test_py_compile import SourceDateEpochTestMeta
+
+
+class CompileallTestsBase:
def setUp(self):
self.directory = tempfile.mkdtemp()
@@ -46,7 +50,7 @@
with open(self.bad_source_path, 'w') as file:
file.write('x (\n')
- def data(self):
+ def timestamp_metadata(self):
with open(self.bc_path, 'rb') as file:
data = file.read(12)
mtime = int(os.stat(self.source_path).st_mtime)
@@ -57,16 +61,18 @@
def recreation_check(self, metadata):
"""Check that compileall recreates bytecode when the new metadata is
used."""
+ if os.environ.get('SOURCE_DATE_EPOCH'):
+ raise unittest.SkipTest('SOURCE_DATE_EPOCH is set')
py_compile.compile(self.source_path)
- self.assertEqual(*self.data())
+ self.assertEqual(*self.timestamp_metadata())
with open(self.bc_path, 'rb') as file:
bc = file.read()[len(metadata):]
with open(self.bc_path, 'wb') as file:
file.write(metadata)
file.write(bc)
- self.assertNotEqual(*self.data())
+ self.assertNotEqual(*self.timestamp_metadata())
compileall.compile_dir(self.directory, force=False, quiet=True)
- self.assertTrue(*self.data())
+ self.assertTrue(*self.timestamp_metadata())
def test_mtime(self):
# Test a change in mtime leads to a new .pyc.
@@ -189,6 +195,21 @@
compileall.compile_dir(self.directory, quiet=True, workers=5)
self.assertTrue(compile_file_mock.called)
+
+class CompileallTestsWithSourceEpoch(CompileallTestsBase,
+ unittest.TestCase,
+ metaclass=SourceDateEpochTestMeta,
+ source_date_epoch=True):
+ pass
+
+
+class CompileallTestsWithoutSourceEpoch(CompileallTestsBase,
+ unittest.TestCase,
+ metaclass=SourceDateEpochTestMeta,
+ source_date_epoch=False):
+ pass
+
+
class EncodingTest(unittest.TestCase):
"""Issue 6716: compileall should escape source code when printing errors
to stdout."""
@@ -212,7 +233,7 @@
sys.stdout = orig_stdout
-class CommandLineTests(unittest.TestCase):
+class CommandLineTestsBase:
"""Test compileall's CLI."""
@classmethod
@@ -285,6 +306,7 @@
self.assertNotCompiled(self.initfn)
self.assertNotCompiled(self.barfn)
+ @without_source_date_epoch # timestamp invalidation test
def test_no_args_respects_force_flag(self):
self._skip_if_sys_path_not_writable()
bazfn = script_helper.make_script(self.directory, 'baz', '')
@@ -353,6 +375,7 @@
self.assertTrue(os.path.exists(self.pkgdir_cachedir))
self.assertFalse(os.path.exists(cachecachedir))
+ @without_source_date_epoch # timestamp invalidation test
def test_force(self):
self.assertRunOK('-q', self.pkgdir)
pycpath = importlib.util.cache_from_source(self.barfn)
@@ -556,5 +579,20 @@
self.assertEqual(compile_dir.call_args[-1]['workers'], None)
+class CommmandLineTestsWithSourceEpoch(CommandLineTestsBase,
+ unittest.TestCase,
+ metaclass=SourceDateEpochTestMeta,
+ source_date_epoch=True):
+ pass
+
+
+class CommmandLineTestsNoSourceEpoch(CommandLineTestsBase,
+ unittest.TestCase,
+ metaclass=SourceDateEpochTestMeta,
+ source_date_epoch=False):
+ pass
+
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py
index cc80f26..c916d7c 100644
--- a/Lib/test/test_importlib/source/test_file_loader.py
+++ b/Lib/test/test_importlib/source/test_file_loader.py
@@ -19,6 +19,9 @@
from test.support import make_legacy_pyc, unload
+from test.test_py_compile import without_source_date_epoch
+from test.test_py_compile import SourceDateEpochTestMeta
+
class SimpleTest(abc.LoaderTests):
@@ -359,6 +362,17 @@
abc=importlib_abc, util=importlib_util)
+class SourceDateEpochTestMeta(SourceDateEpochTestMeta,
+ type(Source_SimpleTest)):
+ pass
+
+
+class SourceDateEpoch_SimpleTest(Source_SimpleTest,
+ metaclass=SourceDateEpochTestMeta,
+ source_date_epoch=True):
+ pass
+
+
class BadBytecodeTest:
def import_(self, file, module_name):
@@ -617,6 +631,7 @@
# [bad timestamp]
@util.writes_bytecode_files
+ @without_source_date_epoch
def test_old_timestamp(self):
# When the timestamp is older than the source, bytecode should be
# regenerated.
diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py
index 8fc0b33..f86abe2 100644
--- a/Lib/test/test_py_compile.py
+++ b/Lib/test/test_py_compile.py
@@ -1,3 +1,4 @@
+import functools
import importlib.util
import os
import py_compile
@@ -10,7 +11,44 @@
from test import support
-class PyCompileTests(unittest.TestCase):
+def without_source_date_epoch(fxn):
+ """Runs function with SOURCE_DATE_EPOCH unset."""
+ @functools.wraps(fxn)
+ def wrapper(*args, **kwargs):
+ with support.EnvironmentVarGuard() as env:
+ env.unset('SOURCE_DATE_EPOCH')
+ return fxn(*args, **kwargs)
+ return wrapper
+
+
+def with_source_date_epoch(fxn):
+ """Runs function with SOURCE_DATE_EPOCH set."""
+ @functools.wraps(fxn)
+ def wrapper(*args, **kwargs):
+ with support.EnvironmentVarGuard() as env:
+ env['SOURCE_DATE_EPOCH'] = '123456789'
+ return fxn(*args, **kwargs)
+ return wrapper
+
+
+# Run tests with SOURCE_DATE_EPOCH set or unset explicitly.
+class SourceDateEpochTestMeta(type(unittest.TestCase)):
+ def __new__(mcls, name, bases, dct, *, source_date_epoch):
+ cls = super().__new__(mcls, name, bases, dct)
+
+ for attr in dir(cls):
+ if attr.startswith('test_'):
+ meth = getattr(cls, attr)
+ if source_date_epoch:
+ wrapper = with_source_date_epoch(meth)
+ else:
+ wrapper = without_source_date_epoch(meth)
+ setattr(cls, attr, wrapper)
+
+ return cls
+
+
+class PyCompileTestsBase:
def setUp(self):
self.directory = tempfile.mkdtemp()
@@ -99,16 +137,18 @@
importlib.util.cache_from_source(bad_coding)))
def test_source_date_epoch(self):
- testtime = 123456789
- with support.EnvironmentVarGuard() as env:
- env["SOURCE_DATE_EPOCH"] = str(testtime)
- py_compile.compile(self.source_path, self.pyc_path)
+ py_compile.compile(self.source_path, self.pyc_path)
self.assertTrue(os.path.exists(self.pyc_path))
self.assertFalse(os.path.exists(self.cache_path))
with open(self.pyc_path, 'rb') as fp:
flags = importlib._bootstrap_external._classify_pyc(
fp.read(), 'test', {})
- self.assertEqual(flags, 0b11)
+ if os.environ.get('SOURCE_DATE_EPOCH'):
+ expected_flags = 0b11
+ else:
+ expected_flags = 0b00
+
+ self.assertEqual(flags, expected_flags)
@unittest.skipIf(sys.flags.optimize > 0, 'test does not work with -O')
def test_double_dot_no_clobber(self):
@@ -153,5 +193,19 @@
self.assertEqual(flags, 0b1)
+class PyCompileTestsWithSourceEpoch(PyCompileTestsBase,
+ unittest.TestCase,
+ metaclass=SourceDateEpochTestMeta,
+ source_date_epoch=True):
+ pass
+
+
+class PyCompileTestsWithoutSourceEpoch(PyCompileTestsBase,
+ unittest.TestCase,
+ metaclass=SourceDateEpochTestMeta,
+ source_date_epoch=False):
+ pass
+
+
if __name__ == "__main__":
unittest.main()