Issue #19700: set __spec__ appropriately in runpy
Note that __spec__.name is not currently guaranteed to be in
sys.modules when the code is running, only __name__ is.
The "running module is in sys.modules" invariant will be
expanded to also cover __spec__.name in a subsequent patch.
diff --git a/Lib/test/script_helper.py b/Lib/test/script_helper.py
index 4d5c1f1..af0545b 100644
--- a/Lib/test/script_helper.py
+++ b/Lib/test/script_helper.py
@@ -101,8 +101,10 @@
subprocess._cleanup()
return data
-def make_script(script_dir, script_basename, source):
- script_filename = script_basename+os.extsep+'py'
+def make_script(script_dir, script_basename, source, omit_suffix=False):
+ script_filename = script_basename
+ if not omit_suffix:
+ script_filename += os.extsep + 'py'
script_name = os.path.join(script_dir, script_filename)
# The script should be encoded to UTF-8, the default string encoding
script_file = open(script_name, 'w', encoding='utf-8')
diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py
index f804d86..03c071e 100644
--- a/Lib/test/test_cmd_line_script.py
+++ b/Lib/test/test_cmd_line_script.py
@@ -41,11 +41,28 @@
_loader = __loader__ if __loader__ is BuiltinImporter else type(__loader__)
print('__loader__==%a' % _loader)
print('__file__==%a' % __file__)
-assertEqual(__cached__, None)
+print('__cached__==%a' % __cached__)
print('__package__==%r' % __package__)
+# Check PEP 451 details
+import os.path
+if __package__ is not None:
+ print('__main__ was located through the import system')
+ assertIdentical(__spec__.loader, __loader__)
+ expected_spec_name = os.path.splitext(os.path.basename(__file__))[0]
+ if __package__:
+ expected_spec_name = __package__ + "." + expected_spec_name
+ assertEqual(__spec__.name, expected_spec_name)
+ assertEqual(__spec__.parent, __package__)
+ assertIdentical(__spec__.submodule_search_locations, None)
+ assertEqual(__spec__.origin, __file__)
+ if __spec__.cached is not None:
+ assertEqual(__spec__.cached, __cached__)
# Check the sys module
import sys
assertIdentical(globals(), sys.modules[__name__].__dict__)
+if __spec__ is not None:
+ # XXX: We're not currently making __main__ available under its real name
+ pass # assertIdentical(globals(), sys.modules[__spec__.name].__dict__)
from test import test_cmd_line_script
example_args_list = test_cmd_line_script.example_args
assertEqual(sys.argv[1:], example_args_list)
diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py
index f19c4ab..786b813 100644
--- a/Lib/test/test_runpy.py
+++ b/Lib/test/test_runpy.py
@@ -5,7 +5,7 @@
import sys
import re
import tempfile
-import importlib
+import importlib, importlib.machinery, importlib.util
import py_compile
from test.support import (
forget, make_legacy_pyc, run_unittest, unload, verbose, no_tracing,
@@ -47,7 +47,7 @@
"__cached__": None,
"__package__": None,
"__doc__": None,
-# "__spec__": None, # XXX Uncomment.
+ "__spec__": None
}
example_namespace = {
"sys": sys,
@@ -57,7 +57,7 @@
"run_name_in_sys_modules": False,
"module_in_sys_modules": False,
"nested": dict(implicit_namespace,
- x=1, __name__="<run>", __loader__=None, __spec__=None),
+ x=1, __name__="<run>", __loader__=None),
}
example_namespace.update(implicit_namespace)
@@ -68,11 +68,19 @@
# testing occurs at those upper layers as well, not just at the utility
# layer
+ # Figuring out the loader details in advance is hard to do, so we skip
+ # checking the full details of loader and loader_state
+ CHECKED_SPEC_ATTRIBUTES = ["name", "parent", "origin", "cached",
+ "has_location", "submodule_search_locations"]
+
def assertNamespaceMatches(self, result_ns, expected_ns):
"""Check two namespaces match.
Ignores any unspecified interpreter created names
"""
+ # Avoid side effects
+ result_ns = result_ns.copy()
+ expected_ns = expected_ns.copy()
# Impls are permitted to add extra names, so filter them out
for k in list(result_ns):
if k.startswith("__") and k.endswith("__"):
@@ -80,7 +88,25 @@
result_ns.pop(k)
if k not in expected_ns["nested"]:
result_ns["nested"].pop(k)
- # Don't use direct dict comparison - the diffs are too hard to debug
+ # Spec equality includes the loader, so we take the spec out of the
+ # result namespace and check that separately
+ result_spec = result_ns.pop("__spec__")
+ expected_spec = expected_ns.pop("__spec__")
+ if expected_spec is None:
+ self.assertIsNone(result_spec)
+ else:
+ # If an expected loader is set, we just check we got the right
+ # type, rather than checking for full equality
+ if expected_spec.loader is not None:
+ self.assertEqual(type(result_spec.loader),
+ type(expected_spec.loader))
+ for attr in self.CHECKED_SPEC_ATTRIBUTES:
+ k = "__spec__." + attr
+ actual = (k, getattr(result_spec, attr))
+ expected = (k, getattr(expected_spec, attr))
+ self.assertEqual(actual, expected)
+ # For the rest, we still don't use direct dict comparison on the
+ # namespace, as the diffs are too hard to debug if anything breaks
self.assertEqual(set(result_ns), set(expected_ns))
for k in result_ns:
actual = (k, result_ns[k])
@@ -131,12 +157,16 @@
mod_fname = "Some other nonsense"
mod_loader = "Now you're just being silly"
mod_package = '' # Treat as a top level module
+ mod_spec = importlib.machinery.ModuleSpec(mod_name,
+ origin=mod_fname,
+ loader=mod_loader)
expected_ns = example_namespace.copy()
expected_ns.update({
"__name__": mod_name,
"__file__": mod_fname,
"__loader__": mod_loader,
"__package__": mod_package,
+ "__spec__": mod_spec,
"run_argv0": mod_fname,
"run_name_in_sys_modules": True,
"module_in_sys_modules": True,
@@ -145,9 +175,7 @@
return _run_module_code(example_source,
init_globals,
mod_name,
- mod_fname,
- mod_loader,
- mod_package)
+ mod_spec)
self.check_code_execution(create_ns, expected_ns)
# TODO: Use self.addCleanup to get rid of a lot of try-finally blocks
@@ -177,31 +205,43 @@
def test_library_module(self):
self.assertEqual(run_module("runpy")["__name__"], "runpy")
- def _add_pkg_dir(self, pkg_dir):
+ def _add_pkg_dir(self, pkg_dir, namespace=False):
os.mkdir(pkg_dir)
+ if namespace:
+ return None
pkg_fname = os.path.join(pkg_dir, "__init__.py")
create_empty_file(pkg_fname)
return pkg_fname
- def _make_pkg(self, source, depth, mod_base="runpy_test"):
+ def _make_pkg(self, source, depth, mod_base="runpy_test",
+ *, namespace=False, parent_namespaces=False):
+ # Enforce a couple of internal sanity checks on test cases
+ if (namespace or parent_namespaces) and not depth:
+ raise RuntimeError("Can't mark top level module as a "
+ "namespace package")
pkg_name = "__runpy_pkg__"
test_fname = mod_base+os.extsep+"py"
pkg_dir = sub_dir = os.path.realpath(tempfile.mkdtemp())
if verbose > 1: print(" Package tree in:", sub_dir)
sys.path.insert(0, pkg_dir)
if verbose > 1: print(" Updated sys.path:", sys.path[0])
- for i in range(depth):
- sub_dir = os.path.join(sub_dir, pkg_name)
- pkg_fname = self._add_pkg_dir(sub_dir)
- if verbose > 1: print(" Next level in:", sub_dir)
- if verbose > 1: print(" Created:", pkg_fname)
+ if depth:
+ namespace_flags = [parent_namespaces] * depth
+ namespace_flags[-1] = namespace
+ for namespace_flag in namespace_flags:
+ sub_dir = os.path.join(sub_dir, pkg_name)
+ pkg_fname = self._add_pkg_dir(sub_dir, namespace_flag)
+ if verbose > 1: print(" Next level in:", sub_dir)
+ if verbose > 1: print(" Created:", pkg_fname)
mod_fname = os.path.join(sub_dir, test_fname)
mod_file = open(mod_fname, "w")
mod_file.write(source)
mod_file.close()
if verbose > 1: print(" Created:", mod_fname)
mod_name = (pkg_name+".")*depth + mod_base
- return pkg_dir, mod_fname, mod_name
+ mod_spec = importlib.util.spec_from_file_location(mod_name,
+ mod_fname)
+ return pkg_dir, mod_fname, mod_name, mod_spec
def _del_pkg(self, top, depth, mod_name):
for entry in list(sys.modules):
@@ -231,20 +271,29 @@
def _fix_ns_for_legacy_pyc(self, ns, alter_sys):
char_to_add = "c" if __debug__ else "o"
ns["__file__"] += char_to_add
+ ns["__cached__"] = ns["__file__"]
+ spec = ns["__spec__"]
+ new_spec = importlib.util.spec_from_file_location(spec.name,
+ ns["__file__"])
+ ns["__spec__"] = new_spec
if alter_sys:
ns["run_argv0"] += char_to_add
- def _check_module(self, depth, alter_sys=False):
- pkg_dir, mod_fname, mod_name = (
- self._make_pkg(example_source, depth))
+ def _check_module(self, depth, alter_sys=False,
+ *, namespace=False, parent_namespaces=False):
+ pkg_dir, mod_fname, mod_name, mod_spec = (
+ self._make_pkg(example_source, depth,
+ namespace=namespace,
+ parent_namespaces=parent_namespaces))
forget(mod_name)
expected_ns = example_namespace.copy()
expected_ns.update({
"__name__": mod_name,
"__file__": mod_fname,
+ "__cached__": mod_spec.cached,
"__package__": mod_name.rpartition(".")[0],
-# "__spec__": None, # XXX Needs to be set.
+ "__spec__": mod_spec,
})
if alter_sys:
expected_ns.update({
@@ -271,17 +320,21 @@
self._del_pkg(pkg_dir, depth, mod_name)
if verbose > 1: print("Module executed successfully")
- def _check_package(self, depth, alter_sys=False):
- pkg_dir, mod_fname, mod_name = (
- self._make_pkg(example_source, depth, "__main__"))
+ def _check_package(self, depth, alter_sys=False,
+ *, namespace=False, parent_namespaces=False):
+ pkg_dir, mod_fname, mod_name, mod_spec = (
+ self._make_pkg(example_source, depth, "__main__",
+ namespace=namespace,
+ parent_namespaces=parent_namespaces))
pkg_name = mod_name.rpartition(".")[0]
forget(mod_name)
expected_ns = example_namespace.copy()
expected_ns.update({
"__name__": mod_name,
"__file__": mod_fname,
+ "__cached__": importlib.util.cache_from_source(mod_fname),
"__package__": pkg_name,
-# "__spec__": None, # XXX Needs to be set.
+ "__spec__": mod_spec,
})
if alter_sys:
expected_ns.update({
@@ -337,7 +390,7 @@
from . import sibling
from ..uncle.cousin import nephew
"""
- pkg_dir, mod_fname, mod_name = (
+ pkg_dir, mod_fname, mod_name, mod_spec = (
self._make_pkg(contents, depth))
if run_name is None:
expected_name = mod_name
@@ -376,11 +429,31 @@
if verbose > 1: print("Testing package depth:", depth)
self._check_module(depth)
+ def test_run_module_in_namespace_package(self):
+ for depth in range(1, 4):
+ if verbose > 1: print("Testing package depth:", depth)
+ self._check_module(depth, namespace=True, parent_namespaces=True)
+
def test_run_package(self):
for depth in range(1, 4):
if verbose > 1: print("Testing package depth:", depth)
self._check_package(depth)
+ def test_run_package_in_namespace_package(self):
+ for depth in range(1, 4):
+ if verbose > 1: print("Testing package depth:", depth)
+ self._check_package(depth, parent_namespaces=True)
+
+ def test_run_namespace_package(self):
+ for depth in range(1, 4):
+ if verbose > 1: print("Testing package depth:", depth)
+ self._check_package(depth, namespace=True)
+
+ def test_run_namespace_package_in_namespace_package(self):
+ for depth in range(1, 4):
+ if verbose > 1: print("Testing package depth:", depth)
+ self._check_package(depth, namespace=True, parent_namespaces=True)
+
def test_run_module_alter_sys(self):
for depth in range(4):
if verbose > 1: print("Testing package depth:", depth)
@@ -404,14 +477,16 @@
def test_run_name(self):
depth = 1
run_name = "And now for something completely different"
- pkg_dir, mod_fname, mod_name = (
+ pkg_dir, mod_fname, mod_name, mod_spec = (
self._make_pkg(example_source, depth))
forget(mod_name)
expected_ns = example_namespace.copy()
expected_ns.update({
"__name__": run_name,
"__file__": mod_fname,
+ "__cached__": importlib.util.cache_from_source(mod_fname),
"__package__": mod_name.rpartition(".")[0],
+ "__spec__": mod_spec,
})
def create_ns(init_globals):
return run_module(mod_name, init_globals, run_name)
@@ -440,7 +515,7 @@
pkg_name = ".".join([base_name] * max_depth)
expected_packages.add(pkg_name)
expected_modules.add(pkg_name + ".runpy_test")
- pkg_dir, mod_fname, mod_name = (
+ pkg_dir, mod_fname, mod_name, mod_spec = (
self._make_pkg("", max_depth))
self.addCleanup(self._del_pkg, pkg_dir, max_depth, mod_name)
for depth in range(2, max_depth+1):
@@ -458,21 +533,39 @@
class RunPathTestCase(unittest.TestCase, CodeExecutionMixin):
"""Unit tests for runpy.run_path"""
- def _make_test_script(self, script_dir, script_basename, source=None):
+ def _make_test_script(self, script_dir, script_basename,
+ source=None, omit_suffix=False):
if source is None:
source = example_source
- return make_script(script_dir, script_basename, source)
+ return make_script(script_dir, script_basename,
+ source, omit_suffix)
def _check_script(self, script_name, expected_name, expected_file,
- expected_argv0):
+ expected_argv0, mod_name=None,
+ expect_spec=True, check_loader=True):
# First check is without run_name
def create_ns(init_globals):
return run_path(script_name, init_globals)
expected_ns = example_namespace.copy()
+ if mod_name is None:
+ spec_name = expected_name
+ else:
+ spec_name = mod_name
+ if expect_spec:
+ mod_spec = importlib.util.spec_from_file_location(spec_name,
+ expected_file)
+ mod_cached = mod_spec.cached
+ if not check_loader:
+ mod_spec.loader = None
+ else:
+ mod_spec = mod_cached = None
+
expected_ns.update({
"__name__": expected_name,
"__file__": expected_file,
+ "__cached__": mod_cached,
"__package__": "",
+ "__spec__": mod_spec,
"run_argv0": expected_argv0,
"run_name_in_sys_modules": True,
"module_in_sys_modules": True,
@@ -482,6 +575,12 @@
run_name = "prove.issue15230.is.fixed"
def create_ns(init_globals):
return run_path(script_name, init_globals, run_name)
+ if expect_spec and mod_name is None:
+ mod_spec = importlib.util.spec_from_file_location(run_name,
+ expected_file)
+ if not check_loader:
+ mod_spec.loader = None
+ expected_ns["__spec__"] = mod_spec
expected_ns["__name__"] = run_name
expected_ns["__package__"] = run_name.rpartition(".")[0]
self.check_code_execution(create_ns, expected_ns)
@@ -495,7 +594,15 @@
mod_name = 'script'
script_name = self._make_test_script(script_dir, mod_name)
self._check_script(script_name, "<run_path>", script_name,
- script_name)
+ script_name, expect_spec=False)
+
+ def test_basic_script_no_suffix(self):
+ with temp_dir() as script_dir:
+ mod_name = 'script'
+ script_name = self._make_test_script(script_dir, mod_name,
+ omit_suffix=True)
+ self._check_script(script_name, "<run_path>", script_name,
+ script_name, expect_spec=False)
def test_script_compiled(self):
with temp_dir() as script_dir:
@@ -504,14 +611,14 @@
compiled_name = py_compile.compile(script_name, doraise=True)
os.remove(script_name)
self._check_script(compiled_name, "<run_path>", compiled_name,
- compiled_name)
+ compiled_name, expect_spec=False)
def test_directory(self):
with temp_dir() as script_dir:
mod_name = '__main__'
script_name = self._make_test_script(script_dir, mod_name)
self._check_script(script_dir, "<run_path>", script_name,
- script_dir)
+ script_dir, mod_name=mod_name)
def test_directory_compiled(self):
with temp_dir() as script_dir:
@@ -522,7 +629,7 @@
if not sys.dont_write_bytecode:
legacy_pyc = make_legacy_pyc(script_name)
self._check_script(script_dir, "<run_path>", legacy_pyc,
- script_dir)
+ script_dir, mod_name=mod_name)
def test_directory_error(self):
with temp_dir() as script_dir:
@@ -536,7 +643,8 @@
mod_name = '__main__'
script_name = self._make_test_script(script_dir, mod_name)
zip_name, fname = make_zip_script(script_dir, 'test_zip', script_name)
- self._check_script(zip_name, "<run_path>", fname, zip_name)
+ self._check_script(zip_name, "<run_path>", fname, zip_name,
+ mod_name=mod_name, check_loader=False)
def test_zipfile_compiled(self):
with temp_dir() as script_dir:
@@ -545,7 +653,8 @@
compiled_name = py_compile.compile(script_name, doraise=True)
zip_name, fname = make_zip_script(script_dir, 'test_zip',
compiled_name)
- self._check_script(zip_name, "<run_path>", fname, zip_name)
+ self._check_script(zip_name, "<run_path>", fname, zip_name,
+ mod_name=mod_name, check_loader=False)
def test_zipfile_error(self):
with temp_dir() as script_dir: