bpo-37369: Fix initialization of sys members when launched via an app container (GH-14428)
sys._base_executable is now always defined on all platforms, and can be overridden through configuration.
Also adds test.support.PythonSymlink to encapsulate platform-specific logic for symlinking sys.executable
diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py
index de4c5ec..ea9c555 100644
--- a/Lib/multiprocessing/popen_spawn_win32.py
+++ b/Lib/multiprocessing/popen_spawn_win32.py
@@ -22,8 +22,7 @@
def _path_eq(p1, p2):
return p1 == p2 or os.path.normcase(p1) == os.path.normcase(p2)
-WINENV = (hasattr(sys, '_base_executable') and
- not _path_eq(sys.executable, sys._base_executable))
+WINENV = not _path_eq(sys.executable, sys._base_executable)
def _close_handles(*handles):
diff --git a/Lib/site.py b/Lib/site.py
index e7aafb7..a065ab0 100644
--- a/Lib/site.py
+++ b/Lib/site.py
@@ -459,13 +459,6 @@
env = os.environ
if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env:
executable = sys._base_executable = os.environ['__PYVENV_LAUNCHER__']
- elif sys.platform == 'win32' and '__PYVENV_LAUNCHER__' in env:
- executable = sys.executable
- import _winapi
- sys._base_executable = _winapi.GetModuleFileName(0)
- # bpo-35873: Clear the environment variable to avoid it being
- # inherited by child processes.
- del os.environ['__PYVENV_LAUNCHER__']
else:
executable = sys.executable
exe_dir, _ = os.path.split(os.path.abspath(executable))
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index b5538d2..19ea976 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -12,6 +12,7 @@
import fnmatch
import functools
import gc
+import glob
import importlib
import importlib.util
import io
@@ -2500,6 +2501,84 @@
msg = "Requires functional symlink implementation"
return test if ok else unittest.skip(msg)(test)
+class PythonSymlink:
+ """Creates a symlink for the current Python executable"""
+ def __init__(self, link=None):
+ self.link = link or os.path.abspath(TESTFN)
+ self._linked = []
+ self.real = os.path.realpath(sys.executable)
+ self._also_link = []
+
+ self._env = None
+
+ self._platform_specific()
+
+ def _platform_specific(self):
+ pass
+
+ if sys.platform == "win32":
+ def _platform_specific(self):
+ import _winapi
+
+ if os.path.lexists(self.real) and not os.path.exists(self.real):
+ # App symlink appears to not exist, but we want the
+ # real executable here anyway
+ self.real = _winapi.GetModuleFileName(0)
+
+ dll = _winapi.GetModuleFileName(sys.dllhandle)
+ src_dir = os.path.dirname(dll)
+ dest_dir = os.path.dirname(self.link)
+ self._also_link.append((
+ dll,
+ os.path.join(dest_dir, os.path.basename(dll))
+ ))
+ for runtime in glob.glob(os.path.join(src_dir, "vcruntime*.dll")):
+ self._also_link.append((
+ runtime,
+ os.path.join(dest_dir, os.path.basename(runtime))
+ ))
+
+ self._env = {k.upper(): os.getenv(k) for k in os.environ}
+ self._env["PYTHONHOME"] = os.path.dirname(self.real)
+ if sysconfig.is_python_build(True):
+ self._env["PYTHONPATH"] = os.path.dirname(os.__file__)
+
+ def __enter__(self):
+ os.symlink(self.real, self.link)
+ self._linked.append(self.link)
+ for real, link in self._also_link:
+ os.symlink(real, link)
+ self._linked.append(link)
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_tb):
+ for link in self._linked:
+ try:
+ os.remove(link)
+ except IOError as ex:
+ if verbose:
+ print("failed to clean up {}: {}".format(link, ex))
+
+ def _call(self, python, args, env, returncode):
+ cmd = [python, *args]
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, env=env)
+ r = p.communicate()
+ if p.returncode != returncode:
+ if verbose:
+ print(repr(r[0]))
+ print(repr(r[1]), file=sys.stderr)
+ raise RuntimeError(
+ 'unexpected return code: {0} (0x{0:08X})'.format(p.returncode))
+ return r
+
+ def call_real(self, *args, returncode=0):
+ return self._call(self.real, args, None, returncode)
+
+ def call_link(self, *args, returncode=0):
+ return self._call(self.link, args, self._env, returncode)
+
+
_can_xattr = None
def can_xattr():
global _can_xattr
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index b897489..9c78aa0 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -362,6 +362,7 @@
'pythonpath_env': None,
'home': None,
'executable': GET_DEFAULT_CONFIG,
+ 'base_executable': GET_DEFAULT_CONFIG,
'prefix': GET_DEFAULT_CONFIG,
'base_prefix': GET_DEFAULT_CONFIG,
@@ -534,14 +535,16 @@
if expected['stdio_errors'] is self.GET_DEFAULT_CONFIG:
expected['stdio_errors'] = 'surrogateescape'
+ if sys.platform == 'win32':
+ default_executable = self.test_exe
+ elif expected['program_name'] is not self.GET_DEFAULT_CONFIG:
+ default_executable = os.path.abspath(expected['program_name'])
+ else:
+ default_executable = os.path.join(os.getcwd(), '_testembed')
if expected['executable'] is self.GET_DEFAULT_CONFIG:
- if sys.platform == 'win32':
- expected['executable'] = self.test_exe
- else:
- if expected['program_name'] is not self.GET_DEFAULT_CONFIG:
- expected['executable'] = os.path.abspath(expected['program_name'])
- else:
- expected['executable'] = os.path.join(os.getcwd(), '_testembed')
+ expected['executable'] = default_executable
+ if expected['base_executable'] is self.GET_DEFAULT_CONFIG:
+ expected['base_executable'] = default_executable
if expected['program_name'] is self.GET_DEFAULT_CONFIG:
expected['program_name'] = './_testembed'
diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py
index 8357ee9..87d4924 100644
--- a/Lib/test/test_httpservers.py
+++ b/Lib/test/test_httpservers.py
@@ -610,9 +610,10 @@
# The shebang line should be pure ASCII: use symlink if possible.
# See issue #7668.
+ self._pythonexe_symlink = None
if support.can_symlink():
self.pythonexe = os.path.join(self.parent_dir, 'python')
- os.symlink(sys.executable, self.pythonexe)
+ self._pythonexe_symlink = support.PythonSymlink(self.pythonexe).__enter__()
else:
self.pythonexe = sys.executable
@@ -655,8 +656,8 @@
def tearDown(self):
try:
os.chdir(self.cwd)
- if self.pythonexe != sys.executable:
- os.remove(self.pythonexe)
+ if self._pythonexe_symlink:
+ self._pythonexe_symlink.__exit__(None, None, None)
if self.nocgi_path:
os.remove(self.nocgi_path)
if self.file1_path:
diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py
index 9cf1772..8b64923 100644
--- a/Lib/test/test_platform.py
+++ b/Lib/test/test_platform.py
@@ -20,37 +20,9 @@
@support.skip_unless_symlink
def test_architecture_via_symlink(self): # issue3762
- # On Windows, the EXE needs to know where pythonXY.dll and *.pyd is at
- # so we add the directory to the path, PYTHONHOME and PYTHONPATH.
- env = None
- if sys.platform == "win32":
- env = {k.upper(): os.environ[k] for k in os.environ}
- env["PATH"] = "{};{}".format(
- os.path.dirname(sys.executable), env.get("PATH", ""))
- env["PYTHONHOME"] = os.path.dirname(sys.executable)
- if sysconfig.is_python_build(True):
- env["PYTHONPATH"] = os.path.dirname(os.__file__)
-
- def get(python, env=None):
- cmd = [python, '-c',
- 'import platform; print(platform.architecture())']
- p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE, env=env)
- r = p.communicate()
- if p.returncode:
- print(repr(r[0]))
- print(repr(r[1]), file=sys.stderr)
- self.fail('unexpected return code: {0} (0x{0:08X})'
- .format(p.returncode))
- return r
-
- real = os.path.realpath(sys.executable)
- link = os.path.abspath(support.TESTFN)
- os.symlink(real, link)
- try:
- self.assertEqual(get(real), get(link, env=env))
- finally:
- os.remove(link)
+ with support.PythonSymlink() as py:
+ cmd = "-c", "import platform; print(platform.architecture())"
+ self.assertEqual(py.call_real(*cmd), py.call_link(*cmd))
def test_platform(self):
for aliased in (False, True):
@@ -275,6 +247,11 @@
os.path.exists(sys.executable+'.exe'):
# Cygwin horror
executable = sys.executable + '.exe'
+ elif sys.platform == "win32" and not os.path.exists(sys.executable):
+ # App symlink appears to not exist, but we want the
+ # real executable here anyway
+ import _winapi
+ executable = _winapi.GetModuleFileName(0)
else:
executable = sys.executable
platform.libc_ver(executable)
diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py
index 1b19298..44e44bf 100644
--- a/Lib/test/test_sysconfig.py
+++ b/Lib/test/test_sysconfig.py
@@ -6,7 +6,8 @@
from copy import copy
from test.support import (import_module, TESTFN, unlink, check_warnings,
- captured_stdout, skip_unless_symlink, change_cwd)
+ captured_stdout, skip_unless_symlink, change_cwd,
+ PythonSymlink)
import sysconfig
from sysconfig import (get_paths, get_platform, get_config_vars,
@@ -232,39 +233,10 @@
self.assertEqual(get_scheme_names(), wanted)
@skip_unless_symlink
- def test_symlink(self):
- # On Windows, the EXE needs to know where pythonXY.dll is at so we have
- # to add the directory to the path.
- env = None
- if sys.platform == "win32":
- env = {k.upper(): os.environ[k] for k in os.environ}
- env["PATH"] = "{};{}".format(
- os.path.dirname(sys.executable), env.get("PATH", ""))
- # Requires PYTHONHOME as well since we locate stdlib from the
- # EXE path and not the DLL path (which should be fixed)
- env["PYTHONHOME"] = os.path.dirname(sys.executable)
- if sysconfig.is_python_build(True):
- env["PYTHONPATH"] = os.path.dirname(os.__file__)
-
- # Issue 7880
- def get(python, env=None):
- cmd = [python, '-c',
- 'import sysconfig; print(sysconfig.get_platform())']
- p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE, env=env)
- out, err = p.communicate()
- if p.returncode:
- print((out, err))
- self.fail('Non-zero return code {0} (0x{0:08X})'
- .format(p.returncode))
- return out, err
- real = os.path.realpath(sys.executable)
- link = os.path.abspath(TESTFN)
- os.symlink(real, link)
- try:
- self.assertEqual(get(real), get(link, env))
- finally:
- unlink(link)
+ def test_symlink(self): # Issue 7880
+ with PythonSymlink() as py:
+ cmd = "-c", "import sysconfig; print(sysconfig.get_platform())"
+ self.assertEqual(py.call_real(*cmd), py.call_link(*cmd))
def test_user_similar(self):
# Issue #8759: make sure the posix scheme for the users
diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py
index 4f6c11b..ea016b5 100644
--- a/Lib/test/test_venv.py
+++ b/Lib/test/test_venv.py
@@ -28,8 +28,8 @@
# Platforms that set sys._base_executable can create venvs from within
# another venv, so no need to skip tests that require venv.create().
requireVenvCreate = unittest.skipUnless(
- hasattr(sys, '_base_executable')
- or sys.prefix == sys.base_prefix,
+ sys.prefix == sys.base_prefix
+ or sys._base_executable != sys.executable,
'cannot run venv.create from within a venv on this platform')
def check_output(cmd, encoding=None):
@@ -57,8 +57,14 @@
self.bindir = 'bin'
self.lib = ('lib', 'python%d.%d' % sys.version_info[:2])
self.include = 'include'
- executable = getattr(sys, '_base_executable', sys.executable)
+ executable = sys._base_executable
self.exe = os.path.split(executable)[-1]
+ if (sys.platform == 'win32'
+ and os.path.lexists(executable)
+ and not os.path.exists(executable)):
+ self.cannot_link_exe = True
+ else:
+ self.cannot_link_exe = False
def tearDown(self):
rmtree(self.env_dir)
@@ -102,7 +108,7 @@
else:
self.assertFalse(os.path.exists(p))
data = self.get_text_file_contents('pyvenv.cfg')
- executable = getattr(sys, '_base_executable', sys.executable)
+ executable = sys._base_executable
path = os.path.dirname(executable)
self.assertIn('home = %s' % path, data)
fn = self.get_env_file(self.bindir, self.exe)
@@ -158,10 +164,6 @@
"""
Test that the prefix values are as expected.
"""
- #check our prefixes
- self.assertEqual(sys.base_prefix, sys.prefix)
- self.assertEqual(sys.base_exec_prefix, sys.exec_prefix)
-
# check a venv's prefixes
rmtree(self.env_dir)
self.run_with_capture(venv.create, self.env_dir)
@@ -169,9 +171,9 @@
cmd = [envpy, '-c', None]
for prefix, expected in (
('prefix', self.env_dir),
- ('prefix', self.env_dir),
- ('base_prefix', sys.prefix),
- ('base_exec_prefix', sys.exec_prefix)):
+ ('exec_prefix', self.env_dir),
+ ('base_prefix', sys.base_prefix),
+ ('base_exec_prefix', sys.base_exec_prefix)):
cmd[2] = 'import sys; print(sys.%s)' % prefix
out, err = check_output(cmd)
self.assertEqual(out.strip(), expected.encode())
@@ -283,7 +285,12 @@
# symlinked to 'python3.3' in the env, even when symlinking in
# general isn't wanted.
if usl:
- self.assertTrue(os.path.islink(fn))
+ if self.cannot_link_exe:
+ # Symlinking is skipped when our executable is already a
+ # special app symlink
+ self.assertFalse(os.path.islink(fn))
+ else:
+ self.assertTrue(os.path.islink(fn))
# If a venv is created from a source build and that venv is used to
# run the test, the pyvenv.cfg in the venv created in the test will
diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py
index b64125f..4ab9cc6 100644
--- a/Lib/venv/__init__.py
+++ b/Lib/venv/__init__.py
@@ -112,7 +112,7 @@
prompt = self.prompt if self.prompt is not None else context.env_name
context.prompt = '(%s) ' % prompt
create_if_needed(env_dir)
- executable = getattr(sys, '_base_executable', sys.executable)
+ executable = sys._base_executable
dirname, exename = os.path.split(os.path.abspath(executable))
context.executable = executable
context.python_dir = dirname
@@ -163,47 +163,66 @@
if self.prompt is not None:
f.write(f'prompt = {self.prompt!r}\n')
- def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
- """
- Try symlinking a file, and if that fails, fall back to copying.
- """
- force_copy = not self.symlinks
- if not force_copy:
- try:
- if not os.path.islink(dst): # can't link to itself!
+ if os.name != 'nt':
+ def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
+ """
+ Try symlinking a file, and if that fails, fall back to copying.
+ """
+ force_copy = not self.symlinks
+ if not force_copy:
+ try:
+ if not os.path.islink(dst): # can't link to itself!
+ if relative_symlinks_ok:
+ assert os.path.dirname(src) == os.path.dirname(dst)
+ os.symlink(os.path.basename(src), dst)
+ else:
+ os.symlink(src, dst)
+ except Exception: # may need to use a more specific exception
+ logger.warning('Unable to symlink %r to %r', src, dst)
+ force_copy = True
+ if force_copy:
+ shutil.copyfile(src, dst)
+ else:
+ def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
+ """
+ Try symlinking a file, and if that fails, fall back to copying.
+ """
+ bad_src = os.path.lexists(src) and not os.path.exists(src)
+ if self.symlinks and not bad_src and not os.path.islink(dst):
+ try:
if relative_symlinks_ok:
assert os.path.dirname(src) == os.path.dirname(dst)
os.symlink(os.path.basename(src), dst)
else:
os.symlink(src, dst)
- except Exception: # may need to use a more specific exception
- logger.warning('Unable to symlink %r to %r', src, dst)
- force_copy = True
- if force_copy:
- if os.name == 'nt':
- # On Windows, we rewrite symlinks to our base python.exe into
- # copies of venvlauncher.exe
- basename, ext = os.path.splitext(os.path.basename(src))
- srcfn = os.path.join(os.path.dirname(__file__),
- "scripts",
- "nt",
- basename + ext)
- # Builds or venv's from builds need to remap source file
- # locations, as we do not put them into Lib/venv/scripts
- if sysconfig.is_python_build(True) or not os.path.isfile(srcfn):
- if basename.endswith('_d'):
- ext = '_d' + ext
- basename = basename[:-2]
- if basename == 'python':
- basename = 'venvlauncher'
- elif basename == 'pythonw':
- basename = 'venvwlauncher'
- src = os.path.join(os.path.dirname(src), basename + ext)
- else:
- src = srcfn
- if not os.path.exists(src):
- logger.warning('Unable to copy %r', src)
return
+ except Exception: # may need to use a more specific exception
+ logger.warning('Unable to symlink %r to %r', src, dst)
+
+ # On Windows, we rewrite symlinks to our base python.exe into
+ # copies of venvlauncher.exe
+ basename, ext = os.path.splitext(os.path.basename(src))
+ srcfn = os.path.join(os.path.dirname(__file__),
+ "scripts",
+ "nt",
+ basename + ext)
+ # Builds or venv's from builds need to remap source file
+ # locations, as we do not put them into Lib/venv/scripts
+ if sysconfig.is_python_build(True) or not os.path.isfile(srcfn):
+ if basename.endswith('_d'):
+ ext = '_d' + ext
+ basename = basename[:-2]
+ if basename == 'python':
+ basename = 'venvlauncher'
+ elif basename == 'pythonw':
+ basename = 'venvwlauncher'
+ src = os.path.join(os.path.dirname(src), basename + ext)
+ else:
+ src = srcfn
+ if not os.path.exists(src):
+ if not bad_src:
+ logger.warning('Unable to copy %r', src)
+ return
shutil.copyfile(src, dst)
@@ -251,7 +270,7 @@
for suffix in suffixes:
src = os.path.join(dirname, suffix)
- if os.path.exists(src):
+ if os.path.lexists(src):
copier(src, os.path.join(binpath, suffix))
if sysconfig.is_python_build(True):