[3.8] bpo-38234: Backport init path config changes from master (GH-16423)
* bpo-38234: Py_SetPath() uses the program full path (GH-16357)
Py_SetPath() now sets sys.executable to the program full path
(Py_GetProgramFullPath()), rather than to the program name
(Py_GetProgramName()).
Fix also memory leaks in pathconfig_set_from_config().
(cherry picked from commit 1ce152a42eaa917d7763bce93f1e1ca72530d7ca)
* bpo-38234: Add tests for Python init path config (GH-16358)
(cherry picked from commit bb6bf7d342b4503a6227fd209fac934905b6a1aa)
* bpo-38234: test_embed: test pyvenv.cfg and pybuilddir.txt (GH-16366)
Add test_init_pybuilddir() and test_init_pyvenv_cfg() to test_embed
to test pyvenv.cfg and pybuilddir.txt configuration files.
Fix sysconfig._generate_posix_vars(): pybuilddir.txt uses UTF-8
encoding, not ASCII.
(cherry picked from commit 52ad33abbfb6637d74932617c7013bae0ccf6e32)
* bpo-38234: Cleanup getpath.c (GH-16367)
* search_for_prefix() directly calls reduce() if found is greater
than 0.
* Add calculate_pybuilddir() subfunction.
* search_for_prefix(): add path string buffer for readability.
* Fix some error handling code paths: release resources on error.
* calculate_read_pyenv(): rename tmpbuffer to filename.
* test.pythoninfo now also logs windows.dll_path
(cherry picked from commit 221fd84703c545408bbb4a6e0b58459651331f5c)
* bpo-38234: Fix test_embed pathconfig tests (GH-16390)
bpo-38234: On macOS and FreeBSD, the temporary directory can be
symbolic link. For example, /tmp can be a symbolic link to /var/tmp.
Call realpath() to resolve all symbolic links.
(cherry picked from commit 00508a7407d7d300b487532e2271534b20e378a7)
* bpo-38234: Add test_init_setpath_config() to test_embed (GH-16402)
* Add test_embed.test_init_setpath_config(): test Py_SetPath()
with PyConfig.
* test_init_setpath() and test_init_setpythonhome() no longer call
Py_SetProgramName(), but use the default program name.
* _PyPathConfig: isolated, site_import and base_executable
fields are now only available on Windows.
* If executable is set explicitly in the configuration, ignore
calculated base_executable: _PyConfig_InitPathConfig() copies
executable to base_executable.
* Complete path config documentation.
(cherry picked from commit 8bf39b606ef7b02c0279a80789f3c4824b0da5e9)
* bpo-38234: Complete init config documentation (GH-16404)
(cherry picked from commit 88feaecd46a8f427e30ef7ad8cfcddfe392a2402)
* bpo-38234: Fix test_embed.test_init_setpath_config() on FreeBSD (GH-16406)
Explicitly preinitializes with a Python preconfiguration to avoid
Py_SetPath() implicit preinitialization with a compat
preconfiguration.
Fix also test_init_setpath() and test_init_setpythonhome() on macOS:
use self.test_exe as the executable (and base_executable), rather
than shutil.which('python3').
(cherry picked from commit 49d99f01e6e51acec5ca57a02e857f0796bc418b)
* bpo-38234: Py_Initialize() sets global path configuration (GH-16421)
* Py_InitializeFromConfig() now writes PyConfig path configuration to
the global path configuration (_Py_path_config).
* Add test_embed.test_get_pathconfig().
* Fix typo in _PyWideStringList_Join().
(cherry picked from commit 12f2f177fc483723406d7917194e7f655a20631b)
diff --git a/Modules/getpath.c b/Modules/getpath.c
index de32c3d..b727f66 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -370,12 +370,15 @@
const wchar_t *argv0_path,
wchar_t *prefix, size_t prefix_len, int *found)
{
+ wchar_t path[MAXPATHLEN+1];
+ memset(path, 0, sizeof(path));
+ size_t path_len = Py_ARRAY_LENGTH(path);
+
PyStatus status;
- size_t n;
- wchar_t *vpath;
/* If PYTHONHOME is set, we believe it unconditionally */
if (pathconfig->home) {
+ /* Path: <home> / <lib_python> */
if (safe_wcscpy(prefix, pathconfig->home, prefix_len) < 0) {
return PATHLEN_ERR();
}
@@ -387,27 +390,25 @@
if (_PyStatus_EXCEPTION(status)) {
return status;
}
- status = joinpath(prefix, LANDMARK, prefix_len);
- if (_PyStatus_EXCEPTION(status)) {
- return status;
- }
*found = 1;
return _PyStatus_OK();
}
/* Check to see if argv[0] is in the build directory */
- if (safe_wcscpy(prefix, argv0_path, prefix_len) < 0) {
+ if (safe_wcscpy(path, argv0_path, path_len) < 0) {
return PATHLEN_ERR();
}
- status = joinpath(prefix, L"Modules/Setup.local", prefix_len);
+ status = joinpath(path, L"Modules/Setup.local", path_len);
if (_PyStatus_EXCEPTION(status)) {
return status;
}
- if (isfile(prefix)) {
- /* Check VPATH to see if argv0_path is in the build directory. */
- vpath = Py_DecodeLocale(VPATH, NULL);
+ if (isfile(path)) {
+ /* Check VPATH to see if argv0_path is in the build directory.
+ VPATH can be empty. */
+ wchar_t *vpath = Py_DecodeLocale(VPATH, NULL);
if (vpath != NULL) {
+ /* Path: <argv0_path> / <vpath> / Lib / LANDMARK */
if (safe_wcscpy(prefix, argv0_path, prefix_len) < 0) {
return PATHLEN_ERR();
}
@@ -428,6 +429,7 @@
if (ismodule(prefix, prefix_len)) {
*found = -1;
+ reduce(prefix);
return _PyStatus_OK();
}
}
@@ -440,7 +442,8 @@
}
do {
- n = wcslen(prefix);
+ /* Path: <argv0_path or substring> / <lib_python> / LANDMARK */
+ size_t n = wcslen(prefix);
status = joinpath(prefix, calculate->lib_python, prefix_len);
if (_PyStatus_EXCEPTION(status)) {
return status;
@@ -452,13 +455,15 @@
if (ismodule(prefix, prefix_len)) {
*found = 1;
+ reduce(prefix);
return _PyStatus_OK();
}
prefix[n] = L'\0';
reduce(prefix);
} while (prefix[0]);
- /* Look at configure's PREFIX */
+ /* Look at configure's PREFIX.
+ Path: <PREFIX macro> / <lib_python> / LANDMARK */
if (safe_wcscpy(prefix, calculate->prefix, prefix_len) < 0) {
return PATHLEN_ERR();
}
@@ -473,6 +478,7 @@
if (ismodule(prefix, prefix_len)) {
*found = 1;
+ reduce(prefix);
return _PyStatus_OK();
}
@@ -509,9 +515,6 @@
return status;
}
}
- else {
- reduce(prefix);
- }
return _PyStatus_OK();
}
@@ -546,6 +549,67 @@
}
+static PyStatus
+calculate_pybuilddir(const wchar_t *argv0_path,
+ wchar_t *exec_prefix, size_t exec_prefix_len,
+ int *found)
+{
+ PyStatus status;
+
+ wchar_t filename[MAXPATHLEN+1];
+ memset(filename, 0, sizeof(filename));
+ size_t filename_len = Py_ARRAY_LENGTH(filename);
+
+ /* Check to see if argv[0] is in the build directory. "pybuilddir.txt"
+ is written by setup.py and contains the relative path to the location
+ of shared library modules.
+
+ Filename: <argv0_path> / "pybuilddir.txt" */
+ if (safe_wcscpy(filename, argv0_path, filename_len) < 0) {
+ return PATHLEN_ERR();
+ }
+ status = joinpath(filename, L"pybuilddir.txt", filename_len);
+ if (_PyStatus_EXCEPTION(status)) {
+ return status;
+ }
+
+ if (!isfile(filename)) {
+ return _PyStatus_OK();
+ }
+
+ FILE *fp = _Py_wfopen(filename, L"rb");
+ if (fp == NULL) {
+ errno = 0;
+ return _PyStatus_OK();
+ }
+
+ char buf[MAXPATHLEN + 1];
+ size_t n = fread(buf, 1, Py_ARRAY_LENGTH(buf) - 1, fp);
+ buf[n] = '\0';
+ fclose(fp);
+
+ size_t dec_len;
+ wchar_t *pybuilddir = _Py_DecodeUTF8_surrogateescape(buf, n, &dec_len);
+ if (!pybuilddir) {
+ return DECODE_LOCALE_ERR("pybuilddir.txt", dec_len);
+ }
+
+ /* Path: <argv0_path> / <pybuilddir content> */
+ if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) {
+ PyMem_RawFree(pybuilddir);
+ return PATHLEN_ERR();
+ }
+ status = joinpath(exec_prefix, pybuilddir, exec_prefix_len);
+ PyMem_RawFree(pybuilddir);
+ if (_PyStatus_EXCEPTION(status)) {
+ return status;
+ }
+
+ *found = -1;
+ return _PyStatus_OK();
+}
+
+
/* search_for_exec_prefix requires that argv0_path be no more than
MAXPATHLEN bytes long.
*/
@@ -556,10 +620,10 @@
int *found)
{
PyStatus status;
- size_t n;
/* If PYTHONHOME is set, we believe it unconditionally */
if (pathconfig->home) {
+ /* Path: <home> / <lib_python> / "lib-dynload" */
wchar_t *delim = wcschr(pathconfig->home, DELIM);
if (delim) {
if (safe_wcscpy(exec_prefix, delim+1, exec_prefix_len) < 0) {
@@ -583,47 +647,15 @@
return _PyStatus_OK();
}
- /* Check to see if argv[0] is in the build directory. "pybuilddir.txt"
- is written by setup.py and contains the relative path to the location
- of shared library modules. */
- if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) {
- return PATHLEN_ERR();
- }
- status = joinpath(exec_prefix, L"pybuilddir.txt", exec_prefix_len);
+ /* Check for pybuilddir.txt */
+ assert(*found == 0);
+ status = calculate_pybuilddir(argv0_path, exec_prefix, exec_prefix_len,
+ found);
if (_PyStatus_EXCEPTION(status)) {
return status;
}
-
- if (isfile(exec_prefix)) {
- FILE *f = _Py_wfopen(exec_prefix, L"rb");
- if (f == NULL) {
- errno = 0;
- }
- else {
- char buf[MAXPATHLEN + 1];
- n = fread(buf, 1, Py_ARRAY_LENGTH(buf) - 1, f);
- buf[n] = '\0';
- fclose(f);
-
- wchar_t *pybuilddir;
- size_t dec_len;
- pybuilddir = _Py_DecodeUTF8_surrogateescape(buf, n, &dec_len);
- if (!pybuilddir) {
- return DECODE_LOCALE_ERR("pybuilddir.txt", dec_len);
- }
-
- if (safe_wcscpy(exec_prefix, argv0_path, exec_prefix_len) < 0) {
- return PATHLEN_ERR();
- }
- status = joinpath(exec_prefix, pybuilddir, exec_prefix_len);
- PyMem_RawFree(pybuilddir );
- if (_PyStatus_EXCEPTION(status)) {
- return status;
- }
-
- *found = -1;
- return _PyStatus_OK();
- }
+ if (*found) {
+ return _PyStatus_OK();
}
/* Search from argv0_path, until root is found */
@@ -633,7 +665,8 @@
}
do {
- n = wcslen(exec_prefix);
+ /* Path: <argv0_path or substring> / <lib_python> / "lib-dynload" */
+ size_t n = wcslen(exec_prefix);
status = joinpath(exec_prefix, calculate->lib_python, exec_prefix_len);
if (_PyStatus_EXCEPTION(status)) {
return status;
@@ -650,7 +683,9 @@
reduce(exec_prefix);
} while (exec_prefix[0]);
- /* Look at configure's EXEC_PREFIX */
+ /* Look at configure's EXEC_PREFIX.
+
+ Path: <EXEC_PREFIX macro> / <lib_python> / "lib-dynload" */
if (safe_wcscpy(exec_prefix, calculate->exec_prefix, exec_prefix_len) < 0) {
return PATHLEN_ERR();
}
@@ -962,43 +997,49 @@
wchar_t *argv0_path, size_t argv0_path_len)
{
PyStatus status;
- wchar_t tmpbuffer[MAXPATHLEN+1];
- const size_t buflen = Py_ARRAY_LENGTH(tmpbuffer);
- wchar_t *env_cfg = L"pyvenv.cfg";
+ const wchar_t *env_cfg = L"pyvenv.cfg";
FILE *env_file;
- if (safe_wcscpy(tmpbuffer, argv0_path, buflen) < 0) {
+ wchar_t filename[MAXPATHLEN+1];
+ const size_t filename_len = Py_ARRAY_LENGTH(filename);
+ memset(filename, 0, sizeof(filename));
+
+ /* Filename: <argv0_path_len> / "pyvenv.cfg" */
+ if (safe_wcscpy(filename, argv0_path, filename_len) < 0) {
return PATHLEN_ERR();
}
- status = joinpath(tmpbuffer, env_cfg, buflen);
+ status = joinpath(filename, env_cfg, filename_len);
if (_PyStatus_EXCEPTION(status)) {
return status;
}
- env_file = _Py_wfopen(tmpbuffer, L"r");
+ env_file = _Py_wfopen(filename, L"r");
if (env_file == NULL) {
errno = 0;
- reduce(tmpbuffer);
- reduce(tmpbuffer);
- status = joinpath(tmpbuffer, env_cfg, buflen);
+ /* Filename: <basename(basename(argv0_path_len))> / "pyvenv.cfg" */
+ reduce(filename);
+ reduce(filename);
+ status = joinpath(filename, env_cfg, filename_len);
if (_PyStatus_EXCEPTION(status)) {
return status;
}
- env_file = _Py_wfopen(tmpbuffer, L"r");
+ env_file = _Py_wfopen(filename, L"r");
if (env_file == NULL) {
errno = 0;
+ return _PyStatus_OK();
}
}
- if (env_file == NULL) {
- return _PyStatus_OK();
- }
-
/* Look for a 'home' variable and set argv0_path to it, if found */
- if (_Py_FindEnvConfigValue(env_file, L"home", tmpbuffer, buflen)) {
- if (safe_wcscpy(argv0_path, tmpbuffer, argv0_path_len) < 0) {
+ wchar_t home[MAXPATHLEN+1];
+ memset(home, 0, sizeof(home));
+
+ if (_Py_FindEnvConfigValue(env_file, L"home",
+ home, Py_ARRAY_LENGTH(home))) {
+ if (safe_wcscpy(argv0_path, home, argv0_path_len) < 0) {
+ fclose(env_file);
return PATHLEN_ERR();
}
}
@@ -1012,12 +1053,12 @@
wchar_t *zip_path, size_t zip_path_len)
{
PyStatus status;
- if (safe_wcscpy(zip_path, prefix, zip_path_len) < 0) {
- return PATHLEN_ERR();
- }
if (calculate->prefix_found > 0) {
/* Use the reduced prefix returned by Py_GetPrefix() */
+ if (safe_wcscpy(zip_path, prefix, zip_path_len) < 0) {
+ return PATHLEN_ERR();
+ }
reduce(zip_path);
reduce(zip_path);
}
@@ -1200,6 +1241,8 @@
return status;
}
+ /* If a pyvenv.cfg configure file is found,
+ argv0_path is overriden with its 'home' variable. */
status = calculate_read_pyenv(calculate,
argv0_path, Py_ARRAY_LENGTH(argv0_path));
if (_PyStatus_EXCEPTION(status)) {