[3.7] bpo-34247: Fix Python 3.7 initialization (#8659)
* -X dev: it is now possible to override the memory allocator using
PYTHONMALLOC even if the developer mode is enabled.
* Add _Py_InitializeFromConfig()
* Add _Py_Initialize_ReadEnvVars() to set global configuration
variables from environment variables
* Fix the code to initialize Python: Py_Initialize() now also reads
environment variables
* _Py_InitializeCore() can now be called twice: the second call
only replaces the configuration.
* Write unit tests on Py_Initialize() and the different ways to
configure Python
* The isolated mode now always sets Py_IgnoreEnvironmentFlag and
Py_NoUserSiteDirectory to 1.
* pymain_read_conf() now saves/restores the configuration
if the encoding changed
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index b1be682..6c35f95 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -1,4 +1,5 @@
#include <Python.h>
+#include "internal/import.h"
#include "pythread.h"
#include <inttypes.h>
#include <stdio.h>
@@ -292,6 +293,293 @@
}
+static void
+dump_config(void)
+{
+#define ASSERT_EQUAL(a, b) \
+ if ((a) != (b)) { \
+ printf("ERROR: %s != %s (%i != %i)\n", #a, #b, (a), (b)); \
+ exit(1); \
+ }
+#define ASSERT_STR_EQUAL(a, b) \
+ if ((a) == NULL || (b == NULL) || wcscmp((a), (b)) != 0) { \
+ printf("ERROR: %s != %s ('%ls' != '%ls')\n", #a, #b, (a), (b)); \
+ exit(1); \
+ }
+
+ PyInterpreterState *interp = PyThreadState_Get()->interp;
+ _PyCoreConfig *config = &interp->core_config;
+
+ printf("install_signal_handlers = %i\n", config->install_signal_handlers);
+
+ printf("Py_IgnoreEnvironmentFlag = %i\n", Py_IgnoreEnvironmentFlag);
+
+ printf("use_hash_seed = %i\n", config->use_hash_seed);
+ printf("hash_seed = %lu\n", config->hash_seed);
+
+ printf("allocator = %s\n", config->allocator);
+
+ printf("dev_mode = %i\n", config->dev_mode);
+ printf("faulthandler = %i\n", config->faulthandler);
+ printf("tracemalloc = %i\n", config->tracemalloc);
+ printf("import_time = %i\n", config->import_time);
+ printf("show_ref_count = %i\n", config->show_ref_count);
+ printf("show_alloc_count = %i\n", config->show_alloc_count);
+ printf("dump_refs = %i\n", config->dump_refs);
+ printf("malloc_stats = %i\n", config->malloc_stats);
+
+ printf("coerce_c_locale = %i\n", config->coerce_c_locale);
+ printf("coerce_c_locale_warn = %i\n", config->coerce_c_locale_warn);
+ printf("utf8_mode = %i\n", config->utf8_mode);
+
+ printf("program_name = %ls\n", config->program_name);
+ ASSERT_STR_EQUAL(config->program_name, Py_GetProgramName());
+
+ printf("argc = %i\n", config->argc);
+ printf("argv = [");
+ for (int i=0; i < config->argc; i++) {
+ if (i) {
+ printf(", ");
+ }
+ printf("\"%ls\"", config->argv[i]);
+ }
+ printf("]\n");
+
+ printf("program = %ls\n", config->program);
+ /* FIXME: test xoptions */
+ /* FIXME: test warnoptions */
+ /* FIXME: test module_search_path_env */
+ /* FIXME: test home */
+ /* FIXME: test module_search_paths */
+ /* FIXME: test executable */
+ /* FIXME: test prefix */
+ /* FIXME: test base_prefix */
+ /* FIXME: test exec_prefix */
+ /* FIXME: test base_exec_prefix */
+ /* FIXME: test dll_path */
+
+ printf("Py_IsolatedFlag = %i\n", Py_IsolatedFlag);
+ printf("Py_NoSiteFlag = %i\n", Py_NoSiteFlag);
+ printf("Py_BytesWarningFlag = %i\n", Py_BytesWarningFlag);
+ printf("Py_InspectFlag = %i\n", Py_InspectFlag);
+ printf("Py_InteractiveFlag = %i\n", Py_InteractiveFlag);
+ printf("Py_OptimizeFlag = %i\n", Py_OptimizeFlag);
+ printf("Py_DebugFlag = %i\n", Py_DebugFlag);
+ printf("Py_DontWriteBytecodeFlag = %i\n", Py_DontWriteBytecodeFlag);
+ printf("Py_VerboseFlag = %i\n", Py_VerboseFlag);
+ printf("Py_QuietFlag = %i\n", Py_QuietFlag);
+ printf("Py_NoUserSiteDirectory = %i\n", Py_NoUserSiteDirectory);
+ printf("Py_UnbufferedStdioFlag = %i\n", Py_UnbufferedStdioFlag);
+ /* FIXME: test legacy_windows_fs_encoding */
+ /* FIXME: test legacy_windows_stdio */
+
+ printf("_disable_importlib = %i\n", config->_disable_importlib);
+ /* cannot test _Py_CheckHashBasedPycsMode: the symbol is not exported */
+ printf("Py_FrozenFlag = %i\n", Py_FrozenFlag);
+
+#undef ASSERT_EQUAL
+#undef ASSERT_STR_EQUAL
+}
+
+
+static int test_init_default_config(void)
+{
+ _testembed_Py_Initialize();
+ dump_config();
+ Py_Finalize();
+ return 0;
+}
+
+
+static int test_init_global_config(void)
+{
+ /* FIXME: test Py_IgnoreEnvironmentFlag */
+
+ putenv("PYTHONUTF8=0");
+ Py_UTF8Mode = 1;
+
+ /* Test initialization from global configuration variables (Py_xxx) */
+ Py_SetProgramName(L"./globalvar");
+
+ /* Py_IsolatedFlag is not tested */
+ Py_NoSiteFlag = 1;
+ Py_BytesWarningFlag = 1;
+
+ putenv("PYTHONINSPECT=");
+ Py_InspectFlag = 1;
+
+ putenv("PYTHONOPTIMIZE=0");
+ Py_InteractiveFlag = 1;
+
+ putenv("PYTHONDEBUG=0");
+ Py_OptimizeFlag = 2;
+
+ /* Py_DebugFlag is not tested */
+
+ putenv("PYTHONDONTWRITEBYTECODE=");
+ Py_DontWriteBytecodeFlag = 1;
+
+ putenv("PYTHONVERBOSE=0");
+ Py_VerboseFlag = 1;
+
+ Py_QuietFlag = 1;
+ Py_NoUserSiteDirectory = 1;
+
+ putenv("PYTHONUNBUFFERED=");
+ Py_UnbufferedStdioFlag = 1;
+
+ Py_FrozenFlag = 1;
+
+ /* FIXME: test Py_LegacyWindowsFSEncodingFlag */
+ /* FIXME: test Py_LegacyWindowsStdioFlag */
+
+ Py_Initialize();
+ dump_config();
+ Py_Finalize();
+ return 0;
+}
+
+
+static int test_init_from_config(void)
+{
+ /* Test _Py_InitializeFromConfig() */
+ _PyCoreConfig config = _PyCoreConfig_INIT;
+ config.install_signal_handlers = 0;
+
+ /* FIXME: test ignore_environment */
+
+ putenv("PYTHONHASHSEED=42");
+ config.use_hash_seed = 1;
+ config.hash_seed = 123;
+
+ putenv("PYTHONMALLOC=malloc");
+ config.allocator = "malloc_debug";
+
+ /* dev_mode=1 is tested in test_init_dev_mode() */
+
+ putenv("PYTHONFAULTHANDLER=");
+ config.faulthandler = 1;
+
+ putenv("PYTHONTRACEMALLOC=0");
+ config.tracemalloc = 2;
+
+ putenv("PYTHONPROFILEIMPORTTIME=0");
+ config.import_time = 1;
+
+ config.show_ref_count = 1;
+ config.show_alloc_count = 1;
+ /* FIXME: test dump_refs: bpo-34223 */
+
+ putenv("PYTHONMALLOCSTATS=0");
+ config.malloc_stats = 1;
+
+ /* FIXME: test coerce_c_locale and coerce_c_locale_warn */
+
+ putenv("PYTHONUTF8=0");
+ Py_UTF8Mode = 0;
+ config.utf8_mode = 1;
+
+ Py_SetProgramName(L"./globalvar");
+ config.program_name = L"./conf_program_name";
+
+ /* FIXME: test argc/argv */
+ config.program = L"conf_program";
+ /* FIXME: test xoptions */
+ /* FIXME: test warnoptions */
+ /* FIXME: test module_search_path_env */
+ /* FIXME: test home */
+ /* FIXME: test path config: module_search_path .. dll_path */
+
+ _PyInitError err = _Py_InitializeFromConfig(&config);
+ /* Don't call _PyCoreConfig_Clear() since all strings are static */
+ if (_Py_INIT_FAILED(err)) {
+ _Py_FatalInitError(err);
+ }
+ dump_config();
+ Py_Finalize();
+ return 0;
+}
+
+
+static void test_init_env_putenvs(void)
+{
+ putenv("PYTHONHASHSEED=42");
+ putenv("PYTHONMALLOC=malloc_debug");
+ putenv("PYTHONTRACEMALLOC=2");
+ putenv("PYTHONPROFILEIMPORTTIME=1");
+ putenv("PYTHONMALLOCSTATS=1");
+ putenv("PYTHONUTF8=1");
+ putenv("PYTHONVERBOSE=1");
+ putenv("PYTHONINSPECT=1");
+ putenv("PYTHONOPTIMIZE=2");
+ putenv("PYTHONDONTWRITEBYTECODE=1");
+ putenv("PYTHONUNBUFFERED=1");
+ putenv("PYTHONNOUSERSITE=1");
+ putenv("PYTHONFAULTHANDLER=1");
+ putenv("PYTHONDEVMODE=1");
+ /* FIXME: test PYTHONWARNINGS */
+ /* FIXME: test PYTHONEXECUTABLE */
+ /* FIXME: test PYTHONHOME */
+ /* FIXME: test PYTHONDEBUG */
+ /* FIXME: test PYTHONDUMPREFS */
+ /* FIXME: test PYTHONCOERCECLOCALE */
+ /* FIXME: test PYTHONPATH */
+}
+
+
+static int test_init_env(void)
+{
+ /* Test initialization from environment variables */
+ Py_IgnoreEnvironmentFlag = 0;
+ test_init_env_putenvs();
+ _testembed_Py_Initialize();
+ dump_config();
+ Py_Finalize();
+ return 0;
+}
+
+
+static int test_init_isolated(void)
+{
+ /* Test _PyCoreConfig.isolated=1 */
+ _PyCoreConfig config = _PyCoreConfig_INIT;
+
+ /* Set coerce_c_locale and utf8_mode to not depend on the locale */
+ config.coerce_c_locale = 0;
+ config.utf8_mode = 0;
+ /* Use path starting with "./" avoids a search along the PATH */
+ config.program_name = L"./_testembed";
+
+ Py_IsolatedFlag = 1;
+
+ test_init_env_putenvs();
+ _PyInitError err = _Py_InitializeFromConfig(&config);
+ if (_Py_INIT_FAILED(err)) {
+ _Py_FatalInitError(err);
+ }
+ dump_config();
+ Py_Finalize();
+ return 0;
+}
+
+
+static int test_init_dev_mode(void)
+{
+ _PyCoreConfig config = _PyCoreConfig_INIT;
+ putenv("PYTHONFAULTHANDLER=");
+ putenv("PYTHONMALLOC=");
+ config.dev_mode = 1;
+ config.program_name = L"./_testembed";
+ _PyInitError err = _Py_InitializeFromConfig(&config);
+ if (_Py_INIT_FAILED(err)) {
+ _Py_FatalInitError(err);
+ }
+ dump_config();
+ Py_Finalize();
+ return 0;
+}
+
+
/* *********************************************************
* List of test cases and the function that implements it.
*
@@ -318,6 +606,12 @@
{ "bpo20891", test_bpo20891 },
{ "initialize_twice", test_initialize_twice },
{ "initialize_pymain", test_initialize_pymain },
+ { "init_default_config", test_init_default_config },
+ { "init_global_config", test_init_global_config },
+ { "init_from_config", test_init_from_config },
+ { "init_env", test_init_env },
+ { "init_dev_mode", test_init_dev_mode },
+ { "init_isolated", test_init_isolated },
{ NULL, NULL }
};