[3.7] bpo-34247: Fix Python 3.7 initialization (#8659)
* -X dev: it is now possible to override the memory allocator using
PYTHONMALLOC even if the developer mode is enabled.
* Add _Py_InitializeFromConfig()
* Add _Py_Initialize_ReadEnvVars() to set global configuration
variables from environment variables
* Fix the code to initialize Python: Py_Initialize() now also reads
environment variables
* _Py_InitializeCore() can now be called twice: the second call
only replaces the configuration.
* Write unit tests on Py_Initialize() and the different ways to
configure Python
* The isolated mode now always sets Py_IgnoreEnvironmentFlag and
Py_NoUserSiteDirectory to 1.
* pymain_read_conf() now saves/restores the configuration
if the encoding changed
diff --git a/Modules/main.c b/Modules/main.c
index 1f1bf53..6a707c4 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -1364,9 +1364,24 @@
}
+void
+_PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config)
+{
+#define COPY_FLAG(ATTR, VALUE) \
+ if (config->ATTR == -1) { \
+ config->ATTR = VALUE; \
+ }
+
+ COPY_FLAG(ignore_environment, Py_IgnoreEnvironmentFlag);
+ COPY_FLAG(utf8_mode, Py_UTF8Mode);
+
+#undef COPY_FLAG
+}
+
+
/* Get Py_xxx global configuration variables */
static void
-pymain_get_global_config(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
+cmdline_get_global_config(_Py_CommandLineDetails *cmdline)
{
cmdline->bytes_warning = Py_BytesWarningFlag;
cmdline->debug = Py_DebugFlag;
@@ -1385,15 +1400,24 @@
cmdline->legacy_windows_stdio = Py_LegacyWindowsStdioFlag;
#endif
cmdline->check_hash_pycs_mode = _Py_CheckHashBasedPycsMode ;
+}
- pymain->config.ignore_environment = Py_IgnoreEnvironmentFlag;
- pymain->config.utf8_mode = Py_UTF8Mode;
+
+void
+_PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config)
+{
+ Py_IgnoreEnvironmentFlag = config->ignore_environment;
+ Py_UTF8Mode = config->utf8_mode;
+
+ /* Random or non-zero hash seed */
+ Py_HashRandomizationFlag = (config->use_hash_seed == 0 ||
+ config->hash_seed != 0);
}
/* Set Py_xxx global configuration variables */
static void
-pymain_set_global_config(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
+cmdline_set_global_config(_Py_CommandLineDetails *cmdline)
{
Py_BytesWarningFlag = cmdline->bytes_warning;
Py_DebugFlag = cmdline->debug;
@@ -1412,13 +1436,6 @@
Py_LegacyWindowsFSEncodingFlag = cmdline->legacy_windows_fs_encoding;
Py_LegacyWindowsStdioFlag = cmdline->legacy_windows_stdio;
#endif
-
- Py_IgnoreEnvironmentFlag = pymain->config.ignore_environment;
- Py_UTF8Mode = pymain->config.utf8_mode;
-
- /* Random or non-zero hash seed */
- Py_HashRandomizationFlag = (pymain->config.use_hash_seed == 0 ||
- pymain->config.hash_seed != 0);
}
@@ -1632,7 +1649,7 @@
static _PyInitError
-pymain_init_tracemalloc(_PyCoreConfig *config)
+config_init_tracemalloc(_PyCoreConfig *config)
{
int nframe;
int valid;
@@ -1714,6 +1731,27 @@
}
+/* Set global variable variables from environment variables */
+void
+_Py_Initialize_ReadEnvVarsNoAlloc(void)
+{
+ _Py_CommandLineDetails cmdline;
+ memset(&cmdline, 0, sizeof(cmdline));
+
+ cmdline_get_global_config(&cmdline);
+ if (cmdline.isolated) {
+ Py_IgnoreEnvironmentFlag = 1;
+ cmdline.no_user_site_directory = 1;
+ }
+ if (!Py_IgnoreEnvironmentFlag) {
+ cmdline_get_env_flags(&cmdline);
+ }
+ cmdline_set_global_config(&cmdline);
+
+ /* no need to call pymain_clear_cmdline(), no memory has been allocated */
+}
+
+
static _PyInitError
config_init_home(_PyCoreConfig *config)
{
@@ -1741,17 +1779,15 @@
static _PyInitError
config_init_hash_seed(_PyCoreConfig *config)
{
- if (config->use_hash_seed < 0) {
- const char *seed_text = config_get_env_var("PYTHONHASHSEED");
- int use_hash_seed;
- unsigned long hash_seed;
- if (_Py_ReadHashSeed(seed_text, &use_hash_seed, &hash_seed) < 0) {
- return _Py_INIT_USER_ERR("PYTHONHASHSEED must be \"random\" "
- "or an integer in range [0; 4294967295]");
- }
- config->use_hash_seed = use_hash_seed;
- config->hash_seed = hash_seed;
+ const char *seed_text = config_get_env_var("PYTHONHASHSEED");
+ int use_hash_seed;
+ unsigned long hash_seed;
+ if (_Py_ReadHashSeed(seed_text, &use_hash_seed, &hash_seed) < 0) {
+ return _Py_INIT_USER_ERR("PYTHONHASHSEED must be \"random\" "
+ "or an integer in range [0; 4294967295]");
}
+ config->use_hash_seed = use_hash_seed;
+ config->hash_seed = hash_seed;
return _Py_INIT_OK();
}
@@ -1759,12 +1795,6 @@
static _PyInitError
config_init_utf8_mode(_PyCoreConfig *config)
{
- /* The option was already set by Py_UTF8Mode,
- Py_LegacyWindowsFSEncodingFlag or PYTHONLEGACYWINDOWSFSENCODING. */
- if (config->utf8_mode >= 0) {
- return _Py_INIT_OK();
- }
-
const wchar_t *xopt = config_get_xoption(config, L"utf8");
if (xopt) {
wchar_t *sep = wcschr(xopt, L'=');
@@ -1808,7 +1838,11 @@
static _PyInitError
config_read_env_vars(_PyCoreConfig *config)
{
- config->allocator = config_get_env_var("PYTHONMALLOC");
+ assert(!config->ignore_environment);
+
+ if (config->allocator == NULL) {
+ config->allocator = config_get_env_var("PYTHONMALLOC");
+ }
if (config_get_env_var("PYTHONDUMPREFS")) {
config->dump_refs = 1;
@@ -1817,16 +1851,18 @@
config->malloc_stats = 1;
}
- const char *env = config_get_env_var("PYTHONCOERCECLOCALE");
- if (env) {
- if (strcmp(env, "0") == 0) {
- config->coerce_c_locale = 0;
- }
- else if (strcmp(env, "warn") == 0) {
- config->coerce_c_locale_warn = 1;
- }
- else {
- config->coerce_c_locale = 1;
+ if (config->coerce_c_locale < 0) {
+ const char *env = config_get_env_var("PYTHONCOERCECLOCALE");
+ if (env) {
+ if (strcmp(env, "0") == 0) {
+ config->coerce_c_locale = 0;
+ }
+ else if (strcmp(env, "warn") == 0) {
+ config->coerce_c_locale_warn = 1;
+ }
+ else {
+ config->coerce_c_locale = 1;
+ }
}
}
@@ -1837,9 +1873,11 @@
}
config->module_search_path_env = path;
- _PyInitError err = config_init_hash_seed(config);
- if (_Py_INIT_FAILED(err)) {
- return err;
+ if (config->use_hash_seed < 0) {
+ _PyInitError err = config_init_hash_seed(config);
+ if (_Py_INIT_FAILED(err)) {
+ return err;
+ }
}
return _Py_INIT_OK();
@@ -1850,9 +1888,11 @@
config_read_complex_options(_PyCoreConfig *config)
{
/* More complex options configured by env var and -X option */
- if (config_get_env_var("PYTHONFAULTHANDLER")
- || config_get_xoption(config, L"faulthandler")) {
- config->faulthandler = 1;
+ if (config->faulthandler < 0) {
+ if (config_get_env_var("PYTHONFAULTHANDLER")
+ || config_get_xoption(config, L"faulthandler")) {
+ config->faulthandler = 1;
+ }
}
if (config_get_env_var("PYTHONPROFILEIMPORTTIME")
|| config_get_xoption(config, L"importtime")) {
@@ -1862,13 +1902,13 @@
config_get_env_var("PYTHONDEVMODE"))
{
config->dev_mode = 1;
- config->faulthandler = 1;
- config->allocator = "debug";
}
- _PyInitError err = pymain_init_tracemalloc(config);
- if (_Py_INIT_FAILED(err)) {
- return err;
+ if (config->tracemalloc < 0) {
+ _PyInitError err = config_init_tracemalloc(config);
+ if (_Py_INIT_FAILED(err)) {
+ return err;
+ }
}
return _Py_INIT_OK();
}
@@ -1892,10 +1932,12 @@
/* Set Py_IgnoreEnvironmentFlag for Py_GETENV() */
_PyCoreConfig *config = &pymain->config;
- Py_IgnoreEnvironmentFlag = config->ignore_environment;
+ Py_IgnoreEnvironmentFlag = config->ignore_environment || cmdline->isolated;
/* Get environment variables */
- cmdline_get_env_flags(cmdline);
+ if (!Py_IgnoreEnvironmentFlag) {
+ cmdline_get_env_flags(cmdline);
+ }
err = cmdline_init_env_warnoptions(cmdline);
if (_Py_INIT_FAILED(err)) {
@@ -1940,6 +1982,8 @@
static int
pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
{
+ _PyCoreConfig *config = &pymain->config;
+ _PyCoreConfig save_config = _PyCoreConfig_INIT;
int res = -1;
char *oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL));
@@ -1953,10 +1997,15 @@
int locale_coerced = 0;
int loops = 0;
- int init_ignore_env = pymain->config.ignore_environment;
+ int init_ignore_env = config->ignore_environment;
+
+ if (_PyCoreConfig_Copy(&save_config, config) < 0) {
+ pymain->err = _Py_INIT_NO_MEMORY();
+ goto done;
+ }
while (1) {
- int utf8_mode = pymain->config.utf8_mode;
+ int init_utf8_mode = config->utf8_mode;
int encoding_changed = 0;
/* Watchdog to prevent an infinite loop */
@@ -1987,20 +2036,20 @@
* See the documentation of the PYTHONCOERCECLOCALE setting for more
* details.
*/
- if (pymain->config.coerce_c_locale == 1 && !locale_coerced) {
+ if (config->coerce_c_locale == 1 && !locale_coerced) {
locale_coerced = 1;
- _Py_CoerceLegacyLocale(&pymain->config);
+ _Py_CoerceLegacyLocale(config);
encoding_changed = 1;
}
- if (utf8_mode == -1) {
- if (pymain->config.utf8_mode == 1) {
+ if (init_utf8_mode == -1) {
+ if (config->utf8_mode == 1) {
/* UTF-8 Mode enabled */
encoding_changed = 1;
}
}
else {
- if (pymain->config.utf8_mode != utf8_mode) {
+ if (config->utf8_mode != init_utf8_mode) {
encoding_changed = 1;
}
}
@@ -2013,12 +2062,18 @@
Py_DecodeLocale(). Reset Py_IgnoreEnvironmentFlag, modified by
pymain_read_conf_impl(). Reset Py_IsolatedFlag and Py_NoSiteFlag
modified by _PyCoreConfig_Read(). */
- Py_UTF8Mode = pymain->config.utf8_mode;
+ int new_utf8_mode = config->utf8_mode;
Py_IgnoreEnvironmentFlag = init_ignore_env;
- _PyCoreConfig_Clear(&pymain->config);
+ if (_PyCoreConfig_Copy(config, &save_config) < 0) {
+ pymain->err = _Py_INIT_NO_MEMORY();
+ goto done;
+ }
pymain_clear_cmdline(pymain, cmdline);
memset(cmdline, 0, sizeof(*cmdline));
- pymain_get_global_config(pymain, cmdline);
+
+ cmdline_get_global_config(cmdline);
+ _PyCoreConfig_GetGlobalConfig(config);
+ config->utf8_mode = new_utf8_mode;
/* The encoding changed: read again the configuration
with the new encoding */
@@ -2026,6 +2081,7 @@
res = 0;
done:
+ _PyCoreConfig_Clear(&save_config);
if (oldloc != NULL) {
setlocale(LC_ALL, oldloc);
PyMem_RawFree(oldloc);
@@ -2038,10 +2094,6 @@
static void
config_init_locale(_PyCoreConfig *config)
{
- if (config->utf8_mode >= 0 && config->coerce_c_locale >= 0) {
- return;
- }
-
if (_Py_LegacyLocaleDetected()) {
/* POSIX locale: enable C locale coercion and UTF-8 Mode */
if (config->utf8_mode < 0) {
@@ -2050,15 +2102,6 @@
if (config->coerce_c_locale < 0) {
config->coerce_c_locale = 1;
}
- return;
- }
-
- /* By default, C locale coercion and UTF-8 Mode are disabled */
- if (config->coerce_c_locale < 0) {
- config->coerce_c_locale = 0;
- }
- if (config->utf8_mode < 0) {
- config->utf8_mode = 0;
}
}
@@ -2175,9 +2218,14 @@
{
_PyInitError err;
- err = config_read_env_vars(config);
- if (_Py_INIT_FAILED(err)) {
- return err;
+ _PyCoreConfig_GetGlobalConfig(config);
+
+ assert(config->ignore_environment >= 0);
+ if (!config->ignore_environment) {
+ err = config_read_env_vars(config);
+ if (_Py_INIT_FAILED(err)) {
+ return err;
+ }
}
/* -X options */
@@ -2193,26 +2241,29 @@
return err;
}
- err = config_init_utf8_mode(config);
- if (_Py_INIT_FAILED(err)) {
- return err;
+ if (config->utf8_mode < 0) {
+ err = config_init_utf8_mode(config);
+ if (_Py_INIT_FAILED(err)) {
+ return err;
+ }
}
- err = config_init_home(config);
- if (_Py_INIT_FAILED(err)) {
- return err;
+ if (config->home == NULL) {
+ err = config_init_home(config);
+ if (_Py_INIT_FAILED(err)) {
+ return err;
+ }
}
- err = config_init_program_name(config);
- if (_Py_INIT_FAILED(err)) {
- return err;
+ if (config->program_name == NULL) {
+ err = config_init_program_name(config);
+ if (_Py_INIT_FAILED(err)) {
+ return err;
+ }
}
- config_init_locale(config);
-
- /* Signal handlers are installed by default */
- if (config->install_signal_handlers < 0) {
- config->install_signal_handlers = 1;
+ if (config->utf8_mode < 0 || config->coerce_c_locale < 0) {
+ config_init_locale(config);
}
if (!config->_disable_importlib) {
@@ -2221,6 +2272,39 @@
return err;
}
}
+
+ /* default values */
+ if (config->dev_mode) {
+ if (config->faulthandler < 0) {
+ config->faulthandler = 1;
+ }
+ if (config->allocator == NULL) {
+ config->allocator = "debug";
+ }
+ }
+ if (config->install_signal_handlers < 0) {
+ config->install_signal_handlers = 1;
+ }
+ if (config->use_hash_seed < 0) {
+ config->use_hash_seed = 0;
+ config->hash_seed = 0;
+ }
+ if (config->faulthandler < 0) {
+ config->faulthandler = 0;
+ }
+ if (config->tracemalloc < 0) {
+ config->tracemalloc = 0;
+ }
+ if (config->coerce_c_locale < 0) {
+ config->coerce_c_locale = 0;
+ }
+ if (config->utf8_mode < 0) {
+ config->utf8_mode = 0;
+ }
+ if (config->argc < 0) {
+ config->argc = 0;
+ }
+
return _Py_INIT_OK();
}
@@ -2289,6 +2373,7 @@
config->LEN = config2->LEN; \
} while (0)
+ COPY_ATTR(install_signal_handlers);
COPY_ATTR(ignore_environment);
COPY_ATTR(use_hash_seed);
COPY_ATTR(hash_seed);
@@ -2302,6 +2387,9 @@
COPY_ATTR(show_alloc_count);
COPY_ATTR(dump_refs);
COPY_ATTR(malloc_stats);
+
+ COPY_ATTR(coerce_c_locale);
+ COPY_ATTR(coerce_c_locale_warn);
COPY_ATTR(utf8_mode);
COPY_STR_ATTR(module_search_path_env);
@@ -2457,14 +2545,14 @@
static int
-pymain_init_python_main(_PyMain *pymain)
+pymain_init_python_main(_PyMain *pymain, PyInterpreterState *interp)
{
_PyInitError err;
_PyMainInterpreterConfig main_config = _PyMainInterpreterConfig_INIT;
- err = _PyMainInterpreterConfig_Read(&main_config, &pymain->config);
+ err = _PyMainInterpreterConfig_Read(&main_config, &interp->core_config);
if (!_Py_INIT_FAILED(err)) {
- err = _Py_InitializeMainInterpreter(&main_config);
+ err = _Py_InitializeMainInterpreter(interp, &main_config);
}
_PyMainInterpreterConfig_Clear(&main_config);
@@ -2615,11 +2703,17 @@
_Py_CommandLineDetails cmdline;
memset(&cmdline, 0, sizeof(cmdline));
- pymain_get_global_config(pymain, &cmdline);
+ cmdline_get_global_config(&cmdline);
+ _PyCoreConfig_GetGlobalConfig(&pymain->config);
int res = pymain_cmdline_impl(pymain, &cmdline);
- pymain_set_global_config(pymain, &cmdline);
+ cmdline_set_global_config(&cmdline);
+ _PyCoreConfig_SetGlobalConfig(&pymain->config);
+ if (Py_IsolatedFlag) {
+ Py_IgnoreEnvironmentFlag = 1;
+ Py_NoUserSiteDirectory = 1;
+ }
pymain_clear_cmdline(pymain, &cmdline);
@@ -2649,16 +2743,13 @@
pymain_init_stdio(pymain);
- /* bpo-34008: For backward compatibility reasons, calling Py_Main() after
- Py_Initialize() ignores the new configuration. */
- if (!_PyRuntime.initialized) {
- pymain->err = _Py_InitializeCore(&pymain->config);
- if (_Py_INIT_FAILED(pymain->err)) {
- _Py_FatalInitError(pymain->err);
- }
+ PyInterpreterState *interp;
+ pymain->err = _Py_InitializeCore(&interp, &pymain->config);
+ if (_Py_INIT_FAILED(pymain->err)) {
+ _Py_FatalInitError(pymain->err);
}
- if (pymain_init_python_main(pymain) < 0) {
+ if (pymain_init_python_main(pymain, interp) < 0) {
_Py_FatalInitError(pymain->err);
}