bpo-36142: Add _PyPreConfig.allocator (GH-12181)
* Move 'allocator' and 'dev_mode' fields from _PyCoreConfig
to _PyPreConfig.
* Fix InitConfigTests of test_embed: dev_mode sets allocator to
"debug", add a new tests for env vars with dev mode enabled.
diff --git a/Include/cpython/coreconfig.h b/Include/cpython/coreconfig.h
index 306577c..2673576 100644
--- a/Include/cpython/coreconfig.h
+++ b/Include/cpython/coreconfig.h
@@ -80,6 +80,9 @@
Set by -X utf8 command line option and PYTHONUTF8 environment variable.
If set to -1 (default), inherit Py_UTF8Mode value. */
int utf8_mode;
+
+ int dev_mode; /* Development mode. PYTHONDEVMODE, -X dev */
+ char *allocator; /* Memory allocator: PYTHONMALLOC */
} _PyPreConfig;
#ifdef MS_WINDOWS
@@ -109,9 +112,6 @@
int use_hash_seed; /* PYTHONHASHSEED=x */
unsigned long hash_seed;
- const char *allocator; /* Memory allocator: PYTHONMALLOC */
- int dev_mode; /* PYTHONDEVMODE, -X dev */
-
/* Enable faulthandler?
Set to 1 by -X faulthandler and PYTHONFAULTHANDLER. -1 means unset. */
int faulthandler;
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
index 6c245eb..1f236a9 100644
--- a/Lib/test/test_embed.py
+++ b/Lib/test/test_embed.py
@@ -561,30 +561,36 @@
}
self.check_config("init_from_config", config)
+ INIT_ENV_CONFIG = {
+ 'use_hash_seed': 1,
+ 'hash_seed': 42,
+ 'allocator': 'malloc_debug',
+ 'tracemalloc': 2,
+ 'import_time': 1,
+ 'malloc_stats': 1,
+ 'utf8_mode': 1,
+ 'filesystem_encoding': 'utf-8',
+ 'filesystem_errors': UTF8_MODE_ERRORS,
+ 'inspect': 1,
+ 'optimization_level': 2,
+ 'pycache_prefix': 'env_pycache_prefix',
+ 'write_bytecode': 0,
+ 'verbose': 1,
+ 'buffered_stdio': 0,
+ 'stdio_encoding': 'iso8859-1',
+ 'stdio_errors': 'replace',
+ 'user_site_directory': 0,
+ 'faulthandler': 1,
+ }
+
def test_init_env(self):
- config = {
- 'use_hash_seed': 1,
- 'hash_seed': 42,
- 'allocator': 'malloc_debug',
- 'tracemalloc': 2,
- 'import_time': 1,
- 'malloc_stats': 1,
- 'utf8_mode': 1,
- 'filesystem_encoding': 'utf-8',
- 'filesystem_errors': self.UTF8_MODE_ERRORS,
- 'inspect': 1,
- 'optimization_level': 2,
- 'pycache_prefix': 'env_pycache_prefix',
- 'write_bytecode': 0,
- 'verbose': 1,
- 'buffered_stdio': 0,
- 'stdio_encoding': 'iso8859-1',
- 'stdio_errors': 'replace',
- 'user_site_directory': 0,
- 'faulthandler': 1,
- 'dev_mode': 1,
- }
- self.check_config("init_env", config)
+ self.check_config("init_env", self.INIT_ENV_CONFIG)
+
+ def test_init_env_dev_mode(self):
+ config = dict(self.INIT_ENV_CONFIG,
+ allocator='debug',
+ dev_mode=1)
+ self.check_config("init_env_dev_mode", config)
def test_init_dev_mode(self):
config = {
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index 9923f8d..70bf960 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -437,7 +437,7 @@
config.hash_seed = 123;
putenv("PYTHONMALLOC=malloc");
- config.allocator = "malloc_debug";
+ config.preconfig.allocator = "malloc_debug";
/* dev_mode=1 is tested in test_init_dev_mode() */
@@ -577,7 +577,6 @@
putenv("PYTHONPYCACHEPREFIX=env_pycache_prefix");
putenv("PYTHONNOUSERSITE=1");
putenv("PYTHONFAULTHANDLER=1");
- putenv("PYTHONDEVMODE=1");
putenv("PYTHONIOENCODING=iso8859-1:replace");
/* FIXME: test PYTHONWARNINGS */
/* FIXME: test PYTHONEXECUTABLE */
@@ -589,6 +588,15 @@
}
+static void test_init_env_dev_mode_putenvs(void)
+{
+ test_init_env_putenvs();
+ putenv("PYTHONMALLOC=malloc");
+ putenv("PYTHONFAULTHANDLER=");
+ putenv("PYTHONDEVMODE=1");
+}
+
+
static int test_init_env(void)
{
/* Test initialization from environment variables */
@@ -601,6 +609,18 @@
}
+static int test_init_env_dev_mode(void)
+{
+ /* Test initialization from environment variables */
+ Py_IgnoreEnvironmentFlag = 0;
+ test_init_env_dev_mode_putenvs();
+ _testembed_Py_Initialize();
+ dump_config();
+ Py_Finalize();
+ return 0;
+}
+
+
static int test_init_isolated(void)
{
/* Test _PyCoreConfig.isolated=1 */
@@ -615,7 +635,7 @@
/* Use path starting with "./" avoids a search along the PATH */
config.program_name = L"./_testembed";
- test_init_env_putenvs();
+ test_init_env_dev_mode_putenvs();
_PyInitError err = _Py_InitializeFromConfig(&config);
if (_Py_INIT_FAILED(err)) {
_Py_ExitInitError(err);
@@ -631,7 +651,7 @@
_PyCoreConfig config = _PyCoreConfig_INIT;
putenv("PYTHONFAULTHANDLER=");
putenv("PYTHONMALLOC=");
- config.dev_mode = 1;
+ config.preconfig.dev_mode = 1;
config.program_name = L"./_testembed";
_PyInitError err = _Py_InitializeFromConfig(&config);
if (_Py_INIT_FAILED(err)) {
@@ -673,6 +693,7 @@
{ "init_global_config", test_init_global_config },
{ "init_from_config", test_init_from_config },
{ "init_env", test_init_env },
+ { "init_env_dev_mode", test_init_env_dev_mode },
{ "init_dev_mode", test_init_dev_mode },
{ "init_isolated", test_init_isolated },
{ NULL, NULL }
diff --git a/Python/coreconfig.c b/Python/coreconfig.c
index e372de4..42441e2 100644
--- a/Python/coreconfig.c
+++ b/Python/coreconfig.c
@@ -521,8 +521,6 @@
COPY_ATTR(use_hash_seed);
COPY_ATTR(hash_seed);
COPY_ATTR(_install_importlib);
- COPY_ATTR(allocator);
- COPY_ATTR(dev_mode);
COPY_ATTR(faulthandler);
COPY_ATTR(tracemalloc);
COPY_ATTR(import_time);
@@ -931,10 +929,6 @@
"PYTHONLEGACYWINDOWSSTDIO");
#endif
- if (config->allocator == NULL) {
- config->allocator = _PyCoreConfig_GetEnv(config, "PYTHONMALLOC");
- }
-
if (_PyCoreConfig_GetEnv(config, "PYTHONDUMPREFS")) {
config->dump_refs = 1;
}
@@ -1059,11 +1053,6 @@
|| config_get_xoption(config, L"importtime")) {
config->import_time = 1;
}
- if (config_get_xoption(config, L"dev" ) ||
- _PyCoreConfig_GetEnv(config, "PYTHONDEVMODE"))
- {
- config->dev_mode = 1;
- }
_PyInitError err;
if (config->tracemalloc < 0) {
@@ -1427,13 +1416,10 @@
}
/* default values */
- if (config->dev_mode) {
+ if (config->preconfig.dev_mode) {
if (config->faulthandler < 0) {
config->faulthandler = 1;
}
- if (config->allocator == NULL) {
- config->allocator = "debug";
- }
}
if (config->use_hash_seed < 0) {
config->use_hash_seed = 0;
@@ -1572,8 +1558,6 @@
SET_ITEM_INT(install_signal_handlers);
SET_ITEM_INT(use_hash_seed);
SET_ITEM_UINT(hash_seed);
- SET_ITEM_STR(allocator);
- SET_ITEM_INT(dev_mode);
SET_ITEM_INT(faulthandler);
SET_ITEM_INT(tracemalloc);
SET_ITEM_INT(import_time);
@@ -1950,7 +1934,7 @@
* the lowest precedence entries first so that later entries override them.
*/
- if (config->dev_mode) {
+ if (config->preconfig.dev_mode) {
err = _Py_wstrlist_append(&config->nwarnoption,
&config->warnoptions,
L"default");
diff --git a/Python/preconfig.c b/Python/preconfig.c
index 3befecf..98e0ede 100644
--- a/Python/preconfig.c
+++ b/Python/preconfig.c
@@ -125,6 +125,15 @@
void
_PyPreConfig_Clear(_PyPreConfig *config)
{
+#define CLEAR(ATTR) \
+ do { \
+ PyMem_RawFree(ATTR); \
+ ATTR = NULL; \
+ } while (0)
+
+ CLEAR(config->allocator);
+
+#undef CLEAR
}
@@ -134,6 +143,15 @@
_PyPreConfig_Clear(config);
#define COPY_ATTR(ATTR) config->ATTR = config2->ATTR
+#define COPY_STR_ATTR(ATTR) \
+ do { \
+ if (config2->ATTR != NULL) { \
+ config->ATTR = _PyMem_RawStrdup(config2->ATTR); \
+ if (config->ATTR == NULL) { \
+ return -1; \
+ } \
+ } \
+ } while (0)
COPY_ATTR(isolated);
COPY_ATTR(use_environment);
@@ -143,8 +161,11 @@
COPY_ATTR(legacy_windows_fs_encoding);
#endif
COPY_ATTR(utf8_mode);
+ COPY_ATTR(dev_mode);
+ COPY_STR_ATTR(allocator);
#undef COPY_ATTR
+#undef COPY_STR_ATTR
return 0;
}
@@ -345,6 +366,7 @@
{
_PyPreConfig_GetGlobalConfig(config);
+ /* isolated and use_environment */
if (config->isolated > 0) {
config->use_environment = 0;
}
@@ -354,6 +376,7 @@
config->use_environment = 0;
}
+ /* legacy_windows_fs_encoding, utf8_mode, coerce_c_locale */
if (config->use_environment) {
#ifdef MS_WINDOWS
_Py_get_env_flag(config, &config->legacy_windows_fs_encoding,
@@ -414,11 +437,43 @@
if (config->utf8_mode < 0) {
config->utf8_mode = 0;
}
+ if (config->coerce_c_locale < 0) {
+ config->coerce_c_locale = 0;
+ }
+
+ /* dev_mode */
+ if ((cmdline && _Py_get_xoption(cmdline->nxoption, cmdline->xoptions, L"dev"))
+ || _PyPreConfig_GetEnv(config, "PYTHONDEVMODE"))
+ {
+ config->dev_mode = 1;
+ }
+ if (config->dev_mode < 0) {
+ config->dev_mode = 0;
+ }
+
+ /* allocator */
+ if (config->dev_mode && config->allocator == NULL) {
+ config->allocator = _PyMem_RawStrdup("debug");
+ if (config->allocator == NULL) {
+ return _Py_INIT_NO_MEMORY();
+ }
+ }
+
+ if (config->allocator == NULL) {
+ const char *allocator = _PyPreConfig_GetEnv(config, "PYTHONMALLOC");
+ if (allocator) {
+ config->allocator = _PyMem_RawStrdup(allocator);
+ if (config->allocator == NULL) {
+ return _Py_INIT_NO_MEMORY();
+ }
+ }
+ }
assert(config->coerce_c_locale >= 0);
assert(config->utf8_mode >= 0);
assert(config->isolated >= 0);
assert(config->use_environment >= 0);
+ assert(config->dev_mode >= 0);
return _Py_INIT_OK();
}
@@ -448,6 +503,12 @@
} while (0)
#define SET_ITEM_INT(ATTR) \
SET_ITEM(#ATTR, PyLong_FromLong(config->ATTR))
+#define FROM_STRING(STR) \
+ ((STR != NULL) ? \
+ PyUnicode_FromString(STR) \
+ : (Py_INCREF(Py_None), Py_None))
+#define SET_ITEM_STR(ATTR) \
+ SET_ITEM(#ATTR, FROM_STRING(config->ATTR))
SET_ITEM_INT(isolated);
SET_ITEM_INT(use_environment);
@@ -457,13 +518,17 @@
#ifdef MS_WINDOWS
SET_ITEM_INT(legacy_windows_fs_encoding);
#endif
+ SET_ITEM_INT(dev_mode);
+ SET_ITEM_STR(allocator);
return 0;
fail:
return -1;
+#undef FROM_STRING
#undef SET_ITEM
#undef SET_ITEM_INT
+#undef SET_ITEM_STR
}
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index dec8904..c955a1d 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -482,9 +482,9 @@
/* bpo-34008: For backward compatibility reasons, calling Py_Main() after
Py_Initialize() ignores the new configuration. */
- if (core_config->allocator != NULL) {
+ if (core_config->preconfig.allocator != NULL) {
const char *allocator = _PyMem_GetAllocatorsName();
- if (allocator == NULL || strcmp(core_config->allocator, allocator) != 0) {
+ if (allocator == NULL || strcmp(core_config->preconfig.allocator, allocator) != 0) {
return _Py_INIT_USER_ERR("cannot modify memory allocator "
"after first Py_Initialize()");
}
@@ -521,8 +521,8 @@
return err;
}
- if (core_config->allocator != NULL) {
- if (_PyMem_SetupAllocators(core_config->allocator) < 0) {
+ if (core_config->preconfig.allocator != NULL) {
+ if (_PyMem_SetupAllocators(core_config->preconfig.allocator) < 0) {
return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator");
}
}
diff --git a/Python/sysmodule.c b/Python/sysmodule.c
index 50ba1a7..99fd460 100644
--- a/Python/sysmodule.c
+++ b/Python/sysmodule.c
@@ -2180,7 +2180,7 @@
SetFlag(config->quiet);
SetFlag(config->use_hash_seed == 0 || config->hash_seed != 0);
SetFlag(config->preconfig.isolated);
- PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->dev_mode));
+ PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->preconfig.dev_mode));
SetFlag(config->preconfig.utf8_mode);
#undef SetFlag