bpo-32030: Enhance Py_Main() (#4412)
Parse more env vars in Py_Main():
* Add more options to _PyCoreConfig:
* faulthandler
* tracemalloc
* importtime
* Move code to parse environment variables from _Py_InitializeCore()
to Py_Main(). This change fixes a regression from Python 3.6:
PYTHONUNBUFFERED is now read before calling pymain_init_stdio().
* _PyFaulthandler_Init() and _PyTraceMalloc_Init() now take an
argument to decide if the module has to be enabled at startup.
* tracemalloc_start() is now responsible to check the maximum number
of frames.
Other changes:
* Cleanup Py_Main():
* Rename some pymain_xxx() subfunctions
* Add pymain_run_python() subfunction
* Cleanup Py_NewInterpreter()
* _PyInterpreterState_Enable() now reports failure
* init_hash_secret() now considers pyurandom() failure as an "user
error": don't fail with abort().
* pymain_optlist_append() and pymain_strdup() now sets err on memory
allocation failure.
diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c
index e667fa9..610541d 100644
--- a/Python/bootstrap_hash.c
+++ b/Python/bootstrap_hash.c
@@ -594,8 +594,8 @@
pyurandom() is non-blocking mode (blocking=0): see the PEP 524. */
res = pyurandom(secret, secret_size, 0, 0);
if (res < 0) {
- return _Py_INIT_ERR("failed to get random numbers "
- "to initialize Python");
+ return _Py_INIT_USER_ERR("failed to get random numbers "
+ "to initialize Python");
}
}
return _Py_INIT_OK();
diff --git a/Python/import.c b/Python/import.c
index fe60844..fef6398 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -1675,10 +1675,9 @@
}
else {
/* 1 -- true, 0 -- false, -1 -- not initialized */
- static int ximporttime = -1;
+ int importtime = interp->core_config.importtime;
static int import_level;
static _PyTime_t accumulated;
- _Py_IDENTIFIER(importtime);
_PyTime_t t1 = 0, accumulated_copy = accumulated;
@@ -1687,32 +1686,14 @@
* Anyway, importlib._find_and_load is much slower than
* _PyDict_GetItemIdWithError().
*/
- if (ximporttime < 0) {
- const char *envoption = Py_GETENV("PYTHONPROFILEIMPORTTIME");
- if (envoption != NULL && *envoption != '\0') {
- ximporttime = 1;
- }
- else {
- PyObject *xoptions = PySys_GetXOptions();
- PyObject *value = NULL;
- if (xoptions) {
- value = _PyDict_GetItemIdWithError(
- xoptions, &PyId_importtime);
- }
- if (value == NULL && PyErr_Occurred()) {
- goto error;
- }
- if (value != NULL || Py_IsInitialized()) {
- ximporttime = (value == Py_True);
- }
- }
- if (ximporttime > 0) {
+ if (importtime) {
+ static int header = 1;
+ if (header) {
fputs("import time: self [us] | cumulative | imported package\n",
stderr);
+ header = 0;
}
- }
- if (ximporttime > 0) {
import_level++;
t1 = _PyTime_GetPerfCounter();
accumulated = 0;
@@ -1731,7 +1712,7 @@
PyDTrace_IMPORT_FIND_LOAD_DONE(PyUnicode_AsUTF8(abs_name),
mod != NULL);
- if (ximporttime > 0) {
+ if (importtime) {
_PyTime_t cum = _PyTime_GetPerfCounter() - t1;
import_level--;
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 66cc711..14fe75e 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -56,7 +56,7 @@
static _PyInitError add_main_module(PyInterpreterState *interp);
static _PyInitError initfsencoding(PyInterpreterState *interp);
static _PyInitError initsite(void);
-static int initstdio(void);
+static _PyInitError init_sys_streams(void);
static _PyInitError initsigs(void);
static void call_py_exitfuncs(void);
static void wait_for_thread_shutdown(void);
@@ -66,10 +66,10 @@
extern void _PyUnicode_Fini(void);
extern int _PyLong_Init(void);
extern void PyLong_Fini(void);
-extern _PyInitError _PyFaulthandler_Init(void);
+extern _PyInitError _PyFaulthandler_Init(int enable);
extern void _PyFaulthandler_Fini(void);
extern void _PyHash_Fini(void);
-extern int _PyTraceMalloc_Init(void);
+extern int _PyTraceMalloc_Init(int enable);
extern int _PyTraceMalloc_Fini(void);
extern void _Py_ReadyTypes(void);
@@ -219,20 +219,6 @@
*/
-static void
-set_flag(int *flag, const char *envs)
-{
- /* Helper to set flag variables from environment variables:
- * - uses the higher of the two values if they're both set
- * - otherwise sets the flag to 1
- */
- int env = atoi(envs);
- if (*flag < env)
- *flag = env;
- if (*flag < 1)
- *flag = 1;
-}
-
static char*
get_codec_name(const char *encoding)
{
@@ -284,7 +270,6 @@
#endif
}
-/* Return NULL on success, or return an error message on failure */
static _PyInitError
initimport(PyInterpreterState *interp, PyObject *sysmod)
{
@@ -346,7 +331,6 @@
return _Py_INIT_OK();
}
-/* Return NULL on success, or return an error message on failure */
static _PyInitError
initexternalimport(PyInterpreterState *interp)
{
@@ -623,8 +607,6 @@
* Any code invoked from this function should *not* assume it has access
* to the Python C API (unless the API is explicitly listed as being
* safe to call without calling Py_Initialize first)
- *
- * Return NULL on success, or return an error message on failure.
*/
/* TODO: Progressively move functionality from Py_BeginInitialization to
@@ -637,7 +619,6 @@
PyInterpreterState *interp;
PyThreadState *tstate;
PyObject *bimod, *sysmod, *pstderr;
- char *p;
_PyCoreConfig core_config = _PyCoreConfig_INIT;
_PyMainInterpreterConfig preinit_config = _PyMainInterpreterConfig_INIT;
_PyInitError err;
@@ -681,31 +662,6 @@
_emit_stderr_warning_for_legacy_locale();
#endif
- if ((p = Py_GETENV("PYTHONDEBUG")) && *p != '\0')
- set_flag(&Py_DebugFlag, p);
- if ((p = Py_GETENV("PYTHONVERBOSE")) && *p != '\0')
- set_flag(&Py_VerboseFlag, p);
- if ((p = Py_GETENV("PYTHONOPTIMIZE")) && *p != '\0')
- set_flag(&Py_OptimizeFlag, p);
- if ((p = Py_GETENV("PYTHONINSPECT")) && *p != '\0')
- set_flag(&Py_InspectFlag, p);
- if ((p = Py_GETENV("PYTHONDONTWRITEBYTECODE")) && *p != '\0')
- set_flag(&Py_DontWriteBytecodeFlag, p);
- if ((p = Py_GETENV("PYTHONNOUSERSITE")) && *p != '\0')
- set_flag(&Py_NoUserSiteDirectory, p);
- if ((p = Py_GETENV("PYTHONUNBUFFERED")) && *p != '\0')
- set_flag(&Py_UnbufferedStdioFlag, p);
- /* The variable is only tested for existence here;
- _Py_HashRandomization_Init will check its value further. */
- if ((p = Py_GETENV("PYTHONHASHSEED")) && *p != '\0')
- set_flag(&Py_HashRandomizationFlag, p);
-#ifdef MS_WINDOWS
- if ((p = Py_GETENV("PYTHONLEGACYWINDOWSFSENCODING")) && *p != '\0')
- set_flag(&Py_LegacyWindowsFSEncodingFlag, p);
- if ((p = Py_GETENV("PYTHONLEGACYWINDOWSSTDIO")) && *p != '\0')
- set_flag(&Py_LegacyWindowsStdioFlag, p);
-#endif
-
err = _Py_HashRandomization_Init(&core_config);
if (_Py_INIT_FAILED(err)) {
return err;
@@ -716,7 +672,11 @@
Py_HashRandomizationFlag = 1;
}
- _PyInterpreterState_Enable(&_PyRuntime);
+ err = _PyInterpreterState_Enable(&_PyRuntime);
+ if (_Py_INIT_FAILED(err)) {
+ return err;
+ }
+
interp = PyInterpreterState_New();
if (interp == NULL)
return _Py_INIT_ERR("can't make main interpreter");
@@ -834,8 +794,6 @@
*
* More advanced selective initialization tricks are possible by calling
* this function multiple times with various preconfigured settings.
- *
- * Return NULL on success, or return an error message on failure.
*/
_PyInitError
@@ -858,8 +816,6 @@
* initialized or without a valid current thread state is a fatal error.
* Other errors should be reported as normal Python exceptions with a
* non-zero return code.
- *
- * Return NULL on success, or return an error message on failure.
*/
_PyInitError
_Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config)
@@ -907,13 +863,14 @@
PySys_SetPath(Py_GetPath());
if (_PySys_EndInit(interp->sysdict) < 0)
return _Py_INIT_ERR("can't finish initializing sys");
+
err = initexternalimport(interp);
if (_Py_INIT_FAILED(err)) {
return err;
}
/* initialize the faulthandler module */
- err = _PyFaulthandler_Init();
+ err = _PyFaulthandler_Init(interp->core_config.faulthandler);
if (_Py_INIT_FAILED(err)) {
return err;
}
@@ -930,15 +887,17 @@
}
}
- if (_PyTraceMalloc_Init() < 0)
+ if (_PyTraceMalloc_Init(interp->core_config.tracemalloc) < 0)
return _Py_INIT_ERR("can't initialize tracemalloc");
err = add_main_module(interp);
if (_Py_INIT_FAILED(err)) {
return err;
}
- if (initstdio() < 0) {
- return _Py_INIT_ERR("can't initialize sys standard streams");
+
+ err = init_sys_streams();
+ if (_Py_INIT_FAILED(err)) {
+ return err;
}
/* Initialize warnings. */
@@ -1302,29 +1261,32 @@
*/
-PyThreadState *
-Py_NewInterpreter(void)
+static _PyInitError
+new_interpreter(PyThreadState **tstate_p)
{
PyInterpreterState *interp;
PyThreadState *tstate, *save_tstate;
PyObject *bimod, *sysmod;
- _PyInitError err = _Py_INIT_OK();
- if (!_PyRuntime.initialized)
- Py_FatalError("Py_NewInterpreter: call Py_Initialize first");
+ if (!_PyRuntime.initialized) {
+ return _Py_INIT_ERR("Py_Initialize must be called first");
+ }
/* Issue #10915, #15751: The GIL API doesn't work with multiple
interpreters: disable PyGILState_Check(). */
_PyGILState_check_enabled = 0;
interp = PyInterpreterState_New();
- if (interp == NULL)
- return NULL;
+ if (interp == NULL) {
+ *tstate_p = NULL;
+ return _Py_INIT_OK();
+ }
tstate = PyThreadState_New(interp);
if (tstate == NULL) {
PyInterpreterState_Delete(interp);
- return NULL;
+ *tstate_p = NULL;
+ return _Py_INIT_OK();
}
save_tstate = PyThreadState_Swap(tstate);
@@ -1343,8 +1305,9 @@
/* XXX The following is lax in error checking */
PyObject *modules = PyDict_New();
- if (modules == NULL)
- Py_FatalError("Py_NewInterpreter: can't make modules dictionary");
+ if (modules == NULL) {
+ return _Py_INIT_ERR("can't make modules dictionary");
+ }
interp->modules = modules;
sysmod = _PyImport_FindBuiltin("sys", modules);
@@ -1371,59 +1334,62 @@
if (bimod != NULL && sysmod != NULL) {
PyObject *pstderr;
+ _PyInitError err;
/* Set up a preliminary stderr printer until we have enough
infrastructure for the io module in place. */
pstderr = PyFile_NewStdPrinter(fileno(stderr));
- if (pstderr == NULL)
- Py_FatalError("Py_NewInterpreter: can't set preliminary stderr");
+ if (pstderr == NULL) {
+ return _Py_INIT_ERR("can't set preliminary stderr");
+ }
_PySys_SetObjectId(&PyId_stderr, pstderr);
PySys_SetObject("__stderr__", pstderr);
Py_DECREF(pstderr);
err = _PyImportHooks_Init();
if (_Py_INIT_FAILED(err)) {
- goto init_failed;
+ return err;
}
err = initimport(interp, sysmod);
if (_Py_INIT_FAILED(err)) {
- goto init_failed;
+ return err;
}
err = initexternalimport(interp);
if (_Py_INIT_FAILED(err)) {
- goto init_failed;
+ return err;
}
err = initfsencoding(interp);
if (_Py_INIT_FAILED(err)) {
- goto init_failed;
+ return err;
}
- if (initstdio() < 0) {
- err = _Py_INIT_ERR("can't initialize sys standard streams");
- goto init_failed;
+ err = init_sys_streams();
+ if (_Py_INIT_FAILED(err)) {
+ return err;
}
err = add_main_module(interp);
if (_Py_INIT_FAILED(err)) {
- goto init_failed;
+ return err;
}
if (!Py_NoSiteFlag) {
err = initsite();
if (_Py_INIT_FAILED(err)) {
- goto init_failed;
+ return err;
}
}
}
- if (!PyErr_Occurred())
- return tstate;
+ if (PyErr_Occurred()) {
+ goto handle_error;
+ }
-init_failed:
- _Py_FatalInitError(err);
+ *tstate_p = tstate;
+ return _Py_INIT_OK();
handle_error:
/* Oops, it didn't work. Undo it all. */
@@ -1434,7 +1400,20 @@
PyThreadState_Delete(tstate);
PyInterpreterState_Delete(interp);
- return NULL;
+ *tstate_p = NULL;
+ return _Py_INIT_OK();
+}
+
+PyThreadState *
+Py_NewInterpreter(void)
+{
+ PyThreadState *tstate;
+ _PyInitError err = new_interpreter(&tstate);
+ if (_Py_INIT_FAILED(err)) {
+ _Py_FatalInitError(err);
+ }
+ return tstate;
+
}
/* Delete an interpreter and its last thread. This requires that the
@@ -1770,16 +1749,17 @@
}
/* Initialize sys.stdin, stdout, stderr and builtins.open */
-static int
-initstdio(void)
+static _PyInitError
+init_sys_streams(void)
{
PyObject *iomod = NULL, *wrapper;
PyObject *bimod = NULL;
PyObject *m;
PyObject *std = NULL;
- int status = 0, fd;
+ int fd;
PyObject * encoding_attr;
char *pythonioencoding = NULL, *encoding, *errors;
+ _PyInitError res = _Py_INIT_OK();
/* Hack to avoid a nasty recursion issue when Python is invoked
in verbose mode: pre-import the Latin-1 and UTF-8 codecs */
@@ -1893,11 +1873,12 @@
Py_DECREF(std);
#endif
- if (0) {
- error:
- status = -1;
- }
+ goto done;
+error:
+ res = _Py_INIT_ERR("can't initialize sys standard streams");
+
+done:
/* We won't need them anymore. */
if (_Py_StandardStreamEncoding) {
PyMem_RawFree(_Py_StandardStreamEncoding);
@@ -1910,7 +1891,7 @@
PyMem_Free(pythonioencoding);
Py_XDECREF(bimod);
Py_XDECREF(iomod);
- return status;
+ return res;
}
diff --git a/Python/pystate.c b/Python/pystate.c
index 30f214e..4544de9 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -76,7 +76,7 @@
static void _PyGILState_NoteThreadState(PyThreadState* tstate);
-void
+_PyInitError
_PyInterpreterState_Enable(_PyRuntimeState *runtime)
{
runtime->interpreters.next_id = 0;
@@ -85,9 +85,11 @@
initialized here. */
if (runtime->interpreters.mutex == NULL) {
runtime->interpreters.mutex = PyThread_allocate_lock();
- if (runtime->interpreters.mutex == NULL)
- Py_FatalError("Can't initialize threads for interpreter");
+ if (runtime->interpreters.mutex == NULL) {
+ return _Py_INIT_ERR("Can't initialize threads for interpreter");
+ }
}
+ return _Py_INIT_OK();
}
PyInterpreterState *