blob: c88e945f8e27bdf6d4aae271fa832985421069fe [file] [log] [blame]
Nick Coghland6009512014-11-20 21:39:37 +10001/* Python interpreter top-level routines, including init/exit */
2
3#include "Python.h"
4
5#include "Python-ast.h"
6#undef Yield /* undefine macro conflicting with winbase.h */
Yury Selivanovf23746a2018-01-22 19:11:18 -05007#include "internal/context.h"
8#include "internal/hamt.h"
Eric Snow2ebc5ce2017-09-07 23:51:28 -06009#include "internal/pystate.h"
Nick Coghland6009512014-11-20 21:39:37 +100010#include "grammar.h"
11#include "node.h"
12#include "token.h"
13#include "parsetok.h"
14#include "errcode.h"
15#include "code.h"
16#include "symtable.h"
17#include "ast.h"
18#include "marshal.h"
19#include "osdefs.h"
20#include <locale.h>
21
22#ifdef HAVE_SIGNAL_H
23#include <signal.h>
24#endif
25
26#ifdef MS_WINDOWS
27#include "malloc.h" /* for alloca */
28#endif
29
30#ifdef HAVE_LANGINFO_H
31#include <langinfo.h>
32#endif
33
34#ifdef MS_WINDOWS
35#undef BYTE
36#include "windows.h"
Steve Dower39294992016-08-30 21:22:36 -070037
38extern PyTypeObject PyWindowsConsoleIO_Type;
39#define PyWindowsConsoleIO_Check(op) (PyObject_TypeCheck((op), &PyWindowsConsoleIO_Type))
Nick Coghland6009512014-11-20 21:39:37 +100040#endif
41
42_Py_IDENTIFIER(flush);
43_Py_IDENTIFIER(name);
44_Py_IDENTIFIER(stdin);
45_Py_IDENTIFIER(stdout);
46_Py_IDENTIFIER(stderr);
Eric Snow3f9eee62017-09-15 16:35:20 -060047_Py_IDENTIFIER(threading);
Nick Coghland6009512014-11-20 21:39:37 +100048
49#ifdef __cplusplus
50extern "C" {
51#endif
52
Nick Coghland6009512014-11-20 21:39:37 +100053extern grammar _PyParser_Grammar; /* From graminit.c */
54
55/* Forward */
Victor Stinnerf7e5b562017-11-15 15:48:08 -080056static _PyInitError add_main_module(PyInterpreterState *interp);
57static _PyInitError initfsencoding(PyInterpreterState *interp);
58static _PyInitError initsite(void);
Victor Stinner91106cd2017-12-13 12:29:09 +010059static _PyInitError init_sys_streams(PyInterpreterState *interp);
Victor Stinnerf7e5b562017-11-15 15:48:08 -080060static _PyInitError initsigs(void);
Marcel Plch776407f2017-12-20 11:17:58 +010061static void call_py_exitfuncs(PyInterpreterState *);
Nick Coghland6009512014-11-20 21:39:37 +100062static void wait_for_thread_shutdown(void);
63static void call_ll_exitfuncs(void);
64extern int _PyUnicode_Init(void);
65extern int _PyStructSequence_Init(void);
66extern void _PyUnicode_Fini(void);
67extern int _PyLong_Init(void);
68extern void PyLong_Fini(void);
Victor Stinnera7368ac2017-11-15 18:11:45 -080069extern _PyInitError _PyFaulthandler_Init(int enable);
Nick Coghland6009512014-11-20 21:39:37 +100070extern void _PyFaulthandler_Fini(void);
71extern void _PyHash_Fini(void);
Victor Stinnera7368ac2017-11-15 18:11:45 -080072extern int _PyTraceMalloc_Init(int enable);
Nick Coghland6009512014-11-20 21:39:37 +100073extern int _PyTraceMalloc_Fini(void);
Eric Snowc7ec9982017-05-23 23:00:52 -070074extern void _Py_ReadyTypes(void);
Nick Coghland6009512014-11-20 21:39:37 +100075
Nick Coghland6009512014-11-20 21:39:37 +100076extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *);
77extern void _PyGILState_Fini(void);
Nick Coghland6009512014-11-20 21:39:37 +100078
Victor Stinnerf7e5b562017-11-15 15:48:08 -080079_PyRuntimeState _PyRuntime = _PyRuntimeState_INIT;
Eric Snow2ebc5ce2017-09-07 23:51:28 -060080
Victor Stinnerf7e5b562017-11-15 15:48:08 -080081_PyInitError
Eric Snow2ebc5ce2017-09-07 23:51:28 -060082_PyRuntime_Initialize(void)
83{
84 /* XXX We only initialize once in the process, which aligns with
85 the static initialization of the former globals now found in
86 _PyRuntime. However, _PyRuntime *should* be initialized with
87 every Py_Initialize() call, but doing so breaks the runtime.
88 This is because the runtime state is not properly finalized
89 currently. */
90 static int initialized = 0;
Victor Stinnerf7e5b562017-11-15 15:48:08 -080091 if (initialized) {
92 return _Py_INIT_OK();
93 }
Eric Snow2ebc5ce2017-09-07 23:51:28 -060094 initialized = 1;
Victor Stinnerf7e5b562017-11-15 15:48:08 -080095
96 return _PyRuntimeState_Init(&_PyRuntime);
Eric Snow2ebc5ce2017-09-07 23:51:28 -060097}
98
99void
100_PyRuntime_Finalize(void)
101{
102 _PyRuntimeState_Fini(&_PyRuntime);
103}
104
105int
106_Py_IsFinalizing(void)
107{
108 return _PyRuntime.finalizing != NULL;
109}
110
Nick Coghland6009512014-11-20 21:39:37 +1000111/* Global configuration variable declarations are in pydebug.h */
112/* XXX (ncoghlan): move those declarations to pylifecycle.h? */
Benjamin Peterson0a37a302017-12-31 10:04:13 -0800113int Py_DebugFlag = 0; /* Needed by parser.c */
114int Py_VerboseFlag = 0; /* Needed by import.c */
115int Py_QuietFlag = 0; /* Needed by sysmodule.c */
116int Py_InteractiveFlag = 0; /* Needed by Py_FdIsInteractive() below */
117int Py_InspectFlag = 0; /* Needed to determine whether to exit at SystemExit */
Nick Coghland6009512014-11-20 21:39:37 +1000118int Py_OptimizeFlag = 0; /* Needed by compile.c */
Benjamin Peterson0a37a302017-12-31 10:04:13 -0800119int Py_NoSiteFlag = 0; /* Suppress 'import site' */
120int Py_BytesWarningFlag = 0; /* Warn on str(bytes) and str(buffer) */
121int Py_FrozenFlag = 0; /* Needed by getpath.c */
122int Py_IgnoreEnvironmentFlag = 0; /* e.g. PYTHONPATH, PYTHONHOME */
123int Py_DontWriteBytecodeFlag = 0; /* Suppress writing bytecode files (*.pyc) */
Nick Coghland6009512014-11-20 21:39:37 +1000124int Py_NoUserSiteDirectory = 0; /* for -s and site.py */
125int Py_UnbufferedStdioFlag = 0; /* Unbuffered binary std{in,out,err} */
126int Py_HashRandomizationFlag = 0; /* for -R and PYTHONHASHSEED */
127int Py_IsolatedFlag = 0; /* for -I, isolate from user's env */
Steve Dowercc16be82016-09-08 10:35:16 -0700128#ifdef MS_WINDOWS
129int Py_LegacyWindowsFSEncodingFlag = 0; /* Uses mbcs instead of utf-8 */
Steve Dower39294992016-08-30 21:22:36 -0700130int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */
Steve Dowercc16be82016-09-08 10:35:16 -0700131#endif
Nick Coghland6009512014-11-20 21:39:37 +1000132
Nick Coghland6009512014-11-20 21:39:37 +1000133/* Hack to force loading of object files */
134int (*_PyOS_mystrnicmp_hack)(const char *, const char *, Py_ssize_t) = \
135 PyOS_mystrnicmp; /* Python/pystrcmp.o */
136
137/* PyModule_GetWarningsModule is no longer necessary as of 2.6
138since _warnings is builtin. This API should not be used. */
139PyObject *
140PyModule_GetWarningsModule(void)
141{
142 return PyImport_ImportModule("warnings");
143}
144
Eric Snowc7ec9982017-05-23 23:00:52 -0700145
Eric Snow1abcf672017-05-23 21:46:51 -0700146/* APIs to access the initialization flags
147 *
148 * Can be called prior to Py_Initialize.
149 */
Nick Coghland6009512014-11-20 21:39:37 +1000150
Eric Snow1abcf672017-05-23 21:46:51 -0700151int
152_Py_IsCoreInitialized(void)
153{
Eric Snow2ebc5ce2017-09-07 23:51:28 -0600154 return _PyRuntime.core_initialized;
Eric Snow1abcf672017-05-23 21:46:51 -0700155}
Nick Coghland6009512014-11-20 21:39:37 +1000156
157int
158Py_IsInitialized(void)
159{
Eric Snow2ebc5ce2017-09-07 23:51:28 -0600160 return _PyRuntime.initialized;
Nick Coghland6009512014-11-20 21:39:37 +1000161}
162
163/* Helper to allow an embedding application to override the normal
164 * mechanism that attempts to figure out an appropriate IO encoding
165 */
166
167static char *_Py_StandardStreamEncoding = NULL;
168static char *_Py_StandardStreamErrors = NULL;
169
170int
171Py_SetStandardStreamEncoding(const char *encoding, const char *errors)
172{
173 if (Py_IsInitialized()) {
174 /* This is too late to have any effect */
175 return -1;
176 }
Victor Stinner31e99082017-12-20 23:41:38 +0100177
178 int res = 0;
179
180 /* Py_SetStandardStreamEncoding() can be called before Py_Initialize(),
181 but Py_Initialize() can change the allocator. Use a known allocator
182 to be able to release the memory later. */
183 PyMemAllocatorEx old_alloc;
184 _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
185
Nick Coghland6009512014-11-20 21:39:37 +1000186 /* Can't call PyErr_NoMemory() on errors, as Python hasn't been
187 * initialised yet.
188 *
189 * However, the raw memory allocators are initialised appropriately
190 * as C static variables, so _PyMem_RawStrdup is OK even though
191 * Py_Initialize hasn't been called yet.
192 */
193 if (encoding) {
194 _Py_StandardStreamEncoding = _PyMem_RawStrdup(encoding);
195 if (!_Py_StandardStreamEncoding) {
Victor Stinner31e99082017-12-20 23:41:38 +0100196 res = -2;
197 goto done;
Nick Coghland6009512014-11-20 21:39:37 +1000198 }
199 }
200 if (errors) {
201 _Py_StandardStreamErrors = _PyMem_RawStrdup(errors);
202 if (!_Py_StandardStreamErrors) {
203 if (_Py_StandardStreamEncoding) {
204 PyMem_RawFree(_Py_StandardStreamEncoding);
205 }
Victor Stinner31e99082017-12-20 23:41:38 +0100206 res = -3;
207 goto done;
Nick Coghland6009512014-11-20 21:39:37 +1000208 }
209 }
Steve Dower39294992016-08-30 21:22:36 -0700210#ifdef MS_WINDOWS
211 if (_Py_StandardStreamEncoding) {
212 /* Overriding the stream encoding implies legacy streams */
213 Py_LegacyWindowsStdioFlag = 1;
214 }
215#endif
Victor Stinner31e99082017-12-20 23:41:38 +0100216
217done:
218 PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
219
220 return res;
Nick Coghland6009512014-11-20 21:39:37 +1000221}
222
Nick Coghlan6ea41862017-06-11 13:16:15 +1000223
Martin Panterb4ce1fc2015-11-30 03:18:29 +0000224/* Global initializations. Can be undone by Py_FinalizeEx(). Don't
225 call this twice without an intervening Py_FinalizeEx() call. When
Nick Coghland6009512014-11-20 21:39:37 +1000226 initializations fail, a fatal error is issued and the function does
227 not return. On return, the first thread and interpreter state have
228 been created.
229
230 Locking: you must hold the interpreter lock while calling this.
231 (If the lock has not yet been initialized, that's equivalent to
232 having the lock, but you cannot use multiple threads.)
233
234*/
235
Nick Coghland6009512014-11-20 21:39:37 +1000236static char*
237get_codec_name(const char *encoding)
238{
Serhiy Storchaka85b0f5b2016-11-20 10:16:47 +0200239 const char *name_utf8;
240 char *name_str;
Nick Coghland6009512014-11-20 21:39:37 +1000241 PyObject *codec, *name = NULL;
242
243 codec = _PyCodec_Lookup(encoding);
244 if (!codec)
245 goto error;
246
247 name = _PyObject_GetAttrId(codec, &PyId_name);
248 Py_CLEAR(codec);
249 if (!name)
250 goto error;
251
Serhiy Storchaka06515832016-11-20 09:13:07 +0200252 name_utf8 = PyUnicode_AsUTF8(name);
Nick Coghland6009512014-11-20 21:39:37 +1000253 if (name_utf8 == NULL)
254 goto error;
255 name_str = _PyMem_RawStrdup(name_utf8);
256 Py_DECREF(name);
257 if (name_str == NULL) {
258 PyErr_NoMemory();
259 return NULL;
260 }
261 return name_str;
262
263error:
264 Py_XDECREF(codec);
265 Py_XDECREF(name);
266 return NULL;
267}
268
269static char*
270get_locale_encoding(void)
271{
Benjamin Petersondb610e92017-09-08 14:30:07 -0700272#if defined(HAVE_LANGINFO_H) && defined(CODESET)
Nick Coghland6009512014-11-20 21:39:37 +1000273 char* codeset = nl_langinfo(CODESET);
274 if (!codeset || codeset[0] == '\0') {
275 PyErr_SetString(PyExc_ValueError, "CODESET is not set or empty");
276 return NULL;
277 }
278 return get_codec_name(codeset);
Stefan Krah144da4e2016-04-26 01:56:50 +0200279#elif defined(__ANDROID__)
280 return get_codec_name("UTF-8");
Nick Coghland6009512014-11-20 21:39:37 +1000281#else
282 PyErr_SetNone(PyExc_NotImplementedError);
283 return NULL;
284#endif
285}
286
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800287static _PyInitError
Eric Snow1abcf672017-05-23 21:46:51 -0700288initimport(PyInterpreterState *interp, PyObject *sysmod)
Nick Coghland6009512014-11-20 21:39:37 +1000289{
290 PyObject *importlib;
291 PyObject *impmod;
Nick Coghland6009512014-11-20 21:39:37 +1000292 PyObject *value;
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800293 _PyInitError err;
Nick Coghland6009512014-11-20 21:39:37 +1000294
295 /* Import _importlib through its frozen version, _frozen_importlib. */
296 if (PyImport_ImportFrozenModule("_frozen_importlib") <= 0) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800297 return _Py_INIT_ERR("can't import _frozen_importlib");
Nick Coghland6009512014-11-20 21:39:37 +1000298 }
299 else if (Py_VerboseFlag) {
300 PySys_FormatStderr("import _frozen_importlib # frozen\n");
301 }
302 importlib = PyImport_AddModule("_frozen_importlib");
303 if (importlib == NULL) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800304 return _Py_INIT_ERR("couldn't get _frozen_importlib from sys.modules");
Nick Coghland6009512014-11-20 21:39:37 +1000305 }
306 interp->importlib = importlib;
307 Py_INCREF(interp->importlib);
308
Serhiy Storchaka133138a2016-08-02 22:51:21 +0300309 interp->import_func = PyDict_GetItemString(interp->builtins, "__import__");
310 if (interp->import_func == NULL)
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800311 return _Py_INIT_ERR("__import__ not found");
Serhiy Storchaka133138a2016-08-02 22:51:21 +0300312 Py_INCREF(interp->import_func);
313
Victor Stinnercd6e6942015-09-18 09:11:57 +0200314 /* Import the _imp module */
Benjamin Petersonc65ef772018-01-29 11:33:57 -0800315 impmod = PyInit__imp();
Nick Coghland6009512014-11-20 21:39:37 +1000316 if (impmod == NULL) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800317 return _Py_INIT_ERR("can't import _imp");
Nick Coghland6009512014-11-20 21:39:37 +1000318 }
319 else if (Py_VerboseFlag) {
Victor Stinnercd6e6942015-09-18 09:11:57 +0200320 PySys_FormatStderr("import _imp # builtin\n");
Nick Coghland6009512014-11-20 21:39:37 +1000321 }
Eric Snow3f9eee62017-09-15 16:35:20 -0600322 if (_PyImport_SetModuleString("_imp", impmod) < 0) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800323 return _Py_INIT_ERR("can't save _imp to sys.modules");
Nick Coghland6009512014-11-20 21:39:37 +1000324 }
325
Victor Stinnercd6e6942015-09-18 09:11:57 +0200326 /* Install importlib as the implementation of import */
Nick Coghland6009512014-11-20 21:39:37 +1000327 value = PyObject_CallMethod(importlib, "_install", "OO", sysmod, impmod);
328 if (value == NULL) {
329 PyErr_Print();
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800330 return _Py_INIT_ERR("importlib install failed");
Nick Coghland6009512014-11-20 21:39:37 +1000331 }
332 Py_DECREF(value);
333 Py_DECREF(impmod);
334
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800335 err = _PyImportZip_Init();
336 if (_Py_INIT_FAILED(err)) {
337 return err;
338 }
339
340 return _Py_INIT_OK();
Nick Coghland6009512014-11-20 21:39:37 +1000341}
342
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800343static _PyInitError
Eric Snow1abcf672017-05-23 21:46:51 -0700344initexternalimport(PyInterpreterState *interp)
345{
346 PyObject *value;
347 value = PyObject_CallMethod(interp->importlib,
348 "_install_external_importers", "");
349 if (value == NULL) {
350 PyErr_Print();
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800351 return _Py_INIT_ERR("external importer setup failed");
Eric Snow1abcf672017-05-23 21:46:51 -0700352 }
Stéphane Wirtelab1cb802017-06-08 13:13:20 +0200353 Py_DECREF(value);
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800354 return _Py_INIT_OK();
Eric Snow1abcf672017-05-23 21:46:51 -0700355}
Nick Coghland6009512014-11-20 21:39:37 +1000356
Nick Coghlan6ea41862017-06-11 13:16:15 +1000357/* Helper functions to better handle the legacy C locale
358 *
359 * The legacy C locale assumes ASCII as the default text encoding, which
360 * causes problems not only for the CPython runtime, but also other
361 * components like GNU readline.
362 *
363 * Accordingly, when the CLI detects it, it attempts to coerce it to a
364 * more capable UTF-8 based alternative as follows:
365 *
366 * if (_Py_LegacyLocaleDetected()) {
367 * _Py_CoerceLegacyLocale();
368 * }
369 *
370 * See the documentation of the PYTHONCOERCECLOCALE setting for more details.
371 *
372 * Locale coercion also impacts the default error handler for the standard
373 * streams: while the usual default is "strict", the default for the legacy
374 * C locale and for any of the coercion target locales is "surrogateescape".
375 */
376
377int
378_Py_LegacyLocaleDetected(void)
379{
380#ifndef MS_WINDOWS
381 /* On non-Windows systems, the C locale is considered a legacy locale */
Nick Coghlaneb817952017-06-18 12:29:42 +1000382 /* XXX (ncoghlan): some platforms (notably Mac OS X) don't appear to treat
383 * the POSIX locale as a simple alias for the C locale, so
384 * we may also want to check for that explicitly.
385 */
Nick Coghlan6ea41862017-06-11 13:16:15 +1000386 const char *ctype_loc = setlocale(LC_CTYPE, NULL);
387 return ctype_loc != NULL && strcmp(ctype_loc, "C") == 0;
388#else
389 /* Windows uses code pages instead of locales, so no locale is legacy */
390 return 0;
391#endif
392}
393
Nick Coghlaneb817952017-06-18 12:29:42 +1000394static const char *_C_LOCALE_WARNING =
395 "Python runtime initialized with LC_CTYPE=C (a locale with default ASCII "
396 "encoding), which may cause Unicode compatibility problems. Using C.UTF-8, "
397 "C.utf8, or UTF-8 (if available) as alternative Unicode-compatible "
398 "locales is recommended.\n";
399
Nick Coghlaneb817952017-06-18 12:29:42 +1000400static void
Victor Stinner94540602017-12-16 04:54:22 +0100401_emit_stderr_warning_for_legacy_locale(const _PyCoreConfig *core_config)
Nick Coghlaneb817952017-06-18 12:29:42 +1000402{
Victor Stinner94540602017-12-16 04:54:22 +0100403 if (core_config->coerce_c_locale_warn) {
Nick Coghlaneb817952017-06-18 12:29:42 +1000404 if (_Py_LegacyLocaleDetected()) {
405 fprintf(stderr, "%s", _C_LOCALE_WARNING);
406 }
407 }
408}
409
Nick Coghlan6ea41862017-06-11 13:16:15 +1000410typedef struct _CandidateLocale {
411 const char *locale_name; /* The locale to try as a coercion target */
412} _LocaleCoercionTarget;
413
414static _LocaleCoercionTarget _TARGET_LOCALES[] = {
415 {"C.UTF-8"},
416 {"C.utf8"},
Nick Coghlan18974c32017-06-30 00:48:14 +1000417 {"UTF-8"},
Nick Coghlan6ea41862017-06-11 13:16:15 +1000418 {NULL}
419};
420
Serhiy Storchaka4ae06c52017-12-12 13:55:04 +0200421static const char *
Nick Coghlan6ea41862017-06-11 13:16:15 +1000422get_default_standard_stream_error_handler(void)
423{
424 const char *ctype_loc = setlocale(LC_CTYPE, NULL);
425 if (ctype_loc != NULL) {
426 /* "surrogateescape" is the default in the legacy C locale */
427 if (strcmp(ctype_loc, "C") == 0) {
428 return "surrogateescape";
429 }
430
431#ifdef PY_COERCE_C_LOCALE
432 /* "surrogateescape" is the default in locale coercion target locales */
433 const _LocaleCoercionTarget *target = NULL;
434 for (target = _TARGET_LOCALES; target->locale_name; target++) {
435 if (strcmp(ctype_loc, target->locale_name) == 0) {
436 return "surrogateescape";
437 }
438 }
439#endif
440 }
441
442 /* Otherwise return NULL to request the typical default error handler */
443 return NULL;
444}
445
446#ifdef PY_COERCE_C_LOCALE
Victor Stinner94540602017-12-16 04:54:22 +0100447static const char C_LOCALE_COERCION_WARNING[] =
Nick Coghlan6ea41862017-06-11 13:16:15 +1000448 "Python detected LC_CTYPE=C: LC_CTYPE coerced to %.20s (set another locale "
449 "or PYTHONCOERCECLOCALE=0 to disable this locale coercion behavior).\n";
450
451static void
Victor Stinner94540602017-12-16 04:54:22 +0100452_coerce_default_locale_settings(const _PyCoreConfig *config, const _LocaleCoercionTarget *target)
Nick Coghlan6ea41862017-06-11 13:16:15 +1000453{
454 const char *newloc = target->locale_name;
455
456 /* Reset locale back to currently configured defaults */
xdegaye1588be62017-11-12 12:45:59 +0100457 _Py_SetLocaleFromEnv(LC_ALL);
Nick Coghlan6ea41862017-06-11 13:16:15 +1000458
459 /* Set the relevant locale environment variable */
460 if (setenv("LC_CTYPE", newloc, 1)) {
461 fprintf(stderr,
462 "Error setting LC_CTYPE, skipping C locale coercion\n");
463 return;
464 }
Victor Stinner94540602017-12-16 04:54:22 +0100465 if (config->coerce_c_locale_warn) {
466 fprintf(stderr, C_LOCALE_COERCION_WARNING, newloc);
Nick Coghlaneb817952017-06-18 12:29:42 +1000467 }
Nick Coghlan6ea41862017-06-11 13:16:15 +1000468
469 /* Reconfigure with the overridden environment variables */
xdegaye1588be62017-11-12 12:45:59 +0100470 _Py_SetLocaleFromEnv(LC_ALL);
Nick Coghlan6ea41862017-06-11 13:16:15 +1000471}
472#endif
473
474void
Victor Stinner94540602017-12-16 04:54:22 +0100475_Py_CoerceLegacyLocale(const _PyCoreConfig *config)
Nick Coghlan6ea41862017-06-11 13:16:15 +1000476{
477#ifdef PY_COERCE_C_LOCALE
Victor Stinner94540602017-12-16 04:54:22 +0100478 const char *locale_override = getenv("LC_ALL");
479 if (locale_override == NULL || *locale_override == '\0') {
480 /* LC_ALL is also not set (or is set to an empty string) */
481 const _LocaleCoercionTarget *target = NULL;
482 for (target = _TARGET_LOCALES; target->locale_name; target++) {
483 const char *new_locale = setlocale(LC_CTYPE,
484 target->locale_name);
485 if (new_locale != NULL) {
xdegaye1588be62017-11-12 12:45:59 +0100486#if !defined(__APPLE__) && !defined(__ANDROID__) && \
Victor Stinner94540602017-12-16 04:54:22 +0100487defined(HAVE_LANGINFO_H) && defined(CODESET)
488 /* Also ensure that nl_langinfo works in this locale */
489 char *codeset = nl_langinfo(CODESET);
490 if (!codeset || *codeset == '\0') {
491 /* CODESET is not set or empty, so skip coercion */
492 new_locale = NULL;
493 _Py_SetLocaleFromEnv(LC_CTYPE);
494 continue;
Nick Coghlan6ea41862017-06-11 13:16:15 +1000495 }
Victor Stinner94540602017-12-16 04:54:22 +0100496#endif
497 /* Successfully configured locale, so make it the default */
498 _coerce_default_locale_settings(config, target);
499 return;
Nick Coghlan6ea41862017-06-11 13:16:15 +1000500 }
501 }
502 }
503 /* No C locale warning here, as Py_Initialize will emit one later */
504#endif
505}
506
xdegaye1588be62017-11-12 12:45:59 +0100507/* _Py_SetLocaleFromEnv() is a wrapper around setlocale(category, "") to
508 * isolate the idiosyncrasies of different libc implementations. It reads the
509 * appropriate environment variable and uses its value to select the locale for
510 * 'category'. */
511char *
512_Py_SetLocaleFromEnv(int category)
513{
514#ifdef __ANDROID__
515 const char *locale;
516 const char **pvar;
517#ifdef PY_COERCE_C_LOCALE
518 const char *coerce_c_locale;
519#endif
520 const char *utf8_locale = "C.UTF-8";
521 const char *env_var_set[] = {
522 "LC_ALL",
523 "LC_CTYPE",
524 "LANG",
525 NULL,
526 };
527
528 /* Android setlocale(category, "") doesn't check the environment variables
529 * and incorrectly sets the "C" locale at API 24 and older APIs. We only
530 * check the environment variables listed in env_var_set. */
531 for (pvar=env_var_set; *pvar; pvar++) {
532 locale = getenv(*pvar);
533 if (locale != NULL && *locale != '\0') {
534 if (strcmp(locale, utf8_locale) == 0 ||
535 strcmp(locale, "en_US.UTF-8") == 0) {
536 return setlocale(category, utf8_locale);
537 }
538 return setlocale(category, "C");
539 }
540 }
541
542 /* Android uses UTF-8, so explicitly set the locale to C.UTF-8 if none of
543 * LC_ALL, LC_CTYPE, or LANG is set to a non-empty string.
544 * Quote from POSIX section "8.2 Internationalization Variables":
545 * "4. If the LANG environment variable is not set or is set to the empty
546 * string, the implementation-defined default locale shall be used." */
547
548#ifdef PY_COERCE_C_LOCALE
549 coerce_c_locale = getenv("PYTHONCOERCECLOCALE");
550 if (coerce_c_locale == NULL || strcmp(coerce_c_locale, "0") != 0) {
551 /* Some other ported code may check the environment variables (e.g. in
552 * extension modules), so we make sure that they match the locale
553 * configuration */
554 if (setenv("LC_CTYPE", utf8_locale, 1)) {
555 fprintf(stderr, "Warning: failed setting the LC_CTYPE "
556 "environment variable to %s\n", utf8_locale);
557 }
558 }
559#endif
560 return setlocale(category, utf8_locale);
561#else /* __ANDROID__ */
562 return setlocale(category, "");
563#endif /* __ANDROID__ */
564}
565
Nick Coghlan6ea41862017-06-11 13:16:15 +1000566
Eric Snow1abcf672017-05-23 21:46:51 -0700567/* Global initializations. Can be undone by Py_Finalize(). Don't
568 call this twice without an intervening Py_Finalize() call.
569
570 Every call to Py_InitializeCore, Py_Initialize or Py_InitializeEx
571 must have a corresponding call to Py_Finalize.
572
573 Locking: you must hold the interpreter lock while calling these APIs.
574 (If the lock has not yet been initialized, that's equivalent to
575 having the lock, but you cannot use multiple threads.)
576
577*/
578
579/* Begin interpreter initialization
580 *
581 * On return, the first thread and interpreter state have been created,
582 * but the compiler, signal handling, multithreading and
583 * multiple interpreter support, and codec infrastructure are not yet
584 * available.
585 *
586 * The import system will support builtin and frozen modules only.
587 * The only supported io is writing to sys.stderr
588 *
589 * If any operation invoked by this function fails, a fatal error is
590 * issued and the function does not return.
591 *
592 * Any code invoked from this function should *not* assume it has access
593 * to the Python C API (unless the API is explicitly listed as being
594 * safe to call without calling Py_Initialize first)
595 */
596
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800597_PyInitError
Victor Stinnerda273412017-12-15 01:46:02 +0100598_Py_InitializeCore(const _PyCoreConfig *core_config)
Nick Coghland6009512014-11-20 21:39:37 +1000599{
Victor Stinnerda273412017-12-15 01:46:02 +0100600 assert(core_config != NULL);
601
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200602 _PyCoreConfig_SetGlobalConfig(core_config);
Nick Coghland6009512014-11-20 21:39:37 +1000603
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200604 _PyInitError err = _PyRuntime_Initialize();
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800605 if (_Py_INIT_FAILED(err)) {
606 return err;
607 }
Eric Snow2ebc5ce2017-09-07 23:51:28 -0600608
Victor Stinner31e99082017-12-20 23:41:38 +0100609 if (core_config->allocator != NULL) {
610 if (_PyMem_SetupAllocators(core_config->allocator) < 0) {
611 return _Py_INIT_USER_ERR("Unknown PYTHONMALLOC allocator");
612 }
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800613 }
614
Eric Snow2ebc5ce2017-09-07 23:51:28 -0600615 if (_PyRuntime.initialized) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800616 return _Py_INIT_ERR("main interpreter already initialized");
Eric Snow1abcf672017-05-23 21:46:51 -0700617 }
Eric Snow2ebc5ce2017-09-07 23:51:28 -0600618 if (_PyRuntime.core_initialized) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800619 return _Py_INIT_ERR("runtime core already initialized");
Eric Snow1abcf672017-05-23 21:46:51 -0700620 }
621
622 /* Py_Finalize leaves _Py_Finalizing set in order to help daemon
623 * threads behave a little more gracefully at interpreter shutdown.
624 * We clobber it here so the new interpreter can start with a clean
625 * slate.
626 *
627 * However, this may still lead to misbehaviour if there are daemon
628 * threads still hanging around from a previous Py_Initialize/Finalize
629 * pair :(
630 */
Eric Snow2ebc5ce2017-09-07 23:51:28 -0600631 _PyRuntime.finalizing = NULL;
632
Nick Coghlan6ea41862017-06-11 13:16:15 +1000633#ifndef MS_WINDOWS
Nick Coghland6009512014-11-20 21:39:37 +1000634 /* Set up the LC_CTYPE locale, so we can obtain
635 the locale's charset without having to switch
636 locales. */
xdegaye1588be62017-11-12 12:45:59 +0100637 _Py_SetLocaleFromEnv(LC_CTYPE);
Victor Stinner94540602017-12-16 04:54:22 +0100638 _emit_stderr_warning_for_legacy_locale(core_config);
Nick Coghlan6ea41862017-06-11 13:16:15 +1000639#endif
Nick Coghland6009512014-11-20 21:39:37 +1000640
Victor Stinnerda273412017-12-15 01:46:02 +0100641 err = _Py_HashRandomization_Init(core_config);
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800642 if (_Py_INIT_FAILED(err)) {
643 return err;
644 }
645
Victor Stinnera7368ac2017-11-15 18:11:45 -0800646 err = _PyInterpreterState_Enable(&_PyRuntime);
647 if (_Py_INIT_FAILED(err)) {
648 return err;
649 }
650
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200651 PyInterpreterState *interp = PyInterpreterState_New();
Victor Stinnerda273412017-12-15 01:46:02 +0100652 if (interp == NULL) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800653 return _Py_INIT_ERR("can't make main interpreter");
Victor Stinnerda273412017-12-15 01:46:02 +0100654 }
655
656 if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) {
657 return _Py_INIT_ERR("failed to copy core config");
658 }
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200659 core_config = &interp->core_config;
Nick Coghland6009512014-11-20 21:39:37 +1000660
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200661 PyThreadState *tstate = PyThreadState_New(interp);
Nick Coghland6009512014-11-20 21:39:37 +1000662 if (tstate == NULL)
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800663 return _Py_INIT_ERR("can't make first thread");
Nick Coghland6009512014-11-20 21:39:37 +1000664 (void) PyThreadState_Swap(tstate);
665
Martin Panterb4ce1fc2015-11-30 03:18:29 +0000666 /* We can't call _PyEval_FiniThreads() in Py_FinalizeEx because
Nick Coghland6009512014-11-20 21:39:37 +1000667 destroying the GIL might fail when it is being referenced from
668 another running thread (see issue #9901).
669 Instead we destroy the previously created GIL here, which ensures
Martin Panterb4ce1fc2015-11-30 03:18:29 +0000670 that we can call Py_Initialize / Py_FinalizeEx multiple times. */
Nick Coghland6009512014-11-20 21:39:37 +1000671 _PyEval_FiniThreads();
Victor Stinner2914bb32018-01-29 11:57:45 +0100672
Nick Coghland6009512014-11-20 21:39:37 +1000673 /* Auto-thread-state API */
674 _PyGILState_Init(interp, tstate);
Nick Coghland6009512014-11-20 21:39:37 +1000675
Victor Stinner2914bb32018-01-29 11:57:45 +0100676 /* Create the GIL */
677 PyEval_InitThreads();
678
Nick Coghland6009512014-11-20 21:39:37 +1000679 _Py_ReadyTypes();
680
Nick Coghland6009512014-11-20 21:39:37 +1000681 if (!_PyLong_Init())
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800682 return _Py_INIT_ERR("can't init longs");
Nick Coghland6009512014-11-20 21:39:37 +1000683
684 if (!PyByteArray_Init())
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800685 return _Py_INIT_ERR("can't init bytearray");
Nick Coghland6009512014-11-20 21:39:37 +1000686
687 if (!_PyFloat_Init())
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800688 return _Py_INIT_ERR("can't init float");
Nick Coghland6009512014-11-20 21:39:37 +1000689
Eric Snowd393c1b2017-09-14 12:18:12 -0600690 PyObject *modules = PyDict_New();
691 if (modules == NULL)
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800692 return _Py_INIT_ERR("can't make modules dictionary");
Eric Snowd393c1b2017-09-14 12:18:12 -0600693 interp->modules = modules;
694
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200695 PyObject *sysmod;
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800696 err = _PySys_BeginInit(&sysmod);
697 if (_Py_INIT_FAILED(err)) {
698 return err;
699 }
700
Eric Snowd393c1b2017-09-14 12:18:12 -0600701 interp->sysdict = PyModule_GetDict(sysmod);
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800702 if (interp->sysdict == NULL) {
703 return _Py_INIT_ERR("can't initialize sys dict");
704 }
705
Eric Snowd393c1b2017-09-14 12:18:12 -0600706 Py_INCREF(interp->sysdict);
707 PyDict_SetItemString(interp->sysdict, "modules", modules);
708 _PyImport_FixupBuiltin(sysmod, "sys", modules);
Nick Coghland6009512014-11-20 21:39:37 +1000709
710 /* Init Unicode implementation; relies on the codec registry */
711 if (_PyUnicode_Init() < 0)
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800712 return _Py_INIT_ERR("can't initialize unicode");
Eric Snow1abcf672017-05-23 21:46:51 -0700713
Nick Coghland6009512014-11-20 21:39:37 +1000714 if (_PyStructSequence_Init() < 0)
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800715 return _Py_INIT_ERR("can't initialize structseq");
Nick Coghland6009512014-11-20 21:39:37 +1000716
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200717 PyObject *bimod = _PyBuiltin_Init();
Nick Coghland6009512014-11-20 21:39:37 +1000718 if (bimod == NULL)
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800719 return _Py_INIT_ERR("can't initialize builtins modules");
Eric Snowd393c1b2017-09-14 12:18:12 -0600720 _PyImport_FixupBuiltin(bimod, "builtins", modules);
Nick Coghland6009512014-11-20 21:39:37 +1000721 interp->builtins = PyModule_GetDict(bimod);
722 if (interp->builtins == NULL)
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800723 return _Py_INIT_ERR("can't initialize builtins dict");
Nick Coghland6009512014-11-20 21:39:37 +1000724 Py_INCREF(interp->builtins);
725
726 /* initialize builtin exceptions */
727 _PyExc_Init(bimod);
728
Nick Coghland6009512014-11-20 21:39:37 +1000729 /* Set up a preliminary stderr printer until we have enough
730 infrastructure for the io module in place. */
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200731 PyObject *pstderr = PyFile_NewStdPrinter(fileno(stderr));
Nick Coghland6009512014-11-20 21:39:37 +1000732 if (pstderr == NULL)
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800733 return _Py_INIT_ERR("can't set preliminary stderr");
Nick Coghland6009512014-11-20 21:39:37 +1000734 _PySys_SetObjectId(&PyId_stderr, pstderr);
735 PySys_SetObject("__stderr__", pstderr);
736 Py_DECREF(pstderr);
737
Victor Stinner672b6ba2017-12-06 17:25:50 +0100738 err = _PyImport_Init(interp);
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800739 if (_Py_INIT_FAILED(err)) {
740 return err;
741 }
Nick Coghland6009512014-11-20 21:39:37 +1000742
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800743 err = _PyImportHooks_Init();
744 if (_Py_INIT_FAILED(err)) {
745 return err;
746 }
Nick Coghland6009512014-11-20 21:39:37 +1000747
748 /* Initialize _warnings. */
Victor Stinner5d862462017-12-19 11:35:58 +0100749 if (_PyWarnings_Init() == NULL) {
Victor Stinner1f151112017-11-23 10:43:14 +0100750 return _Py_INIT_ERR("can't initialize warnings");
751 }
Nick Coghland6009512014-11-20 21:39:37 +1000752
Yury Selivanovf23746a2018-01-22 19:11:18 -0500753 if (!_PyContext_Init())
754 return _Py_INIT_ERR("can't init context");
755
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200756 if (core_config->_install_importlib) {
Victor Stinnerb1147e42018-07-21 02:06:16 +0200757 err = _PyCoreConfig_SetPathConfig(core_config);
758 if (_Py_INIT_FAILED(err)) {
759 return err;
760 }
761 }
762
Eric Snow1abcf672017-05-23 21:46:51 -0700763 /* This call sets up builtin and frozen import support */
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200764 if (core_config->_install_importlib) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800765 err = initimport(interp, sysmod);
766 if (_Py_INIT_FAILED(err)) {
767 return err;
768 }
Eric Snow1abcf672017-05-23 21:46:51 -0700769 }
770
771 /* Only when we get here is the runtime core fully initialized */
Eric Snow2ebc5ce2017-09-07 23:51:28 -0600772 _PyRuntime.core_initialized = 1;
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800773 return _Py_INIT_OK();
Eric Snow1abcf672017-05-23 21:46:51 -0700774}
775
Victor Stinnerfb47bca2018-07-20 17:34:23 +0200776/* Py_Initialize() has already been called: update the main interpreter
777 configuration. Example of bpo-34008: Py_Main() called after
778 Py_Initialize(). */
779static _PyInitError
780_Py_ReconfigureMainInterpreter(PyInterpreterState *interp,
781 const _PyMainInterpreterConfig *config)
782{
783 if (config->argv != NULL) {
784 int res = PyDict_SetItemString(interp->sysdict, "argv", config->argv);
785 if (res < 0) {
786 return _Py_INIT_ERR("fail to set sys.argv");
787 }
788 }
789 return _Py_INIT_OK();
790}
791
Eric Snowc7ec9982017-05-23 23:00:52 -0700792/* Update interpreter state based on supplied configuration settings
793 *
794 * After calling this function, most of the restrictions on the interpreter
795 * are lifted. The only remaining incomplete settings are those related
796 * to the main module (sys.argv[0], __main__ metadata)
797 *
798 * Calling this when the interpreter is not initializing, is already
799 * initialized or without a valid current thread state is a fatal error.
800 * Other errors should be reported as normal Python exceptions with a
801 * non-zero return code.
802 */
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800803_PyInitError
804_Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config)
Eric Snow1abcf672017-05-23 21:46:51 -0700805{
Eric Snow2ebc5ce2017-09-07 23:51:28 -0600806 if (!_PyRuntime.core_initialized) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800807 return _Py_INIT_ERR("runtime core not initialized");
Eric Snowc7ec9982017-05-23 23:00:52 -0700808 }
Eric Snowc7ec9982017-05-23 23:00:52 -0700809
Eric Snow1abcf672017-05-23 21:46:51 -0700810 /* Get current thread state and interpreter pointer */
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200811 PyThreadState *tstate = PyThreadState_GET();
812 if (!tstate) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800813 return _Py_INIT_ERR("failed to read thread state");
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200814 }
815 PyInterpreterState *interp = tstate->interp;
816 if (!interp) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800817 return _Py_INIT_ERR("failed to get interpreter");
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200818 }
819 _PyCoreConfig *core_config = &interp->core_config;
Eric Snow1abcf672017-05-23 21:46:51 -0700820
821 /* Now finish configuring the main interpreter */
Victor Stinnerda273412017-12-15 01:46:02 +0100822 if (_PyMainInterpreterConfig_Copy(&interp->config, config) < 0) {
823 return _Py_INIT_ERR("failed to copy main interpreter config");
824 }
Eric Snowc7ec9982017-05-23 23:00:52 -0700825
Victor Stinnerfb47bca2018-07-20 17:34:23 +0200826 if (_PyRuntime.initialized) {
827 return _Py_ReconfigureMainInterpreter(interp, config);
828 }
829
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200830 if (!core_config->_install_importlib) {
Eric Snow1abcf672017-05-23 21:46:51 -0700831 /* Special mode for freeze_importlib: run with no import system
832 *
833 * This means anything which needs support from extension modules
834 * or pure Python code in the standard library won't work.
835 */
Eric Snow2ebc5ce2017-09-07 23:51:28 -0600836 _PyRuntime.initialized = 1;
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800837 return _Py_INIT_OK();
Eric Snow1abcf672017-05-23 21:46:51 -0700838 }
Victor Stinner9316ee42017-11-25 03:17:57 +0100839
Victor Stinner33c377e2017-12-05 15:12:41 +0100840 if (_PyTime_Init() < 0) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800841 return _Py_INIT_ERR("can't initialize time");
Victor Stinner33c377e2017-12-05 15:12:41 +0100842 }
Victor Stinner13019fd2015-04-03 13:10:54 +0200843
Victor Stinner41264f12017-12-15 02:05:29 +0100844 if (_PySys_EndInit(interp->sysdict, &interp->config) < 0) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800845 return _Py_INIT_ERR("can't finish initializing sys");
Victor Stinnerda273412017-12-15 01:46:02 +0100846 }
Victor Stinnera7368ac2017-11-15 18:11:45 -0800847
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200848 _PyInitError err = initexternalimport(interp);
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800849 if (_Py_INIT_FAILED(err)) {
850 return err;
851 }
Nick Coghland6009512014-11-20 21:39:37 +1000852
853 /* initialize the faulthandler module */
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200854 err = _PyFaulthandler_Init(core_config->faulthandler);
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800855 if (_Py_INIT_FAILED(err)) {
856 return err;
857 }
Nick Coghland6009512014-11-20 21:39:37 +1000858
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800859 err = initfsencoding(interp);
860 if (_Py_INIT_FAILED(err)) {
861 return err;
862 }
Nick Coghland6009512014-11-20 21:39:37 +1000863
Victor Stinner1f151112017-11-23 10:43:14 +0100864 if (interp->config.install_signal_handlers) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800865 err = initsigs(); /* Signal handling stuff, including initintr() */
866 if (_Py_INIT_FAILED(err)) {
867 return err;
868 }
869 }
Nick Coghland6009512014-11-20 21:39:37 +1000870
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200871 if (_PyTraceMalloc_Init(core_config->tracemalloc) < 0) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800872 return _Py_INIT_ERR("can't initialize tracemalloc");
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200873 }
Nick Coghland6009512014-11-20 21:39:37 +1000874
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800875 err = add_main_module(interp);
876 if (_Py_INIT_FAILED(err)) {
877 return err;
878 }
Victor Stinnera7368ac2017-11-15 18:11:45 -0800879
Victor Stinner91106cd2017-12-13 12:29:09 +0100880 err = init_sys_streams(interp);
Victor Stinnera7368ac2017-11-15 18:11:45 -0800881 if (_Py_INIT_FAILED(err)) {
882 return err;
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800883 }
Nick Coghland6009512014-11-20 21:39:37 +1000884
885 /* Initialize warnings. */
Victor Stinner5d862462017-12-19 11:35:58 +0100886 if (interp->config.warnoptions != NULL &&
887 PyList_Size(interp->config.warnoptions) > 0)
888 {
Nick Coghland6009512014-11-20 21:39:37 +1000889 PyObject *warnings_module = PyImport_ImportModule("warnings");
890 if (warnings_module == NULL) {
891 fprintf(stderr, "'import warnings' failed; traceback:\n");
892 PyErr_Print();
893 }
894 Py_XDECREF(warnings_module);
895 }
896
Eric Snow2ebc5ce2017-09-07 23:51:28 -0600897 _PyRuntime.initialized = 1;
Eric Snow1abcf672017-05-23 21:46:51 -0700898
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200899 if (core_config->site_import) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800900 err = initsite(); /* Module site */
901 if (_Py_INIT_FAILED(err)) {
902 return err;
903 }
904 }
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800905 return _Py_INIT_OK();
Nick Coghland6009512014-11-20 21:39:37 +1000906}
907
Eric Snowc7ec9982017-05-23 23:00:52 -0700908#undef _INIT_DEBUG_PRINT
909
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800910_PyInitError
Eric Snow1abcf672017-05-23 21:46:51 -0700911_Py_InitializeEx_Private(int install_sigs, int install_importlib)
912{
Victor Stinner209abf72018-06-22 19:14:51 +0200913 if (_PyRuntime.initialized) {
914 /* bpo-33932: Calling Py_Initialize() twice does nothing. */
915 return _Py_INIT_OK();
916 }
917
Victor Stinner9cfc0022017-12-20 19:36:46 +0100918 _PyCoreConfig config = _PyCoreConfig_INIT;
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800919 _PyInitError err;
Eric Snow1abcf672017-05-23 21:46:51 -0700920
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200921 config._install_importlib = install_importlib;
Eric Snowc7ec9982017-05-23 23:00:52 -0700922 config.install_signal_handlers = install_sigs;
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800923
Victor Stinnerd19d8d52018-07-24 13:55:48 +0200924 err = _PyCoreConfig_Read(&config);
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800925 if (_Py_INIT_FAILED(err)) {
Victor Stinnerbc8ac6b2017-11-30 18:03:55 +0100926 goto done;
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800927 }
928
Victor Stinner9cfc0022017-12-20 19:36:46 +0100929 err = _Py_InitializeCore(&config);
Victor Stinnerbc8ac6b2017-11-30 18:03:55 +0100930 if (_Py_INIT_FAILED(err)) {
931 goto done;
932 }
933
Victor Stinner9cfc0022017-12-20 19:36:46 +0100934 _PyMainInterpreterConfig main_config = _PyMainInterpreterConfig_INIT;
935 err = _PyMainInterpreterConfig_Read(&main_config, &config);
936 if (!_Py_INIT_FAILED(err)) {
937 err = _Py_InitializeMainInterpreter(&main_config);
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800938 }
Victor Stinner9cfc0022017-12-20 19:36:46 +0100939 _PyMainInterpreterConfig_Clear(&main_config);
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800940 if (_Py_INIT_FAILED(err)) {
Victor Stinnerbc8ac6b2017-11-30 18:03:55 +0100941 goto done;
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800942 }
943
Victor Stinnerbc8ac6b2017-11-30 18:03:55 +0100944 err = _Py_INIT_OK();
945
946done:
Victor Stinner9cfc0022017-12-20 19:36:46 +0100947 _PyCoreConfig_Clear(&config);
Victor Stinnerbc8ac6b2017-11-30 18:03:55 +0100948 return err;
Eric Snow1abcf672017-05-23 21:46:51 -0700949}
950
951
952void
Nick Coghland6009512014-11-20 21:39:37 +1000953Py_InitializeEx(int install_sigs)
954{
Victor Stinnerf7e5b562017-11-15 15:48:08 -0800955 _PyInitError err = _Py_InitializeEx_Private(install_sigs, 1);
956 if (_Py_INIT_FAILED(err)) {
957 _Py_FatalInitError(err);
958 }
Nick Coghland6009512014-11-20 21:39:37 +1000959}
960
961void
962Py_Initialize(void)
963{
964 Py_InitializeEx(1);
965}
966
967
968#ifdef COUNT_ALLOCS
969extern void dump_counts(FILE*);
970#endif
971
972/* Flush stdout and stderr */
973
974static int
975file_is_closed(PyObject *fobj)
976{
977 int r;
978 PyObject *tmp = PyObject_GetAttrString(fobj, "closed");
979 if (tmp == NULL) {
980 PyErr_Clear();
981 return 0;
982 }
983 r = PyObject_IsTrue(tmp);
984 Py_DECREF(tmp);
985 if (r < 0)
986 PyErr_Clear();
987 return r > 0;
988}
989
Martin Panterb4ce1fc2015-11-30 03:18:29 +0000990static int
Nick Coghland6009512014-11-20 21:39:37 +1000991flush_std_files(void)
992{
993 PyObject *fout = _PySys_GetObjectId(&PyId_stdout);
994 PyObject *ferr = _PySys_GetObjectId(&PyId_stderr);
995 PyObject *tmp;
Martin Panterb4ce1fc2015-11-30 03:18:29 +0000996 int status = 0;
Nick Coghland6009512014-11-20 21:39:37 +1000997
998 if (fout != NULL && fout != Py_None && !file_is_closed(fout)) {
Victor Stinner3466bde2016-09-05 18:16:01 -0700999 tmp = _PyObject_CallMethodId(fout, &PyId_flush, NULL);
Martin Panterb4ce1fc2015-11-30 03:18:29 +00001000 if (tmp == NULL) {
Nick Coghland6009512014-11-20 21:39:37 +10001001 PyErr_WriteUnraisable(fout);
Martin Panterb4ce1fc2015-11-30 03:18:29 +00001002 status = -1;
1003 }
Nick Coghland6009512014-11-20 21:39:37 +10001004 else
1005 Py_DECREF(tmp);
1006 }
1007
1008 if (ferr != NULL && ferr != Py_None && !file_is_closed(ferr)) {
Victor Stinner3466bde2016-09-05 18:16:01 -07001009 tmp = _PyObject_CallMethodId(ferr, &PyId_flush, NULL);
Martin Panterb4ce1fc2015-11-30 03:18:29 +00001010 if (tmp == NULL) {
Nick Coghland6009512014-11-20 21:39:37 +10001011 PyErr_Clear();
Martin Panterb4ce1fc2015-11-30 03:18:29 +00001012 status = -1;
1013 }
Nick Coghland6009512014-11-20 21:39:37 +10001014 else
1015 Py_DECREF(tmp);
1016 }
Martin Panterb4ce1fc2015-11-30 03:18:29 +00001017
1018 return status;
Nick Coghland6009512014-11-20 21:39:37 +10001019}
1020
1021/* Undo the effect of Py_Initialize().
1022
1023 Beware: if multiple interpreter and/or thread states exist, these
1024 are not wiped out; only the current thread and interpreter state
1025 are deleted. But since everything else is deleted, those other
1026 interpreter and thread states should no longer be used.
1027
1028 (XXX We should do better, e.g. wipe out all interpreters and
1029 threads.)
1030
1031 Locking: as above.
1032
1033*/
1034
Martin Panterb4ce1fc2015-11-30 03:18:29 +00001035int
1036Py_FinalizeEx(void)
Nick Coghland6009512014-11-20 21:39:37 +10001037{
1038 PyInterpreterState *interp;
1039 PyThreadState *tstate;
Martin Panterb4ce1fc2015-11-30 03:18:29 +00001040 int status = 0;
Nick Coghland6009512014-11-20 21:39:37 +10001041
Eric Snow2ebc5ce2017-09-07 23:51:28 -06001042 if (!_PyRuntime.initialized)
Martin Panterb4ce1fc2015-11-30 03:18:29 +00001043 return status;
Nick Coghland6009512014-11-20 21:39:37 +10001044
1045 wait_for_thread_shutdown();
1046
Marcel Plch776407f2017-12-20 11:17:58 +01001047 /* Get current thread state and interpreter pointer */
1048 tstate = PyThreadState_GET();
1049 interp = tstate->interp;
1050
Nick Coghland6009512014-11-20 21:39:37 +10001051 /* The interpreter is still entirely intact at this point, and the
1052 * exit funcs may be relying on that. In particular, if some thread
1053 * or exit func is still waiting to do an import, the import machinery
1054 * expects Py_IsInitialized() to return true. So don't say the
1055 * interpreter is uninitialized until after the exit funcs have run.
1056 * Note that Threading.py uses an exit func to do a join on all the
1057 * threads created thru it, so this also protects pending imports in
1058 * the threads created via Threading.
1059 */
Nick Coghland6009512014-11-20 21:39:37 +10001060
Marcel Plch776407f2017-12-20 11:17:58 +01001061 call_py_exitfuncs(interp);
Nick Coghland6009512014-11-20 21:39:37 +10001062
Victor Stinnerda273412017-12-15 01:46:02 +01001063 /* Copy the core config, PyInterpreterState_Delete() free
1064 the core config memory */
Victor Stinner5d862462017-12-19 11:35:58 +01001065#ifdef Py_REF_DEBUG
Victor Stinnerda273412017-12-15 01:46:02 +01001066 int show_ref_count = interp->core_config.show_ref_count;
Victor Stinner5d862462017-12-19 11:35:58 +01001067#endif
1068#ifdef Py_TRACE_REFS
Victor Stinnerda273412017-12-15 01:46:02 +01001069 int dump_refs = interp->core_config.dump_refs;
Victor Stinner5d862462017-12-19 11:35:58 +01001070#endif
1071#ifdef WITH_PYMALLOC
Victor Stinnerda273412017-12-15 01:46:02 +01001072 int malloc_stats = interp->core_config.malloc_stats;
Victor Stinner5d862462017-12-19 11:35:58 +01001073#endif
Victor Stinner6bf992a2017-12-06 17:26:10 +01001074
Nick Coghland6009512014-11-20 21:39:37 +10001075 /* Remaining threads (e.g. daemon threads) will automatically exit
1076 after taking the GIL (in PyEval_RestoreThread()). */
Eric Snow2ebc5ce2017-09-07 23:51:28 -06001077 _PyRuntime.finalizing = tstate;
1078 _PyRuntime.initialized = 0;
1079 _PyRuntime.core_initialized = 0;
Nick Coghland6009512014-11-20 21:39:37 +10001080
Victor Stinnere0deff32015-03-24 13:46:18 +01001081 /* Flush sys.stdout and sys.stderr */
Martin Panterb4ce1fc2015-11-30 03:18:29 +00001082 if (flush_std_files() < 0) {
1083 status = -1;
1084 }
Nick Coghland6009512014-11-20 21:39:37 +10001085
1086 /* Disable signal handling */
1087 PyOS_FiniInterrupts();
1088
1089 /* Collect garbage. This may call finalizers; it's nice to call these
1090 * before all modules are destroyed.
1091 * XXX If a __del__ or weakref callback is triggered here, and tries to
1092 * XXX import a module, bad things can happen, because Python no
1093 * XXX longer believes it's initialized.
1094 * XXX Fatal Python error: Interpreter not initialized (version mismatch?)
1095 * XXX is easy to provoke that way. I've also seen, e.g.,
1096 * XXX Exception exceptions.ImportError: 'No module named sha'
1097 * XXX in <function callback at 0x008F5718> ignored
1098 * XXX but I'm unclear on exactly how that one happens. In any case,
1099 * XXX I haven't seen a real-life report of either of these.
1100 */
Łukasz Langafef7e942016-09-09 21:47:46 -07001101 _PyGC_CollectIfEnabled();
Nick Coghland6009512014-11-20 21:39:37 +10001102#ifdef COUNT_ALLOCS
1103 /* With COUNT_ALLOCS, it helps to run GC multiple times:
1104 each collection might release some types from the type
1105 list, so they become garbage. */
Łukasz Langafef7e942016-09-09 21:47:46 -07001106 while (_PyGC_CollectIfEnabled() > 0)
Nick Coghland6009512014-11-20 21:39:37 +10001107 /* nothing */;
1108#endif
Eric Snowdae02762017-09-14 00:35:58 -07001109
Nick Coghland6009512014-11-20 21:39:37 +10001110 /* Destroy all modules */
1111 PyImport_Cleanup();
1112
Victor Stinnere0deff32015-03-24 13:46:18 +01001113 /* Flush sys.stdout and sys.stderr (again, in case more was printed) */
Martin Panterb4ce1fc2015-11-30 03:18:29 +00001114 if (flush_std_files() < 0) {
1115 status = -1;
1116 }
Nick Coghland6009512014-11-20 21:39:37 +10001117
1118 /* Collect final garbage. This disposes of cycles created by
1119 * class definitions, for example.
1120 * XXX This is disabled because it caused too many problems. If
1121 * XXX a __del__ or weakref callback triggers here, Python code has
1122 * XXX a hard time running, because even the sys module has been
1123 * XXX cleared out (sys.stdout is gone, sys.excepthook is gone, etc).
1124 * XXX One symptom is a sequence of information-free messages
1125 * XXX coming from threads (if a __del__ or callback is invoked,
1126 * XXX other threads can execute too, and any exception they encounter
1127 * XXX triggers a comedy of errors as subsystem after subsystem
1128 * XXX fails to find what it *expects* to find in sys to help report
1129 * XXX the exception and consequent unexpected failures). I've also
1130 * XXX seen segfaults then, after adding print statements to the
1131 * XXX Python code getting called.
1132 */
1133#if 0
Łukasz Langafef7e942016-09-09 21:47:46 -07001134 _PyGC_CollectIfEnabled();
Nick Coghland6009512014-11-20 21:39:37 +10001135#endif
1136
1137 /* Disable tracemalloc after all Python objects have been destroyed,
1138 so it is possible to use tracemalloc in objects destructor. */
1139 _PyTraceMalloc_Fini();
1140
1141 /* Destroy the database used by _PyImport_{Fixup,Find}Extension */
1142 _PyImport_Fini();
1143
1144 /* Cleanup typeobject.c's internal caches. */
1145 _PyType_Fini();
1146
1147 /* unload faulthandler module */
1148 _PyFaulthandler_Fini();
1149
1150 /* Debugging stuff */
1151#ifdef COUNT_ALLOCS
Serhiy Storchaka7e160ce2016-07-03 21:03:53 +03001152 dump_counts(stderr);
Nick Coghland6009512014-11-20 21:39:37 +10001153#endif
1154 /* dump hash stats */
1155 _PyHash_Fini();
1156
Eric Snowdae02762017-09-14 00:35:58 -07001157#ifdef Py_REF_DEBUG
Victor Stinnerda273412017-12-15 01:46:02 +01001158 if (show_ref_count) {
Victor Stinner25420fe2017-11-20 18:12:22 -08001159 _PyDebug_PrintTotalRefs();
1160 }
Eric Snowdae02762017-09-14 00:35:58 -07001161#endif
Nick Coghland6009512014-11-20 21:39:37 +10001162
1163#ifdef Py_TRACE_REFS
1164 /* Display all objects still alive -- this can invoke arbitrary
1165 * __repr__ overrides, so requires a mostly-intact interpreter.
1166 * Alas, a lot of stuff may still be alive now that will be cleaned
1167 * up later.
1168 */
Victor Stinnerda273412017-12-15 01:46:02 +01001169 if (dump_refs) {
Nick Coghland6009512014-11-20 21:39:37 +10001170 _Py_PrintReferences(stderr);
Victor Stinner6bf992a2017-12-06 17:26:10 +01001171 }
Nick Coghland6009512014-11-20 21:39:37 +10001172#endif /* Py_TRACE_REFS */
1173
1174 /* Clear interpreter state and all thread states. */
1175 PyInterpreterState_Clear(interp);
1176
1177 /* Now we decref the exception classes. After this point nothing
1178 can raise an exception. That's okay, because each Fini() method
1179 below has been checked to make sure no exceptions are ever
1180 raised.
1181 */
1182
1183 _PyExc_Fini();
1184
1185 /* Sundry finalizers */
1186 PyMethod_Fini();
1187 PyFrame_Fini();
1188 PyCFunction_Fini();
1189 PyTuple_Fini();
1190 PyList_Fini();
1191 PySet_Fini();
1192 PyBytes_Fini();
1193 PyByteArray_Fini();
1194 PyLong_Fini();
1195 PyFloat_Fini();
1196 PyDict_Fini();
1197 PySlice_Fini();
1198 _PyGC_Fini();
Eric Snow6b4be192017-05-22 21:36:03 -07001199 _Py_HashRandomization_Fini();
Serhiy Storchaka9171a8b2016-08-14 10:52:18 +03001200 _PyArg_Fini();
Yury Selivanoveb636452016-09-08 22:01:51 -07001201 PyAsyncGen_Fini();
Yury Selivanovf23746a2018-01-22 19:11:18 -05001202 _PyContext_Fini();
Nick Coghland6009512014-11-20 21:39:37 +10001203
1204 /* Cleanup Unicode implementation */
1205 _PyUnicode_Fini();
1206
1207 /* reset file system default encoding */
1208 if (!Py_HasFileSystemDefaultEncoding && Py_FileSystemDefaultEncoding) {
1209 PyMem_RawFree((char*)Py_FileSystemDefaultEncoding);
1210 Py_FileSystemDefaultEncoding = NULL;
1211 }
1212
1213 /* XXX Still allocated:
1214 - various static ad-hoc pointers to interned strings
1215 - int and float free list blocks
1216 - whatever various modules and libraries allocate
1217 */
1218
1219 PyGrammar_RemoveAccelerators(&_PyParser_Grammar);
1220
1221 /* Cleanup auto-thread-state */
Nick Coghland6009512014-11-20 21:39:37 +10001222 _PyGILState_Fini();
Nick Coghland6009512014-11-20 21:39:37 +10001223
1224 /* Delete current thread. After this, many C API calls become crashy. */
1225 PyThreadState_Swap(NULL);
Victor Stinner8a1be612016-03-14 22:07:55 +01001226
Nick Coghland6009512014-11-20 21:39:37 +10001227 PyInterpreterState_Delete(interp);
1228
1229#ifdef Py_TRACE_REFS
1230 /* Display addresses (& refcnts) of all objects still alive.
1231 * An address can be used to find the repr of the object, printed
1232 * above by _Py_PrintReferences.
1233 */
Victor Stinnerda273412017-12-15 01:46:02 +01001234 if (dump_refs) {
Nick Coghland6009512014-11-20 21:39:37 +10001235 _Py_PrintReferenceAddresses(stderr);
Victor Stinner6bf992a2017-12-06 17:26:10 +01001236 }
Nick Coghland6009512014-11-20 21:39:37 +10001237#endif /* Py_TRACE_REFS */
Victor Stinner34be807c2016-03-14 12:04:26 +01001238#ifdef WITH_PYMALLOC
Victor Stinnerda273412017-12-15 01:46:02 +01001239 if (malloc_stats) {
Victor Stinner6bf992a2017-12-06 17:26:10 +01001240 _PyObject_DebugMallocStats(stderr);
Victor Stinner34be807c2016-03-14 12:04:26 +01001241 }
Nick Coghland6009512014-11-20 21:39:37 +10001242#endif
1243
1244 call_ll_exitfuncs();
Victor Stinner9316ee42017-11-25 03:17:57 +01001245
Eric Snow2ebc5ce2017-09-07 23:51:28 -06001246 _PyRuntime_Finalize();
Martin Panterb4ce1fc2015-11-30 03:18:29 +00001247 return status;
1248}
1249
1250void
1251Py_Finalize(void)
1252{
1253 Py_FinalizeEx();
Nick Coghland6009512014-11-20 21:39:37 +10001254}
1255
1256/* Create and initialize a new interpreter and thread, and return the
1257 new thread. This requires that Py_Initialize() has been called
1258 first.
1259
1260 Unsuccessful initialization yields a NULL pointer. Note that *no*
1261 exception information is available even in this case -- the
1262 exception information is held in the thread, and there is no
1263 thread.
1264
1265 Locking: as above.
1266
1267*/
1268
Victor Stinnera7368ac2017-11-15 18:11:45 -08001269static _PyInitError
1270new_interpreter(PyThreadState **tstate_p)
Nick Coghland6009512014-11-20 21:39:37 +10001271{
1272 PyInterpreterState *interp;
1273 PyThreadState *tstate, *save_tstate;
1274 PyObject *bimod, *sysmod;
Victor Stinner9316ee42017-11-25 03:17:57 +01001275 _PyInitError err;
Nick Coghland6009512014-11-20 21:39:37 +10001276
Victor Stinnera7368ac2017-11-15 18:11:45 -08001277 if (!_PyRuntime.initialized) {
1278 return _Py_INIT_ERR("Py_Initialize must be called first");
1279 }
Nick Coghland6009512014-11-20 21:39:37 +10001280
Victor Stinner8a1be612016-03-14 22:07:55 +01001281 /* Issue #10915, #15751: The GIL API doesn't work with multiple
1282 interpreters: disable PyGILState_Check(). */
1283 _PyGILState_check_enabled = 0;
1284
Nick Coghland6009512014-11-20 21:39:37 +10001285 interp = PyInterpreterState_New();
Victor Stinnera7368ac2017-11-15 18:11:45 -08001286 if (interp == NULL) {
1287 *tstate_p = NULL;
1288 return _Py_INIT_OK();
1289 }
Nick Coghland6009512014-11-20 21:39:37 +10001290
1291 tstate = PyThreadState_New(interp);
1292 if (tstate == NULL) {
1293 PyInterpreterState_Delete(interp);
Victor Stinnera7368ac2017-11-15 18:11:45 -08001294 *tstate_p = NULL;
1295 return _Py_INIT_OK();
Nick Coghland6009512014-11-20 21:39:37 +10001296 }
1297
1298 save_tstate = PyThreadState_Swap(tstate);
1299
Eric Snow1abcf672017-05-23 21:46:51 -07001300 /* Copy the current interpreter config into the new interpreter */
Victor Stinnerda273412017-12-15 01:46:02 +01001301 _PyCoreConfig *core_config;
1302 _PyMainInterpreterConfig *config;
Eric Snow1abcf672017-05-23 21:46:51 -07001303 if (save_tstate != NULL) {
Victor Stinnerda273412017-12-15 01:46:02 +01001304 core_config = &save_tstate->interp->core_config;
1305 config = &save_tstate->interp->config;
Eric Snow1abcf672017-05-23 21:46:51 -07001306 } else {
1307 /* No current thread state, copy from the main interpreter */
1308 PyInterpreterState *main_interp = PyInterpreterState_Main();
Victor Stinnerda273412017-12-15 01:46:02 +01001309 core_config = &main_interp->core_config;
1310 config = &main_interp->config;
1311 }
1312
1313 if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) {
1314 return _Py_INIT_ERR("failed to copy core config");
1315 }
Victor Stinnerd19d8d52018-07-24 13:55:48 +02001316 core_config = &interp->core_config;
Victor Stinnerda273412017-12-15 01:46:02 +01001317 if (_PyMainInterpreterConfig_Copy(&interp->config, config) < 0) {
1318 return _Py_INIT_ERR("failed to copy main interpreter config");
Eric Snow1abcf672017-05-23 21:46:51 -07001319 }
1320
Nick Coghland6009512014-11-20 21:39:37 +10001321 /* XXX The following is lax in error checking */
Eric Snowd393c1b2017-09-14 12:18:12 -06001322 PyObject *modules = PyDict_New();
Victor Stinnera7368ac2017-11-15 18:11:45 -08001323 if (modules == NULL) {
1324 return _Py_INIT_ERR("can't make modules dictionary");
1325 }
Eric Snowd393c1b2017-09-14 12:18:12 -06001326 interp->modules = modules;
Nick Coghland6009512014-11-20 21:39:37 +10001327
Eric Snowd393c1b2017-09-14 12:18:12 -06001328 sysmod = _PyImport_FindBuiltin("sys", modules);
1329 if (sysmod != NULL) {
1330 interp->sysdict = PyModule_GetDict(sysmod);
1331 if (interp->sysdict == NULL)
1332 goto handle_error;
1333 Py_INCREF(interp->sysdict);
1334 PyDict_SetItemString(interp->sysdict, "modules", modules);
Victor Stinner41264f12017-12-15 02:05:29 +01001335 _PySys_EndInit(interp->sysdict, &interp->config);
Eric Snowd393c1b2017-09-14 12:18:12 -06001336 }
1337
1338 bimod = _PyImport_FindBuiltin("builtins", modules);
Nick Coghland6009512014-11-20 21:39:37 +10001339 if (bimod != NULL) {
1340 interp->builtins = PyModule_GetDict(bimod);
1341 if (interp->builtins == NULL)
1342 goto handle_error;
1343 Py_INCREF(interp->builtins);
1344 }
1345
1346 /* initialize builtin exceptions */
1347 _PyExc_Init(bimod);
1348
Nick Coghland6009512014-11-20 21:39:37 +10001349 if (bimod != NULL && sysmod != NULL) {
1350 PyObject *pstderr;
1351
Nick Coghland6009512014-11-20 21:39:37 +10001352 /* Set up a preliminary stderr printer until we have enough
1353 infrastructure for the io module in place. */
1354 pstderr = PyFile_NewStdPrinter(fileno(stderr));
Victor Stinnera7368ac2017-11-15 18:11:45 -08001355 if (pstderr == NULL) {
1356 return _Py_INIT_ERR("can't set preliminary stderr");
1357 }
Nick Coghland6009512014-11-20 21:39:37 +10001358 _PySys_SetObjectId(&PyId_stderr, pstderr);
1359 PySys_SetObject("__stderr__", pstderr);
1360 Py_DECREF(pstderr);
1361
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001362 err = _PyImportHooks_Init();
1363 if (_Py_INIT_FAILED(err)) {
Victor Stinnera7368ac2017-11-15 18:11:45 -08001364 return err;
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001365 }
Nick Coghland6009512014-11-20 21:39:37 +10001366
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001367 err = initimport(interp, sysmod);
1368 if (_Py_INIT_FAILED(err)) {
Victor Stinnera7368ac2017-11-15 18:11:45 -08001369 return err;
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001370 }
Nick Coghland6009512014-11-20 21:39:37 +10001371
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001372 err = initexternalimport(interp);
1373 if (_Py_INIT_FAILED(err)) {
Victor Stinnera7368ac2017-11-15 18:11:45 -08001374 return err;
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001375 }
Nick Coghland6009512014-11-20 21:39:37 +10001376
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001377 err = initfsencoding(interp);
1378 if (_Py_INIT_FAILED(err)) {
Victor Stinnera7368ac2017-11-15 18:11:45 -08001379 return err;
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001380 }
1381
Victor Stinner91106cd2017-12-13 12:29:09 +01001382 err = init_sys_streams(interp);
Victor Stinnera7368ac2017-11-15 18:11:45 -08001383 if (_Py_INIT_FAILED(err)) {
1384 return err;
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001385 }
1386
1387 err = add_main_module(interp);
1388 if (_Py_INIT_FAILED(err)) {
Victor Stinnera7368ac2017-11-15 18:11:45 -08001389 return err;
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001390 }
1391
Victor Stinnerd19d8d52018-07-24 13:55:48 +02001392 if (core_config->site_import) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001393 err = initsite();
1394 if (_Py_INIT_FAILED(err)) {
Victor Stinnera7368ac2017-11-15 18:11:45 -08001395 return err;
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001396 }
1397 }
Nick Coghland6009512014-11-20 21:39:37 +10001398 }
1399
Victor Stinnera7368ac2017-11-15 18:11:45 -08001400 if (PyErr_Occurred()) {
1401 goto handle_error;
1402 }
Nick Coghland6009512014-11-20 21:39:37 +10001403
Victor Stinnera7368ac2017-11-15 18:11:45 -08001404 *tstate_p = tstate;
1405 return _Py_INIT_OK();
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001406
Nick Coghland6009512014-11-20 21:39:37 +10001407handle_error:
1408 /* Oops, it didn't work. Undo it all. */
1409
1410 PyErr_PrintEx(0);
1411 PyThreadState_Clear(tstate);
1412 PyThreadState_Swap(save_tstate);
1413 PyThreadState_Delete(tstate);
1414 PyInterpreterState_Delete(interp);
1415
Victor Stinnera7368ac2017-11-15 18:11:45 -08001416 *tstate_p = NULL;
1417 return _Py_INIT_OK();
1418}
1419
1420PyThreadState *
1421Py_NewInterpreter(void)
1422{
1423 PyThreadState *tstate;
1424 _PyInitError err = new_interpreter(&tstate);
1425 if (_Py_INIT_FAILED(err)) {
1426 _Py_FatalInitError(err);
1427 }
1428 return tstate;
1429
Nick Coghland6009512014-11-20 21:39:37 +10001430}
1431
1432/* Delete an interpreter and its last thread. This requires that the
1433 given thread state is current, that the thread has no remaining
1434 frames, and that it is its interpreter's only remaining thread.
1435 It is a fatal error to violate these constraints.
1436
Martin Panterb4ce1fc2015-11-30 03:18:29 +00001437 (Py_FinalizeEx() doesn't have these constraints -- it zaps
Nick Coghland6009512014-11-20 21:39:37 +10001438 everything, regardless.)
1439
1440 Locking: as above.
1441
1442*/
1443
1444void
1445Py_EndInterpreter(PyThreadState *tstate)
1446{
1447 PyInterpreterState *interp = tstate->interp;
1448
1449 if (tstate != PyThreadState_GET())
1450 Py_FatalError("Py_EndInterpreter: thread is not current");
1451 if (tstate->frame != NULL)
1452 Py_FatalError("Py_EndInterpreter: thread still has a frame");
1453
1454 wait_for_thread_shutdown();
1455
Marcel Plch776407f2017-12-20 11:17:58 +01001456 call_py_exitfuncs(interp);
1457
Nick Coghland6009512014-11-20 21:39:37 +10001458 if (tstate != interp->tstate_head || tstate->next != NULL)
1459 Py_FatalError("Py_EndInterpreter: not the last thread");
1460
1461 PyImport_Cleanup();
1462 PyInterpreterState_Clear(interp);
1463 PyThreadState_Swap(NULL);
1464 PyInterpreterState_Delete(interp);
1465}
1466
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001467/* Add the __main__ module */
Nick Coghland6009512014-11-20 21:39:37 +10001468
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001469static _PyInitError
1470add_main_module(PyInterpreterState *interp)
Nick Coghland6009512014-11-20 21:39:37 +10001471{
Yury Selivanovf8cb8a12016-09-08 20:50:03 -07001472 PyObject *m, *d, *loader, *ann_dict;
Nick Coghland6009512014-11-20 21:39:37 +10001473 m = PyImport_AddModule("__main__");
1474 if (m == NULL)
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001475 return _Py_INIT_ERR("can't create __main__ module");
1476
Nick Coghland6009512014-11-20 21:39:37 +10001477 d = PyModule_GetDict(m);
Yury Selivanovf8cb8a12016-09-08 20:50:03 -07001478 ann_dict = PyDict_New();
1479 if ((ann_dict == NULL) ||
1480 (PyDict_SetItemString(d, "__annotations__", ann_dict) < 0)) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001481 return _Py_INIT_ERR("Failed to initialize __main__.__annotations__");
Yury Selivanovf8cb8a12016-09-08 20:50:03 -07001482 }
1483 Py_DECREF(ann_dict);
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001484
Nick Coghland6009512014-11-20 21:39:37 +10001485 if (PyDict_GetItemString(d, "__builtins__") == NULL) {
1486 PyObject *bimod = PyImport_ImportModule("builtins");
1487 if (bimod == NULL) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001488 return _Py_INIT_ERR("Failed to retrieve builtins module");
Nick Coghland6009512014-11-20 21:39:37 +10001489 }
1490 if (PyDict_SetItemString(d, "__builtins__", bimod) < 0) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001491 return _Py_INIT_ERR("Failed to initialize __main__.__builtins__");
Nick Coghland6009512014-11-20 21:39:37 +10001492 }
1493 Py_DECREF(bimod);
1494 }
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001495
Nick Coghland6009512014-11-20 21:39:37 +10001496 /* Main is a little special - imp.is_builtin("__main__") will return
1497 * False, but BuiltinImporter is still the most appropriate initial
1498 * setting for its __loader__ attribute. A more suitable value will
1499 * be set if __main__ gets further initialized later in the startup
1500 * process.
1501 */
1502 loader = PyDict_GetItemString(d, "__loader__");
1503 if (loader == NULL || loader == Py_None) {
1504 PyObject *loader = PyObject_GetAttrString(interp->importlib,
1505 "BuiltinImporter");
1506 if (loader == NULL) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001507 return _Py_INIT_ERR("Failed to retrieve BuiltinImporter");
Nick Coghland6009512014-11-20 21:39:37 +10001508 }
1509 if (PyDict_SetItemString(d, "__loader__", loader) < 0) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001510 return _Py_INIT_ERR("Failed to initialize __main__.__loader__");
Nick Coghland6009512014-11-20 21:39:37 +10001511 }
1512 Py_DECREF(loader);
1513 }
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001514 return _Py_INIT_OK();
Nick Coghland6009512014-11-20 21:39:37 +10001515}
1516
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001517static _PyInitError
Nick Coghland6009512014-11-20 21:39:37 +10001518initfsencoding(PyInterpreterState *interp)
1519{
1520 PyObject *codec;
1521
Steve Dowercc16be82016-09-08 10:35:16 -07001522#ifdef MS_WINDOWS
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001523 if (Py_LegacyWindowsFSEncodingFlag) {
Steve Dowercc16be82016-09-08 10:35:16 -07001524 Py_FileSystemDefaultEncoding = "mbcs";
1525 Py_FileSystemDefaultEncodeErrors = "replace";
1526 }
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001527 else {
Steve Dowercc16be82016-09-08 10:35:16 -07001528 Py_FileSystemDefaultEncoding = "utf-8";
1529 Py_FileSystemDefaultEncodeErrors = "surrogatepass";
1530 }
1531#else
Victor Stinner91106cd2017-12-13 12:29:09 +01001532 if (Py_FileSystemDefaultEncoding == NULL &&
1533 interp->core_config.utf8_mode)
1534 {
1535 Py_FileSystemDefaultEncoding = "utf-8";
1536 Py_HasFileSystemDefaultEncoding = 1;
1537 }
1538 else if (Py_FileSystemDefaultEncoding == NULL) {
Nick Coghland6009512014-11-20 21:39:37 +10001539 Py_FileSystemDefaultEncoding = get_locale_encoding();
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001540 if (Py_FileSystemDefaultEncoding == NULL) {
1541 return _Py_INIT_ERR("Unable to get the locale encoding");
1542 }
Nick Coghland6009512014-11-20 21:39:37 +10001543
1544 Py_HasFileSystemDefaultEncoding = 0;
1545 interp->fscodec_initialized = 1;
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001546 return _Py_INIT_OK();
Nick Coghland6009512014-11-20 21:39:37 +10001547 }
Steve Dowercc16be82016-09-08 10:35:16 -07001548#endif
Nick Coghland6009512014-11-20 21:39:37 +10001549
1550 /* the encoding is mbcs, utf-8 or ascii */
1551 codec = _PyCodec_Lookup(Py_FileSystemDefaultEncoding);
1552 if (!codec) {
1553 /* Such error can only occurs in critical situations: no more
1554 * memory, import a module of the standard library failed,
1555 * etc. */
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001556 return _Py_INIT_ERR("unable to load the file system codec");
Nick Coghland6009512014-11-20 21:39:37 +10001557 }
1558 Py_DECREF(codec);
1559 interp->fscodec_initialized = 1;
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001560 return _Py_INIT_OK();
Nick Coghland6009512014-11-20 21:39:37 +10001561}
1562
1563/* Import the site module (not into __main__ though) */
1564
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001565static _PyInitError
Nick Coghland6009512014-11-20 21:39:37 +10001566initsite(void)
1567{
1568 PyObject *m;
1569 m = PyImport_ImportModule("site");
1570 if (m == NULL) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001571 return _Py_INIT_USER_ERR("Failed to import the site module");
Nick Coghland6009512014-11-20 21:39:37 +10001572 }
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001573 Py_DECREF(m);
1574 return _Py_INIT_OK();
Nick Coghland6009512014-11-20 21:39:37 +10001575}
1576
Victor Stinner874dbe82015-09-04 17:29:57 +02001577/* Check if a file descriptor is valid or not.
1578 Return 0 if the file descriptor is invalid, return non-zero otherwise. */
1579static int
1580is_valid_fd(int fd)
1581{
Victor Stinner1c4670e2017-05-04 00:45:56 +02001582#ifdef __APPLE__
1583 /* bpo-30225: On macOS Tiger, when stdout is redirected to a pipe
1584 and the other side of the pipe is closed, dup(1) succeed, whereas
1585 fstat(1, &st) fails with EBADF. Prefer fstat() over dup() to detect
1586 such error. */
1587 struct stat st;
1588 return (fstat(fd, &st) == 0);
1589#else
Victor Stinner874dbe82015-09-04 17:29:57 +02001590 int fd2;
Steve Dower940f33a2016-09-08 11:21:54 -07001591 if (fd < 0)
Victor Stinner874dbe82015-09-04 17:29:57 +02001592 return 0;
1593 _Py_BEGIN_SUPPRESS_IPH
Victor Stinner449b2712015-09-29 13:59:50 +02001594 /* Prefer dup() over fstat(). fstat() can require input/output whereas
1595 dup() doesn't, there is a low risk of EMFILE/ENFILE at Python
1596 startup. */
Victor Stinner874dbe82015-09-04 17:29:57 +02001597 fd2 = dup(fd);
1598 if (fd2 >= 0)
1599 close(fd2);
1600 _Py_END_SUPPRESS_IPH
1601 return fd2 >= 0;
Victor Stinner1c4670e2017-05-04 00:45:56 +02001602#endif
Victor Stinner874dbe82015-09-04 17:29:57 +02001603}
1604
1605/* returns Py_None if the fd is not valid */
Nick Coghland6009512014-11-20 21:39:37 +10001606static PyObject*
1607create_stdio(PyObject* io,
Serhiy Storchakaef1585e2015-12-25 20:01:53 +02001608 int fd, int write_mode, const char* name,
1609 const char* encoding, const char* errors)
Nick Coghland6009512014-11-20 21:39:37 +10001610{
1611 PyObject *buf = NULL, *stream = NULL, *text = NULL, *raw = NULL, *res;
1612 const char* mode;
1613 const char* newline;
Serhiy Storchaka77732be2017-10-04 20:25:40 +03001614 PyObject *line_buffering, *write_through;
Nick Coghland6009512014-11-20 21:39:37 +10001615 int buffering, isatty;
1616 _Py_IDENTIFIER(open);
1617 _Py_IDENTIFIER(isatty);
1618 _Py_IDENTIFIER(TextIOWrapper);
1619 _Py_IDENTIFIER(mode);
1620
Victor Stinner874dbe82015-09-04 17:29:57 +02001621 if (!is_valid_fd(fd))
1622 Py_RETURN_NONE;
1623
Nick Coghland6009512014-11-20 21:39:37 +10001624 /* stdin is always opened in buffered mode, first because it shouldn't
1625 make a difference in common use cases, second because TextIOWrapper
1626 depends on the presence of a read1() method which only exists on
1627 buffered streams.
1628 */
1629 if (Py_UnbufferedStdioFlag && write_mode)
1630 buffering = 0;
1631 else
1632 buffering = -1;
1633 if (write_mode)
1634 mode = "wb";
1635 else
1636 mode = "rb";
1637 buf = _PyObject_CallMethodId(io, &PyId_open, "isiOOOi",
1638 fd, mode, buffering,
Martin Panterb4ce1fc2015-11-30 03:18:29 +00001639 Py_None, Py_None, /* encoding, errors */
1640 Py_None, 0); /* newline, closefd */
Nick Coghland6009512014-11-20 21:39:37 +10001641 if (buf == NULL)
1642 goto error;
1643
1644 if (buffering) {
1645 _Py_IDENTIFIER(raw);
1646 raw = _PyObject_GetAttrId(buf, &PyId_raw);
1647 if (raw == NULL)
1648 goto error;
1649 }
1650 else {
1651 raw = buf;
1652 Py_INCREF(raw);
1653 }
1654
Steve Dower39294992016-08-30 21:22:36 -07001655#ifdef MS_WINDOWS
1656 /* Windows console IO is always UTF-8 encoded */
1657 if (PyWindowsConsoleIO_Check(raw))
1658 encoding = "utf-8";
1659#endif
1660
Nick Coghland6009512014-11-20 21:39:37 +10001661 text = PyUnicode_FromString(name);
1662 if (text == NULL || _PyObject_SetAttrId(raw, &PyId_name, text) < 0)
1663 goto error;
Victor Stinner3466bde2016-09-05 18:16:01 -07001664 res = _PyObject_CallMethodId(raw, &PyId_isatty, NULL);
Nick Coghland6009512014-11-20 21:39:37 +10001665 if (res == NULL)
1666 goto error;
1667 isatty = PyObject_IsTrue(res);
1668 Py_DECREF(res);
1669 if (isatty == -1)
1670 goto error;
Serhiy Storchaka77732be2017-10-04 20:25:40 +03001671 if (Py_UnbufferedStdioFlag)
1672 write_through = Py_True;
1673 else
1674 write_through = Py_False;
1675 if (isatty && !Py_UnbufferedStdioFlag)
Nick Coghland6009512014-11-20 21:39:37 +10001676 line_buffering = Py_True;
1677 else
1678 line_buffering = Py_False;
1679
1680 Py_CLEAR(raw);
1681 Py_CLEAR(text);
1682
1683#ifdef MS_WINDOWS
1684 /* sys.stdin: enable universal newline mode, translate "\r\n" and "\r"
1685 newlines to "\n".
1686 sys.stdout and sys.stderr: translate "\n" to "\r\n". */
1687 newline = NULL;
1688#else
1689 /* sys.stdin: split lines at "\n".
1690 sys.stdout and sys.stderr: don't translate newlines (use "\n"). */
1691 newline = "\n";
1692#endif
1693
Serhiy Storchaka77732be2017-10-04 20:25:40 +03001694 stream = _PyObject_CallMethodId(io, &PyId_TextIOWrapper, "OsssOO",
Nick Coghland6009512014-11-20 21:39:37 +10001695 buf, encoding, errors,
Serhiy Storchaka77732be2017-10-04 20:25:40 +03001696 newline, line_buffering, write_through);
Nick Coghland6009512014-11-20 21:39:37 +10001697 Py_CLEAR(buf);
1698 if (stream == NULL)
1699 goto error;
1700
1701 if (write_mode)
1702 mode = "w";
1703 else
1704 mode = "r";
1705 text = PyUnicode_FromString(mode);
1706 if (!text || _PyObject_SetAttrId(stream, &PyId_mode, text) < 0)
1707 goto error;
1708 Py_CLEAR(text);
1709 return stream;
1710
1711error:
1712 Py_XDECREF(buf);
1713 Py_XDECREF(stream);
1714 Py_XDECREF(text);
1715 Py_XDECREF(raw);
Nick Coghland6009512014-11-20 21:39:37 +10001716
Victor Stinner874dbe82015-09-04 17:29:57 +02001717 if (PyErr_ExceptionMatches(PyExc_OSError) && !is_valid_fd(fd)) {
1718 /* Issue #24891: the file descriptor was closed after the first
1719 is_valid_fd() check was called. Ignore the OSError and set the
1720 stream to None. */
1721 PyErr_Clear();
1722 Py_RETURN_NONE;
1723 }
1724 return NULL;
Nick Coghland6009512014-11-20 21:39:37 +10001725}
1726
1727/* Initialize sys.stdin, stdout, stderr and builtins.open */
Victor Stinnera7368ac2017-11-15 18:11:45 -08001728static _PyInitError
Victor Stinner91106cd2017-12-13 12:29:09 +01001729init_sys_streams(PyInterpreterState *interp)
Nick Coghland6009512014-11-20 21:39:37 +10001730{
1731 PyObject *iomod = NULL, *wrapper;
1732 PyObject *bimod = NULL;
1733 PyObject *m;
1734 PyObject *std = NULL;
Victor Stinnera7368ac2017-11-15 18:11:45 -08001735 int fd;
Nick Coghland6009512014-11-20 21:39:37 +10001736 PyObject * encoding_attr;
Serhiy Storchaka4ae06c52017-12-12 13:55:04 +02001737 char *pythonioencoding = NULL;
1738 const char *encoding, *errors;
Victor Stinnera7368ac2017-11-15 18:11:45 -08001739 _PyInitError res = _Py_INIT_OK();
Nick Coghland6009512014-11-20 21:39:37 +10001740
1741 /* Hack to avoid a nasty recursion issue when Python is invoked
1742 in verbose mode: pre-import the Latin-1 and UTF-8 codecs */
1743 if ((m = PyImport_ImportModule("encodings.utf_8")) == NULL) {
1744 goto error;
1745 }
1746 Py_DECREF(m);
1747
1748 if (!(m = PyImport_ImportModule("encodings.latin_1"))) {
1749 goto error;
1750 }
1751 Py_DECREF(m);
1752
1753 if (!(bimod = PyImport_ImportModule("builtins"))) {
1754 goto error;
1755 }
1756
1757 if (!(iomod = PyImport_ImportModule("io"))) {
1758 goto error;
1759 }
1760 if (!(wrapper = PyObject_GetAttrString(iomod, "OpenWrapper"))) {
1761 goto error;
1762 }
1763
1764 /* Set builtins.open */
1765 if (PyObject_SetAttrString(bimod, "open", wrapper) == -1) {
1766 Py_DECREF(wrapper);
1767 goto error;
1768 }
1769 Py_DECREF(wrapper);
1770
1771 encoding = _Py_StandardStreamEncoding;
1772 errors = _Py_StandardStreamErrors;
1773 if (!encoding || !errors) {
Victor Stinner91106cd2017-12-13 12:29:09 +01001774 char *opt = Py_GETENV("PYTHONIOENCODING");
1775 if (opt && opt[0] != '\0') {
Nick Coghland6009512014-11-20 21:39:37 +10001776 char *err;
Victor Stinner91106cd2017-12-13 12:29:09 +01001777 pythonioencoding = _PyMem_Strdup(opt);
Nick Coghland6009512014-11-20 21:39:37 +10001778 if (pythonioencoding == NULL) {
1779 PyErr_NoMemory();
1780 goto error;
1781 }
1782 err = strchr(pythonioencoding, ':');
1783 if (err) {
1784 *err = '\0';
1785 err++;
Serhiy Storchakafc435112016-04-10 14:34:13 +03001786 if (*err && !errors) {
Nick Coghland6009512014-11-20 21:39:37 +10001787 errors = err;
1788 }
1789 }
1790 if (*pythonioencoding && !encoding) {
1791 encoding = pythonioencoding;
1792 }
1793 }
Victor Stinner91106cd2017-12-13 12:29:09 +01001794 else if (interp->core_config.utf8_mode) {
1795 encoding = "utf-8";
1796 errors = "surrogateescape";
1797 }
1798
1799 if (!errors && !pythonioencoding) {
Nick Coghlan6ea41862017-06-11 13:16:15 +10001800 /* Choose the default error handler based on the current locale */
1801 errors = get_default_standard_stream_error_handler();
Serhiy Storchakafc435112016-04-10 14:34:13 +03001802 }
Nick Coghland6009512014-11-20 21:39:37 +10001803 }
1804
1805 /* Set sys.stdin */
1806 fd = fileno(stdin);
1807 /* Under some conditions stdin, stdout and stderr may not be connected
1808 * and fileno() may point to an invalid file descriptor. For example
1809 * GUI apps don't have valid standard streams by default.
1810 */
Victor Stinner874dbe82015-09-04 17:29:57 +02001811 std = create_stdio(iomod, fd, 0, "<stdin>", encoding, errors);
1812 if (std == NULL)
1813 goto error;
Nick Coghland6009512014-11-20 21:39:37 +10001814 PySys_SetObject("__stdin__", std);
1815 _PySys_SetObjectId(&PyId_stdin, std);
1816 Py_DECREF(std);
1817
1818 /* Set sys.stdout */
1819 fd = fileno(stdout);
Victor Stinner874dbe82015-09-04 17:29:57 +02001820 std = create_stdio(iomod, fd, 1, "<stdout>", encoding, errors);
1821 if (std == NULL)
1822 goto error;
Nick Coghland6009512014-11-20 21:39:37 +10001823 PySys_SetObject("__stdout__", std);
1824 _PySys_SetObjectId(&PyId_stdout, std);
1825 Py_DECREF(std);
1826
1827#if 1 /* Disable this if you have trouble debugging bootstrap stuff */
1828 /* Set sys.stderr, replaces the preliminary stderr */
1829 fd = fileno(stderr);
Victor Stinner874dbe82015-09-04 17:29:57 +02001830 std = create_stdio(iomod, fd, 1, "<stderr>", encoding, "backslashreplace");
1831 if (std == NULL)
1832 goto error;
Nick Coghland6009512014-11-20 21:39:37 +10001833
1834 /* Same as hack above, pre-import stderr's codec to avoid recursion
1835 when import.c tries to write to stderr in verbose mode. */
1836 encoding_attr = PyObject_GetAttrString(std, "encoding");
1837 if (encoding_attr != NULL) {
Serhiy Storchaka85b0f5b2016-11-20 10:16:47 +02001838 const char *std_encoding = PyUnicode_AsUTF8(encoding_attr);
Nick Coghland6009512014-11-20 21:39:37 +10001839 if (std_encoding != NULL) {
1840 PyObject *codec_info = _PyCodec_Lookup(std_encoding);
1841 Py_XDECREF(codec_info);
1842 }
1843 Py_DECREF(encoding_attr);
1844 }
1845 PyErr_Clear(); /* Not a fatal error if codec isn't available */
1846
1847 if (PySys_SetObject("__stderr__", std) < 0) {
1848 Py_DECREF(std);
1849 goto error;
1850 }
1851 if (_PySys_SetObjectId(&PyId_stderr, std) < 0) {
1852 Py_DECREF(std);
1853 goto error;
1854 }
1855 Py_DECREF(std);
1856#endif
1857
Victor Stinnera7368ac2017-11-15 18:11:45 -08001858 goto done;
Nick Coghland6009512014-11-20 21:39:37 +10001859
Victor Stinnera7368ac2017-11-15 18:11:45 -08001860error:
1861 res = _Py_INIT_ERR("can't initialize sys standard streams");
1862
Victor Stinner31e99082017-12-20 23:41:38 +01001863 /* Use the same allocator than Py_SetStandardStreamEncoding() */
1864 PyMemAllocatorEx old_alloc;
Victor Stinnera7368ac2017-11-15 18:11:45 -08001865done:
Victor Stinner31e99082017-12-20 23:41:38 +01001866 _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
1867
Nick Coghland6009512014-11-20 21:39:37 +10001868 /* We won't need them anymore. */
1869 if (_Py_StandardStreamEncoding) {
1870 PyMem_RawFree(_Py_StandardStreamEncoding);
1871 _Py_StandardStreamEncoding = NULL;
1872 }
1873 if (_Py_StandardStreamErrors) {
1874 PyMem_RawFree(_Py_StandardStreamErrors);
1875 _Py_StandardStreamErrors = NULL;
1876 }
Victor Stinner31e99082017-12-20 23:41:38 +01001877
1878 PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
1879
Nick Coghland6009512014-11-20 21:39:37 +10001880 PyMem_Free(pythonioencoding);
1881 Py_XDECREF(bimod);
1882 Py_XDECREF(iomod);
Victor Stinnera7368ac2017-11-15 18:11:45 -08001883 return res;
Nick Coghland6009512014-11-20 21:39:37 +10001884}
1885
1886
Victor Stinner10dc4842015-03-24 12:01:30 +01001887static void
Victor Stinner791da1c2016-03-14 16:53:12 +01001888_Py_FatalError_DumpTracebacks(int fd)
Victor Stinner10dc4842015-03-24 12:01:30 +01001889{
Victor Stinner10dc4842015-03-24 12:01:30 +01001890 fputc('\n', stderr);
1891 fflush(stderr);
1892
1893 /* display the current Python stack */
Victor Stinner861d9ab2016-03-16 22:45:24 +01001894 _Py_DumpTracebackThreads(fd, NULL, NULL);
Victor Stinner10dc4842015-03-24 12:01:30 +01001895}
Victor Stinner791da1c2016-03-14 16:53:12 +01001896
1897/* Print the current exception (if an exception is set) with its traceback,
1898 or display the current Python stack.
1899
1900 Don't call PyErr_PrintEx() and the except hook, because Py_FatalError() is
1901 called on catastrophic cases.
1902
1903 Return 1 if the traceback was displayed, 0 otherwise. */
1904
1905static int
1906_Py_FatalError_PrintExc(int fd)
1907{
1908 PyObject *ferr, *res;
1909 PyObject *exception, *v, *tb;
1910 int has_tb;
1911
1912 if (PyThreadState_GET() == NULL) {
1913 /* The GIL is released: trying to acquire it is likely to deadlock,
1914 just give up. */
1915 return 0;
1916 }
1917
1918 PyErr_Fetch(&exception, &v, &tb);
1919 if (exception == NULL) {
1920 /* No current exception */
1921 return 0;
1922 }
1923
1924 ferr = _PySys_GetObjectId(&PyId_stderr);
1925 if (ferr == NULL || ferr == Py_None) {
1926 /* sys.stderr is not set yet or set to None,
1927 no need to try to display the exception */
1928 return 0;
1929 }
1930
1931 PyErr_NormalizeException(&exception, &v, &tb);
1932 if (tb == NULL) {
1933 tb = Py_None;
1934 Py_INCREF(tb);
1935 }
1936 PyException_SetTraceback(v, tb);
1937 if (exception == NULL) {
1938 /* PyErr_NormalizeException() failed */
1939 return 0;
1940 }
1941
1942 has_tb = (tb != Py_None);
1943 PyErr_Display(exception, v, tb);
1944 Py_XDECREF(exception);
1945 Py_XDECREF(v);
1946 Py_XDECREF(tb);
1947
1948 /* sys.stderr may be buffered: call sys.stderr.flush() */
Victor Stinner3466bde2016-09-05 18:16:01 -07001949 res = _PyObject_CallMethodId(ferr, &PyId_flush, NULL);
Victor Stinner791da1c2016-03-14 16:53:12 +01001950 if (res == NULL)
1951 PyErr_Clear();
1952 else
1953 Py_DECREF(res);
1954
1955 return has_tb;
1956}
1957
Nick Coghland6009512014-11-20 21:39:37 +10001958/* Print fatal error message and abort */
1959
Victor Stinner8d5a3aa2017-10-04 09:50:12 -07001960#ifdef MS_WINDOWS
1961static void
1962fatal_output_debug(const char *msg)
1963{
1964 /* buffer of 256 bytes allocated on the stack */
1965 WCHAR buffer[256 / sizeof(WCHAR)];
1966 size_t buflen = Py_ARRAY_LENGTH(buffer) - 1;
1967 size_t msglen;
1968
1969 OutputDebugStringW(L"Fatal Python error: ");
1970
1971 msglen = strlen(msg);
1972 while (msglen) {
1973 size_t i;
1974
1975 if (buflen > msglen) {
1976 buflen = msglen;
1977 }
1978
1979 /* Convert the message to wchar_t. This uses a simple one-to-one
1980 conversion, assuming that the this error message actually uses
1981 ASCII only. If this ceases to be true, we will have to convert. */
1982 for (i=0; i < buflen; ++i) {
1983 buffer[i] = msg[i];
1984 }
1985 buffer[i] = L'\0';
1986 OutputDebugStringW(buffer);
1987
1988 msg += buflen;
1989 msglen -= buflen;
1990 }
1991 OutputDebugStringW(L"\n");
1992}
1993#endif
1994
Benjamin Petersoncef88b92017-11-25 13:02:55 -08001995static void _Py_NO_RETURN
Victor Stinnerf7e5b562017-11-15 15:48:08 -08001996fatal_error(const char *prefix, const char *msg, int status)
Nick Coghland6009512014-11-20 21:39:37 +10001997{
1998 const int fd = fileno(stderr);
Victor Stinner53345a42015-03-25 01:55:14 +01001999 static int reentrant = 0;
Victor Stinner53345a42015-03-25 01:55:14 +01002000
2001 if (reentrant) {
2002 /* Py_FatalError() caused a second fatal error.
2003 Example: flush_std_files() raises a recursion error. */
2004 goto exit;
2005 }
2006 reentrant = 1;
Nick Coghland6009512014-11-20 21:39:37 +10002007
Victor Stinnerf7e5b562017-11-15 15:48:08 -08002008 fprintf(stderr, "Fatal Python error: ");
2009 if (prefix) {
2010 fputs(prefix, stderr);
2011 fputs(": ", stderr);
2012 }
2013 if (msg) {
2014 fputs(msg, stderr);
2015 }
2016 else {
2017 fprintf(stderr, "<message not set>");
2018 }
2019 fputs("\n", stderr);
Nick Coghland6009512014-11-20 21:39:37 +10002020 fflush(stderr); /* it helps in Windows debug build */
Victor Stinner10dc4842015-03-24 12:01:30 +01002021
Victor Stinnere0deff32015-03-24 13:46:18 +01002022 /* Print the exception (if an exception is set) with its traceback,
2023 * or display the current Python stack. */
Victor Stinner8d5a3aa2017-10-04 09:50:12 -07002024 if (!_Py_FatalError_PrintExc(fd)) {
Victor Stinner791da1c2016-03-14 16:53:12 +01002025 _Py_FatalError_DumpTracebacks(fd);
Victor Stinner8d5a3aa2017-10-04 09:50:12 -07002026 }
Victor Stinner10dc4842015-03-24 12:01:30 +01002027
Victor Stinner8d5a3aa2017-10-04 09:50:12 -07002028 /* The main purpose of faulthandler is to display the traceback.
2029 This function already did its best to display a traceback.
2030 Disable faulthandler to prevent writing a second traceback
2031 on abort(). */
Victor Stinner2025d782016-03-16 23:19:15 +01002032 _PyFaulthandler_Fini();
2033
Victor Stinner791da1c2016-03-14 16:53:12 +01002034 /* Check if the current Python thread hold the GIL */
2035 if (PyThreadState_GET() != NULL) {
2036 /* Flush sys.stdout and sys.stderr */
2037 flush_std_files();
2038 }
Victor Stinnere0deff32015-03-24 13:46:18 +01002039
Nick Coghland6009512014-11-20 21:39:37 +10002040#ifdef MS_WINDOWS
Victor Stinner8d5a3aa2017-10-04 09:50:12 -07002041 fatal_output_debug(msg);
Victor Stinner53345a42015-03-25 01:55:14 +01002042#endif /* MS_WINDOWS */
2043
2044exit:
Victor Stinnerf7e5b562017-11-15 15:48:08 -08002045 if (status < 0) {
Victor Stinner53345a42015-03-25 01:55:14 +01002046#if defined(MS_WINDOWS) && defined(_DEBUG)
Victor Stinnerf7e5b562017-11-15 15:48:08 -08002047 DebugBreak();
Nick Coghland6009512014-11-20 21:39:37 +10002048#endif
Victor Stinnerf7e5b562017-11-15 15:48:08 -08002049 abort();
2050 }
2051 else {
2052 exit(status);
2053 }
2054}
2055
Victor Stinner19760862017-12-20 01:41:59 +01002056void _Py_NO_RETURN
Victor Stinnerf7e5b562017-11-15 15:48:08 -08002057Py_FatalError(const char *msg)
2058{
2059 fatal_error(NULL, msg, -1);
2060}
2061
Victor Stinner19760862017-12-20 01:41:59 +01002062void _Py_NO_RETURN
Victor Stinnerf7e5b562017-11-15 15:48:08 -08002063_Py_FatalInitError(_PyInitError err)
2064{
2065 /* On "user" error: exit with status 1.
2066 For all other errors, call abort(). */
2067 int status = err.user_err ? 1 : -1;
2068 fatal_error(err.prefix, err.msg, status);
Nick Coghland6009512014-11-20 21:39:37 +10002069}
2070
2071/* Clean up and exit */
2072
Victor Stinnerd7292b52016-06-17 12:29:00 +02002073# include "pythread.h"
Nick Coghland6009512014-11-20 21:39:37 +10002074
Nick Coghland6009512014-11-20 21:39:37 +10002075/* For the atexit module. */
Marcel Plch776407f2017-12-20 11:17:58 +01002076void _Py_PyAtExit(void (*func)(PyObject *), PyObject *module)
Nick Coghland6009512014-11-20 21:39:37 +10002077{
Marcel Plch776407f2017-12-20 11:17:58 +01002078 PyThreadState *ts;
2079 PyInterpreterState *is;
Victor Stinnerca719ac2017-12-20 18:00:19 +01002080
Marcel Plch776407f2017-12-20 11:17:58 +01002081 ts = PyThreadState_GET();
2082 is = ts->interp;
2083
Antoine Pitroufc5db952017-12-13 02:29:07 +01002084 /* Guard against API misuse (see bpo-17852) */
Marcel Plch776407f2017-12-20 11:17:58 +01002085 assert(is->pyexitfunc == NULL || is->pyexitfunc == func);
2086
2087 is->pyexitfunc = func;
2088 is->pyexitmodule = module;
Nick Coghland6009512014-11-20 21:39:37 +10002089}
2090
2091static void
Marcel Plch776407f2017-12-20 11:17:58 +01002092call_py_exitfuncs(PyInterpreterState *istate)
Nick Coghland6009512014-11-20 21:39:37 +10002093{
Marcel Plch776407f2017-12-20 11:17:58 +01002094 if (istate->pyexitfunc == NULL)
Nick Coghland6009512014-11-20 21:39:37 +10002095 return;
2096
Marcel Plch776407f2017-12-20 11:17:58 +01002097 (*istate->pyexitfunc)(istate->pyexitmodule);
Nick Coghland6009512014-11-20 21:39:37 +10002098 PyErr_Clear();
2099}
2100
2101/* Wait until threading._shutdown completes, provided
2102 the threading module was imported in the first place.
2103 The shutdown routine will wait until all non-daemon
2104 "threading" threads have completed. */
2105static void
2106wait_for_thread_shutdown(void)
2107{
Nick Coghland6009512014-11-20 21:39:37 +10002108 _Py_IDENTIFIER(_shutdown);
2109 PyObject *result;
Eric Snow3f9eee62017-09-15 16:35:20 -06002110 PyObject *threading = _PyImport_GetModuleId(&PyId_threading);
Nick Coghland6009512014-11-20 21:39:37 +10002111 if (threading == NULL) {
2112 /* threading not imported */
2113 PyErr_Clear();
2114 return;
2115 }
Victor Stinner3466bde2016-09-05 18:16:01 -07002116 result = _PyObject_CallMethodId(threading, &PyId__shutdown, NULL);
Nick Coghland6009512014-11-20 21:39:37 +10002117 if (result == NULL) {
2118 PyErr_WriteUnraisable(threading);
2119 }
2120 else {
2121 Py_DECREF(result);
2122 }
2123 Py_DECREF(threading);
Nick Coghland6009512014-11-20 21:39:37 +10002124}
2125
2126#define NEXITFUNCS 32
Nick Coghland6009512014-11-20 21:39:37 +10002127int Py_AtExit(void (*func)(void))
2128{
Eric Snow2ebc5ce2017-09-07 23:51:28 -06002129 if (_PyRuntime.nexitfuncs >= NEXITFUNCS)
Nick Coghland6009512014-11-20 21:39:37 +10002130 return -1;
Eric Snow2ebc5ce2017-09-07 23:51:28 -06002131 _PyRuntime.exitfuncs[_PyRuntime.nexitfuncs++] = func;
Nick Coghland6009512014-11-20 21:39:37 +10002132 return 0;
2133}
2134
2135static void
2136call_ll_exitfuncs(void)
2137{
Eric Snow2ebc5ce2017-09-07 23:51:28 -06002138 while (_PyRuntime.nexitfuncs > 0)
2139 (*_PyRuntime.exitfuncs[--_PyRuntime.nexitfuncs])();
Nick Coghland6009512014-11-20 21:39:37 +10002140
2141 fflush(stdout);
2142 fflush(stderr);
2143}
2144
2145void
2146Py_Exit(int sts)
2147{
Martin Panterb4ce1fc2015-11-30 03:18:29 +00002148 if (Py_FinalizeEx() < 0) {
2149 sts = 120;
2150 }
Nick Coghland6009512014-11-20 21:39:37 +10002151
2152 exit(sts);
2153}
2154
Victor Stinnerf7e5b562017-11-15 15:48:08 -08002155static _PyInitError
Nick Coghland6009512014-11-20 21:39:37 +10002156initsigs(void)
2157{
2158#ifdef SIGPIPE
2159 PyOS_setsig(SIGPIPE, SIG_IGN);
2160#endif
2161#ifdef SIGXFZ
2162 PyOS_setsig(SIGXFZ, SIG_IGN);
2163#endif
2164#ifdef SIGXFSZ
2165 PyOS_setsig(SIGXFSZ, SIG_IGN);
2166#endif
2167 PyOS_InitInterrupts(); /* May imply initsignal() */
2168 if (PyErr_Occurred()) {
Victor Stinnerf7e5b562017-11-15 15:48:08 -08002169 return _Py_INIT_ERR("can't import signal");
Nick Coghland6009512014-11-20 21:39:37 +10002170 }
Victor Stinnerf7e5b562017-11-15 15:48:08 -08002171 return _Py_INIT_OK();
Nick Coghland6009512014-11-20 21:39:37 +10002172}
2173
2174
2175/* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL.
2176 *
2177 * All of the code in this function must only use async-signal-safe functions,
2178 * listed at `man 7 signal` or
2179 * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html.
2180 */
2181void
2182_Py_RestoreSignals(void)
2183{
2184#ifdef SIGPIPE
2185 PyOS_setsig(SIGPIPE, SIG_DFL);
2186#endif
2187#ifdef SIGXFZ
2188 PyOS_setsig(SIGXFZ, SIG_DFL);
2189#endif
2190#ifdef SIGXFSZ
2191 PyOS_setsig(SIGXFSZ, SIG_DFL);
2192#endif
2193}
2194
2195
2196/*
2197 * The file descriptor fd is considered ``interactive'' if either
2198 * a) isatty(fd) is TRUE, or
2199 * b) the -i flag was given, and the filename associated with
2200 * the descriptor is NULL or "<stdin>" or "???".
2201 */
2202int
2203Py_FdIsInteractive(FILE *fp, const char *filename)
2204{
2205 if (isatty((int)fileno(fp)))
2206 return 1;
2207 if (!Py_InteractiveFlag)
2208 return 0;
2209 return (filename == NULL) ||
2210 (strcmp(filename, "<stdin>") == 0) ||
2211 (strcmp(filename, "???") == 0);
2212}
2213
2214
Nick Coghland6009512014-11-20 21:39:37 +10002215/* Wrappers around sigaction() or signal(). */
2216
2217PyOS_sighandler_t
2218PyOS_getsig(int sig)
2219{
2220#ifdef HAVE_SIGACTION
2221 struct sigaction context;
2222 if (sigaction(sig, NULL, &context) == -1)
2223 return SIG_ERR;
2224 return context.sa_handler;
2225#else
2226 PyOS_sighandler_t handler;
2227/* Special signal handling for the secure CRT in Visual Studio 2005 */
2228#if defined(_MSC_VER) && _MSC_VER >= 1400
2229 switch (sig) {
2230 /* Only these signals are valid */
2231 case SIGINT:
2232 case SIGILL:
2233 case SIGFPE:
2234 case SIGSEGV:
2235 case SIGTERM:
2236 case SIGBREAK:
2237 case SIGABRT:
2238 break;
2239 /* Don't call signal() with other values or it will assert */
2240 default:
2241 return SIG_ERR;
2242 }
2243#endif /* _MSC_VER && _MSC_VER >= 1400 */
2244 handler = signal(sig, SIG_IGN);
2245 if (handler != SIG_ERR)
2246 signal(sig, handler);
2247 return handler;
2248#endif
2249}
2250
2251/*
2252 * All of the code in this function must only use async-signal-safe functions,
2253 * listed at `man 7 signal` or
2254 * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html.
2255 */
2256PyOS_sighandler_t
2257PyOS_setsig(int sig, PyOS_sighandler_t handler)
2258{
2259#ifdef HAVE_SIGACTION
2260 /* Some code in Modules/signalmodule.c depends on sigaction() being
2261 * used here if HAVE_SIGACTION is defined. Fix that if this code
2262 * changes to invalidate that assumption.
2263 */
2264 struct sigaction context, ocontext;
2265 context.sa_handler = handler;
2266 sigemptyset(&context.sa_mask);
2267 context.sa_flags = 0;
2268 if (sigaction(sig, &context, &ocontext) == -1)
2269 return SIG_ERR;
2270 return ocontext.sa_handler;
2271#else
2272 PyOS_sighandler_t oldhandler;
2273 oldhandler = signal(sig, handler);
2274#ifdef HAVE_SIGINTERRUPT
2275 siginterrupt(sig, 1);
2276#endif
2277 return oldhandler;
2278#endif
2279}
2280
2281#ifdef __cplusplus
2282}
2283#endif