Issue #18093: Factor out the programs that embed the runtime
diff --git a/Programs/README b/Programs/README
new file mode 100644
index 0000000..c24578b
--- /dev/null
+++ b/Programs/README
@@ -0,0 +1 @@
+Source files for binary executables (as opposed to shared modules)
diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c
new file mode 100644
index 0000000..57b1ac0
--- /dev/null
+++ b/Programs/_freeze_importlib.c
@@ -0,0 +1,144 @@
+/* This is built as a stand-alone executable by the Makefile, and helps turn
+ Lib/importlib/_bootstrap.py into a frozen module in Python/importlib.h
+*/
+
+#include <Python.h>
+#include <marshal.h>
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#ifndef MS_WINDOWS
+#include <unistd.h>
+#endif
+
+
+/* To avoid a circular dependency on frozen.o, we create our own structure
+ of frozen modules instead, left deliberately blank so as to avoid
+ unintentional import of a stale version of _frozen_importlib. */
+
+const static struct _frozen _PyImport_FrozenModules[] = {
+ {0, 0, 0} /* sentinel */
+};
+
+#ifndef MS_WINDOWS
+/* On Windows, this links with the regular pythonXY.dll, so this variable comes
+ from frozen.obj. In the Makefile, frozen.o is not linked into this executable,
+ so we define the variable here. */
+const struct _frozen *PyImport_FrozenModules;
+#endif
+
+const char header[] = "/* Auto-generated by Modules/_freeze_importlib.c */";
+
+int
+main(int argc, char *argv[])
+{
+ char *inpath, *outpath;
+ FILE *infile = NULL, *outfile = NULL;
+ struct stat st;
+ size_t text_size, data_size, n;
+ char *text = NULL;
+ unsigned char *data;
+ PyObject *code = NULL, *marshalled = NULL;
+
+ PyImport_FrozenModules = _PyImport_FrozenModules;
+
+ if (argc != 3) {
+ fprintf(stderr, "need to specify input and output paths\n");
+ return 2;
+ }
+ inpath = argv[1];
+ outpath = argv[2];
+ infile = fopen(inpath, "rb");
+ if (infile == NULL) {
+ fprintf(stderr, "cannot open '%s' for reading\n", inpath);
+ goto error;
+ }
+ if (fstat(fileno(infile), &st)) {
+ fprintf(stderr, "cannot fstat '%s'\n", inpath);
+ goto error;
+ }
+ text_size = st.st_size;
+ text = (char *) malloc(text_size + 1);
+ if (text == NULL) {
+ fprintf(stderr, "could not allocate %ld bytes\n", (long) text_size);
+ goto error;
+ }
+ n = fread(text, 1, text_size, infile);
+ fclose(infile);
+ infile = NULL;
+ if (n < text_size) {
+ fprintf(stderr, "read too short: got %ld instead of %ld bytes\n",
+ (long) n, (long) text_size);
+ goto error;
+ }
+ text[text_size] = '\0';
+
+ Py_NoUserSiteDirectory++;
+ Py_NoSiteFlag++;
+ Py_IgnoreEnvironmentFlag++;
+
+ Py_SetProgramName(L"./_freeze_importlib");
+ /* Don't install importlib, since it could execute outdated bytecode. */
+ _Py_InitializeEx_Private(1, 0);
+
+ code = Py_CompileStringExFlags(text, "<frozen importlib._bootstrap>",
+ Py_file_input, NULL, 0);
+ if (code == NULL)
+ goto error;
+ free(text);
+ text = NULL;
+
+ marshalled = PyMarshal_WriteObjectToString(code, Py_MARSHAL_VERSION);
+ Py_CLEAR(code);
+ if (marshalled == NULL)
+ goto error;
+
+ assert(PyBytes_CheckExact(marshalled));
+ data = (unsigned char *) PyBytes_AS_STRING(marshalled);
+ data_size = PyBytes_GET_SIZE(marshalled);
+
+ /* Open the file in text mode. The hg checkout should be using the eol extension,
+ which in turn should cause the EOL style match the C library's text mode */
+ outfile = fopen(outpath, "w");
+ if (outfile == NULL) {
+ fprintf(stderr, "cannot open '%s' for writing\n", outpath);
+ goto error;
+ }
+ fprintf(outfile, "%s\n", header);
+ fprintf(outfile, "const unsigned char _Py_M__importlib[] = {\n");
+ for (n = 0; n < data_size; n += 16) {
+ size_t i, end = Py_MIN(n + 16, data_size);
+ fprintf(outfile, " ");
+ for (i = n; i < end; i++) {
+ fprintf(outfile, "%d,", (unsigned int) data[i]);
+ }
+ fprintf(outfile, "\n");
+ }
+ fprintf(outfile, "};\n");
+
+ Py_CLEAR(marshalled);
+
+ Py_Finalize();
+ if (outfile) {
+ if (ferror(outfile)) {
+ fprintf(stderr, "error when writing to '%s'\n", outpath);
+ goto error;
+ }
+ fclose(outfile);
+ }
+ return 0;
+
+error:
+ PyErr_Print();
+ Py_Finalize();
+ if (infile)
+ fclose(infile);
+ if (outfile)
+ fclose(outfile);
+ if (text)
+ free(text);
+ if (marshalled)
+ Py_DECREF(marshalled);
+ return 1;
+}
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
new file mode 100644
index 0000000..39ff097
--- /dev/null
+++ b/Programs/_testembed.c
@@ -0,0 +1,140 @@
+#include <Python.h>
+#include <stdio.h>
+
+/*********************************************************
+ * Embedded interpreter tests that need a custom exe
+ *
+ * Executed via 'EmbeddingTests' in Lib/test/test_capi.py
+ *********************************************************/
+
+static void _testembed_Py_Initialize(void)
+{
+ /* HACK: the "./" at front avoids a search along the PATH in
+ Modules/getpath.c */
+ Py_SetProgramName(L"./_testembed");
+ Py_Initialize();
+}
+
+
+/*****************************************************
+ * Test repeated initalisation and subinterpreters
+ *****************************************************/
+
+static void print_subinterp(void)
+{
+ /* Just output some debug stuff */
+ PyThreadState *ts = PyThreadState_Get();
+ printf("interp %p, thread state %p: ", ts->interp, ts);
+ fflush(stdout);
+ PyRun_SimpleString(
+ "import sys;"
+ "print('id(modules) =', id(sys.modules));"
+ "sys.stdout.flush()"
+ );
+}
+
+static void test_repeated_init_and_subinterpreters(void)
+{
+ PyThreadState *mainstate, *substate;
+#ifdef WITH_THREAD
+ PyGILState_STATE gilstate;
+#endif
+ int i, j;
+
+ for (i=0; i<3; i++) {
+ printf("--- Pass %d ---\n", i);
+ _testembed_Py_Initialize();
+ mainstate = PyThreadState_Get();
+
+#ifdef WITH_THREAD
+ PyEval_InitThreads();
+ PyEval_ReleaseThread(mainstate);
+
+ gilstate = PyGILState_Ensure();
+#endif
+ print_subinterp();
+ PyThreadState_Swap(NULL);
+
+ for (j=0; j<3; j++) {
+ substate = Py_NewInterpreter();
+ print_subinterp();
+ Py_EndInterpreter(substate);
+ }
+
+ PyThreadState_Swap(mainstate);
+ print_subinterp();
+#ifdef WITH_THREAD
+ PyGILState_Release(gilstate);
+#endif
+
+ PyEval_RestoreThread(mainstate);
+ Py_Finalize();
+ }
+}
+
+/*****************************************************
+ * Test forcing a particular IO encoding
+ *****************************************************/
+
+static void check_stdio_details(const char *encoding, const char * errors)
+{
+ /* Output info for the test case to check */
+ if (encoding) {
+ printf("Expected encoding: %s\n", encoding);
+ } else {
+ printf("Expected encoding: default\n");
+ }
+ if (errors) {
+ printf("Expected errors: %s\n", errors);
+ } else {
+ printf("Expected errors: default\n");
+ }
+ fflush(stdout);
+ /* Force the given IO encoding */
+ Py_SetStandardStreamEncoding(encoding, errors);
+ _testembed_Py_Initialize();
+ PyRun_SimpleString(
+ "import sys;"
+ "print('stdin: {0.encoding}:{0.errors}'.format(sys.stdin));"
+ "print('stdout: {0.encoding}:{0.errors}'.format(sys.stdout));"
+ "print('stderr: {0.encoding}:{0.errors}'.format(sys.stderr));"
+ "sys.stdout.flush()"
+ );
+ Py_Finalize();
+}
+
+static void test_forced_io_encoding(void)
+{
+ /* Check various combinations */
+ printf("--- Use defaults ---\n");
+ check_stdio_details(NULL, NULL);
+ printf("--- Set errors only ---\n");
+ check_stdio_details(NULL, "ignore");
+ printf("--- Set encoding only ---\n");
+ check_stdio_details("latin-1", NULL);
+ printf("--- Set encoding and errors ---\n");
+ check_stdio_details("latin-1", "replace");
+
+ /* Check calling after initialization fails */
+ Py_Initialize();
+
+ if (Py_SetStandardStreamEncoding(NULL, NULL) == 0) {
+ printf("Unexpected success calling Py_SetStandardStreamEncoding");
+ }
+ Py_Finalize();
+}
+
+/* Different embedding tests */
+int main(int argc, char *argv[])
+{
+
+ /* TODO: Check the argument string to allow for more test cases */
+ if (argc > 1) {
+ /* For now: assume "forced_io_encoding */
+ test_forced_io_encoding();
+ } else {
+ /* Run the original embedding test case by default */
+ test_repeated_init_and_subinterpreters();
+ }
+ return 0;
+}
diff --git a/Programs/python.c b/Programs/python.c
new file mode 100644
index 0000000..9811c01
--- /dev/null
+++ b/Programs/python.c
@@ -0,0 +1,77 @@
+/* Minimal main program -- everything is loaded from the library */
+
+#include "Python.h"
+#include <locale.h>
+
+#ifdef __FreeBSD__
+#include <floatingpoint.h>
+#endif
+
+#ifdef MS_WINDOWS
+int
+wmain(int argc, wchar_t **argv)
+{
+ return Py_Main(argc, argv);
+}
+#else
+
+int
+main(int argc, char **argv)
+{
+ wchar_t **argv_copy;
+ /* We need a second copy, as Python might modify the first one. */
+ wchar_t **argv_copy2;
+ int i, res;
+ char *oldloc;
+#ifdef __FreeBSD__
+ fp_except_t m;
+#endif
+
+ argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
+ argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
+ if (!argv_copy || !argv_copy2) {
+ fprintf(stderr, "out of memory\n");
+ return 1;
+ }
+
+ /* 754 requires that FP exceptions run in "no stop" mode by default,
+ * and until C vendors implement C99's ways to control FP exceptions,
+ * Python requires non-stop mode. Alas, some platforms enable FP
+ * exceptions by default. Here we disable them.
+ */
+#ifdef __FreeBSD__
+ m = fpgetmask();
+ fpsetmask(m & ~FP_X_OFL);
+#endif
+
+ oldloc = _PyMem_RawStrdup(setlocale(LC_ALL, NULL));
+ if (!oldloc) {
+ fprintf(stderr, "out of memory\n");
+ return 1;
+ }
+
+ setlocale(LC_ALL, "");
+ for (i = 0; i < argc; i++) {
+ argv_copy[i] = _Py_char2wchar(argv[i], NULL);
+ if (!argv_copy[i]) {
+ PyMem_RawFree(oldloc);
+ fprintf(stderr, "Fatal Python error: "
+ "unable to decode the command line argument #%i\n",
+ i + 1);
+ return 1;
+ }
+ argv_copy2[i] = argv_copy[i];
+ }
+ argv_copy2[argc] = argv_copy[argc] = NULL;
+
+ setlocale(LC_ALL, oldloc);
+ PyMem_RawFree(oldloc);
+ res = Py_Main(argc, argv_copy);
+ for (i = 0; i < argc; i++) {
+ PyMem_RawFree(argv_copy2[i]);
+ }
+ PyMem_RawFree(argv_copy);
+ PyMem_RawFree(argv_copy2);
+ return res;
+}
+#endif