<Hermetic> Implement CPython2 Launcher in C++.
The launcher is actually a wrapper which will statically link Python
interpreter within the .par file. It is used to bootstrap embedded
interpreter.
The next step is to change/integrate with Soong to make this hermetic .par
generation process more automatic.
Bug: b/62380596
Test: The launcher has been tested using real files:
zip -r hermetic.zip entry_point.txt Stdlib/ runfiles/
cat launcher | cat - hermetic.zip > executable && chmod u+x executable
Change-Id: I293cae2fe74d46766044f3e3c4b654a54d319b67
diff --git a/Android.bp b/Android.bp
index 69c436e..224f963 100644
--- a/Android.bp
+++ b/Android.bp
@@ -12,6 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+subdirs = [
+ "Lib",
+]
+
cc_defaults {
name: "py2-interp-defaults",
cflags: [
@@ -165,26 +169,23 @@
}
cc_binary {
- name: "python",
+ name: "py2-launcher",
defaults: ["py2-interp-defaults"],
- relative_install_path: "python2.7",
cflags: [
- // Depends on /usr/lib/python2.7 for now to fetch Python standard Libs.
- // This will be changed after std Libs are packed up within hermetic Python.
"-DVERSION=\"2.7\"",
- "-DVPATH=\".\"",
- "-DPREFIX=\".\"",
- "-DEXEC_PREFIX=\".\"",
- "-DPYTHONPATH=\".\"",
+ "-DVPATH=\"\"",
+ "-DPREFIX=\"\"",
+ "-DEXEC_PREFIX=\"\"",
+ "-DPYTHONPATH=\"\"",
],
srcs: [
- "Modules/main.c",
- "Modules/python.c",
+ "Launcher/launcher_main.cpp",
+ "Launcher/launcher_internal.cpp",
"Modules/gcmodule.c",
"Modules/getpath.c",
"Modules/config.c",
"Modules/getbuildinfo.c",
- // Default built-in extension C-modules.
+ // Default built-in extension py2-c-modules.
"Modules/threadmodule.c",
"Modules/signalmodule.c",
"Modules/posixmodule.c",
@@ -196,52 +197,61 @@
"Modules/zipimport.c",
"Modules/symtablemodule.c",
],
+ local_include_dirs: [
+ "Launcher",
+ ],
+ // NOTE: Please update Modules/config.c if new lib get added in the static_libs.
static_libs: [
- "py2-interp-parser",
+ "libbase",
+ // Be careful the order of these three static libraries below due to
+ // missing symbols issues.
"py2-interp-object",
"py2-interp-python",
- "c-module-array",
- "c-module-cmath",
- "c-module-math",
- "c-module-_struct",
- "c-module-time",
- "c-module-operator",
- "c-module-_testcapi",
- "c-module-_random",
- "c-module-_collections",
- "c-module-_heapq",
- "c-module-itertools",
- "c-module-strop",
- "c-module-_functools",
- "c-module-datetime",
- "c-module-_bisect",
- "c-module-unicodedata",
- "c-module-_io",
- "c-module-fcntl",
- "c-module-select",
- "c-module-mmap",
- "c-module-_csv",
- "c-module-_socket",
- "c-module-termios",
- "c-module-_md5",
- "c-module-_sha",
- "c-module-_sha256",
- "c-module-_sha512",
- "c-module-binascii",
- "c-module-parser",
- "c-module-cStringIO",
- "c-module-cPickle",
- "c-module-xxsubtype",
- "c-module-future_builtins",
- "c-module-_json",
- "c-module-_hotshot",
- "c-module-_lsprof",
- "c-module-grp",
- "c-module-syslog",
- "c-module-audioop",
- "c-module-resource",
- "c-module-_multibytecodec",
- "c-module-_multiprocessing",
+ "py2-interp-parser",
+ "py2-c-module-array",
+ "py2-c-module-cmath",
+ "py2-c-module-math",
+ "py2-c-module-_struct",
+ "py2-c-module-time",
+ "py2-c-module-operator",
+ "py2-c-module-_testcapi",
+ "py2-c-module-_random",
+ "py2-c-module-_collections",
+ "py2-c-module-_heapq",
+ "py2-c-module-itertools",
+ "py2-c-module-strop",
+ "py2-c-module-_functools",
+ "py2-c-module-datetime",
+ "py2-c-module-_bisect",
+ "py2-c-module-unicodedata",
+ "py2-c-module-_io",
+ "py2-c-module-fcntl",
+ "py2-c-module-select",
+ "py2-c-module-mmap",
+ "py2-c-module-_csv",
+ "py2-c-module-_socket",
+ "py2-c-module-termios",
+ "py2-c-module-_md5",
+ "py2-c-module-_sha",
+ "py2-c-module-_sha256",
+ "py2-c-module-_sha512",
+ "py2-c-module-binascii",
+ "py2-c-module-parser",
+ "py2-c-module-cStringIO",
+ "py2-c-module-cPickle",
+ "py2-c-module-xxsubtype",
+ "py2-c-module-future_builtins",
+ "py2-c-module-_json",
+ "py2-c-module-_hotshot",
+ "py2-c-module-_lsprof",
+ "py2-c-module-grp",
+ "py2-c-module-syslog",
+ "py2-c-module-audioop",
+ "py2-c-module-resource",
+ "py2-c-module-_multibytecodec",
+ "py2-c-module-_multiprocessing",
+ "py2-c-module-zlib",
+ "libz",
],
host_ldlibs: [
"-lpthread",
@@ -249,13 +259,12 @@
"-lutil",
"-lm"
],
- stl: "none",
}
-// Enabled extension C-modules.
+// Enabled extension py2-c-modules.
cc_library_static {
- name: "c-module-array",
+ name: "py2-c-module-array",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/arraymodule.c",
@@ -263,7 +272,7 @@
}
cc_library_static {
- name: "c-module-cmath",
+ name: "py2-c-module-cmath",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/cmathmodule.c",
@@ -272,7 +281,7 @@
}
cc_library_static {
- name: "c-module-math",
+ name: "py2-c-module-math",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/mathmodule.c",
@@ -281,7 +290,7 @@
}
cc_library_static {
- name: "c-module-_struct",
+ name: "py2-c-module-_struct",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/_struct.c",
@@ -289,7 +298,7 @@
}
cc_library_static {
- name: "c-module-time",
+ name: "py2-c-module-time",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/timemodule.c",
@@ -297,7 +306,7 @@
}
cc_library_static {
- name: "c-module-operator",
+ name: "py2-c-module-operator",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/operator.c",
@@ -305,7 +314,7 @@
}
cc_library_static {
- name: "c-module-_testcapi",
+ name: "py2-c-module-_testcapi",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/_testcapimodule.c",
@@ -313,7 +322,7 @@
}
cc_library_static {
- name: "c-module-_random",
+ name: "py2-c-module-_random",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/_randommodule.c",
@@ -321,7 +330,7 @@
}
cc_library_static {
- name: "c-module-_collections",
+ name: "py2-c-module-_collections",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/_collectionsmodule.c",
@@ -329,7 +338,7 @@
}
cc_library_static {
- name: "c-module-_heapq",
+ name: "py2-c-module-_heapq",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/_heapqmodule.c",
@@ -337,7 +346,7 @@
}
cc_library_static {
- name: "c-module-itertools",
+ name: "py2-c-module-itertools",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/itertoolsmodule.c",
@@ -345,7 +354,7 @@
}
cc_library_static {
- name: "c-module-strop",
+ name: "py2-c-module-strop",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/stropmodule.c",
@@ -353,7 +362,7 @@
}
cc_library_static {
- name: "c-module-_functools",
+ name: "py2-c-module-_functools",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/_functoolsmodule.c",
@@ -361,7 +370,7 @@
}
cc_library_static {
- name: "c-module-datetime",
+ name: "py2-c-module-datetime",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/datetimemodule.c",
@@ -369,7 +378,7 @@
}
cc_library_static {
- name: "c-module-_bisect",
+ name: "py2-c-module-_bisect",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/_bisectmodule.c",
@@ -377,7 +386,7 @@
}
cc_library_static {
- name: "c-module-unicodedata",
+ name: "py2-c-module-unicodedata",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/unicodedata.c",
@@ -385,7 +394,7 @@
}
cc_library_static {
- name: "c-module-_io",
+ name: "py2-c-module-_io",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/_io/bufferedio.c",
@@ -402,7 +411,7 @@
}
cc_library_static {
- name: "c-module-fcntl",
+ name: "py2-c-module-fcntl",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/fcntlmodule.c",
@@ -410,7 +419,7 @@
}
cc_library_static {
- name: "c-module-select",
+ name: "py2-c-module-select",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/selectmodule.c",
@@ -418,7 +427,7 @@
}
cc_library_static {
- name: "c-module-mmap",
+ name: "py2-c-module-mmap",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/mmapmodule.c",
@@ -426,7 +435,7 @@
}
cc_library_static {
- name: "c-module-_csv",
+ name: "py2-c-module-_csv",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/_csv.c",
@@ -434,7 +443,7 @@
}
cc_library_static {
- name: "c-module-_socket",
+ name: "py2-c-module-_socket",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/socketmodule.c",
@@ -443,7 +452,7 @@
}
cc_library_static {
- name: "c-module-termios",
+ name: "py2-c-module-termios",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/termios.c",
@@ -451,7 +460,7 @@
}
cc_library_static {
- name: "c-module-_md5",
+ name: "py2-c-module-_md5",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/md5module.c",
@@ -460,7 +469,7 @@
}
cc_library_static {
- name: "c-module-_sha",
+ name: "py2-c-module-_sha",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/shamodule.c",
@@ -468,7 +477,7 @@
}
cc_library_static {
- name: "c-module-_sha256",
+ name: "py2-c-module-_sha256",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/sha256module.c",
@@ -476,7 +485,7 @@
}
cc_library_static {
- name: "c-module-_sha512",
+ name: "py2-c-module-_sha512",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/sha512module.c",
@@ -484,7 +493,7 @@
}
cc_library_static {
- name: "c-module-binascii",
+ name: "py2-c-module-binascii",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/binascii.c",
@@ -492,7 +501,7 @@
}
cc_library_static {
- name: "c-module-parser",
+ name: "py2-c-module-parser",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/parsermodule.c",
@@ -500,7 +509,7 @@
}
cc_library_static {
- name: "c-module-cStringIO",
+ name: "py2-c-module-cStringIO",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/cStringIO.c",
@@ -508,7 +517,7 @@
}
cc_library_static {
- name: "c-module-cPickle",
+ name: "py2-c-module-cPickle",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/cPickle.c",
@@ -516,7 +525,7 @@
}
cc_library_static {
- name: "c-module-xxsubtype",
+ name: "py2-c-module-xxsubtype",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/xxsubtype.c",
@@ -524,7 +533,7 @@
}
cc_library_static {
- name: "c-module-future_builtins",
+ name: "py2-c-module-future_builtins",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/future_builtins.c",
@@ -532,7 +541,7 @@
}
cc_library_static {
- name: "c-module-_json",
+ name: "py2-c-module-_json",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/_json.c",
@@ -540,7 +549,7 @@
}
cc_library_static {
- name: "c-module-_hotshot",
+ name: "py2-c-module-_hotshot",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/_hotshot.c",
@@ -548,7 +557,7 @@
}
cc_library_static {
- name: "c-module-_lsprof",
+ name: "py2-c-module-_lsprof",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/_lsprof.c",
@@ -557,7 +566,7 @@
}
cc_library_static {
- name: "c-module-grp",
+ name: "py2-c-module-grp",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/grpmodule.c",
@@ -566,7 +575,7 @@
}
cc_library_static {
- name: "c-module-syslog",
+ name: "py2-c-module-syslog",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/syslogmodule.c",
@@ -574,7 +583,7 @@
}
cc_library_static {
- name: "c-module-audioop",
+ name: "py2-c-module-audioop",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/audioop.c",
@@ -582,7 +591,7 @@
}
cc_library_static {
- name: "c-module-resource",
+ name: "py2-c-module-resource",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/resource.c",
@@ -590,7 +599,7 @@
}
cc_library_static {
- name: "c-module-_multibytecodec",
+ name: "py2-c-module-_multibytecodec",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/cjkcodecs/multibytecodec.c",
@@ -598,7 +607,7 @@
}
cc_library_static {
- name: "c-module-_multiprocessing",
+ name: "py2-c-module-_multiprocessing",
defaults: ["py2-interp-defaults"],
srcs: [
"Modules/_multiprocessing/multiprocessing.c",
@@ -606,3 +615,19 @@
"Modules/_multiprocessing/socket_connection.c",
],
}
+
+cc_library_static {
+ name: "py2-c-module-zlib",
+ defaults: [
+ "py2-interp-defaults",
+ ],
+ cflags: [
+ "-DUSE_ZLIB_CRC32",
+ ],
+ srcs: [
+ "Modules/zlibmodule.c",
+ ],
+ static_libs: [
+ "libz",
+ ],
+}
diff --git a/Launcher/launcher_internal.cpp b/Launcher/launcher_internal.cpp
new file mode 100644
index 0000000..be4d48e
--- /dev/null
+++ b/Launcher/launcher_internal.cpp
@@ -0,0 +1,216 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "launcher_internal.h"
+
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+extern "C" {
+// Cpython built-in C functions.
+/*
+ read_directory(archive) -> files dict (new reference)
+
+ Given a path to a Zip archive, build a dict, mapping file names
+ (local to the archive, using SEP as a separator) to toc entries.
+*/
+PyObject *read_directory(const char *archive);
+
+/* Given a path to a Zip file and a toc_entry, return the (uncompressed)
+ data as a new reference. */
+PyObject *get_data(const char *archive, PyObject *toc_entry);
+}
+
+namespace android {
+namespace cpython2 {
+namespace python_launcher {
+namespace internal{
+
+int RunModule(const char *module, int set_argv0) {
+ PyObject *runpy, *runmodule, *runargs, *result;
+ runpy = PyImport_ImportModule("runpy");
+ if (runpy == NULL) {
+ fprintf(stderr, "Could not import runpy module\n");
+ return -1;
+ }
+ runmodule = PyObject_GetAttrString(runpy, "_run_module_as_main");
+ if (runmodule == NULL) {
+ fprintf(stderr, "Could not access runpy._run_module_as_main\n");
+ Py_DECREF(runpy);
+ return -1;
+ }
+ runargs = Py_BuildValue("(si)", module, set_argv0);
+ if (runargs == NULL) {
+ fprintf(stderr,
+ "Could not create arguments for runpy._run_module_as_main\n");
+ Py_DECREF(runpy);
+ Py_DECREF(runmodule);
+ return -1;
+ }
+ result = PyObject_Call(runmodule, runargs, NULL);
+ if (result == NULL) {
+ PyErr_Print();
+ }
+ Py_DECREF(runpy);
+ Py_DECREF(runmodule);
+ Py_DECREF(runargs);
+ if (result == NULL) {
+ return -1;
+ }
+ Py_DECREF(result);
+ return 0;
+}
+
+std::string GetEntryPointFilePath(const char *launcher_path) {
+ PyObject *files;
+ files = read_directory(launcher_path);
+ if (files == NULL) {
+ return std::string();
+ }
+ PyObject *toc_entry;
+ // Return value: Borrowed reference.
+ toc_entry = PyDict_GetItemString(files, ENTRYPOINT_FILE);
+ if (toc_entry == NULL) {
+ Py_DECREF(files);
+ return std::string();
+ }
+ PyObject *py_data;
+ py_data = get_data(launcher_path, toc_entry);
+ if (py_data == NULL) {
+ Py_DECREF(files);
+ return std::string();
+ }
+ // PyString_AsString returns a NUL-terminated representation of the "py_data",
+ // "data" must not be modified in any way. And it must not be deallocated.
+ char *data = PyString_AsString(py_data);
+ if (data == NULL) {
+ Py_DECREF(py_data);
+ Py_DECREF(files);
+ return std::string();
+ }
+
+ char *res = strdup(data); /* deep copy of data */
+ Py_DECREF(py_data);
+ Py_DECREF(files);
+
+ int i = 0;
+ /* Strip newline and other trailing whitespace. */
+ for (i = strlen(res) - 1; i >= 0 && isspace(res[i]); i--) {
+ res[i] = '\0';
+ }
+ /* Check for the file extension. */
+ i = strlen(res);
+ if (i > 3 && strcmp(res + i - 3, ".py") == 0) {
+ res[i - 3] = '\0';
+ } else {
+ PyErr_Format(PyExc_ValueError, "Invalid entrypoint in %s: %s",
+ ENTRYPOINT_FILE, res);
+ return std::string();
+ }
+ return std::string(res);
+}
+
+int RunModuleNameFromEntryPoint(const char *launcher_path, std::string entrypoint) {
+ if (entrypoint.empty()) {
+ return -1;
+ }
+ // Has to pass to free to avoid a memory leak after use.
+ char *arr = strdup(entrypoint.c_str());
+ // Replace file system path seperator with Python package/module seperator.
+ char *ch;
+ for (ch = arr; *ch; ch++) {
+ if (*ch == '/') {
+ *ch = '.';
+ }
+ }
+
+ if (AddPathToPythonSysPath(launcher_path) < 0) {
+ free(arr);
+ return -1;
+ }
+ // Calculate the runfiles path size. Extra space for '\0'.
+ size_t size = snprintf(nullptr, 0, "%s/%s", launcher_path, RUNFILES) + 1;
+ char runfiles_path[size];
+ snprintf(runfiles_path, size, "%s/%s", launcher_path, RUNFILES);
+ if (AddPathToPythonSysPath(runfiles_path) < 0) {
+ free(arr);
+ return -1;
+ }
+ int ret = RunModule(arr, 0);
+ free(arr);
+ return ret;
+}
+
+int AddPathToPythonSysPath(const char *path) {
+ if (path == NULL) {
+ return -1;
+ }
+ PyObject *py_path;
+ py_path = PyString_FromString(path);
+ if (py_path == NULL) {
+ return -1;
+ }
+ PyObject *sys_path;
+ // Return value: Borrowed reference.
+ sys_path = PySys_GetObject(const_cast<char*>("path"));
+ if (sys_path == NULL) {
+ Py_DECREF(py_path);
+ return -1;
+ }
+ PyList_Insert(sys_path, 0, py_path);
+ Py_DECREF(py_path);
+ return 0;
+}
+
+int RunMainFromImporter(const char *launcher_path) {
+ PyObject *py_launcher_path, *importer;
+ py_launcher_path = PyString_FromString(launcher_path);
+ if (py_launcher_path == NULL) {
+ return -1;
+ }
+ importer = PyImport_GetImporter(py_launcher_path);
+ if (importer == NULL) {
+ Py_DECREF(py_launcher_path);
+ return -1;
+ }
+ if (importer != Py_None && importer->ob_type != &PyNullImporter_Type) {
+ /* Launcher path is usable as an import source, so
+ put it in sys.path[0] and import __main__ */
+ if (AddPathToPythonSysPath(launcher_path) < 0) {
+ Py_DECREF(importer);
+ Py_DECREF(py_launcher_path);
+ return -1;
+ }
+ }
+ Py_DECREF(importer);
+ Py_DECREF(py_launcher_path);
+ return RunModule("__main__", 0);
+}
+} // namespace internal
+
+int RunEntryPointOrMainModule(const char *launcher_path) {
+ std::string entrypoint = internal::GetEntryPointFilePath(launcher_path);
+ if (entrypoint.empty()) {
+ // If entry point can not be found or can not be executed, we try to
+ // run __main__.py within the .par file.
+ fprintf(stderr, "Cannot find valid entry point to execute par file!\n");
+ fprintf(stdout, "Start trying to run __main__ module within par file.\n");
+ return internal::RunMainFromImporter(launcher_path);
+ }
+ return internal::RunModuleNameFromEntryPoint(launcher_path, entrypoint);
+}
+} // namespace python_launcher
+} // namespace cpython2
+} // namespace android
diff --git a/Launcher/launcher_internal.h b/Launcher/launcher_internal.h
new file mode 100644
index 0000000..285176a
--- /dev/null
+++ b/Launcher/launcher_internal.h
@@ -0,0 +1,62 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef ANDROID_CPYTHON2_PYTHON_LAUNCHER_INTERNAL_H
+#define ANDROID_CPYTHON2_PYTHON_LAUNCHER_INTERNAL_H
+
+#include <Python.h>
+
+#include <string>
+
+namespace android {
+namespace cpython2 {
+namespace python_launcher {
+
+namespace internal{
+#define ENTRYPOINT_FILE "entry_point.txt"
+#define RUNFILES "runfiles"
+
+// Use "runpy" module to locate and run Python script using Python module
+// namespace rather than the filesystem.
+// The caller owns "module" pointer, which cannot be NULL.
+int RunModule(const char *module, int set_argv0);
+
+// Get valid entrypoint file path.
+// The caller owns "launcher_path" pointer, which cannot be NULL.
+// Return non-empty string as success. Otherwise, return empty string.
+std::string GetEntryPointFilePath(const char *launcher_path);
+
+// Run the Python script embedded in ENTRYPOINT_FILE.
+// The caller owns "launcher_path" pointer, which cannot be NULL.
+int RunModuleNameFromEntryPoint(const char *launcher_path, std::string entrypoint);
+
+// Add path to Python sys.path list.
+// The caller owns "path" pointer, which cannot be NULL.
+int AddPathToPythonSysPath(const char *path);
+
+// Run __main__ module within the hermetic .par file.
+// The caller owns "launcher_path" pointer, which cannot be NULL.
+// Return 0 as success. Otherwise, return -1.
+int RunMainFromImporter(const char *launcher_path);
+} // namespace internal
+
+// Try to run the Python script embedded in ENTRYPOINT_FILE. Otherwise,
+// run __main__ module as fallback.
+// The caller owns "launcher_path" pointer, which cannot be NULL.
+int RunEntryPointOrMainModule(const char *launcher_path);
+} // namespace python_launcher
+} // namespace cpython2
+} // namespace android
+
+#endif // ANDROID_CPYTHON2_PYTHON_LAUNCHER_INTERNAL_H
diff --git a/Launcher/launcher_main.cpp b/Launcher/launcher_main.cpp
new file mode 100644
index 0000000..a49998c
--- /dev/null
+++ b/Launcher/launcher_main.cpp
@@ -0,0 +1,124 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "launcher_internal.h"
+
+#include <Python.h>
+#include <android-base/file.h>
+#include <osdefs.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
+
+int main(int argc, char *argv[]) {
+ int result = 0 /* Used to mark if current program runs with success/failure. */;
+
+ // Clear PYTHONPATH and PYTHONHOME so Python doesn't attempt to check the local
+ // disk for Python modules to load. The value of PYTHONHOME will replace "prefix"
+ // and "exe_prefix" based on the description in getpath.c.
+ // Please don't use PYTHONPATH and PYTHONHOME within user program.
+ // TODO(nanzhang): figure out if unsetenv("PYTHONPATH") is better.
+ unsetenv(const_cast<char *>("PYTHONPATH"));
+ // TODO(nanzhang): figure out if Py_SetPythonHome() is better.
+ unsetenv(const_cast<char *>("PYTHONHOME"));
+ // PYTHONEXECUTABLE is only used on MacOs X, when the Python interpreter
+ // embedded in an application bundle. It is not sure that we have this use case
+ // for Android hermetic Python. So override this environment variable to empty
+ // for now to make our self-contained environment more strict.
+ // For user (.py) program, it can access hermetic .par file path through
+ // sys.argv[0].
+ unsetenv(const_cast<char *>("PYTHONEXECUTABLE"));
+
+ // Resolving absolute path based on argv[0] is not reliable since it may
+ // include something unusable, too bad.
+ // android::base::GetExecutablePath() also handles for Darwin/Windows.
+ std::string executable_path = android::base::GetExecutablePath();
+
+ argv[0] = strdup(executable_path.c_str());
+ // argv[0] is used for setting internal path, and Python sys.argv[0]. It
+ // should not exceed MAXPATHLEN defined for CPython.
+ if (!argv[0] || strlen(argv[0]) > MAXPATHLEN) {
+ fprintf(stderr, "The executable path %s is NULL or of invalid length.\n", argv[0]);
+ return 1;
+ }
+
+ // For debugging/logging purpose, set stdin/stdout/stderr unbuffered through
+ // environment variable.
+ // TODO(nanzhang): Set Py_VerboseFlag if more debugging requests needed.
+ const char *unbuffered_env = getenv("PYTHONUNBUFFERED");
+ if (unbuffered_env && unbuffered_env[0]) {
+ #if defined(MS_WINDOWS) || defined(__CYGWIN__)
+ _setmode(fileno(stdin), O_BINARY);
+ _setmode(fileno(stdout), O_BINARY);
+ #endif
+ #ifdef HAVE_SETVBUF
+ setvbuf(stdin, (char *)NULL, _IONBF, BUFSIZ);
+ setvbuf(stdout, (char *)NULL, _IONBF, BUFSIZ);
+ setvbuf(stderr, (char *)NULL, _IONBF, BUFSIZ);
+ #else /* !HAVE_SETVBUF */
+ setbuf(stdin, (char *)NULL);
+ setbuf(stdout, (char *)NULL);
+ setbuf(stderr, (char *)NULL);
+ #endif /* !HAVE_SETVBUF */
+ }
+ //For debugging/logging purpose, Warning control.
+ //Python’s warning machinery by default prints warning messages to sys.stderr.
+ //The full form of argument is:action:message:category:module:line
+ char *warnings_env = getenv("PYTHONWARNINGS");
+ if (warnings_env && warnings_env[0]) {
+ char *warnings_buf, *warning;
+
+ // Note: "new" operation; we need free this chuck of data after use.
+ warnings_buf = new char[strlen(warnings_env) + 1];
+ if (warnings_buf == NULL)
+ Py_FatalError(
+ "not enough memory to copy PYTHONWARNINGS");
+ strcpy(warnings_buf, warnings_env);
+ for (warning = strtok(warnings_buf, ",");
+ warning != NULL;
+ warning = strtok(NULL, ","))
+ PySys_AddWarnOption(warning);
+ delete[] warnings_buf;
+ }
+
+ // Always enable Python "-s" option. We don't need user-site directories,
+ // everything's supposed to be hermetic.
+ Py_NoUserSiteDirectory = 1;
+
+ Py_SetProgramName(argv[0]);
+ Py_Initialize();
+ PySys_SetArgvEx(argc, argv, 0);
+
+ // Set sys.executable to None. The real executable is available as
+ // sys.argv[0], and too many things assume sys.executable is a regular Python
+ // binary, which isn't available. By setting it to None we get clear errors
+ // when people try to use it.
+ if (PySys_SetObject(const_cast<char *>("executable"), Py_None) < 0) {
+ PyErr_Print();
+ result = 1;
+ goto error;
+ }
+
+ result = android::cpython2::python_launcher::RunEntryPointOrMainModule(argv[0]);
+ if (result < 0) {
+ PyErr_Print();
+ goto error;
+ }
+
+error:
+ Py_Finalize();
+
+ free(argv[0]);
+ exit(abs(result));
+}
diff --git a/Lib/Android.bp b/Lib/Android.bp
new file mode 100644
index 0000000..55971a3
--- /dev/null
+++ b/Lib/Android.bp
@@ -0,0 +1,64 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+python_library_host {
+ name: "py2-stdlib",
+ is_internal: true,
+ pkg_path: "stdlib",
+ srcs: [
+ "*.py",
+ "compiler/**/*.py",
+ "email/**/*.py",
+ "encodings/**/*.py",
+ "hotshot/**/*.py",
+ "importlib/**/*.py",
+ "json/**/*.py",
+ "logging/**/*.py",
+ "multiprocessing/**/*.py",
+ "pydoc_data/**/*.py",
+ "unittest/**/*.py",
+ "wsgiref/**/*.py",
+ "xml/**/*.py",
+ ],
+ // TODO(nanzhang): make exclude_srcs support globs.
+ exclude_srcs: [
+ "__phello__.foo.py",
+ ],
+ version: {
+ py2: {
+ enabled: true,
+ },
+ py3: {
+ enabled: false,
+ },
+ },
+ target: {
+ darwin: {
+ enabled: false,
+ },
+ linux_x86_64: {
+ srcs: [
+ ":py2-plat-linux2",
+ ],
+ },
+ },
+}
+
+filegroup {
+ name: "py2-plat-linux2",
+ srcs: [
+ "plat-linux2/*.py",
+ ],
+ path: "plat-linux2",
+}
diff --git a/Modules/config.c b/Modules/config.c
index 4f0e55e..807d1e9 100644
--- a/Modules/config.c
+++ b/Modules/config.c
@@ -75,6 +75,7 @@
extern void initresource(void);
extern void init_multibytecodec(void);
extern void init_multiprocessing(void);
+extern void initzlib(void);
/* -- ADDMODULE MARKER 1 -- */
@@ -138,6 +139,7 @@
{"resource", initresource},
{"_multibytecodec", init_multibytecodec},
{"_multiprocessing", init_multiprocessing},
+ {"zlib", initzlib},
/* -- ADDMODULE MARKER 2 -- */
diff --git a/Modules/getpath.c b/Modules/getpath.c
index c42ce31..e9d969b 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -104,6 +104,14 @@
#define LANDMARK "os.py"
#endif
+#ifndef INTERNALDIR
+#define INTERNALDIR "internal"
+#endif
+
+#ifndef STDLIBDIR
+#define STDLIBDIR "stdlib"
+#endif
+
static char prefix[MAXPATHLEN+1];
static char exec_prefix[MAXPATHLEN+1];
static char progpath[MAXPATHLEN+1];
@@ -255,8 +263,11 @@
return 1;
}
- /* Check to see if argv[0] is in the build directory */
+ // GOOGLE(nanzhang): Always set prefix with hermetic executable full path.
strcpy(prefix, argv0_path);
+ return 1;
+
+ /* Check to see if argv[0] is in the build directory */
joinpath(prefix, "Modules/Setup");
if (isfile(prefix)) {
/* Check VPATH to see if argv0_path is in the build directory. */
@@ -314,10 +325,13 @@
return 1;
}
+ // GOOGLE(nanzhang): Always set exec_prefix with hermetic executable full path.
+ strcpy(exec_prefix, argv0_path);
+ return 1;
+
/* Check to see if argv[0] is in the build directory. "pybuilddir.txt"
is written by setup.py and contains the relative path to the location
of shared library modules. */
- strcpy(exec_prefix, argv0_path);
joinpath(exec_prefix, "pybuilddir.txt");
if (isfile(exec_prefix)) {
FILE *f = fopen(exec_prefix, "r");
@@ -363,19 +377,62 @@
calculate_path(void)
{
extern char *Py_GetProgramName(void);
-
static char delimiter[2] = {DELIM, '\0'};
static char separator[2] = {SEP, '\0'};
char *pythonpath = PYTHONPATH;
- char *rtpypath = Py_GETENV("PYTHONPATH");
char *home = Py_GetPythonHome();
- char *path = getenv("PATH");
+ // We have overrided argv[0] using the full path to the hermetic Python
+ // launcher itself. And then Py_SetProgramName(argv[0]) was invoked at
+ // launcher_main.cpp. The launcher_main.cpp has guaranteed that
+ // strlen(Py_GetProgramName()) must not exceed MAXPATHLEN.
char *prog = Py_GetProgramName();
char argv0_path[MAXPATHLEN+1];
- char zip_path[MAXPATHLEN+1];
- int pfound, efound; /* 1 if found; -1 if found build directory */
char *buf;
size_t bufsz;
+
+ strncpy(progpath, prog, MAXPATHLEN);
+ progpath[MAXPATHLEN] = '\0' /* In case of no NUL-termination. */;
+ strncpy(argv0_path, prog, MAXPATHLEN);
+ argv0_path[MAXPATHLEN] = '\0' /* In case of no NUL-termination. */;
+
+ // We don't reduce the path of prefix, and exec_prefix.
+ search_for_prefix(argv0_path, home);
+ search_for_exec_prefix(argv0_path, home);
+
+ // Calculate size of return buffer.
+ bufsz = strlen(prog) + 1 + sizeof(INTERNALDIR) /* 1 is for SEP */;
+ bufsz += strlen(prog) + 1 + sizeof(INTERNALDIR) + 1 + sizeof(STDLIBDIR) /* 1 is for SEP */;
+
+ /* This is the only malloc call in this file */
+ buf = (char *)PyMem_Malloc(bufsz);
+
+ if (buf == NULL) {
+ /* We can't exit, so print a warning and limp along */
+ fprintf(stderr, "Not enough memory for dynamic PYTHONPATH.\n");
+ fprintf(stderr, "Using default static PYTHONPATH.\n");
+ exit(1);
+ } else {
+ buf[0] = '\0';
+ strcat(buf, prefix);
+ strcat(buf, separator);
+ strcat(buf, INTERNALDIR);
+ strcat(buf, delimiter);
+
+ strcat(buf, prefix);
+ strcat(buf, separator);
+ strcat(buf, INTERNALDIR);
+ strcat(buf, separator);
+ strcat(buf, STDLIBDIR);
+
+ module_search_path = buf;
+ }
+ // GOOGLE(nanzhang): Don't need all the code below for embedded Python launcher.
+ return;
+
+ char *path = getenv("PATH");
+ char *rtpypath = Py_GETENV("PYTHONPATH");
+ char zip_path[MAXPATHLEN+1];
+ int pfound, efound; /* 1 if found; -1 if found build directory */
size_t prefixsz;
char *defpath = pythonpath;
#ifdef WITH_NEXT_FRAMEWORK
diff --git a/Modules/zipimport.c b/Modules/zipimport.c
index 1691773..0a7cd0f 100644
--- a/Modules/zipimport.c
+++ b/Modules/zipimport.c
@@ -43,9 +43,11 @@
static PyObject *ZipImportError;
static PyObject *zip_directory_cache = NULL;
+// GOOGLE(nanzhang): Changed two functions below to be visible to launcher so
+// that launcher can access the zip metadata section.
/* forward decls */
-static PyObject *read_directory(const char *archive);
-static PyObject *get_data(const char *archive, PyObject *toc_entry);
+PyObject *read_directory(const char *archive);
+PyObject *get_data(const char *archive, PyObject *toc_entry);
static PyObject *get_module_code(ZipImporter *self, char *fullname,
int *p_ispackage, char **p_modpath);
@@ -702,7 +704,7 @@
Directories can be recognized by the trailing SEP in the name,
data_size and file_offset are 0.
*/
-static PyObject *
+PyObject *
read_directory(const char *archive)
{
PyObject *files = NULL;
@@ -912,7 +914,7 @@
/* Given a path to a Zip file and a toc_entry, return the (uncompressed)
data as a new reference. */
-static PyObject *
+PyObject *
get_data(const char *archive, PyObject *toc_entry)
{
PyObject *raw_data = NULL, *data, *decompress;