Merge 035797dff09959fd2e0137d9fcde173273288832 on remote branch
Change-Id: Ibcb2f09fc544a4ba9b85af138fd6d29d70ac95b7
diff --git a/bpfloader/BpfLoader.cpp b/bpfloader/BpfLoader.cpp
index 5390fb4..e67c469 100644
--- a/bpfloader/BpfLoader.cpp
+++ b/bpfloader/BpfLoader.cpp
@@ -53,7 +53,8 @@
#define BPF_PROG_PATH "/system/etc/bpf/"
-void loadAllElfObjects(void) {
+int loadAllElfObjects(void) {
+ int retVal = 0;
DIR* dir;
struct dirent* ent;
@@ -64,21 +65,35 @@
string progPath = BPF_PROG_PATH + s;
- int ret = android::bpf::loadProg(progPath.c_str());
- ALOGI("Attempted load object: %s, ret: %s", progPath.c_str(), std::strerror(-ret));
+ bool critical;
+ int ret = android::bpf::loadProg(progPath.c_str(), &critical);
+ if (ret) {
+ if (critical) retVal = ret;
+ ALOGE("Failed to load object: %s, ret: %s", progPath.c_str(), std::strerror(-ret));
+ } else {
+ ALOGI("Loaded object: %s", progPath.c_str());
+ }
}
closedir(dir);
}
+ return retVal;
}
int main() {
- if (android::bpf::isBpfSupported()) {
- // Load all ELF objects, create programs and maps, and pin them
- loadAllElfObjects();
+ if (!android::bpf::isBpfSupported()) return 0;
+
+ // Load all ELF objects, create programs and maps, and pin them
+ if (loadAllElfObjects() != 0) {
+ ALOGE("=== CRITICAL FAILURE LOADING BPF PROGRAMS ===");
+ ALOGE("If this triggers reliably, you're probably missing kernel options or patches.");
+ ALOGE("If this triggers randomly, you might be hitting some memory allocation problems or "
+ "startup script race.");
+ ALOGE("--- DO NOT EXPECT SYSTEM TO BOOT SUCCESSFULLY ---");
+ return 2;
}
if (android::base::SetProperty("bpf.progs_loaded", "1") == false) {
- ALOGE("Failed to set bpf.progs_loaded property\n");
+ ALOGE("Failed to set bpf.progs_loaded property");
return 1;
}
diff --git a/bpfloader/bpfloader.rc b/bpfloader/bpfloader.rc
index 836ec63..04d9b81 100644
--- a/bpfloader/bpfloader.rc
+++ b/bpfloader/bpfloader.rc
@@ -1,5 +1,31 @@
+# zygote-start is what officially starts netd (see //system/core/rootdir/init.rc)
+# However, on some hardware it's started from post-fs-data as well, which is just
+# a tad earlier. There's no benefit to that though, since on 4.9+ P+ devices netd
+# will just block until bpfloader finishes and sets the bpf.progs_loaded property.
+#
+# It is important that we start bpfloader after:
+# - /sys/fs/bpf is already mounted,
+# - apex (incl. rollback) is initialized (so that in the future we can load bpf
+# programs shipped as part of apex mainline modules)
+# - system properties have been set, this is because isBpfSupported() calls
+# getUncachedBpfSupportLevel() which depends on
+# ro.kernel.ebpf.supported, ro.product.first_api_level & ro.build.version.sdk
+# - logd is ready for us to log stuff
+#
+# At the same time we want to be as early as possible to reduce races and thus
+# failures (before memory is fragmented, and cpu is busy running tons of other
+# stuff) and we absolutely want to be before netd and the system boot slot is
+# considered to have booted successfully.
+#
+on load_bpf_programs
+ # Enable the eBPF JIT -- but do note that on 64-bit kernels it is likely
+ # already force enabled by the kernel config option BPF_JIT_ALWAYS_ON
+ write /proc/sys/net/core/bpf_jit_enable 1
+ # Enable JIT kallsyms export for privileged users only
+ write /proc/sys/net/core/bpf_jit_kallsyms 1
+ start bpfloader
+
service bpfloader /system/bin/bpfloader
- class main
capabilities CHOWN SYS_ADMIN
#
# Set RLIMIT_MEMLOCK to 1GiB for bpfloader
@@ -31,11 +57,6 @@
#
rlimit memlock 1073741824 1073741824
oneshot
-
-# Need to make sure this runs *before* the bpfloader.
-on early-init
- # Enable the eBPF JIT -- but do note that it is likely already force enabled
- # by the kernel config option BPF_JIT_ALWAYS_ON
- write /proc/sys/net/core/bpf_jit_enable 1
- # Enable JIT kallsyms export for privileged users only
- write /proc/sys/net/core/bpf_jit_kallsyms 1
+ reboot_on_failure reboot,bpfloader-failed
+ # we're not really updatable, but want to be able to load bpf programs shipped in apexes
+ updatable
diff --git a/libbpf_android/BpfLoadTest.cpp b/libbpf_android/BpfLoadTest.cpp
index 349e997..b384c11 100644
--- a/libbpf_android/BpfLoadTest.cpp
+++ b/libbpf_android/BpfLoadTest.cpp
@@ -43,7 +43,9 @@
unlink(tp_prog_path);
unlink(tp_map_path);
- EXPECT_EQ(android::bpf::loadProg("/system/etc/bpf/bpf_load_tp_prog.o"), 0);
+ bool critical = true;
+ EXPECT_EQ(android::bpf::loadProg("/system/etc/bpf/bpf_load_tp_prog.o", &critical), 0);
+ EXPECT_EQ(false, critical);
mProgFd = bpf_obj_get(tp_prog_path);
EXPECT_GT(mProgFd, 0);
diff --git a/libbpf_android/Loader.cpp b/libbpf_android/Loader.cpp
index f808dc6..da6a109 100644
--- a/libbpf_android/Loader.cpp
+++ b/libbpf_android/Loader.cpp
@@ -564,8 +564,9 @@
progPinLoc += '_';
progPinLoc += name;
if (access(progPinLoc.c_str(), F_OK) == 0) {
- fd = bpf_obj_get(progPinLoc.c_str());
- ALOGD("New bpf prog load reusing prog %s, ret: %d\n", progPinLoc.c_str(), fd);
+ fd = retrieveProgram(progPinLoc.c_str());
+ ALOGD("New bpf prog load reusing prog %s, ret: %d (%s)\n", progPinLoc.c_str(), fd,
+ (fd < 0 ? std::strerror(errno) : "no error"));
reuse = true;
} else {
vector<char> log_buf(BPF_LOAD_LOG_SZ, 0);
@@ -579,9 +580,15 @@
if (fd < 0) {
std::vector<std::string> lines = android::base::Split(log_buf.data(), "\n");
- ALOGE("bpf_prog_load - BEGIN log_buf contents:");
- for (const auto& line : lines) ALOGE("%s", line.c_str());
- ALOGE("bpf_prog_load - END log_buf contents.");
+ ALOGW("bpf_prog_load - BEGIN log_buf contents:");
+ for (const auto& line : lines) ALOGW("%s", line.c_str());
+ ALOGW("bpf_prog_load - END log_buf contents.");
+
+ if (cs[i].prog_def->optional) {
+ ALOGW("failed program is marked optional - continuing...");
+ continue;
+ }
+ ALOGE("non-optional program failed to load.");
}
}
@@ -606,21 +613,30 @@
return 0;
}
-int loadProg(const char* elfPath) {
+int loadProg(const char* elfPath, bool* isCritical) {
vector<char> license;
+ vector<char> critical;
vector<codeSection> cs;
vector<unique_fd> mapFds;
int ret;
+ if (!isCritical) return -1;
+ *isCritical = false;
+
ifstream elfFile(elfPath, ios::in | ios::binary);
if (!elfFile.is_open()) return -1;
+ ret = readSectionByName("critical", elfFile, critical);
+ *isCritical = !ret;
+
ret = readSectionByName("license", elfFile, license);
if (ret) {
ALOGE("Couldn't find license in %s\n", elfPath);
return ret;
} else {
- ALOGD("Loading ELF object %s with license %s\n", elfPath, (char*)license.data());
+ ALOGD("Loading %s%s ELF object %s with license %s\n",
+ *isCritical ? "critical for " : "optional", *isCritical ? (char*)critical.data() : "",
+ elfPath, (char*)license.data());
}
ret = readCodeSections(elfFile, cs);
@@ -649,10 +665,21 @@
return ret;
}
+static bool waitSecondsForProgsLoaded(int seconds) {
+ bool ok =
+ android::base::WaitForProperty("bpf.progs_loaded", "1", std::chrono::seconds(seconds));
+ if (!ok) ALOGW("Waited %ds for bpf.progs_loaded, still waiting...", seconds);
+ return ok;
+}
+
void waitForProgsLoaded() {
- while (!android::base::WaitForProperty("bpf.progs_loaded", "1", std::chrono::seconds(5))) {
- ALOGW("Waited 5s for bpf.progs_loaded, still waiting...");
- }
+ if (!android::bpf::isBpfSupported()) return;
+
+ if (waitSecondsForProgsLoaded(5)) return;
+ if (waitSecondsForProgsLoaded(10)) return;
+ if (waitSecondsForProgsLoaded(20)) return;
+ while (!waitSecondsForProgsLoaded(60))
+ ; // loop until success
}
} // namespace bpf
diff --git a/libbpf_android/include/bpf/BpfMap.h b/libbpf_android/include/bpf/BpfMap.h
index c92f989..3e7413e 100644
--- a/libbpf_android/include/bpf/BpfMap.h
+++ b/libbpf_android/include/bpf/BpfMap.h
@@ -179,7 +179,7 @@
template <class Key, class Value>
base::Result<void> BpfMap<Key, Value>::init(const char* path) {
- mMapFd = base::unique_fd(mapRetrieve(path, 0));
+ mMapFd = base::unique_fd(mapRetrieveRW(path));
if (mMapFd == -1) {
return ErrnoErrorf("Pinned map not accessible or does not exist: ({})", path);
}
diff --git a/libbpf_android/include/bpf/BpfUtils.h b/libbpf_android/include/bpf/BpfUtils.h
index f724f3b..6814f94 100644
--- a/libbpf_android/include/bpf/BpfUtils.h
+++ b/libbpf_android/include/bpf/BpfUtils.h
@@ -129,6 +129,22 @@
return bpfFdGet(pathname, flag);
}
+inline int mapRetrieveRW(const char* pathname) {
+ return mapRetrieve(pathname, 0);
+}
+
+inline int mapRetrieveRO(const char* pathname) {
+ return mapRetrieve(pathname, BPF_F_RDONLY);
+}
+
+inline int mapRetrieveWO(const char* pathname) {
+ return mapRetrieve(pathname, BPF_F_WRONLY);
+}
+
+inline int retrieveProgram(const char* pathname) {
+ return bpfFdGet(pathname, BPF_F_RDONLY);
+}
+
inline int attachProgram(bpf_attach_type type, const base::unique_fd& prog_fd,
const base::unique_fd& cg_fd) {
return bpf(BPF_PROG_ATTACH, {
diff --git a/libbpf_android/include/libbpf_android.h b/libbpf_android/include/libbpf_android.h
index 0a01c85..3810d07 100644
--- a/libbpf_android/include/libbpf_android.h
+++ b/libbpf_android/include/libbpf_android.h
@@ -25,7 +25,7 @@
namespace bpf {
// BPF loader implementation. Loads an eBPF ELF object
-int loadProg(const char* elfpath);
+int loadProg(const char* elfPath, bool* isCritical);
// Wait for bpfloader to load BPF programs.
void waitForProgsLoaded();
diff --git a/progs/include/bpf_helpers.h b/progs/include/bpf_helpers.h
index 8ff155f..9d22584 100644
--- a/progs/include/bpf_helpers.h
+++ b/progs/include/bpf_helpers.h
@@ -12,6 +12,12 @@
/* Example use: LICENSE("GPL"); or LICENSE("Apache 2.0"); */
#define LICENSE(NAME) char _license[] SEC("license") = (NAME)
+/* flag the resulting bpf .o file as critical to system functionality,
+ * loading all kernel version appropriate programs in it must succeed
+ * for bpfloader success
+ */
+#define CRITICAL(REASON) char _critical[] SEC("critical") = (REASON)
+
/*
* Helper functions called from eBPF programs written in C. These are
* implemented in the kernel sources.
@@ -105,21 +111,47 @@
#define KVER(a, b, c) ((a)*65536 + (b)*256 + (c))
#define KVER_INF 0xFFFFFFFF
-// programs requiring a kernel version >= min_kv && < max_kv
-#define DEFINE_BPF_PROG_KVER_RANGE(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv) \
- const struct bpf_prog_def SEC("progs") the_prog##_def = { \
- .uid = (prog_uid), \
- .gid = (prog_gid), \
- .min_kver = (min_kv), \
- .max_kver = (max_kv), \
- }; \
- SEC(SECTION_NAME) \
+#define DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv, \
+ opt) \
+ const struct bpf_prog_def SEC("progs") the_prog##_def = { \
+ .uid = (prog_uid), \
+ .gid = (prog_gid), \
+ .min_kver = (min_kv), \
+ .max_kver = (max_kv), \
+ .optional = (opt), \
+ }; \
+ SEC(SECTION_NAME) \
int the_prog
+// Programs (here used in the sense of functions/sections) marked optional are allowed to fail
+// to load (for example due to missing kernel patches).
+// The bpfloader will just ignore these failures and continue processing the next section.
+//
+// A non-optional program (function/section) failing to load causes a failure and aborts
+// processing of the entire .o, if the .o is additionally marked critical, this will result
+// in the entire bpfloader process terminating with a failure and not setting the bpf.progs_loaded
+// system property. This in turn results in waitForProgsLoaded() never finishing.
+//
+// ie. a non-optional program in a critical .o is mandatory for kernels matching the min/max kver.
+
+// programs requiring a kernel version >= min_kv && < max_kv
+#define DEFINE_BPF_PROG_KVER_RANGE(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv) \
+ DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv, \
+ false)
+#define DEFINE_OPTIONAL_BPF_PROG_KVER_RANGE(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, \
+ max_kv) \
+ DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, max_kv, true)
+
// programs requiring a kernel version >= min_kv
-#define DEFINE_BPF_PROG_KVER(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv) \
- DEFINE_BPF_PROG_KVER_RANGE(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, KVER_INF)
+#define DEFINE_BPF_PROG_KVER(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv) \
+ DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, KVER_INF, \
+ false)
+#define DEFINE_OPTIONAL_BPF_PROG_KVER(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv) \
+ DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, min_kv, KVER_INF, \
+ true)
// programs with no kernel version requirements
#define DEFINE_BPF_PROG(SECTION_NAME, prog_uid, prog_gid, the_prog) \
- DEFINE_BPF_PROG_KVER_RANGE(SECTION_NAME, prog_uid, prog_gid, the_prog, 0, KVER_INF)
+ DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, 0, KVER_INF, false)
+#define DEFINE_OPTIONAL_BPF_PROG(SECTION_NAME, prog_uid, prog_gid, the_prog) \
+ DEFINE_BPF_PROG_KVER_RANGE_OPT(SECTION_NAME, prog_uid, prog_gid, the_prog, 0, KVER_INF, true)
diff --git a/progs/include/bpf_map_def.h b/progs/include/bpf_map_def.h
index 01d9b8e..452b682 100644
--- a/progs/include/bpf_map_def.h
+++ b/progs/include/bpf_map_def.h
@@ -60,4 +60,6 @@
unsigned int min_kver; // KERNEL_MAJOR * 65536 + KERNEL_MINOR * 256 + KERNEL_SUB
unsigned int max_kver; // ie. 0x40900 for Linux 4.9 - but beware of hexadecimal for >= 10
+
+ bool optional; // program section (ie. function) may fail to load, continue onto next func.
};