Merge all bpf programs into one
Multiple bpf program can be stored the same binary and we can extract
them one by one according to their function name elf header at run time.
Store them in one file can reduce the load time by only open one binary
file once. Rewrite the elf parser with Slice support.
Test: phone boot and eBPF program is loaded and pinned.
Bug: 78132446
Change-Id: I96dba91a69654fcac2c022100e954d8b0c4e0718
diff --git a/bpfloader/Android.bp b/bpfloader/Android.bp
index 9a759d1..46b8a1f 100644
--- a/bpfloader/Android.bp
+++ b/bpfloader/Android.bp
@@ -41,10 +41,7 @@
],
required: [
- "cgroup_bpf_ingress_prog.o",
- "cgroup_bpf_egress_prog.o",
- "xt_bpf_ingress_prog.o",
- "xt_bpf_egress_prog.o",
+ "bpf_kern.o",
],
}
diff --git a/bpfloader/Android.mk b/bpfloader/Android.mk
index 005abef..1393376 100644
--- a/bpfloader/Android.mk
+++ b/bpfloader/Android.mk
@@ -1,43 +1,10 @@
LOCAL_PATH:= $(call my-dir)
#######################################
-# bpf_ingress.o
+# bpf_kern.o
include $(CLEAR_VARS)
-LOCAL_MODULE := cgroup_bpf_ingress_prog.o
-LOCAL_SRC_FILES := $(LOCAL_MODULE)
-LOCAL_MODULE_CLASS := ETC
-LOCAL_MODULE_PATH := $(TARGET_OUT_ETC)/bpf
-
-include $(BUILD_PREBUILT)
-
-#######################################
-# bpf_egress.o
-include $(CLEAR_VARS)
-
-LOCAL_MODULE := cgroup_bpf_egress_prog.o
-LOCAL_SRC_FILES := $(LOCAL_MODULE)
-LOCAL_MODULE_CLASS := ETC
-LOCAL_MODULE_PATH := $(TARGET_OUT_ETC)/bpf
-
-include $(BUILD_PREBUILT)
-
-#######################################
-# xt_bpf_ingress_prog.o
-include $(CLEAR_VARS)
-
-LOCAL_MODULE := xt_bpf_ingress_prog.o
-LOCAL_SRC_FILES := $(LOCAL_MODULE)
-LOCAL_MODULE_CLASS := ETC
-LOCAL_MODULE_PATH := $(TARGET_OUT_ETC)/bpf
-
-include $(BUILD_PREBUILT)
-
-#######################################
-# xt_bpf_egress_prog.o
-include $(CLEAR_VARS)
-
-LOCAL_MODULE := xt_bpf_egress_prog.o
+LOCAL_MODULE := bpf_kern.o
LOCAL_SRC_FILES := $(LOCAL_MODULE)
LOCAL_MODULE_CLASS := ETC
LOCAL_MODULE_PATH := $(TARGET_OUT_ETC)/bpf
diff --git a/bpfloader/BpfLoader.cpp b/bpfloader/BpfLoader.cpp
index 90d3d6a..c83ee1f 100644
--- a/bpfloader/BpfLoader.cpp
+++ b/bpfloader/BpfLoader.cpp
@@ -50,11 +50,7 @@
using android::netdutils::Slice;
#define BPF_PROG_PATH "/system/etc/bpf"
-
-#define INGRESS_PROG BPF_PROG_PATH"/cgroup_bpf_ingress_prog.o"
-#define EGRESS_PROG BPF_PROG_PATH"/cgroup_bpf_egress_prog.o"
-#define XT_BPF_INGRESS_PROG BPF_PROG_PATH "/xt_bpf_ingress_prog.o"
-#define XT_BPF_EGRESS_PROG BPF_PROG_PATH "/xt_bpf_egress_prog.o"
+#define BPF_PROG_SRC BPF_PROG_PATH "/bpf_kern.o"
#define MAP_LD_CMD_HEAD 0x18
#define FAIL(...) \
@@ -112,7 +108,24 @@
}
};
-int loadProg(const char* path, const std::vector<ReplacePattern> &mapPatterns) {
+Slice cgroupIngressProg;
+Slice cgroupEgressProg;
+Slice xtIngressProg;
+Slice xtEgressProg;
+
+Slice getProgFromMem(Slice buffer, Elf64_Shdr* section) {
+ uint64_t progSize = (uint64_t)section->sh_size;
+ Slice progSection = take(drop(buffer, section->sh_offset), progSize);
+ if (progSection.size() < progSize) FAIL("programSection out of bound\n");
+ char* progArray = new char[progSize];
+ Slice progCopy(progArray, progSize);
+ if (copy(progCopy, progSection) != progSize) {
+ FAIL("program cannot be extracted");
+ }
+ return progCopy;
+}
+
+void parseProgramsFromFile(const char* path) {
int fd = open(path, O_RDONLY);
if (fd == -1) {
FAIL("Failed to open %s program: %s", path, strerror(errno));
@@ -127,86 +140,99 @@
if ((uint32_t)fileLen < sizeof(Elf64_Ehdr)) FAIL("file size too small for Elf64_Ehdr");
- Elf64_Ehdr* elf = (Elf64_Ehdr*)baseAddr;
+ Slice buffer(baseAddr, fileLen);
+ Slice elfHeader = take(buffer, sizeof(Elf64_Ehdr));
+
+ if (elfHeader.size() < sizeof(Elf64_Ehdr)) FAIL("bpf buffer does not have complete elf header");
+
+ Elf64_Ehdr* elf = (Elf64_Ehdr*)elfHeader.base();
// Find section names string table. This is the section whose index is e_shstrndx.
- if (elf->e_shstrndx == SHN_UNDEF ||
- elf->e_shoff + (elf->e_shstrndx + 1) * sizeof(Elf64_Shdr) > (uint32_t)fileLen) {
+ if (elf->e_shstrndx == SHN_UNDEF) {
FAIL("cannot locate namesSection\n");
}
-
- Elf64_Shdr* sections = (Elf64_Shdr*)(baseAddr + elf->e_shoff);
-
- Elf64_Shdr* namesSection = sections + elf->e_shstrndx;
-
- if (namesSection->sh_offset + namesSection->sh_size > (uint32_t)fileLen)
- FAIL("namesSection out of bound\n");
-
- const char* strTab = baseAddr + namesSection->sh_offset;
- void* progSection = nullptr;
- uint64_t progSize = 0;
- for (int i = 0; i < elf->e_shnum; i++) {
- Elf64_Shdr* section = sections + i;
- if (((char*)section - baseAddr) + sizeof(Elf64_Shdr) > (uint32_t)fileLen) {
- FAIL("next section is out of bound\n");
- }
-
- if (!strcmp(strTab + section->sh_name, BPF_PROG_SEC_NAME)) {
- progSection = baseAddr + section->sh_offset;
- progSize = (uint64_t)section->sh_size;
- break;
- }
+ size_t totalSectionSize = (elf->e_shnum) * sizeof(Elf64_Shdr);
+ Slice sections = take(drop(buffer, elf->e_shoff), totalSectionSize);
+ if (sections.size() < totalSectionSize) {
+ FAIL("sections corrupted");
}
- if (!progSection) FAIL("program section not found");
- if ((char*)progSection - baseAddr + progSize > (uint32_t)fileLen)
- FAIL("programSection out of bound\n");
+ Slice namesSection = take(drop(sections, elf->e_shstrndx * sizeof(Elf64_Shdr)),
+ sizeof(Elf64_Shdr));
+ if (namesSection.size() != sizeof(Elf64_Shdr)) {
+ FAIL("namesSection corrupted");
+ }
+ size_t strTabOffset = ((Elf64_Shdr*) namesSection.base())->sh_offset;
+ size_t strTabSize = ((Elf64_Shdr*) namesSection.base())->sh_size;
- char* prog = new char[progSize]();
- memcpy(prog, progSection, progSize);
+ Slice strTab = take(drop(buffer, strTabOffset), strTabSize);
+ if (strTab.size() < strTabSize) {
+ FAIL("string table out of bound\n");
+ }
+ for (int i = 0; i < elf->e_shnum; i++) {
+ Slice section = take(drop(sections, i * sizeof(Elf64_Shdr)), sizeof(Elf64_Shdr));
+ if (section.size() < sizeof(Elf64_Shdr)) {
+ FAIL("section %d is out of bound, section size: %zu, header size: %zu, total size: %zu",
+ i, section.size(), sizeof(Elf64_Shdr), sections.size());
+ }
+ Elf64_Shdr* sectionPtr = (Elf64_Shdr*)section.base();
+ Slice nameSlice = drop(strTab, sectionPtr->sh_name);
+ if (nameSlice.size() == 0) {
+ FAIL("nameSlice out of bound, i: %d, strTabSize: %zu, sh_name: %u", i, strTabSize,
+ sectionPtr->sh_name);
+ }
+ if (!strcmp((char *)nameSlice.base(), BPF_CGROUP_INGRESS_PROG_NAME)) {
+ cgroupIngressProg = getProgFromMem(buffer, sectionPtr);
+ } else if (!strcmp((char *)nameSlice.base(), BPF_CGROUP_EGRESS_PROG_NAME)) {
+ cgroupEgressProg = getProgFromMem(buffer, sectionPtr);
+ } else if (!strcmp((char *)nameSlice.base(), XT_BPF_INGRESS_PROG_NAME)) {
+ xtIngressProg = getProgFromMem(buffer, sectionPtr);
+ } else if (!strcmp((char *)nameSlice.base(), XT_BPF_EGRESS_PROG_NAME)) {
+ xtEgressProg = getProgFromMem(buffer, sectionPtr);
+ }
+ }
+}
- char* mapHead = prog;
- while ((uint64_t)(mapHead - prog + MAP_CMD_SIZE) < progSize) {
+int loadProg(Slice prog, bpf_prog_type type, const std::vector<ReplacePattern>& mapPatterns) {
+ if (prog.size() == 0) {
+ FAIL("Couldn't find or parse program type %d", type);
+ }
+ Slice remaining = prog;
+ while (remaining.size() >= MAP_CMD_SIZE) {
// Scan the program, examining all possible places that might be the start of a map load
- // operation (i.e., all byes of value MAP_LD_CMD_HEAD).
- //
+ // operation (i.e., all bytes of value MAP_LD_CMD_HEAD).
// In each of these places, check whether it is the start of one of the patterns we want to
// replace, and if so, replace it.
- mapHead = (char*)memchr(mapHead, MAP_LD_CMD_HEAD, progSize);
- if (!mapHead) break;
+ Slice mapHead = findFirstMatching(remaining, MAP_LD_CMD_HEAD);
+ if (mapHead.size() < MAP_CMD_SIZE) break;
+ bool replaced = false;
for (const auto& pattern : mapPatterns) {
- if (!memcmp(mapHead, pattern.search.data(), MAP_CMD_SIZE)) {
- memcpy(mapHead, pattern.replace.data(), MAP_CMD_SIZE);
+ if (!memcmp(mapHead.base(), pattern.search.data(), MAP_CMD_SIZE)) {
+ memcpy(mapHead.base(), pattern.replace.data(), MAP_CMD_SIZE);
+ replaced = true;
+ break;
}
}
- mapHead++;
+ remaining = drop(mapHead, replaced ? MAP_CMD_SIZE : sizeof(uint8_t));
}
- Slice insns = Slice(prog, progSize);
char bpf_log_buf[LOG_BUF_SIZE];
Slice bpfLog = Slice(bpf_log_buf, sizeof(bpf_log_buf));
- if (strcmp(path, XT_BPF_INGRESS_PROG) && strcmp(path, XT_BPF_EGRESS_PROG)) {
- return bpfProgLoad(BPF_PROG_TYPE_CGROUP_SKB, insns, "Apache 2.0", 0, bpfLog);
- }
- return bpfProgLoad(BPF_PROG_TYPE_SOCKET_FILTER, insns, "Apache 2.0", 0, bpfLog);
+ return bpfProgLoad(type, prog, "Apache 2.0", 0, bpfLog);
}
int loadAndAttachProgram(bpf_attach_type type, const char* path, const char* name,
std::vector<ReplacePattern> mapPatterns) {
- unique_fd cg_fd(open(CGROUP_ROOT_PATH, O_DIRECTORY | O_RDONLY | O_CLOEXEC));
- if (cg_fd < 0) {
- FAIL("Failed to open the cgroup directory");
- }
unique_fd fd;
if (type == BPF_CGROUP_INET_INGRESS) {
- fd.reset(loadProg(INGRESS_PROG, mapPatterns));
+ fd.reset(loadProg(cgroupIngressProg, BPF_PROG_TYPE_CGROUP_SKB, mapPatterns));
} else if (type == BPF_CGROUP_INET_EGRESS) {
- fd.reset(loadProg(EGRESS_PROG, mapPatterns));
- } else if (!strcmp(name, "xt_bpf_ingress_prog")) {
- fd.reset(loadProg(XT_BPF_INGRESS_PROG, mapPatterns));
- } else if (!strcmp(name, "xt_bpf_egress_prog")) {
- fd.reset(loadProg(XT_BPF_EGRESS_PROG, mapPatterns));
+ fd.reset(loadProg(cgroupEgressProg, BPF_PROG_TYPE_CGROUP_SKB, mapPatterns));
+ } else if (!strcmp(name, XT_BPF_INGRESS_PROG_NAME)) {
+ fd.reset(loadProg(xtIngressProg, BPF_PROG_TYPE_SOCKET_FILTER, mapPatterns));
+ } else if (!strcmp(name, XT_BPF_EGRESS_PROG_NAME)) {
+ fd.reset(loadProg(xtEgressProg, BPF_PROG_TYPE_SOCKET_FILTER, mapPatterns));
} else {
FAIL("Unrecognized program type: %s", name);
}
@@ -216,6 +242,10 @@
}
int ret = 0;
if (type == BPF_CGROUP_INET_EGRESS || type == BPF_CGROUP_INET_INGRESS) {
+ unique_fd cg_fd(open(CGROUP_ROOT_PATH, O_DIRECTORY | O_RDONLY | O_CLOEXEC));
+ if (cg_fd < 0) {
+ FAIL("Failed to open the cgroup directory");
+ }
ret = attachProgram(type, fd, cg_fd);
if (ret) {
FAIL("%s attach failed: %s", name, strerror(errno));
@@ -245,6 +275,7 @@
using android::bpf::XT_BPF_EGRESS_PROG_PATH;
using android::bpf::XT_BPF_INGRESS_PROG_PATH;
using android::bpf::ReplacePattern;
+using android::bpf::loadAndAttachProgram;
static void usage(void) {
ALOGE( "Usage: ./bpfloader [-i] [-e]\n"
@@ -297,30 +328,32 @@
FAIL("unknown argument %c", opt);
}
}
+ android::bpf::parseProgramsFromFile(BPF_PROG_SRC);
+
if (doIngress) {
- ret = android::bpf::loadAndAttachProgram(BPF_CGROUP_INET_INGRESS, BPF_INGRESS_PROG_PATH,
- "ingress_prog", mapPatterns);
+ ret = loadAndAttachProgram(BPF_CGROUP_INET_INGRESS, BPF_INGRESS_PROG_PATH,
+ BPF_CGROUP_INGRESS_PROG_NAME, mapPatterns);
if (ret) {
FAIL("Failed to set up ingress program");
}
}
if (doEgress) {
- ret = android::bpf::loadAndAttachProgram(BPF_CGROUP_INET_EGRESS, BPF_EGRESS_PROG_PATH,
- "egress_prog", mapPatterns);
+ ret = loadAndAttachProgram(BPF_CGROUP_INET_EGRESS, BPF_EGRESS_PROG_PATH,
+ BPF_CGROUP_EGRESS_PROG_NAME, mapPatterns);
if (ret) {
FAIL("Failed to set up ingress program");
}
}
if (doPrerouting) {
- ret = android::bpf::loadAndAttachProgram(
- MAX_BPF_ATTACH_TYPE, XT_BPF_INGRESS_PROG_PATH, "xt_bpf_ingress_prog", mapPatterns);
+ ret = loadAndAttachProgram(MAX_BPF_ATTACH_TYPE, XT_BPF_INGRESS_PROG_PATH,
+ XT_BPF_INGRESS_PROG_NAME, mapPatterns);
if (ret) {
FAIL("Failed to set up xt_bpf program");
}
}
if (doMangle) {
- ret = android::bpf::loadAndAttachProgram(
- MAX_BPF_ATTACH_TYPE, XT_BPF_EGRESS_PROG_PATH, "xt_bpf_egress_prog", mapPatterns);
+ ret = loadAndAttachProgram(MAX_BPF_ATTACH_TYPE, XT_BPF_EGRESS_PROG_PATH,
+ XT_BPF_EGRESS_PROG_NAME, mapPatterns);
if (ret) {
FAIL("Failed to set up xt_bpf program");
}
diff --git a/bpfloader/bpf_egress.c b/bpfloader/bpf_egress.c
deleted file mode 100644
index 5cf648c..0000000
--- a/bpfloader/bpf_egress.c
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <linux/bpf.h>
-#include "bpf_kern.h"
-
-ELF_SEC(BPF_PROG_SEC_NAME)
-int bpf_cgroup_egress(struct __sk_buff* skb) {
- return bpf_traffic_account(skb, BPF_EGRESS);
-}
diff --git a/bpfloader/bpf_ingress.c b/bpfloader/bpf_ingress.c
deleted file mode 100644
index 4bd5170..0000000
--- a/bpfloader/bpf_ingress.c
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <linux/bpf.h>
-#include "bpf_kern.h"
-
-ELF_SEC(BPF_PROG_SEC_NAME)
-int bpf_cgroup_ingress(struct __sk_buff* skb) {
- return bpf_traffic_account(skb, BPF_INGRESS);
-}
diff --git a/bpfloader/xt_bpf_ingress_prog.c b/bpfloader/bpf_kern.c
similarity index 61%
rename from bpfloader/xt_bpf_ingress_prog.c
rename to bpfloader/bpf_kern.c
index 5f10255..3a1668f 100644
--- a/bpfloader/xt_bpf_ingress_prog.c
+++ b/bpfloader/bpf_kern.c
@@ -17,7 +17,25 @@
#include <linux/bpf.h>
#include "bpf_kern.h"
-ELF_SEC(BPF_PROG_SEC_NAME)
+
+ELF_SEC(BPF_CGROUP_INGRESS_PROG_NAME)
+int bpf_cgroup_ingress(struct __sk_buff* skb) {
+ return bpf_traffic_account(skb, BPF_INGRESS);
+}
+
+ELF_SEC(BPF_CGROUP_EGRESS_PROG_NAME)
+int bpf_cgroup_egress(struct __sk_buff* skb) {
+ return bpf_traffic_account(skb, BPF_EGRESS);
+}
+
+ELF_SEC(XT_BPF_EGRESS_PROG_NAME)
+int xt_bpf_egress_prog(struct __sk_buff* skb) {
+ uint32_t key = skb->ifindex;
+ bpf_update_stats(skb, IFACE_STATS_MAP, BPF_EGRESS, &key);
+ return BPF_PASS;
+}
+
+ELF_SEC(XT_BPF_INGRESS_PROG_NAME)
int xt_bpf_ingress_prog(struct __sk_buff* skb) {
uint32_t key = skb->ifindex;
bpf_update_stats(skb, IFACE_STATS_MAP, BPF_INGRESS, &key);
diff --git a/bpfloader/bpf_kern.o b/bpfloader/bpf_kern.o
new file mode 100644
index 0000000..32d8e87
--- /dev/null
+++ b/bpfloader/bpf_kern.o
Binary files differ
diff --git a/bpfloader/cgroup_bpf_egress_prog.o b/bpfloader/cgroup_bpf_egress_prog.o
deleted file mode 100644
index f68fd02..0000000
--- a/bpfloader/cgroup_bpf_egress_prog.o
+++ /dev/null
Binary files differ
diff --git a/bpfloader/cgroup_bpf_ingress_prog.o b/bpfloader/cgroup_bpf_ingress_prog.o
deleted file mode 100644
index 18cfbbc..0000000
--- a/bpfloader/cgroup_bpf_ingress_prog.o
+++ /dev/null
Binary files differ
diff --git a/bpfloader/xt_bpf_egress_prog.c b/bpfloader/xt_bpf_egress_prog.c
deleted file mode 100644
index 3f4bace..0000000
--- a/bpfloader/xt_bpf_egress_prog.c
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Copyright (C) 2018 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <linux/bpf.h>
-#include "bpf_kern.h"
-
-ELF_SEC(BPF_PROG_SEC_NAME)
-int xt_bpf_egress_prog(struct __sk_buff* skb) {
- uint32_t key = skb->ifindex;
- bpf_update_stats(skb, IFACE_STATS_MAP, BPF_EGRESS, &key);
- return BPF_PASS;
-}
diff --git a/bpfloader/xt_bpf_egress_prog.o b/bpfloader/xt_bpf_egress_prog.o
deleted file mode 100644
index 739b3da..0000000
--- a/bpfloader/xt_bpf_egress_prog.o
+++ /dev/null
Binary files differ
diff --git a/bpfloader/xt_bpf_ingress_prog.o b/bpfloader/xt_bpf_ingress_prog.o
deleted file mode 100644
index 20e71bc..0000000
--- a/bpfloader/xt_bpf_ingress_prog.o
+++ /dev/null
Binary files differ