Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 1 | #include <stdio.h> |
| 2 | #include <sys/types.h> |
| 3 | #include <sys/stat.h> |
| 4 | #include <fcntl.h> |
| 5 | #include <libelf.h> |
| 6 | #include <gelf.h> |
| 7 | #include <errno.h> |
| 8 | #include <unistd.h> |
| 9 | #include <string.h> |
| 10 | #include <stdbool.h> |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 11 | #include <stdlib.h> |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 12 | #include <linux/bpf.h> |
| 13 | #include <linux/filter.h> |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 14 | #include <linux/perf_event.h> |
| 15 | #include <sys/syscall.h> |
| 16 | #include <sys/ioctl.h> |
| 17 | #include <sys/mman.h> |
| 18 | #include <poll.h> |
Alexei Starovoitov | 5bacd78 | 2015-05-19 16:59:05 -0700 | [diff] [blame] | 19 | #include <ctype.h> |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 20 | #include "libbpf.h" |
| 21 | #include "bpf_helpers.h" |
| 22 | #include "bpf_load.h" |
| 23 | |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 24 | #define DEBUGFS "/sys/kernel/debug/tracing/" |
| 25 | |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 26 | static char license[128]; |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 27 | static int kern_version; |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 28 | static bool processed_sec[128]; |
| 29 | int map_fd[MAX_MAPS]; |
| 30 | int prog_fd[MAX_PROGS]; |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 31 | int event_fd[MAX_PROGS]; |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 32 | int prog_cnt; |
Alexei Starovoitov | 5bacd78 | 2015-05-19 16:59:05 -0700 | [diff] [blame] | 33 | int prog_array_fd = -1; |
| 34 | |
| 35 | static int populate_prog_array(const char *event, int prog_fd) |
| 36 | { |
| 37 | int ind = atoi(event), err; |
| 38 | |
| 39 | err = bpf_update_elem(prog_array_fd, &ind, &prog_fd, BPF_ANY); |
| 40 | if (err < 0) { |
| 41 | printf("failed to store prog_fd in prog_array\n"); |
| 42 | return -1; |
| 43 | } |
| 44 | return 0; |
| 45 | } |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 46 | |
| 47 | static int load_and_attach(const char *event, struct bpf_insn *prog, int size) |
| 48 | { |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 49 | bool is_socket = strncmp(event, "socket", 6) == 0; |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 50 | bool is_kprobe = strncmp(event, "kprobe/", 7) == 0; |
| 51 | bool is_kretprobe = strncmp(event, "kretprobe/", 10) == 0; |
| 52 | enum bpf_prog_type prog_type; |
| 53 | char buf[256]; |
| 54 | int fd, efd, err, id; |
| 55 | struct perf_event_attr attr = {}; |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 56 | |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 57 | attr.type = PERF_TYPE_TRACEPOINT; |
| 58 | attr.sample_type = PERF_SAMPLE_RAW; |
| 59 | attr.sample_period = 1; |
| 60 | attr.wakeup_events = 1; |
| 61 | |
| 62 | if (is_socket) { |
| 63 | prog_type = BPF_PROG_TYPE_SOCKET_FILTER; |
| 64 | } else if (is_kprobe || is_kretprobe) { |
| 65 | prog_type = BPF_PROG_TYPE_KPROBE; |
| 66 | } else { |
| 67 | printf("Unknown event '%s'\n", event); |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 68 | return -1; |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 69 | } |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 70 | |
Alexei Starovoitov | 5bacd78 | 2015-05-19 16:59:05 -0700 | [diff] [blame] | 71 | fd = bpf_prog_load(prog_type, prog, size, license, kern_version); |
| 72 | if (fd < 0) { |
| 73 | printf("bpf_prog_load() err=%d\n%s", errno, bpf_log_buf); |
| 74 | return -1; |
| 75 | } |
| 76 | |
| 77 | prog_fd[prog_cnt++] = fd; |
| 78 | |
| 79 | if (is_socket) { |
| 80 | event += 6; |
| 81 | if (*event != '/') |
| 82 | return 0; |
| 83 | event++; |
| 84 | if (!isdigit(*event)) { |
| 85 | printf("invalid prog number\n"); |
| 86 | return -1; |
| 87 | } |
| 88 | return populate_prog_array(event, fd); |
| 89 | } |
| 90 | |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 91 | if (is_kprobe || is_kretprobe) { |
| 92 | if (is_kprobe) |
| 93 | event += 7; |
| 94 | else |
| 95 | event += 10; |
| 96 | |
Alexei Starovoitov | 5bacd78 | 2015-05-19 16:59:05 -0700 | [diff] [blame] | 97 | if (*event == 0) { |
| 98 | printf("event name cannot be empty\n"); |
| 99 | return -1; |
| 100 | } |
| 101 | |
| 102 | if (isdigit(*event)) |
| 103 | return populate_prog_array(event, fd); |
| 104 | |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 105 | snprintf(buf, sizeof(buf), |
| 106 | "echo '%c:%s %s' >> /sys/kernel/debug/tracing/kprobe_events", |
| 107 | is_kprobe ? 'p' : 'r', event, event); |
| 108 | err = system(buf); |
| 109 | if (err < 0) { |
| 110 | printf("failed to create kprobe '%s' error '%s'\n", |
| 111 | event, strerror(errno)); |
| 112 | return -1; |
| 113 | } |
| 114 | } |
| 115 | |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 116 | strcpy(buf, DEBUGFS); |
| 117 | strcat(buf, "events/kprobes/"); |
| 118 | strcat(buf, event); |
| 119 | strcat(buf, "/id"); |
| 120 | |
| 121 | efd = open(buf, O_RDONLY, 0); |
| 122 | if (efd < 0) { |
| 123 | printf("failed to open event %s\n", event); |
| 124 | return -1; |
| 125 | } |
| 126 | |
| 127 | err = read(efd, buf, sizeof(buf)); |
| 128 | if (err < 0 || err >= sizeof(buf)) { |
| 129 | printf("read from '%s' failed '%s'\n", event, strerror(errno)); |
| 130 | return -1; |
| 131 | } |
| 132 | |
| 133 | close(efd); |
| 134 | |
| 135 | buf[err] = 0; |
| 136 | id = atoi(buf); |
| 137 | attr.config = id; |
| 138 | |
| 139 | efd = perf_event_open(&attr, -1/*pid*/, 0/*cpu*/, -1/*group_fd*/, 0); |
| 140 | if (efd < 0) { |
| 141 | printf("event %d fd %d err %s\n", id, efd, strerror(errno)); |
| 142 | return -1; |
| 143 | } |
| 144 | event_fd[prog_cnt - 1] = efd; |
| 145 | ioctl(efd, PERF_EVENT_IOC_ENABLE, 0); |
| 146 | ioctl(efd, PERF_EVENT_IOC_SET_BPF, fd); |
| 147 | |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 148 | return 0; |
| 149 | } |
| 150 | |
| 151 | static int load_maps(struct bpf_map_def *maps, int len) |
| 152 | { |
| 153 | int i; |
| 154 | |
| 155 | for (i = 0; i < len / sizeof(struct bpf_map_def); i++) { |
| 156 | |
| 157 | map_fd[i] = bpf_create_map(maps[i].type, |
| 158 | maps[i].key_size, |
| 159 | maps[i].value_size, |
Alexei Starovoitov | 89b9760 | 2016-03-07 21:57:20 -0800 | [diff] [blame] | 160 | maps[i].max_entries, |
| 161 | maps[i].map_flags); |
Alexei Starovoitov | 618ec9a7 | 2016-03-07 21:57:18 -0800 | [diff] [blame] | 162 | if (map_fd[i] < 0) { |
| 163 | printf("failed to create a map: %d %s\n", |
| 164 | errno, strerror(errno)); |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 165 | return 1; |
Alexei Starovoitov | 618ec9a7 | 2016-03-07 21:57:18 -0800 | [diff] [blame] | 166 | } |
Alexei Starovoitov | 5bacd78 | 2015-05-19 16:59:05 -0700 | [diff] [blame] | 167 | |
| 168 | if (maps[i].type == BPF_MAP_TYPE_PROG_ARRAY) |
| 169 | prog_array_fd = map_fd[i]; |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 170 | } |
| 171 | return 0; |
| 172 | } |
| 173 | |
| 174 | static int get_sec(Elf *elf, int i, GElf_Ehdr *ehdr, char **shname, |
| 175 | GElf_Shdr *shdr, Elf_Data **data) |
| 176 | { |
| 177 | Elf_Scn *scn; |
| 178 | |
| 179 | scn = elf_getscn(elf, i); |
| 180 | if (!scn) |
| 181 | return 1; |
| 182 | |
| 183 | if (gelf_getshdr(scn, shdr) != shdr) |
| 184 | return 2; |
| 185 | |
| 186 | *shname = elf_strptr(elf, ehdr->e_shstrndx, shdr->sh_name); |
| 187 | if (!*shname || !shdr->sh_size) |
| 188 | return 3; |
| 189 | |
| 190 | *data = elf_getdata(scn, 0); |
| 191 | if (!*data || elf_getdata(scn, *data) != NULL) |
| 192 | return 4; |
| 193 | |
| 194 | return 0; |
| 195 | } |
| 196 | |
| 197 | static int parse_relo_and_apply(Elf_Data *data, Elf_Data *symbols, |
| 198 | GElf_Shdr *shdr, struct bpf_insn *insn) |
| 199 | { |
| 200 | int i, nrels; |
| 201 | |
| 202 | nrels = shdr->sh_size / shdr->sh_entsize; |
| 203 | |
| 204 | for (i = 0; i < nrels; i++) { |
| 205 | GElf_Sym sym; |
| 206 | GElf_Rel rel; |
| 207 | unsigned int insn_idx; |
| 208 | |
| 209 | gelf_getrel(data, i, &rel); |
| 210 | |
| 211 | insn_idx = rel.r_offset / sizeof(struct bpf_insn); |
| 212 | |
| 213 | gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym); |
| 214 | |
| 215 | if (insn[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { |
| 216 | printf("invalid relo for insn[%d].code 0x%x\n", |
| 217 | insn_idx, insn[insn_idx].code); |
| 218 | return 1; |
| 219 | } |
| 220 | insn[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; |
| 221 | insn[insn_idx].imm = map_fd[sym.st_value / sizeof(struct bpf_map_def)]; |
| 222 | } |
| 223 | |
| 224 | return 0; |
| 225 | } |
| 226 | |
| 227 | int load_bpf_file(char *path) |
| 228 | { |
| 229 | int fd, i; |
| 230 | Elf *elf; |
| 231 | GElf_Ehdr ehdr; |
| 232 | GElf_Shdr shdr, shdr_prog; |
| 233 | Elf_Data *data, *data_prog, *symbols = NULL; |
| 234 | char *shname, *shname_prog; |
| 235 | |
| 236 | if (elf_version(EV_CURRENT) == EV_NONE) |
| 237 | return 1; |
| 238 | |
| 239 | fd = open(path, O_RDONLY, 0); |
| 240 | if (fd < 0) |
| 241 | return 1; |
| 242 | |
| 243 | elf = elf_begin(fd, ELF_C_READ, NULL); |
| 244 | |
| 245 | if (!elf) |
| 246 | return 1; |
| 247 | |
| 248 | if (gelf_getehdr(elf, &ehdr) != &ehdr) |
| 249 | return 1; |
| 250 | |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 251 | /* clear all kprobes */ |
| 252 | i = system("echo \"\" > /sys/kernel/debug/tracing/kprobe_events"); |
| 253 | |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 254 | /* scan over all elf sections to get license and map info */ |
| 255 | for (i = 1; i < ehdr.e_shnum; i++) { |
| 256 | |
| 257 | if (get_sec(elf, i, &ehdr, &shname, &shdr, &data)) |
| 258 | continue; |
| 259 | |
| 260 | if (0) /* helpful for llvm debugging */ |
| 261 | printf("section %d:%s data %p size %zd link %d flags %d\n", |
| 262 | i, shname, data->d_buf, data->d_size, |
| 263 | shdr.sh_link, (int) shdr.sh_flags); |
| 264 | |
| 265 | if (strcmp(shname, "license") == 0) { |
| 266 | processed_sec[i] = true; |
| 267 | memcpy(license, data->d_buf, data->d_size); |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 268 | } else if (strcmp(shname, "version") == 0) { |
| 269 | processed_sec[i] = true; |
| 270 | if (data->d_size != sizeof(int)) { |
| 271 | printf("invalid size of version section %zd\n", |
| 272 | data->d_size); |
| 273 | return 1; |
| 274 | } |
| 275 | memcpy(&kern_version, data->d_buf, sizeof(int)); |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 276 | } else if (strcmp(shname, "maps") == 0) { |
| 277 | processed_sec[i] = true; |
| 278 | if (load_maps(data->d_buf, data->d_size)) |
| 279 | return 1; |
| 280 | } else if (shdr.sh_type == SHT_SYMTAB) { |
| 281 | symbols = data; |
| 282 | } |
| 283 | } |
| 284 | |
| 285 | /* load programs that need map fixup (relocations) */ |
| 286 | for (i = 1; i < ehdr.e_shnum; i++) { |
| 287 | |
| 288 | if (get_sec(elf, i, &ehdr, &shname, &shdr, &data)) |
| 289 | continue; |
| 290 | if (shdr.sh_type == SHT_REL) { |
| 291 | struct bpf_insn *insns; |
| 292 | |
| 293 | if (get_sec(elf, shdr.sh_info, &ehdr, &shname_prog, |
| 294 | &shdr_prog, &data_prog)) |
| 295 | continue; |
| 296 | |
| 297 | insns = (struct bpf_insn *) data_prog->d_buf; |
| 298 | |
| 299 | processed_sec[shdr.sh_info] = true; |
| 300 | processed_sec[i] = true; |
| 301 | |
| 302 | if (parse_relo_and_apply(data, symbols, &shdr, insns)) |
| 303 | continue; |
| 304 | |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 305 | if (memcmp(shname_prog, "kprobe/", 7) == 0 || |
| 306 | memcmp(shname_prog, "kretprobe/", 10) == 0 || |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 307 | memcmp(shname_prog, "socket", 6) == 0) |
| 308 | load_and_attach(shname_prog, insns, data_prog->d_size); |
| 309 | } |
| 310 | } |
| 311 | |
| 312 | /* load programs that don't use maps */ |
| 313 | for (i = 1; i < ehdr.e_shnum; i++) { |
| 314 | |
| 315 | if (processed_sec[i]) |
| 316 | continue; |
| 317 | |
| 318 | if (get_sec(elf, i, &ehdr, &shname, &shdr, &data)) |
| 319 | continue; |
| 320 | |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 321 | if (memcmp(shname, "kprobe/", 7) == 0 || |
| 322 | memcmp(shname, "kretprobe/", 10) == 0 || |
Alexei Starovoitov | 249b812 | 2014-12-01 15:06:37 -0800 | [diff] [blame] | 323 | memcmp(shname, "socket", 6) == 0) |
| 324 | load_and_attach(shname, data->d_buf, data->d_size); |
| 325 | } |
| 326 | |
| 327 | close(fd); |
| 328 | return 0; |
| 329 | } |
Alexei Starovoitov | b896c4f | 2015-03-25 12:49:23 -0700 | [diff] [blame] | 330 | |
| 331 | void read_trace_pipe(void) |
| 332 | { |
| 333 | int trace_fd; |
| 334 | |
| 335 | trace_fd = open(DEBUGFS "trace_pipe", O_RDONLY, 0); |
| 336 | if (trace_fd < 0) |
| 337 | return; |
| 338 | |
| 339 | while (1) { |
| 340 | static char buf[4096]; |
| 341 | ssize_t sz; |
| 342 | |
| 343 | sz = read(trace_fd, buf, sizeof(buf)); |
| 344 | if (sz > 0) { |
| 345 | buf[sz] = 0; |
| 346 | puts(buf); |
| 347 | } |
| 348 | } |
| 349 | } |
Alexei Starovoitov | 3622e7e | 2016-03-07 21:57:19 -0800 | [diff] [blame] | 350 | |
| 351 | #define MAX_SYMS 300000 |
| 352 | static struct ksym syms[MAX_SYMS]; |
| 353 | static int sym_cnt; |
| 354 | |
| 355 | static int ksym_cmp(const void *p1, const void *p2) |
| 356 | { |
| 357 | return ((struct ksym *)p1)->addr - ((struct ksym *)p2)->addr; |
| 358 | } |
| 359 | |
| 360 | int load_kallsyms(void) |
| 361 | { |
| 362 | FILE *f = fopen("/proc/kallsyms", "r"); |
| 363 | char func[256], buf[256]; |
| 364 | char symbol; |
| 365 | void *addr; |
| 366 | int i = 0; |
| 367 | |
| 368 | if (!f) |
| 369 | return -ENOENT; |
| 370 | |
| 371 | while (!feof(f)) { |
| 372 | if (!fgets(buf, sizeof(buf), f)) |
| 373 | break; |
| 374 | if (sscanf(buf, "%p %c %s", &addr, &symbol, func) != 3) |
| 375 | break; |
| 376 | if (!addr) |
| 377 | continue; |
| 378 | syms[i].addr = (long) addr; |
| 379 | syms[i].name = strdup(func); |
| 380 | i++; |
| 381 | } |
| 382 | sym_cnt = i; |
| 383 | qsort(syms, sym_cnt, sizeof(struct ksym), ksym_cmp); |
| 384 | return 0; |
| 385 | } |
| 386 | |
| 387 | struct ksym *ksym_search(long key) |
| 388 | { |
| 389 | int start = 0, end = sym_cnt; |
| 390 | int result; |
| 391 | |
| 392 | while (start < end) { |
| 393 | size_t mid = start + (end - start) / 2; |
| 394 | |
| 395 | result = key - syms[mid].addr; |
| 396 | if (result < 0) |
| 397 | end = mid; |
| 398 | else if (result > 0) |
| 399 | start = mid + 1; |
| 400 | else |
| 401 | return &syms[mid]; |
| 402 | } |
| 403 | |
| 404 | if (start >= 1 && syms[start - 1].addr < key && |
| 405 | key < syms[start].addr) |
| 406 | /* valid ksym */ |
| 407 | return &syms[start - 1]; |
| 408 | |
| 409 | /* out of range. return _stext */ |
| 410 | return &syms[0]; |
| 411 | } |