Jiri Pirko | 1d129d1 | 2015-01-19 16:56:29 +0100 | [diff] [blame] | 1 | /* |
| 2 | * tc_bpf.c BPF common code |
| 3 | * |
| 4 | * This program is free software; you can distribute it and/or |
| 5 | * modify it under the terms of the GNU General Public License |
| 6 | * as published by the Free Software Foundation; either version |
| 7 | * 2 of the License, or (at your option) any later version. |
| 8 | * |
| 9 | * Authors: Daniel Borkmann <dborkman@redhat.com> |
| 10 | * Jiri Pirko <jiri@resnulli.us> |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 11 | * Alexei Starovoitov <ast@plumgrid.com> |
Jiri Pirko | 1d129d1 | 2015-01-19 16:56:29 +0100 | [diff] [blame] | 12 | */ |
| 13 | |
| 14 | #include <stdio.h> |
| 15 | #include <stdlib.h> |
| 16 | #include <unistd.h> |
| 17 | #include <string.h> |
| 18 | #include <stdbool.h> |
Daniel Borkmann | 473d784 | 2015-05-29 15:15:44 +0200 | [diff] [blame] | 19 | #include <stdint.h> |
Jiri Pirko | 1d129d1 | 2015-01-19 16:56:29 +0100 | [diff] [blame] | 20 | #include <errno.h> |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 21 | #include <fcntl.h> |
| 22 | #include <stdarg.h> |
Jiri Pirko | 1d129d1 | 2015-01-19 16:56:29 +0100 | [diff] [blame] | 23 | |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 24 | #ifdef HAVE_ELF |
| 25 | #include <libelf.h> |
| 26 | #include <gelf.h> |
| 27 | #endif |
| 28 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 29 | #include <sys/types.h> |
| 30 | #include <sys/stat.h> |
| 31 | #include <sys/un.h> |
| 32 | #include <sys/vfs.h> |
| 33 | #include <sys/mount.h> |
| 34 | #include <sys/syscall.h> |
| 35 | #include <sys/sendfile.h> |
| 36 | #include <sys/resource.h> |
| 37 | |
| 38 | #include <linux/bpf.h> |
| 39 | #include <linux/filter.h> |
| 40 | #include <linux/if_alg.h> |
| 41 | |
Jiri Pirko | 1d129d1 | 2015-01-19 16:56:29 +0100 | [diff] [blame] | 42 | #include "utils.h" |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 43 | |
| 44 | #include "bpf_elf.h" |
| 45 | #include "bpf_scm.h" |
| 46 | |
Jiri Pirko | 1d129d1 | 2015-01-19 16:56:29 +0100 | [diff] [blame] | 47 | #include "tc_util.h" |
| 48 | #include "tc_bpf.h" |
| 49 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 50 | #ifdef HAVE_ELF |
| 51 | static int bpf_obj_open(const char *path, enum bpf_prog_type type, |
| 52 | const char *sec, bool verbose); |
| 53 | #else |
| 54 | static int bpf_obj_open(const char *path, enum bpf_prog_type type, |
| 55 | const char *sec, bool verbose) |
| 56 | { |
| 57 | fprintf(stderr, "No ELF library support compiled in.\n"); |
| 58 | errno = ENOSYS; |
| 59 | return -1; |
| 60 | } |
| 61 | #endif |
| 62 | |
| 63 | static inline __u64 bpf_ptr_to_u64(const void *ptr) |
| 64 | { |
| 65 | return (__u64)(unsigned long)ptr; |
| 66 | } |
| 67 | |
| 68 | static int bpf(int cmd, union bpf_attr *attr, unsigned int size) |
| 69 | { |
| 70 | #ifdef __NR_bpf |
| 71 | return syscall(__NR_bpf, cmd, attr, size); |
| 72 | #else |
| 73 | fprintf(stderr, "No bpf syscall, kernel headers too old?\n"); |
| 74 | errno = ENOSYS; |
| 75 | return -1; |
| 76 | #endif |
| 77 | } |
| 78 | |
| 79 | static int bpf_obj_get(const char *pathname) |
| 80 | { |
| 81 | union bpf_attr attr = { |
| 82 | .pathname = bpf_ptr_to_u64(pathname), |
| 83 | }; |
| 84 | |
| 85 | return bpf(BPF_OBJ_GET, &attr, sizeof(attr)); |
| 86 | } |
| 87 | |
| 88 | static int bpf_parse_string(char *arg, bool from_file, __u16 *bpf_len, |
| 89 | char **bpf_string, bool *need_release, |
| 90 | const char separator) |
Jiri Pirko | 1d129d1 | 2015-01-19 16:56:29 +0100 | [diff] [blame] | 91 | { |
| 92 | char sp; |
| 93 | |
| 94 | if (from_file) { |
| 95 | size_t tmp_len, op_len = sizeof("65535 255 255 4294967295,"); |
| 96 | char *tmp_string; |
| 97 | FILE *fp; |
| 98 | |
| 99 | tmp_len = sizeof("4096,") + BPF_MAXINSNS * op_len; |
| 100 | tmp_string = malloc(tmp_len); |
| 101 | if (tmp_string == NULL) |
| 102 | return -ENOMEM; |
| 103 | |
| 104 | memset(tmp_string, 0, tmp_len); |
| 105 | |
| 106 | fp = fopen(arg, "r"); |
| 107 | if (fp == NULL) { |
| 108 | perror("Cannot fopen"); |
| 109 | free(tmp_string); |
| 110 | return -ENOENT; |
| 111 | } |
| 112 | |
| 113 | if (!fgets(tmp_string, tmp_len, fp)) { |
| 114 | free(tmp_string); |
| 115 | fclose(fp); |
| 116 | return -EIO; |
| 117 | } |
| 118 | |
| 119 | fclose(fp); |
| 120 | |
| 121 | *need_release = true; |
| 122 | *bpf_string = tmp_string; |
| 123 | } else { |
| 124 | *need_release = false; |
| 125 | *bpf_string = arg; |
| 126 | } |
| 127 | |
| 128 | if (sscanf(*bpf_string, "%hu%c", bpf_len, &sp) != 2 || |
| 129 | sp != separator) { |
| 130 | if (*need_release) |
| 131 | free(*bpf_string); |
| 132 | return -EINVAL; |
| 133 | } |
| 134 | |
| 135 | return 0; |
| 136 | } |
| 137 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 138 | static int bpf_ops_parse(int argc, char **argv, struct sock_filter *bpf_ops, |
| 139 | bool from_file) |
Jiri Pirko | 1d129d1 | 2015-01-19 16:56:29 +0100 | [diff] [blame] | 140 | { |
| 141 | char *bpf_string, *token, separator = ','; |
| 142 | int ret = 0, i = 0; |
| 143 | bool need_release; |
| 144 | __u16 bpf_len = 0; |
| 145 | |
| 146 | if (argc < 1) |
| 147 | return -EINVAL; |
| 148 | if (bpf_parse_string(argv[0], from_file, &bpf_len, &bpf_string, |
| 149 | &need_release, separator)) |
| 150 | return -EINVAL; |
| 151 | if (bpf_len == 0 || bpf_len > BPF_MAXINSNS) { |
| 152 | ret = -EINVAL; |
| 153 | goto out; |
| 154 | } |
| 155 | |
| 156 | token = bpf_string; |
| 157 | while ((token = strchr(token, separator)) && (++token)[0]) { |
| 158 | if (i >= bpf_len) { |
| 159 | fprintf(stderr, "Real program length exceeds encoded " |
| 160 | "length parameter!\n"); |
| 161 | ret = -EINVAL; |
| 162 | goto out; |
| 163 | } |
| 164 | |
| 165 | if (sscanf(token, "%hu %hhu %hhu %u,", |
| 166 | &bpf_ops[i].code, &bpf_ops[i].jt, |
| 167 | &bpf_ops[i].jf, &bpf_ops[i].k) != 4) { |
| 168 | fprintf(stderr, "Error at instruction %d!\n", i); |
| 169 | ret = -EINVAL; |
| 170 | goto out; |
| 171 | } |
| 172 | |
| 173 | i++; |
| 174 | } |
| 175 | |
| 176 | if (i != bpf_len) { |
| 177 | fprintf(stderr, "Parsed program length is less than encoded" |
| 178 | "length parameter!\n"); |
| 179 | ret = -EINVAL; |
| 180 | goto out; |
| 181 | } |
| 182 | ret = bpf_len; |
Jiri Pirko | 1d129d1 | 2015-01-19 16:56:29 +0100 | [diff] [blame] | 183 | out: |
| 184 | if (need_release) |
| 185 | free(bpf_string); |
| 186 | |
| 187 | return ret; |
| 188 | } |
| 189 | |
| 190 | void bpf_print_ops(FILE *f, struct rtattr *bpf_ops, __u16 len) |
| 191 | { |
| 192 | struct sock_filter *ops = (struct sock_filter *) RTA_DATA(bpf_ops); |
| 193 | int i; |
| 194 | |
| 195 | if (len == 0) |
| 196 | return; |
| 197 | |
| 198 | fprintf(f, "bytecode \'%u,", len); |
| 199 | |
| 200 | for (i = 0; i < len - 1; i++) |
| 201 | fprintf(f, "%hu %hhu %hhu %u,", ops[i].code, ops[i].jt, |
| 202 | ops[i].jf, ops[i].k); |
| 203 | |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 204 | fprintf(f, "%hu %hhu %hhu %u\'", ops[i].code, ops[i].jt, |
Jiri Pirko | 1d129d1 | 2015-01-19 16:56:29 +0100 | [diff] [blame] | 205 | ops[i].jf, ops[i].k); |
| 206 | } |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 207 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 208 | static int bpf_valid_mntpt(const char *mnt, unsigned long magic) |
| 209 | { |
| 210 | struct statfs st_fs; |
| 211 | |
| 212 | if (statfs(mnt, &st_fs) < 0) |
| 213 | return -ENOENT; |
| 214 | if ((unsigned long)st_fs.f_type != magic) |
| 215 | return -ENOENT; |
| 216 | |
| 217 | return 0; |
| 218 | } |
| 219 | |
| 220 | static const char *bpf_find_mntpt(const char *fstype, unsigned long magic, |
| 221 | char *mnt, int len, |
| 222 | const char * const *known_mnts) |
| 223 | { |
| 224 | const char * const *ptr; |
| 225 | char type[100]; |
| 226 | FILE *fp; |
| 227 | |
| 228 | if (known_mnts) { |
| 229 | ptr = known_mnts; |
| 230 | while (*ptr) { |
| 231 | if (bpf_valid_mntpt(*ptr, magic) == 0) { |
| 232 | strncpy(mnt, *ptr, len - 1); |
| 233 | mnt[len - 1] = 0; |
| 234 | return mnt; |
| 235 | } |
| 236 | ptr++; |
| 237 | } |
| 238 | } |
| 239 | |
| 240 | fp = fopen("/proc/mounts", "r"); |
| 241 | if (fp == NULL || len != PATH_MAX) |
| 242 | return NULL; |
| 243 | |
| 244 | while (fscanf(fp, "%*s %" textify(PATH_MAX) "s %99s %*s %*d %*d\n", |
| 245 | mnt, type) == 2) { |
| 246 | if (strcmp(type, fstype) == 0) |
| 247 | break; |
| 248 | } |
| 249 | |
| 250 | fclose(fp); |
| 251 | if (strcmp(type, fstype) != 0) |
| 252 | return NULL; |
| 253 | |
| 254 | return mnt; |
| 255 | } |
| 256 | |
| 257 | int bpf_trace_pipe(void) |
| 258 | { |
| 259 | char tracefs_mnt[PATH_MAX] = TRACE_DIR_MNT; |
| 260 | static const char * const tracefs_known_mnts[] = { |
| 261 | TRACE_DIR_MNT, |
| 262 | "/sys/kernel/debug/tracing", |
| 263 | "/tracing", |
| 264 | "/trace", |
| 265 | 0, |
| 266 | }; |
| 267 | char tpipe[PATH_MAX]; |
| 268 | const char *mnt; |
| 269 | int fd; |
| 270 | |
| 271 | mnt = bpf_find_mntpt("tracefs", TRACEFS_MAGIC, tracefs_mnt, |
| 272 | sizeof(tracefs_mnt), tracefs_known_mnts); |
| 273 | if (!mnt) { |
| 274 | fprintf(stderr, "tracefs not mounted?\n"); |
| 275 | return -1; |
| 276 | } |
| 277 | |
| 278 | snprintf(tpipe, sizeof(tpipe), "%s/trace_pipe", mnt); |
| 279 | |
| 280 | fd = open(tpipe, O_RDONLY); |
| 281 | if (fd < 0) |
| 282 | return -1; |
| 283 | |
| 284 | fprintf(stderr, "Running! Hang up with ^C!\n\n"); |
| 285 | while (1) { |
| 286 | static char buff[4096]; |
| 287 | ssize_t ret; |
| 288 | |
| 289 | ret = read(fd, buff, sizeof(buff) - 1); |
| 290 | if (ret > 0) { |
| 291 | write(2, buff, ret); |
| 292 | fflush(stderr); |
| 293 | } |
| 294 | } |
| 295 | |
| 296 | return 0; |
| 297 | } |
| 298 | |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 299 | const char *bpf_default_section(const enum bpf_prog_type type) |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 300 | { |
| 301 | switch (type) { |
| 302 | case BPF_PROG_TYPE_SCHED_CLS: |
| 303 | return ELF_SECTION_CLASSIFIER; |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 304 | case BPF_PROG_TYPE_SCHED_ACT: |
| 305 | return ELF_SECTION_ACTION; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 306 | default: |
| 307 | return NULL; |
| 308 | } |
| 309 | } |
| 310 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 311 | int bpf_parse_common(int *ptr_argc, char ***ptr_argv, const int *nla_tbl, |
| 312 | enum bpf_prog_type type, const char **ptr_object, |
| 313 | const char **ptr_uds_name, struct nlmsghdr *n) |
| 314 | { |
| 315 | struct sock_filter opcodes[BPF_MAXINSNS]; |
| 316 | const char *file, *section, *uds_name; |
| 317 | char **argv = *ptr_argv; |
| 318 | int argc = *ptr_argc; |
| 319 | char annotation[256]; |
| 320 | bool verbose = false; |
| 321 | int ret; |
| 322 | enum bpf_mode { |
| 323 | CBPF_BYTECODE, |
| 324 | CBPF_FILE, |
| 325 | EBPF_OBJECT, |
| 326 | EBPF_PINNED, |
| 327 | } mode; |
| 328 | |
| 329 | if (matches(*argv, "bytecode") == 0 || |
| 330 | strcmp(*argv, "bc") == 0) { |
| 331 | mode = CBPF_BYTECODE; |
| 332 | } else if (matches(*argv, "bytecode-file") == 0 || |
| 333 | strcmp(*argv, "bcf") == 0) { |
| 334 | mode = CBPF_FILE; |
| 335 | } else if (matches(*argv, "object-file") == 0 || |
| 336 | strcmp(*argv, "obj") == 0) { |
| 337 | mode = EBPF_OBJECT; |
| 338 | } else if (matches(*argv, "object-pinned") == 0 || |
| 339 | matches(*argv, "pinned") == 0 || |
| 340 | matches(*argv, "fd") == 0) { |
| 341 | mode = EBPF_PINNED; |
| 342 | } else { |
| 343 | fprintf(stderr, "What mode is \"%s\"?\n", *argv); |
| 344 | return -1; |
| 345 | } |
| 346 | |
| 347 | NEXT_ARG(); |
| 348 | file = section = uds_name = NULL; |
| 349 | if (mode == EBPF_OBJECT || mode == EBPF_PINNED) { |
| 350 | file = *argv; |
| 351 | NEXT_ARG_FWD(); |
| 352 | |
| 353 | section = bpf_default_section(type); |
| 354 | if (argc > 0 && matches(*argv, "section") == 0) { |
| 355 | NEXT_ARG(); |
| 356 | section = *argv; |
| 357 | NEXT_ARG_FWD(); |
| 358 | } |
| 359 | |
| 360 | uds_name = getenv(BPF_ENV_UDS); |
| 361 | if (argc > 0 && !uds_name && |
| 362 | matches(*argv, "export") == 0) { |
| 363 | NEXT_ARG(); |
| 364 | uds_name = *argv; |
| 365 | NEXT_ARG_FWD(); |
| 366 | } |
| 367 | |
| 368 | if (argc > 0 && matches(*argv, "verbose") == 0) { |
| 369 | verbose = true; |
| 370 | NEXT_ARG_FWD(); |
| 371 | } |
| 372 | |
| 373 | PREV_ARG(); |
| 374 | } |
| 375 | |
| 376 | if (mode == CBPF_BYTECODE || mode == CBPF_FILE) |
| 377 | ret = bpf_ops_parse(argc, argv, opcodes, mode == CBPF_FILE); |
| 378 | else if (mode == EBPF_OBJECT) |
| 379 | ret = bpf_obj_open(file, type, section, verbose); |
| 380 | else if (mode == EBPF_PINNED) |
| 381 | ret = bpf_obj_get(file); |
| 382 | if (ret < 0) |
| 383 | return -1; |
| 384 | |
| 385 | if (mode == CBPF_BYTECODE || mode == CBPF_FILE) { |
| 386 | addattr16(n, MAX_MSG, nla_tbl[BPF_NLA_OPS_LEN], ret); |
| 387 | addattr_l(n, MAX_MSG, nla_tbl[BPF_NLA_OPS], opcodes, |
| 388 | ret * sizeof(struct sock_filter)); |
| 389 | } else if (mode == EBPF_OBJECT || mode == EBPF_PINNED) { |
| 390 | snprintf(annotation, sizeof(annotation), "%s:[%s]", |
| 391 | basename(file), mode == EBPF_PINNED ? "*fsobj" : |
| 392 | section); |
| 393 | |
| 394 | addattr32(n, MAX_MSG, nla_tbl[BPF_NLA_FD], ret); |
| 395 | addattrstrz(n, MAX_MSG, nla_tbl[BPF_NLA_NAME], annotation); |
| 396 | } |
| 397 | |
| 398 | *ptr_object = file; |
| 399 | *ptr_uds_name = uds_name; |
| 400 | |
| 401 | *ptr_argc = argc; |
| 402 | *ptr_argv = argv; |
| 403 | |
| 404 | return 0; |
| 405 | } |
| 406 | |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 407 | #ifdef HAVE_ELF |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 408 | struct bpf_elf_prog { |
| 409 | enum bpf_prog_type type; |
| 410 | const struct bpf_insn *insns; |
| 411 | size_t size; |
| 412 | const char *license; |
| 413 | }; |
| 414 | |
| 415 | struct bpf_elf_ctx { |
| 416 | Elf *elf_fd; |
| 417 | GElf_Ehdr elf_hdr; |
| 418 | Elf_Data *sym_tab; |
| 419 | Elf_Data *str_tab; |
| 420 | int obj_fd; |
| 421 | int map_fds[ELF_MAX_MAPS]; |
| 422 | struct bpf_elf_map maps[ELF_MAX_MAPS]; |
| 423 | int sym_num; |
| 424 | int map_num; |
| 425 | bool *sec_done; |
| 426 | int sec_maps; |
| 427 | char license[ELF_MAX_LICENSE_LEN]; |
| 428 | enum bpf_prog_type type; |
| 429 | bool verbose; |
| 430 | struct bpf_elf_st stat; |
| 431 | }; |
| 432 | |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 433 | struct bpf_elf_sec_data { |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 434 | GElf_Shdr sec_hdr; |
| 435 | Elf_Data *sec_data; |
| 436 | const char *sec_name; |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 437 | }; |
| 438 | |
| 439 | struct bpf_map_data { |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 440 | int *fds; |
| 441 | const char *obj; |
| 442 | struct bpf_elf_st *st; |
| 443 | struct bpf_elf_map *ent; |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 444 | }; |
| 445 | |
| 446 | /* If we provide a small buffer with log level enabled, the kernel |
| 447 | * could fail program load as no buffer space is available for the |
| 448 | * log and thus verifier fails. In case something doesn't pass the |
| 449 | * verifier we still want to hand something descriptive to the user. |
| 450 | */ |
| 451 | static char bpf_log_buf[65536]; |
| 452 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 453 | static __check_format_string(1, 2) void bpf_dump_error(const char *format, ...) |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 454 | { |
| 455 | va_list vl; |
| 456 | |
| 457 | va_start(vl, format); |
| 458 | vfprintf(stderr, format, vl); |
| 459 | va_end(vl); |
| 460 | |
Daniel Borkmann | d937a74 | 2015-04-28 13:37:42 +0200 | [diff] [blame] | 461 | if (bpf_log_buf[0]) { |
| 462 | fprintf(stderr, "%s\n", bpf_log_buf); |
| 463 | memset(bpf_log_buf, 0, sizeof(bpf_log_buf)); |
| 464 | } |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 465 | } |
| 466 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 467 | static int bpf_map_create(enum bpf_map_type type, unsigned int size_key, |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 468 | unsigned int size_value, unsigned int max_elem) |
| 469 | { |
| 470 | union bpf_attr attr = { |
| 471 | .map_type = type, |
| 472 | .key_size = size_key, |
| 473 | .value_size = size_value, |
| 474 | .max_entries = max_elem, |
| 475 | }; |
| 476 | |
| 477 | return bpf(BPF_MAP_CREATE, &attr, sizeof(attr)); |
| 478 | } |
| 479 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 480 | static int bpf_map_update(int fd, const void *key, const void *value, |
Daniel Borkmann | 473d784 | 2015-05-29 15:15:44 +0200 | [diff] [blame] | 481 | uint64_t flags) |
| 482 | { |
| 483 | union bpf_attr attr = { |
| 484 | .map_fd = fd, |
| 485 | .key = bpf_ptr_to_u64(key), |
| 486 | .value = bpf_ptr_to_u64(value), |
| 487 | .flags = flags, |
| 488 | }; |
| 489 | |
| 490 | return bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr)); |
| 491 | } |
| 492 | |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 493 | static int bpf_prog_load(enum bpf_prog_type type, const struct bpf_insn *insns, |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 494 | size_t size, const char *license) |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 495 | { |
| 496 | union bpf_attr attr = { |
| 497 | .prog_type = type, |
| 498 | .insns = bpf_ptr_to_u64(insns), |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 499 | .insn_cnt = size / sizeof(struct bpf_insn), |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 500 | .license = bpf_ptr_to_u64(license), |
| 501 | .log_buf = bpf_ptr_to_u64(bpf_log_buf), |
| 502 | .log_size = sizeof(bpf_log_buf), |
| 503 | .log_level = 1, |
| 504 | }; |
| 505 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 506 | if (getenv(BPF_ENV_NOLOG)) { |
| 507 | attr.log_buf = 0; |
| 508 | attr.log_size = 0; |
| 509 | attr.log_level = 0; |
| 510 | } |
| 511 | |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 512 | return bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); |
| 513 | } |
| 514 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 515 | static int bpf_obj_pin(int fd, const char *pathname) |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 516 | { |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 517 | union bpf_attr attr = { |
| 518 | .pathname = bpf_ptr_to_u64(pathname), |
| 519 | .bpf_fd = fd, |
| 520 | }; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 521 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 522 | return bpf(BPF_OBJ_PIN, &attr, sizeof(attr)); |
| 523 | } |
| 524 | |
| 525 | static int bpf_obj_hash(const char *object, uint8_t *out, size_t len) |
| 526 | { |
| 527 | struct sockaddr_alg alg = { |
| 528 | .salg_family = AF_ALG, |
| 529 | .salg_type = "hash", |
| 530 | .salg_name = "sha1", |
| 531 | }; |
| 532 | int ret, cfd, ofd, ffd; |
| 533 | struct stat stbuff; |
| 534 | ssize_t size; |
| 535 | |
| 536 | if (!object || len != 20) |
| 537 | return -EINVAL; |
| 538 | |
| 539 | cfd = socket(AF_ALG, SOCK_SEQPACKET, 0); |
| 540 | if (cfd < 0) { |
| 541 | fprintf(stderr, "Cannot get AF_ALG socket: %s\n", |
| 542 | strerror(errno)); |
| 543 | return cfd; |
Daniel Borkmann | d937a74 | 2015-04-28 13:37:42 +0200 | [diff] [blame] | 544 | } |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 545 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 546 | ret = bind(cfd, (struct sockaddr *)&alg, sizeof(alg)); |
| 547 | if (ret < 0) { |
| 548 | fprintf(stderr, "Error binding socket: %s\n", strerror(errno)); |
| 549 | goto out_cfd; |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 550 | } |
| 551 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 552 | ofd = accept(cfd, NULL, 0); |
| 553 | if (ofd < 0) { |
| 554 | fprintf(stderr, "Error accepting socket: %s\n", |
| 555 | strerror(errno)); |
| 556 | ret = ofd; |
| 557 | goto out_cfd; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 558 | } |
| 559 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 560 | ffd = open(object, O_RDONLY); |
| 561 | if (ffd < 0) { |
| 562 | fprintf(stderr, "Error opening object %s: %s\n", |
| 563 | object, strerror(errno)); |
| 564 | ret = ffd; |
| 565 | goto out_ofd; |
| 566 | } |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 567 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 568 | ret = fstat(ffd, &stbuff); |
| 569 | if (ret < 0) { |
| 570 | fprintf(stderr, "Error doing fstat: %s\n", |
| 571 | strerror(errno)); |
| 572 | goto out_ffd; |
| 573 | } |
| 574 | |
| 575 | size = sendfile(ofd, ffd, NULL, stbuff.st_size); |
| 576 | if (size != stbuff.st_size) { |
| 577 | fprintf(stderr, "Error from sendfile (%zd vs %zu bytes): %s\n", |
| 578 | size, stbuff.st_size, strerror(errno)); |
| 579 | ret = -1; |
| 580 | goto out_ffd; |
| 581 | } |
| 582 | |
| 583 | size = read(ofd, out, len); |
| 584 | if (size != len) { |
| 585 | fprintf(stderr, "Error from read (%zd vs %zu bytes): %s\n", |
| 586 | size, len, strerror(errno)); |
| 587 | ret = -1; |
| 588 | } else { |
| 589 | ret = 0; |
| 590 | } |
| 591 | out_ffd: |
| 592 | close(ffd); |
| 593 | out_ofd: |
| 594 | close(ofd); |
| 595 | out_cfd: |
| 596 | close(cfd); |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 597 | return ret; |
| 598 | } |
| 599 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 600 | static const char *bpf_get_obj_uid(const char *pathname) |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 601 | { |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 602 | static bool bpf_uid_cached = false; |
| 603 | static char bpf_uid[64]; |
| 604 | uint8_t tmp[20]; |
| 605 | int ret; |
| 606 | |
| 607 | if (bpf_uid_cached) |
| 608 | goto done; |
| 609 | |
| 610 | ret = bpf_obj_hash(pathname, tmp, sizeof(tmp)); |
| 611 | if (ret) { |
| 612 | fprintf(stderr, "Object hashing failed!\n"); |
| 613 | return NULL; |
| 614 | } |
| 615 | |
| 616 | hexstring_n2a(tmp, sizeof(tmp), bpf_uid, sizeof(bpf_uid)); |
| 617 | bpf_uid_cached = true; |
| 618 | done: |
| 619 | return bpf_uid; |
| 620 | } |
| 621 | |
| 622 | static int bpf_mnt_fs(const char *target) |
| 623 | { |
| 624 | bool bind_done = false; |
| 625 | |
| 626 | while (mount("", target, "none", MS_PRIVATE | MS_REC, NULL)) { |
| 627 | if (errno != EINVAL || bind_done) { |
| 628 | fprintf(stderr, "mount --make-private %s failed: %s\n", |
| 629 | target, strerror(errno)); |
| 630 | return -1; |
| 631 | } |
| 632 | |
| 633 | if (mount(target, target, "none", MS_BIND, NULL)) { |
| 634 | fprintf(stderr, "mount --bind %s %s failed: %s\n", |
| 635 | target, target, strerror(errno)); |
| 636 | return -1; |
| 637 | } |
| 638 | |
| 639 | bind_done = true; |
| 640 | } |
| 641 | |
| 642 | if (mount("bpf", target, "bpf", 0, NULL)) { |
| 643 | fprintf(stderr, "mount -t bpf bpf %s failed: %s\n", |
| 644 | target, strerror(errno)); |
| 645 | return -1; |
| 646 | } |
| 647 | |
| 648 | return 0; |
| 649 | } |
| 650 | |
| 651 | static const char *bpf_get_tc_dir(void) |
| 652 | { |
| 653 | static bool bpf_mnt_cached = false; |
| 654 | static char bpf_tc_dir[PATH_MAX]; |
| 655 | static const char *mnt; |
| 656 | static const char * const bpf_known_mnts[] = { |
| 657 | BPF_DIR_MNT, |
| 658 | 0, |
| 659 | }; |
| 660 | char bpf_mnt[PATH_MAX] = BPF_DIR_MNT; |
| 661 | char bpf_glo_dir[PATH_MAX]; |
| 662 | int ret; |
| 663 | |
| 664 | if (bpf_mnt_cached) |
| 665 | goto done; |
| 666 | |
| 667 | mnt = bpf_find_mntpt("bpf", BPF_FS_MAGIC, bpf_mnt, sizeof(bpf_mnt), |
| 668 | bpf_known_mnts); |
| 669 | if (!mnt) { |
| 670 | mnt = getenv(BPF_ENV_MNT); |
| 671 | if (!mnt) |
| 672 | mnt = BPF_DIR_MNT; |
| 673 | ret = bpf_mnt_fs(mnt); |
| 674 | if (ret) { |
| 675 | mnt = NULL; |
| 676 | goto out; |
| 677 | } |
| 678 | } |
| 679 | |
| 680 | snprintf(bpf_tc_dir, sizeof(bpf_tc_dir), "%s/%s", mnt, BPF_DIR_TC); |
| 681 | ret = mkdir(bpf_tc_dir, S_IRWXU); |
| 682 | if (ret && errno != EEXIST) { |
| 683 | fprintf(stderr, "mkdir %s failed: %s\n", bpf_tc_dir, |
| 684 | strerror(errno)); |
| 685 | mnt = NULL; |
| 686 | goto out; |
| 687 | } |
| 688 | |
| 689 | snprintf(bpf_glo_dir, sizeof(bpf_glo_dir), "%s/%s", |
| 690 | bpf_tc_dir, BPF_DIR_GLOBALS); |
| 691 | ret = mkdir(bpf_glo_dir, S_IRWXU); |
| 692 | if (ret && errno != EEXIST) { |
| 693 | fprintf(stderr, "mkdir %s failed: %s\n", bpf_glo_dir, |
| 694 | strerror(errno)); |
| 695 | mnt = NULL; |
| 696 | goto out; |
| 697 | } |
| 698 | |
| 699 | mnt = bpf_tc_dir; |
| 700 | out: |
| 701 | bpf_mnt_cached = true; |
| 702 | done: |
| 703 | return mnt; |
| 704 | } |
| 705 | |
| 706 | static int bpf_init_env(const char *pathname) |
| 707 | { |
| 708 | struct rlimit limit = { |
| 709 | .rlim_cur = RLIM_INFINITY, |
| 710 | .rlim_max = RLIM_INFINITY, |
| 711 | }; |
| 712 | |
| 713 | /* Don't bother in case we fail! */ |
| 714 | setrlimit(RLIMIT_MEMLOCK, &limit); |
| 715 | |
| 716 | if (!bpf_get_tc_dir()) { |
| 717 | fprintf(stderr, "Continuing without mounted eBPF fs. " |
| 718 | "Too old kernel?\n"); |
| 719 | return 0; |
| 720 | } |
| 721 | |
| 722 | if (!bpf_get_obj_uid(pathname)) |
| 723 | return -1; |
| 724 | |
| 725 | return 0; |
| 726 | } |
| 727 | |
| 728 | static bool bpf_no_pinning(int pinning) |
| 729 | { |
| 730 | switch (pinning) { |
| 731 | case PIN_OBJECT_NS: |
| 732 | case PIN_GLOBAL_NS: |
| 733 | return false; |
| 734 | case PIN_NONE: |
| 735 | default: |
| 736 | return true; |
| 737 | } |
| 738 | } |
| 739 | |
| 740 | static void bpf_make_pathname(char *pathname, size_t len, const char *name, |
| 741 | int pinning) |
| 742 | { |
| 743 | switch (pinning) { |
| 744 | case PIN_OBJECT_NS: |
| 745 | snprintf(pathname, len, "%s/%s/%s", bpf_get_tc_dir(), |
| 746 | bpf_get_obj_uid(NULL), name); |
| 747 | break; |
| 748 | case PIN_GLOBAL_NS: |
| 749 | snprintf(pathname, len, "%s/%s/%s", bpf_get_tc_dir(), |
| 750 | BPF_DIR_GLOBALS, name); |
| 751 | break; |
| 752 | } |
| 753 | } |
| 754 | |
| 755 | static int bpf_probe_pinned(const char *name, int pinning) |
| 756 | { |
| 757 | char pathname[PATH_MAX]; |
| 758 | |
| 759 | if (bpf_no_pinning(pinning) || !bpf_get_tc_dir()) |
| 760 | return 0; |
| 761 | |
| 762 | bpf_make_pathname(pathname, sizeof(pathname), name, pinning); |
| 763 | return bpf_obj_get(pathname); |
| 764 | } |
| 765 | |
| 766 | static int bpf_place_pinned(int fd, const char *name, int pinning) |
| 767 | { |
| 768 | char pathname[PATH_MAX]; |
| 769 | int ret; |
| 770 | |
| 771 | if (bpf_no_pinning(pinning) || !bpf_get_tc_dir()) |
| 772 | return 0; |
| 773 | |
| 774 | if (pinning == PIN_OBJECT_NS) { |
| 775 | snprintf(pathname, sizeof(pathname), "%s/%s", |
| 776 | bpf_get_tc_dir(), bpf_get_obj_uid(NULL)); |
| 777 | |
| 778 | ret = mkdir(pathname, S_IRWXU); |
| 779 | if (ret && errno != EEXIST) { |
| 780 | fprintf(stderr, "mkdir %s failed: %s\n", pathname, |
| 781 | strerror(errno)); |
| 782 | return ret; |
| 783 | } |
| 784 | } |
| 785 | |
| 786 | bpf_make_pathname(pathname, sizeof(pathname), name, pinning); |
| 787 | return bpf_obj_pin(fd, pathname); |
| 788 | } |
| 789 | |
| 790 | static int bpf_prog_attach(const char *section, |
| 791 | const struct bpf_elf_prog *prog, bool verbose) |
| 792 | { |
| 793 | int fd; |
| 794 | |
| 795 | /* We can add pinning here later as well, same as bpf_map_attach(). */ |
| 796 | errno = 0; |
| 797 | fd = bpf_prog_load(prog->type, prog->insns, prog->size, |
| 798 | prog->license); |
| 799 | if (fd < 0 || verbose) { |
| 800 | bpf_dump_error("Prog section \'%s\' (type:%u insns:%zu " |
| 801 | "license:\'%s\') %s%s (%d)!\n\n", |
| 802 | section, prog->type, |
| 803 | prog->size / sizeof(struct bpf_insn), |
| 804 | prog->license, fd < 0 ? "rejected :" : |
| 805 | "loaded", fd < 0 ? strerror(errno) : "", |
| 806 | fd < 0 ? errno : fd); |
| 807 | } |
| 808 | |
| 809 | return fd; |
| 810 | } |
| 811 | |
| 812 | static int bpf_map_attach(const char *name, const struct bpf_elf_map *map, |
| 813 | bool verbose) |
| 814 | { |
| 815 | int fd, ret; |
| 816 | |
| 817 | fd = bpf_probe_pinned(name, map->pinning); |
| 818 | if (fd > 0) { |
| 819 | if (verbose) |
| 820 | fprintf(stderr, "Map \'%s\' loaded as pinned!\n", |
| 821 | name); |
| 822 | return fd; |
| 823 | } |
| 824 | |
| 825 | errno = 0; |
| 826 | fd = bpf_map_create(map->type, map->size_key, map->size_value, |
| 827 | map->max_elem); |
| 828 | if (fd < 0 || verbose) { |
| 829 | bpf_dump_error("Map \'%s\' (type:%u id:%u pinning:%u " |
| 830 | "ksize:%u vsize:%u max-elems:%u) %s%s (%d)!\n", |
| 831 | name, map->type, map->id, map->pinning, |
| 832 | map->size_key, map->size_value, map->max_elem, |
| 833 | fd < 0 ? "rejected: " : "loaded", fd < 0 ? |
| 834 | strerror(errno) : "", fd < 0 ? errno : fd); |
| 835 | if (fd < 0) |
| 836 | return fd; |
| 837 | } |
| 838 | |
| 839 | ret = bpf_place_pinned(fd, name, map->pinning); |
| 840 | if (ret < 0 && errno != EEXIST) { |
| 841 | fprintf(stderr, "Could not pin %s map: %s\n", name, |
| 842 | strerror(errno)); |
| 843 | close(fd); |
| 844 | return ret; |
| 845 | } |
| 846 | |
| 847 | return fd; |
| 848 | } |
| 849 | |
| 850 | #define __ELF_ST_BIND(x) ((x) >> 4) |
| 851 | #define __ELF_ST_TYPE(x) (((unsigned int) x) & 0xf) |
| 852 | |
| 853 | static const char *bpf_str_tab_name(const struct bpf_elf_ctx *ctx, |
| 854 | const GElf_Sym *sym) |
| 855 | { |
| 856 | return ctx->str_tab->d_buf + sym->st_name; |
| 857 | } |
| 858 | |
| 859 | static const char *bpf_map_fetch_name(struct bpf_elf_ctx *ctx, int which) |
| 860 | { |
| 861 | GElf_Sym sym; |
| 862 | int i; |
| 863 | |
| 864 | for (i = 0; i < ctx->sym_num; i++) { |
| 865 | if (gelf_getsym(ctx->sym_tab, i, &sym) != &sym) |
| 866 | continue; |
| 867 | |
| 868 | if (__ELF_ST_BIND(sym.st_info) != STB_GLOBAL || |
| 869 | __ELF_ST_TYPE(sym.st_info) != STT_NOTYPE || |
| 870 | sym.st_shndx != ctx->sec_maps || |
| 871 | sym.st_value / sizeof(struct bpf_elf_map) != which) |
| 872 | continue; |
| 873 | |
| 874 | return bpf_str_tab_name(ctx, &sym); |
| 875 | } |
| 876 | |
| 877 | return NULL; |
| 878 | } |
| 879 | |
| 880 | static int bpf_maps_attach_all(struct bpf_elf_ctx *ctx) |
| 881 | { |
| 882 | const char *map_name; |
| 883 | int i, fd; |
| 884 | |
| 885 | for (i = 0; i < ctx->map_num; i++) { |
| 886 | map_name = bpf_map_fetch_name(ctx, i); |
| 887 | if (!map_name) |
| 888 | return -EIO; |
| 889 | |
| 890 | fd = bpf_map_attach(map_name, &ctx->maps[i], ctx->verbose); |
| 891 | if (fd < 0) |
| 892 | return fd; |
| 893 | |
| 894 | ctx->map_fds[i] = fd; |
| 895 | } |
| 896 | |
| 897 | return 0; |
| 898 | } |
| 899 | |
| 900 | static int bpf_fill_section_data(struct bpf_elf_ctx *ctx, int section, |
| 901 | struct bpf_elf_sec_data *data) |
| 902 | { |
| 903 | Elf_Data *sec_edata; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 904 | GElf_Shdr sec_hdr; |
| 905 | Elf_Scn *sec_fd; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 906 | char *sec_name; |
| 907 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 908 | memset(data, 0, sizeof(*data)); |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 909 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 910 | sec_fd = elf_getscn(ctx->elf_fd, section); |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 911 | if (!sec_fd) |
| 912 | return -EINVAL; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 913 | if (gelf_getshdr(sec_fd, &sec_hdr) != &sec_hdr) |
| 914 | return -EIO; |
| 915 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 916 | sec_name = elf_strptr(ctx->elf_fd, ctx->elf_hdr.e_shstrndx, |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 917 | sec_hdr.sh_name); |
| 918 | if (!sec_name || !sec_hdr.sh_size) |
| 919 | return -ENOENT; |
| 920 | |
| 921 | sec_edata = elf_getdata(sec_fd, NULL); |
| 922 | if (!sec_edata || elf_getdata(sec_fd, sec_edata)) |
| 923 | return -EIO; |
| 924 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 925 | memcpy(&data->sec_hdr, &sec_hdr, sizeof(sec_hdr)); |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 926 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 927 | data->sec_name = sec_name; |
| 928 | data->sec_data = sec_edata; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 929 | return 0; |
| 930 | } |
| 931 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 932 | static int bpf_fetch_maps(struct bpf_elf_ctx *ctx, int section, |
| 933 | struct bpf_elf_sec_data *data) |
| 934 | { |
| 935 | if (data->sec_data->d_size % sizeof(struct bpf_elf_map) != 0) |
| 936 | return -EINVAL; |
| 937 | |
| 938 | ctx->map_num = data->sec_data->d_size / sizeof(struct bpf_elf_map); |
| 939 | ctx->sec_maps = section; |
| 940 | ctx->sec_done[section] = true; |
| 941 | |
| 942 | if (ctx->map_num > ARRAY_SIZE(ctx->map_fds)) { |
| 943 | fprintf(stderr, "Too many BPF maps in ELF section!\n"); |
| 944 | return -ENOMEM; |
| 945 | } |
| 946 | |
| 947 | memcpy(ctx->maps, data->sec_data->d_buf, data->sec_data->d_size); |
| 948 | return 0; |
| 949 | } |
| 950 | |
| 951 | static int bpf_fetch_license(struct bpf_elf_ctx *ctx, int section, |
| 952 | struct bpf_elf_sec_data *data) |
| 953 | { |
| 954 | if (data->sec_data->d_size > sizeof(ctx->license)) |
| 955 | return -ENOMEM; |
| 956 | |
| 957 | memcpy(ctx->license, data->sec_data->d_buf, data->sec_data->d_size); |
| 958 | ctx->sec_done[section] = true; |
| 959 | return 0; |
| 960 | } |
| 961 | |
| 962 | static int bpf_fetch_symtab(struct bpf_elf_ctx *ctx, int section, |
| 963 | struct bpf_elf_sec_data *data) |
| 964 | { |
| 965 | ctx->sym_tab = data->sec_data; |
| 966 | ctx->sym_num = data->sec_hdr.sh_size / data->sec_hdr.sh_entsize; |
| 967 | ctx->sec_done[section] = true; |
| 968 | return 0; |
| 969 | } |
| 970 | |
| 971 | static int bpf_fetch_strtab(struct bpf_elf_ctx *ctx, int section, |
| 972 | struct bpf_elf_sec_data *data) |
| 973 | { |
| 974 | ctx->str_tab = data->sec_data; |
| 975 | ctx->sec_done[section] = true; |
| 976 | return 0; |
| 977 | } |
| 978 | |
| 979 | static int bpf_fetch_ancillary(struct bpf_elf_ctx *ctx) |
| 980 | { |
| 981 | struct bpf_elf_sec_data data; |
| 982 | int i, ret = -1; |
| 983 | |
| 984 | for (i = 1; i < ctx->elf_hdr.e_shnum; i++) { |
| 985 | ret = bpf_fill_section_data(ctx, i, &data); |
| 986 | if (ret < 0) |
| 987 | continue; |
| 988 | |
| 989 | if (!strcmp(data.sec_name, ELF_SECTION_MAPS)) |
| 990 | ret = bpf_fetch_maps(ctx, i, &data); |
| 991 | else if (!strcmp(data.sec_name, ELF_SECTION_LICENSE)) |
| 992 | ret = bpf_fetch_license(ctx, i, &data); |
| 993 | else if (data.sec_hdr.sh_type == SHT_SYMTAB) |
| 994 | ret = bpf_fetch_symtab(ctx, i, &data); |
| 995 | else if (data.sec_hdr.sh_type == SHT_STRTAB && |
| 996 | i != ctx->elf_hdr.e_shstrndx) |
| 997 | ret = bpf_fetch_strtab(ctx, i, &data); |
| 998 | if (ret < 0) { |
| 999 | fprintf(stderr, "Error parsing section %d! Perhaps" |
| 1000 | "check with readelf -a?\n", i); |
| 1001 | break; |
| 1002 | } |
| 1003 | } |
| 1004 | |
| 1005 | if (ctx->sym_tab && ctx->str_tab && ctx->sec_maps) { |
| 1006 | ret = bpf_maps_attach_all(ctx); |
| 1007 | if (ret < 0) { |
| 1008 | fprintf(stderr, "Error loading maps into kernel!\n"); |
| 1009 | return ret; |
| 1010 | } |
| 1011 | } |
| 1012 | |
| 1013 | return ret; |
| 1014 | } |
| 1015 | |
| 1016 | static int bpf_fetch_prog(struct bpf_elf_ctx *ctx, const char *section) |
| 1017 | { |
| 1018 | struct bpf_elf_sec_data data; |
| 1019 | struct bpf_elf_prog prog; |
| 1020 | int ret, i, fd = -1; |
| 1021 | |
| 1022 | for (i = 1; i < ctx->elf_hdr.e_shnum; i++) { |
| 1023 | if (ctx->sec_done[i]) |
| 1024 | continue; |
| 1025 | |
| 1026 | ret = bpf_fill_section_data(ctx, i, &data); |
| 1027 | if (ret < 0 || strcmp(data.sec_name, section)) |
| 1028 | continue; |
| 1029 | |
| 1030 | memset(&prog, 0, sizeof(prog)); |
| 1031 | prog.type = ctx->type; |
| 1032 | prog.insns = data.sec_data->d_buf; |
| 1033 | prog.size = data.sec_data->d_size; |
| 1034 | prog.license = ctx->license; |
| 1035 | |
| 1036 | fd = bpf_prog_attach(section, &prog, ctx->verbose); |
| 1037 | if (fd < 0) |
| 1038 | continue; |
| 1039 | |
| 1040 | ctx->sec_done[i] = true; |
| 1041 | break; |
| 1042 | } |
| 1043 | |
| 1044 | return fd; |
| 1045 | } |
| 1046 | |
| 1047 | static int bpf_apply_relo_data(struct bpf_elf_ctx *ctx, |
| 1048 | struct bpf_elf_sec_data *data_relo, |
| 1049 | struct bpf_elf_sec_data *data_insn) |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1050 | { |
| 1051 | Elf_Data *idata = data_insn->sec_data; |
| 1052 | GElf_Shdr *rhdr = &data_relo->sec_hdr; |
| 1053 | int relo_ent, relo_num = rhdr->sh_size / rhdr->sh_entsize; |
| 1054 | struct bpf_insn *insns = idata->d_buf; |
| 1055 | unsigned int num_insns = idata->d_size / sizeof(*insns); |
| 1056 | |
| 1057 | for (relo_ent = 0; relo_ent < relo_num; relo_ent++) { |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1058 | unsigned int ioff, rmap; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1059 | GElf_Rel relo; |
| 1060 | GElf_Sym sym; |
| 1061 | |
| 1062 | if (gelf_getrel(data_relo->sec_data, relo_ent, &relo) != &relo) |
| 1063 | return -EIO; |
| 1064 | |
| 1065 | ioff = relo.r_offset / sizeof(struct bpf_insn); |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1066 | if (ioff >= num_insns || |
| 1067 | insns[ioff].code != (BPF_LD | BPF_IMM | BPF_DW)) |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1068 | return -EINVAL; |
| 1069 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1070 | if (gelf_getsym(ctx->sym_tab, GELF_R_SYM(relo.r_info), &sym) != &sym) |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1071 | return -EIO; |
| 1072 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1073 | rmap = sym.st_value / sizeof(struct bpf_elf_map); |
| 1074 | if (rmap >= ARRAY_SIZE(ctx->map_fds)) |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1075 | return -EINVAL; |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1076 | if (!ctx->map_fds[rmap]) |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1077 | return -EINVAL; |
| 1078 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1079 | if (ctx->verbose) |
| 1080 | fprintf(stderr, "Map \'%s\' (%d) injected into prog " |
| 1081 | "section \'%s\' at offset %u!\n", |
| 1082 | bpf_str_tab_name(ctx, &sym), ctx->map_fds[rmap], |
| 1083 | data_insn->sec_name, ioff); |
| 1084 | |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1085 | insns[ioff].src_reg = BPF_PSEUDO_MAP_FD; |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1086 | insns[ioff].imm = ctx->map_fds[rmap]; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1087 | } |
| 1088 | |
| 1089 | return 0; |
| 1090 | } |
| 1091 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1092 | static int bpf_fetch_prog_relo(struct bpf_elf_ctx *ctx, const char *section) |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1093 | { |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1094 | struct bpf_elf_sec_data data_relo, data_insn; |
| 1095 | struct bpf_elf_prog prog; |
| 1096 | int ret, idx, i, fd = -1; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1097 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1098 | for (i = 1; i < ctx->elf_hdr.e_shnum; i++) { |
| 1099 | ret = bpf_fill_section_data(ctx, i, &data_relo); |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1100 | if (ret < 0 || data_relo.sec_hdr.sh_type != SHT_REL) |
| 1101 | continue; |
| 1102 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1103 | idx = data_relo.sec_hdr.sh_info; |
| 1104 | ret = bpf_fill_section_data(ctx, idx, &data_insn); |
| 1105 | if (ret < 0 || strcmp(data_insn.sec_name, section)) |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1106 | continue; |
| 1107 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1108 | ret = bpf_apply_relo_data(ctx, &data_relo, &data_insn); |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1109 | if (ret < 0) |
| 1110 | continue; |
| 1111 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1112 | memset(&prog, 0, sizeof(prog)); |
| 1113 | prog.type = ctx->type; |
| 1114 | prog.insns = data_insn.sec_data->d_buf; |
| 1115 | prog.size = data_insn.sec_data->d_size; |
| 1116 | prog.license = ctx->license; |
| 1117 | |
| 1118 | fd = bpf_prog_attach(section, &prog, ctx->verbose); |
| 1119 | if (fd < 0) |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1120 | continue; |
| 1121 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1122 | ctx->sec_done[i] = true; |
| 1123 | ctx->sec_done[idx] = true; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1124 | break; |
| 1125 | } |
| 1126 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1127 | return fd; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1128 | } |
| 1129 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1130 | static int bpf_fetch_prog_sec(struct bpf_elf_ctx *ctx, const char *section) |
Daniel Borkmann | 473d784 | 2015-05-29 15:15:44 +0200 | [diff] [blame] | 1131 | { |
| 1132 | int ret = -1; |
| 1133 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1134 | if (ctx->sym_tab) |
| 1135 | ret = bpf_fetch_prog_relo(ctx, section); |
Daniel Borkmann | 473d784 | 2015-05-29 15:15:44 +0200 | [diff] [blame] | 1136 | if (ret < 0) |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1137 | ret = bpf_fetch_prog(ctx, section); |
| 1138 | |
Daniel Borkmann | 473d784 | 2015-05-29 15:15:44 +0200 | [diff] [blame] | 1139 | return ret; |
| 1140 | } |
| 1141 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1142 | static int bpf_fill_prog_arrays(struct bpf_elf_ctx *ctx) |
Daniel Borkmann | 473d784 | 2015-05-29 15:15:44 +0200 | [diff] [blame] | 1143 | { |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1144 | struct bpf_elf_sec_data data; |
| 1145 | uint32_t map_id, key_id; |
| 1146 | int fd, i, ret; |
Daniel Borkmann | 473d784 | 2015-05-29 15:15:44 +0200 | [diff] [blame] | 1147 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1148 | for (i = 1; i < ctx->elf_hdr.e_shnum; i++) { |
| 1149 | if (ctx->sec_done[i]) |
Daniel Borkmann | 473d784 | 2015-05-29 15:15:44 +0200 | [diff] [blame] | 1150 | continue; |
| 1151 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1152 | ret = bpf_fill_section_data(ctx, i, &data); |
Daniel Borkmann | 473d784 | 2015-05-29 15:15:44 +0200 | [diff] [blame] | 1153 | if (ret < 0) |
| 1154 | continue; |
| 1155 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1156 | ret = sscanf(data.sec_name, "%u/%u", &map_id, &key_id); |
| 1157 | if (ret != 2 || map_id >= ARRAY_SIZE(ctx->map_fds) || |
| 1158 | !ctx->map_fds[map_id]) |
| 1159 | continue; |
| 1160 | if (ctx->maps[map_id].type != BPF_MAP_TYPE_PROG_ARRAY || |
| 1161 | ctx->maps[map_id].max_elem <= key_id) |
Daniel Borkmann | 473d784 | 2015-05-29 15:15:44 +0200 | [diff] [blame] | 1162 | continue; |
| 1163 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1164 | fd = bpf_fetch_prog_sec(ctx, data.sec_name); |
| 1165 | if (fd < 0) |
Daniel Borkmann | 473d784 | 2015-05-29 15:15:44 +0200 | [diff] [blame] | 1166 | return -EIO; |
| 1167 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1168 | ret = bpf_map_update(ctx->map_fds[map_id], &key_id, |
| 1169 | &fd, BPF_NOEXIST); |
Daniel Borkmann | 473d784 | 2015-05-29 15:15:44 +0200 | [diff] [blame] | 1170 | if (ret < 0) |
| 1171 | return -ENOENT; |
| 1172 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1173 | ctx->sec_done[i] = true; |
Daniel Borkmann | 473d784 | 2015-05-29 15:15:44 +0200 | [diff] [blame] | 1174 | } |
| 1175 | |
| 1176 | return 0; |
| 1177 | } |
| 1178 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1179 | static void bpf_save_finfo(struct bpf_elf_ctx *ctx) |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1180 | { |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1181 | struct stat st; |
| 1182 | int ret; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1183 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1184 | memset(&ctx->stat, 0, sizeof(ctx->stat)); |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1185 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1186 | ret = fstat(ctx->obj_fd, &st); |
| 1187 | if (ret < 0) { |
| 1188 | fprintf(stderr, "Stat of elf file failed: %s\n", |
| 1189 | strerror(errno)); |
| 1190 | return; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1191 | } |
| 1192 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1193 | ctx->stat.st_dev = st.st_dev; |
| 1194 | ctx->stat.st_ino = st.st_ino; |
| 1195 | } |
| 1196 | |
| 1197 | static int bpf_elf_ctx_init(struct bpf_elf_ctx *ctx, const char *pathname, |
| 1198 | enum bpf_prog_type type, bool verbose) |
| 1199 | { |
| 1200 | int ret = -EINVAL; |
| 1201 | |
| 1202 | if (elf_version(EV_CURRENT) == EV_NONE || |
| 1203 | bpf_init_env(pathname)) |
| 1204 | return ret; |
| 1205 | |
| 1206 | memset(ctx, 0, sizeof(*ctx)); |
| 1207 | ctx->verbose = verbose; |
| 1208 | ctx->type = type; |
| 1209 | |
| 1210 | ctx->obj_fd = open(pathname, O_RDONLY); |
| 1211 | if (ctx->obj_fd < 0) |
| 1212 | return ctx->obj_fd; |
| 1213 | |
| 1214 | ctx->elf_fd = elf_begin(ctx->obj_fd, ELF_C_READ, NULL); |
| 1215 | if (!ctx->elf_fd) { |
| 1216 | ret = -EINVAL; |
| 1217 | goto out_fd; |
| 1218 | } |
| 1219 | |
| 1220 | if (gelf_getehdr(ctx->elf_fd, &ctx->elf_hdr) != |
| 1221 | &ctx->elf_hdr) { |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1222 | ret = -EIO; |
| 1223 | goto out_elf; |
| 1224 | } |
| 1225 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1226 | ctx->sec_done = calloc(ctx->elf_hdr.e_shnum, |
| 1227 | sizeof(*(ctx->sec_done))); |
| 1228 | if (!ctx->sec_done) { |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1229 | ret = -ENOMEM; |
| 1230 | goto out_elf; |
| 1231 | } |
| 1232 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1233 | bpf_save_finfo(ctx); |
| 1234 | return 0; |
| 1235 | out_elf: |
| 1236 | elf_end(ctx->elf_fd); |
| 1237 | out_fd: |
| 1238 | close(ctx->obj_fd); |
| 1239 | return ret; |
| 1240 | } |
Daniel Borkmann | d937a74 | 2015-04-28 13:37:42 +0200 | [diff] [blame] | 1241 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1242 | static int bpf_maps_count(struct bpf_elf_ctx *ctx) |
| 1243 | { |
| 1244 | int i, count = 0; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1245 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1246 | for (i = 0; i < ARRAY_SIZE(ctx->map_fds); i++) { |
| 1247 | if (!ctx->map_fds[i]) |
| 1248 | break; |
| 1249 | count++; |
Daniel Borkmann | 473d784 | 2015-05-29 15:15:44 +0200 | [diff] [blame] | 1250 | } |
| 1251 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1252 | return count; |
| 1253 | } |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1254 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1255 | static void bpf_maps_teardown(struct bpf_elf_ctx *ctx) |
| 1256 | { |
| 1257 | int i; |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1258 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1259 | for (i = 0; i < ARRAY_SIZE(ctx->map_fds); i++) { |
| 1260 | if (ctx->map_fds[i]) |
| 1261 | close(ctx->map_fds[i]); |
| 1262 | } |
| 1263 | } |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1264 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1265 | static void bpf_elf_ctx_destroy(struct bpf_elf_ctx *ctx, bool failure) |
| 1266 | { |
| 1267 | if (failure) |
| 1268 | bpf_maps_teardown(ctx); |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1269 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1270 | free(ctx->sec_done); |
| 1271 | elf_end(ctx->elf_fd); |
| 1272 | close(ctx->obj_fd); |
| 1273 | } |
| 1274 | |
| 1275 | static struct bpf_elf_ctx __ctx; |
| 1276 | |
| 1277 | static int bpf_obj_open(const char *pathname, enum bpf_prog_type type, |
| 1278 | const char *section, bool verbose) |
| 1279 | { |
| 1280 | struct bpf_elf_ctx *ctx = &__ctx; |
| 1281 | int fd = 0, ret; |
| 1282 | |
| 1283 | ret = bpf_elf_ctx_init(ctx, pathname, type, verbose); |
| 1284 | if (ret < 0) { |
| 1285 | fprintf(stderr, "Cannot initialize ELF context!\n"); |
| 1286 | return ret; |
| 1287 | } |
| 1288 | |
| 1289 | ret = bpf_fetch_ancillary(ctx); |
| 1290 | if (ret < 0) { |
| 1291 | fprintf(stderr, "Error fetching ELF ancillary data!\n"); |
| 1292 | goto out; |
| 1293 | } |
| 1294 | |
| 1295 | fd = bpf_fetch_prog_sec(ctx, section); |
| 1296 | if (fd < 0) { |
| 1297 | fprintf(stderr, "Error fetching program/map!\n"); |
| 1298 | ret = fd; |
| 1299 | goto out; |
| 1300 | } |
| 1301 | |
| 1302 | ret = bpf_fill_prog_arrays(ctx); |
| 1303 | if (ret < 0) |
| 1304 | fprintf(stderr, "Error filling program arrays!\n"); |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1305 | out: |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1306 | bpf_elf_ctx_destroy(ctx, ret < 0); |
| 1307 | if (ret < 0) { |
| 1308 | if (fd) |
| 1309 | close(fd); |
| 1310 | return ret; |
| 1311 | } |
| 1312 | |
| 1313 | return fd; |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1314 | } |
| 1315 | |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1316 | static int |
Daniel Borkmann | 4bd6244 | 2015-04-16 21:20:06 +0200 | [diff] [blame] | 1317 | bpf_map_set_send(int fd, struct sockaddr_un *addr, unsigned int addr_len, |
| 1318 | const struct bpf_map_data *aux, unsigned int entries) |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1319 | { |
| 1320 | struct bpf_map_set_msg msg; |
| 1321 | int *cmsg_buf, min_fd; |
| 1322 | char *amsg_buf; |
| 1323 | int i; |
| 1324 | |
| 1325 | memset(&msg, 0, sizeof(msg)); |
| 1326 | |
| 1327 | msg.aux.uds_ver = BPF_SCM_AUX_VER; |
Daniel Borkmann | 4bd6244 | 2015-04-16 21:20:06 +0200 | [diff] [blame] | 1328 | msg.aux.num_ent = entries; |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1329 | |
| 1330 | strncpy(msg.aux.obj_name, aux->obj, sizeof(msg.aux.obj_name)); |
| 1331 | memcpy(&msg.aux.obj_st, aux->st, sizeof(msg.aux.obj_st)); |
| 1332 | |
| 1333 | cmsg_buf = bpf_map_set_init(&msg, addr, addr_len); |
| 1334 | amsg_buf = (char *)msg.aux.ent; |
| 1335 | |
Daniel Borkmann | 4bd6244 | 2015-04-16 21:20:06 +0200 | [diff] [blame] | 1336 | for (i = 0; i < entries; i += min_fd) { |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1337 | int ret; |
| 1338 | |
Daniel Borkmann | 4bd6244 | 2015-04-16 21:20:06 +0200 | [diff] [blame] | 1339 | min_fd = min(BPF_SCM_MAX_FDS * 1U, entries - i); |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1340 | bpf_map_set_init_single(&msg, min_fd); |
| 1341 | |
| 1342 | memcpy(cmsg_buf, &aux->fds[i], sizeof(aux->fds[0]) * min_fd); |
| 1343 | memcpy(amsg_buf, &aux->ent[i], sizeof(aux->ent[0]) * min_fd); |
| 1344 | |
| 1345 | ret = sendmsg(fd, &msg.hdr, 0); |
| 1346 | if (ret <= 0) |
| 1347 | return ret ? : -1; |
| 1348 | } |
| 1349 | |
| 1350 | return 0; |
| 1351 | } |
| 1352 | |
Daniel Borkmann | 4bd6244 | 2015-04-16 21:20:06 +0200 | [diff] [blame] | 1353 | static int |
| 1354 | bpf_map_set_recv(int fd, int *fds, struct bpf_map_aux *aux, |
| 1355 | unsigned int entries) |
| 1356 | { |
| 1357 | struct bpf_map_set_msg msg; |
| 1358 | int *cmsg_buf, min_fd; |
| 1359 | char *amsg_buf, *mmsg_buf; |
| 1360 | unsigned int needed = 1; |
| 1361 | int i; |
| 1362 | |
| 1363 | cmsg_buf = bpf_map_set_init(&msg, NULL, 0); |
| 1364 | amsg_buf = (char *)msg.aux.ent; |
| 1365 | mmsg_buf = (char *)&msg.aux; |
| 1366 | |
| 1367 | for (i = 0; i < min(entries, needed); i += min_fd) { |
| 1368 | struct cmsghdr *cmsg; |
| 1369 | int ret; |
| 1370 | |
| 1371 | min_fd = min(entries, entries - i); |
| 1372 | bpf_map_set_init_single(&msg, min_fd); |
| 1373 | |
| 1374 | ret = recvmsg(fd, &msg.hdr, 0); |
| 1375 | if (ret <= 0) |
| 1376 | return ret ? : -1; |
| 1377 | |
| 1378 | cmsg = CMSG_FIRSTHDR(&msg.hdr); |
| 1379 | if (!cmsg || cmsg->cmsg_type != SCM_RIGHTS) |
| 1380 | return -EINVAL; |
| 1381 | if (msg.hdr.msg_flags & MSG_CTRUNC) |
| 1382 | return -EIO; |
| 1383 | if (msg.aux.uds_ver != BPF_SCM_AUX_VER) |
| 1384 | return -ENOSYS; |
| 1385 | |
| 1386 | min_fd = (cmsg->cmsg_len - sizeof(*cmsg)) / sizeof(fd); |
| 1387 | if (min_fd > entries || min_fd <= 0) |
| 1388 | return -EINVAL; |
| 1389 | |
| 1390 | memcpy(&fds[i], cmsg_buf, sizeof(fds[0]) * min_fd); |
| 1391 | memcpy(&aux->ent[i], amsg_buf, sizeof(aux->ent[0]) * min_fd); |
| 1392 | memcpy(aux, mmsg_buf, offsetof(struct bpf_map_aux, ent)); |
| 1393 | |
| 1394 | needed = aux->num_ent; |
| 1395 | } |
| 1396 | |
| 1397 | return 0; |
| 1398 | } |
| 1399 | |
| 1400 | int bpf_send_map_fds(const char *path, const char *obj) |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1401 | { |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1402 | struct bpf_elf_ctx *ctx = &__ctx; |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1403 | struct sockaddr_un addr; |
| 1404 | struct bpf_map_data bpf_aux; |
| 1405 | int fd, ret; |
| 1406 | |
| 1407 | fd = socket(AF_UNIX, SOCK_DGRAM, 0); |
| 1408 | if (fd < 0) { |
| 1409 | fprintf(stderr, "Cannot open socket: %s\n", |
| 1410 | strerror(errno)); |
| 1411 | return -1; |
| 1412 | } |
| 1413 | |
| 1414 | memset(&addr, 0, sizeof(addr)); |
| 1415 | addr.sun_family = AF_UNIX; |
| 1416 | strncpy(addr.sun_path, path, sizeof(addr.sun_path)); |
| 1417 | |
| 1418 | ret = connect(fd, (struct sockaddr *)&addr, sizeof(addr)); |
| 1419 | if (ret < 0) { |
| 1420 | fprintf(stderr, "Cannot connect to %s: %s\n", |
| 1421 | path, strerror(errno)); |
| 1422 | return -1; |
| 1423 | } |
| 1424 | |
| 1425 | memset(&bpf_aux, 0, sizeof(bpf_aux)); |
| 1426 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1427 | bpf_aux.fds = ctx->map_fds; |
| 1428 | bpf_aux.ent = ctx->maps; |
| 1429 | bpf_aux.st = &ctx->stat; |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1430 | bpf_aux.obj = obj; |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1431 | |
Daniel Borkmann | 4bd6244 | 2015-04-16 21:20:06 +0200 | [diff] [blame] | 1432 | ret = bpf_map_set_send(fd, &addr, sizeof(addr), &bpf_aux, |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1433 | bpf_maps_count(ctx)); |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1434 | if (ret < 0) |
Daniel Borkmann | 4bd6244 | 2015-04-16 21:20:06 +0200 | [diff] [blame] | 1435 | fprintf(stderr, "Cannot send fds to %s: %s\n", |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1436 | path, strerror(errno)); |
| 1437 | |
Daniel Borkmann | 32e93fb | 2015-11-13 00:39:29 +0100 | [diff] [blame^] | 1438 | bpf_maps_teardown(ctx); |
Daniel Borkmann | 6256f8c | 2015-04-01 17:57:44 +0200 | [diff] [blame] | 1439 | close(fd); |
| 1440 | return ret; |
| 1441 | } |
Daniel Borkmann | 4bd6244 | 2015-04-16 21:20:06 +0200 | [diff] [blame] | 1442 | |
| 1443 | int bpf_recv_map_fds(const char *path, int *fds, struct bpf_map_aux *aux, |
| 1444 | unsigned int entries) |
| 1445 | { |
| 1446 | struct sockaddr_un addr; |
| 1447 | int fd, ret; |
| 1448 | |
| 1449 | fd = socket(AF_UNIX, SOCK_DGRAM, 0); |
| 1450 | if (fd < 0) { |
| 1451 | fprintf(stderr, "Cannot open socket: %s\n", |
| 1452 | strerror(errno)); |
| 1453 | return -1; |
| 1454 | } |
| 1455 | |
| 1456 | memset(&addr, 0, sizeof(addr)); |
| 1457 | addr.sun_family = AF_UNIX; |
| 1458 | strncpy(addr.sun_path, path, sizeof(addr.sun_path)); |
| 1459 | |
| 1460 | ret = bind(fd, (struct sockaddr *)&addr, sizeof(addr)); |
| 1461 | if (ret < 0) { |
| 1462 | fprintf(stderr, "Cannot bind to socket: %s\n", |
| 1463 | strerror(errno)); |
| 1464 | return -1; |
| 1465 | } |
| 1466 | |
| 1467 | ret = bpf_map_set_recv(fd, fds, aux, entries); |
| 1468 | if (ret < 0) |
| 1469 | fprintf(stderr, "Cannot recv fds from %s: %s\n", |
| 1470 | path, strerror(errno)); |
| 1471 | |
| 1472 | unlink(addr.sun_path); |
| 1473 | close(fd); |
| 1474 | return ret; |
| 1475 | } |
Daniel Borkmann | 11c39b5 | 2015-03-16 19:37:41 +0100 | [diff] [blame] | 1476 | #endif /* HAVE_ELF */ |