blob: 7fabad8dc3fc5104f8721ab6a2dc1d3165881979 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Linux Socket Filter Data Structures
3 */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004#ifndef __LINUX_FILTER_H__
5#define __LINUX_FILTER_H__
6
Daniel Borkmannb954d832014-09-10 15:01:02 +02007#include <stdarg.h>
8
Arun Sharma600634972011-07-26 16:09:06 -07009#include <linux/atomic.h>
Will Drewry0c5fe1b2012-04-12 16:47:53 -050010#include <linux/compat.h>
Zi Shen Lim9f12fbe2014-07-03 07:56:54 -070011#include <linux/skbuff.h>
Daniel Borkmannb954d832014-09-10 15:01:02 +020012#include <linux/linkage.h>
13#include <linux/printk.h>
Alexei Starovoitovd45ed4a2013-10-04 00:14:06 -070014#include <linux/workqueue.h>
Daniel Borkmannb13138e2015-07-30 12:42:49 +020015#include <linux/sched.h>
Daniel Borkmann4f3446b2016-05-13 19:08:32 +020016#include <linux/capability.h>
17
Alexei Starovoitovff936a02015-10-07 10:55:41 -070018#include <net/sch_generic.h>
Daniel Borkmannb954d832014-09-10 15:01:02 +020019
Daniel Borkmann60a3b222014-09-02 22:53:44 +020020#include <asm/cacheflush.h>
Daniel Borkmannb954d832014-09-10 15:01:02 +020021
22#include <uapi/linux/filter.h>
Alexei Starovoitovdaedfb22014-09-04 22:17:18 -070023#include <uapi/linux/bpf.h>
Daniel Borkmann60a3b222014-09-02 22:53:44 +020024
25struct sk_buff;
26struct sock;
27struct seccomp_data;
Alexei Starovoitov09756af2014-09-26 00:17:00 -070028struct bpf_prog_aux;
Heiko Carstens792d4b52011-05-22 07:08:11 +000029
Daniel Borkmann30743832014-05-01 18:34:19 +020030/* ArgX, context and stack frame pointer register positions. Note,
31 * Arg1, Arg2, Arg3, etc are used as argument mappings of function
32 * calls in BPF_CALL instruction.
33 */
34#define BPF_REG_ARG1 BPF_REG_1
35#define BPF_REG_ARG2 BPF_REG_2
36#define BPF_REG_ARG3 BPF_REG_3
37#define BPF_REG_ARG4 BPF_REG_4
38#define BPF_REG_ARG5 BPF_REG_5
39#define BPF_REG_CTX BPF_REG_6
40#define BPF_REG_FP BPF_REG_10
41
42/* Additional register mappings for converted user programs. */
43#define BPF_REG_A BPF_REG_0
44#define BPF_REG_X BPF_REG_7
45#define BPF_REG_TMP BPF_REG_8
Alexei Starovoitovbd4cf0e2014-03-28 18:58:25 +010046
Daniel Borkmann4f3446b2016-05-13 19:08:32 +020047/* Kernel hidden auxiliary/helper register for hardening step.
48 * Only used by eBPF JITs. It's nothing more than a temporary
49 * register that JITs use internally, only that here it's part
50 * of eBPF instructions that have been rewritten for blinding
51 * constants. See JIT pre-step in bpf_jit_blind_constants().
52 */
53#define BPF_REG_AX MAX_BPF_REG
54#define MAX_BPF_JIT_REG (MAX_BPF_REG + 1)
55
Alexei Starovoitovbd4cf0e2014-03-28 18:58:25 +010056/* BPF program can access up to 512 bytes of stack space. */
57#define MAX_BPF_STACK 512
58
Daniel Borkmannf8f6d672014-05-29 10:22:51 +020059/* Helper macros for filter block array initializers. */
Alexei Starovoitov9739eef2014-05-08 14:10:51 -070060
Alexei Starovoitove430f342014-06-06 14:46:06 -070061/* ALU ops on registers, bpf_add|sub|...: dst_reg += src_reg */
Alexei Starovoitov9739eef2014-05-08 14:10:51 -070062
Alexei Starovoitove430f342014-06-06 14:46:06 -070063#define BPF_ALU64_REG(OP, DST, SRC) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -070064 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +020065 .code = BPF_ALU64 | BPF_OP(OP) | BPF_X, \
Alexei Starovoitove430f342014-06-06 14:46:06 -070066 .dst_reg = DST, \
67 .src_reg = SRC, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +020068 .off = 0, \
69 .imm = 0 })
Alexei Starovoitov9739eef2014-05-08 14:10:51 -070070
Alexei Starovoitove430f342014-06-06 14:46:06 -070071#define BPF_ALU32_REG(OP, DST, SRC) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -070072 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +020073 .code = BPF_ALU | BPF_OP(OP) | BPF_X, \
Alexei Starovoitove430f342014-06-06 14:46:06 -070074 .dst_reg = DST, \
75 .src_reg = SRC, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +020076 .off = 0, \
77 .imm = 0 })
Alexei Starovoitov9739eef2014-05-08 14:10:51 -070078
Alexei Starovoitove430f342014-06-06 14:46:06 -070079/* ALU ops on immediates, bpf_add|sub|...: dst_reg += imm32 */
Alexei Starovoitov9739eef2014-05-08 14:10:51 -070080
Alexei Starovoitove430f342014-06-06 14:46:06 -070081#define BPF_ALU64_IMM(OP, DST, IMM) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -070082 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +020083 .code = BPF_ALU64 | BPF_OP(OP) | BPF_K, \
Alexei Starovoitove430f342014-06-06 14:46:06 -070084 .dst_reg = DST, \
85 .src_reg = 0, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +020086 .off = 0, \
87 .imm = IMM })
Alexei Starovoitov9739eef2014-05-08 14:10:51 -070088
Alexei Starovoitove430f342014-06-06 14:46:06 -070089#define BPF_ALU32_IMM(OP, DST, IMM) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -070090 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +020091 .code = BPF_ALU | BPF_OP(OP) | BPF_K, \
Alexei Starovoitove430f342014-06-06 14:46:06 -070092 .dst_reg = DST, \
93 .src_reg = 0, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +020094 .off = 0, \
95 .imm = IMM })
Alexei Starovoitov9739eef2014-05-08 14:10:51 -070096
Daniel Borkmannf8f6d672014-05-29 10:22:51 +020097/* Endianess conversion, cpu_to_{l,b}e(), {l,b}e_to_cpu() */
Alexei Starovoitov9739eef2014-05-08 14:10:51 -070098
Alexei Starovoitove430f342014-06-06 14:46:06 -070099#define BPF_ENDIAN(TYPE, DST, LEN) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700100 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200101 .code = BPF_ALU | BPF_END | BPF_SRC(TYPE), \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700102 .dst_reg = DST, \
103 .src_reg = 0, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200104 .off = 0, \
105 .imm = LEN })
106
Alexei Starovoitove430f342014-06-06 14:46:06 -0700107/* Short form of mov, dst_reg = src_reg */
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200108
Alexei Starovoitove430f342014-06-06 14:46:06 -0700109#define BPF_MOV64_REG(DST, SRC) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700110 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200111 .code = BPF_ALU64 | BPF_MOV | BPF_X, \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700112 .dst_reg = DST, \
113 .src_reg = SRC, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200114 .off = 0, \
115 .imm = 0 })
116
Alexei Starovoitove430f342014-06-06 14:46:06 -0700117#define BPF_MOV32_REG(DST, SRC) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700118 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200119 .code = BPF_ALU | BPF_MOV | BPF_X, \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700120 .dst_reg = DST, \
121 .src_reg = SRC, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200122 .off = 0, \
123 .imm = 0 })
124
Alexei Starovoitove430f342014-06-06 14:46:06 -0700125/* Short form of mov, dst_reg = imm32 */
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200126
Alexei Starovoitove430f342014-06-06 14:46:06 -0700127#define BPF_MOV64_IMM(DST, IMM) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700128 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200129 .code = BPF_ALU64 | BPF_MOV | BPF_K, \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700130 .dst_reg = DST, \
131 .src_reg = 0, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200132 .off = 0, \
133 .imm = IMM })
134
Alexei Starovoitove430f342014-06-06 14:46:06 -0700135#define BPF_MOV32_IMM(DST, IMM) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700136 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200137 .code = BPF_ALU | BPF_MOV | BPF_K, \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700138 .dst_reg = DST, \
139 .src_reg = 0, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200140 .off = 0, \
141 .imm = IMM })
142
Alexei Starovoitov02ab6952014-09-04 22:17:17 -0700143/* BPF_LD_IMM64 macro encodes single 'load 64-bit immediate' insn */
144#define BPF_LD_IMM64(DST, IMM) \
145 BPF_LD_IMM64_RAW(DST, 0, IMM)
146
147#define BPF_LD_IMM64_RAW(DST, SRC, IMM) \
148 ((struct bpf_insn) { \
149 .code = BPF_LD | BPF_DW | BPF_IMM, \
150 .dst_reg = DST, \
151 .src_reg = SRC, \
152 .off = 0, \
153 .imm = (__u32) (IMM) }), \
154 ((struct bpf_insn) { \
155 .code = 0, /* zero is reserved opcode */ \
156 .dst_reg = 0, \
157 .src_reg = 0, \
158 .off = 0, \
159 .imm = ((__u64) (IMM)) >> 32 })
160
Alexei Starovoitov0246e642014-09-26 00:17:04 -0700161/* pseudo BPF_LD_IMM64 insn used to refer to process-local map_fd */
162#define BPF_LD_MAP_FD(DST, MAP_FD) \
163 BPF_LD_IMM64_RAW(DST, BPF_PSEUDO_MAP_FD, MAP_FD)
164
Alexei Starovoitove430f342014-06-06 14:46:06 -0700165/* Short form of mov based on type, BPF_X: dst_reg = src_reg, BPF_K: dst_reg = imm32 */
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200166
Alexei Starovoitove430f342014-06-06 14:46:06 -0700167#define BPF_MOV64_RAW(TYPE, DST, SRC, IMM) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700168 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200169 .code = BPF_ALU64 | BPF_MOV | BPF_SRC(TYPE), \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700170 .dst_reg = DST, \
171 .src_reg = SRC, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200172 .off = 0, \
173 .imm = IMM })
174
Alexei Starovoitove430f342014-06-06 14:46:06 -0700175#define BPF_MOV32_RAW(TYPE, DST, SRC, IMM) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700176 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200177 .code = BPF_ALU | BPF_MOV | BPF_SRC(TYPE), \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700178 .dst_reg = DST, \
179 .src_reg = SRC, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200180 .off = 0, \
181 .imm = IMM })
182
Alexei Starovoitove430f342014-06-06 14:46:06 -0700183/* Direct packet access, R0 = *(uint *) (skb->data + imm32) */
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200184
Alexei Starovoitove430f342014-06-06 14:46:06 -0700185#define BPF_LD_ABS(SIZE, IMM) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700186 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200187 .code = BPF_LD | BPF_SIZE(SIZE) | BPF_ABS, \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700188 .dst_reg = 0, \
189 .src_reg = 0, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200190 .off = 0, \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700191 .imm = IMM })
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200192
Alexei Starovoitove430f342014-06-06 14:46:06 -0700193/* Indirect packet access, R0 = *(uint *) (skb->data + src_reg + imm32) */
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200194
Alexei Starovoitove430f342014-06-06 14:46:06 -0700195#define BPF_LD_IND(SIZE, SRC, IMM) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700196 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200197 .code = BPF_LD | BPF_SIZE(SIZE) | BPF_IND, \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700198 .dst_reg = 0, \
199 .src_reg = SRC, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200200 .off = 0, \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700201 .imm = IMM })
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200202
Alexei Starovoitove430f342014-06-06 14:46:06 -0700203/* Memory load, dst_reg = *(uint *) (src_reg + off16) */
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200204
Alexei Starovoitove430f342014-06-06 14:46:06 -0700205#define BPF_LDX_MEM(SIZE, DST, SRC, OFF) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700206 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200207 .code = BPF_LDX | BPF_SIZE(SIZE) | BPF_MEM, \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700208 .dst_reg = DST, \
209 .src_reg = SRC, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200210 .off = OFF, \
211 .imm = 0 })
212
Alexei Starovoitove430f342014-06-06 14:46:06 -0700213/* Memory store, *(uint *) (dst_reg + off16) = src_reg */
214
215#define BPF_STX_MEM(SIZE, DST, SRC, OFF) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700216 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200217 .code = BPF_STX | BPF_SIZE(SIZE) | BPF_MEM, \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700218 .dst_reg = DST, \
219 .src_reg = SRC, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200220 .off = OFF, \
221 .imm = 0 })
222
Michael Holzheucffc6422015-05-11 22:22:44 -0700223/* Atomic memory add, *(uint *)(dst_reg + off16) += src_reg */
224
225#define BPF_STX_XADD(SIZE, DST, SRC, OFF) \
226 ((struct bpf_insn) { \
227 .code = BPF_STX | BPF_SIZE(SIZE) | BPF_XADD, \
228 .dst_reg = DST, \
229 .src_reg = SRC, \
230 .off = OFF, \
231 .imm = 0 })
232
Alexei Starovoitove430f342014-06-06 14:46:06 -0700233/* Memory store, *(uint *) (dst_reg + off16) = imm32 */
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200234
Alexei Starovoitove430f342014-06-06 14:46:06 -0700235#define BPF_ST_MEM(SIZE, DST, OFF, IMM) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700236 ((struct bpf_insn) { \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700237 .code = BPF_ST | BPF_SIZE(SIZE) | BPF_MEM, \
238 .dst_reg = DST, \
239 .src_reg = 0, \
240 .off = OFF, \
241 .imm = IMM })
242
243/* Conditional jumps against registers, if (dst_reg 'op' src_reg) goto pc + off16 */
244
245#define BPF_JMP_REG(OP, DST, SRC, OFF) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700246 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200247 .code = BPF_JMP | BPF_OP(OP) | BPF_X, \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700248 .dst_reg = DST, \
249 .src_reg = SRC, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200250 .off = OFF, \
251 .imm = 0 })
252
Alexei Starovoitove430f342014-06-06 14:46:06 -0700253/* Conditional jumps against immediates, if (dst_reg 'op' imm32) goto pc + off16 */
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200254
Alexei Starovoitove430f342014-06-06 14:46:06 -0700255#define BPF_JMP_IMM(OP, DST, IMM, OFF) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700256 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200257 .code = BPF_JMP | BPF_OP(OP) | BPF_K, \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700258 .dst_reg = DST, \
259 .src_reg = 0, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200260 .off = OFF, \
261 .imm = IMM })
262
263/* Function call */
264
265#define BPF_EMIT_CALL(FUNC) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700266 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200267 .code = BPF_JMP | BPF_CALL, \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700268 .dst_reg = 0, \
269 .src_reg = 0, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200270 .off = 0, \
271 .imm = ((FUNC) - __bpf_call_base) })
272
273/* Raw code statement block */
274
Alexei Starovoitove430f342014-06-06 14:46:06 -0700275#define BPF_RAW_INSN(CODE, DST, SRC, OFF, IMM) \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700276 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200277 .code = CODE, \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700278 .dst_reg = DST, \
279 .src_reg = SRC, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200280 .off = OFF, \
281 .imm = IMM })
282
283/* Program exit */
284
285#define BPF_EXIT_INSN() \
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700286 ((struct bpf_insn) { \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200287 .code = BPF_JMP | BPF_EXIT, \
Alexei Starovoitove430f342014-06-06 14:46:06 -0700288 .dst_reg = 0, \
289 .src_reg = 0, \
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200290 .off = 0, \
291 .imm = 0 })
292
Daniel Borkmanna4afd37b2015-05-13 13:12:43 +0200293/* Internal classic blocks for direct assignment */
294
295#define __BPF_STMT(CODE, K) \
296 ((struct sock_filter) BPF_STMT(CODE, K))
297
298#define __BPF_JUMP(CODE, K, JT, JF) \
299 ((struct sock_filter) BPF_JUMP(CODE, K, JT, JF))
300
Daniel Borkmannf8f6d672014-05-29 10:22:51 +0200301#define bytes_to_bpf_size(bytes) \
302({ \
303 int bpf_size = -EINVAL; \
304 \
305 if (bytes == sizeof(u8)) \
306 bpf_size = BPF_B; \
307 else if (bytes == sizeof(u16)) \
308 bpf_size = BPF_H; \
309 else if (bytes == sizeof(u32)) \
310 bpf_size = BPF_W; \
311 else if (bytes == sizeof(u64)) \
312 bpf_size = BPF_DW; \
313 \
314 bpf_size; \
315})
Alexei Starovoitov9739eef2014-05-08 14:10:51 -0700316
Daniel Borkmannf035a512016-09-09 02:45:29 +0200317#define BPF_SIZEOF(type) \
318 ({ \
319 const int __size = bytes_to_bpf_size(sizeof(type)); \
320 BUILD_BUG_ON(__size < 0); \
321 __size; \
322 })
323
324#define BPF_FIELD_SIZEOF(type, field) \
325 ({ \
326 const int __size = bytes_to_bpf_size(FIELD_SIZEOF(type, field)); \
327 BUILD_BUG_ON(__size < 0); \
328 __size; \
329 })
330
Alexei Starovoitovbd4cf0e2014-03-28 18:58:25 +0100331#ifdef CONFIG_COMPAT
332/* A struct sock_filter is architecture independent. */
Will Drewry0c5fe1b2012-04-12 16:47:53 -0500333struct compat_sock_fprog {
334 u16 len;
Alexei Starovoitovbd4cf0e2014-03-28 18:58:25 +0100335 compat_uptr_t filter; /* struct sock_filter * */
Will Drewry0c5fe1b2012-04-12 16:47:53 -0500336};
337#endif
338
Daniel Borkmanna3ea2692014-03-28 18:58:19 +0100339struct sock_fprog_kern {
340 u16 len;
341 struct sock_filter *filter;
342};
343
Daniel Borkmann738cbe72014-09-08 08:04:47 +0200344struct bpf_binary_header {
345 unsigned int pages;
346 u8 image[];
347};
348
Alexei Starovoitov7ae457c2014-07-30 20:34:16 -0700349struct bpf_prog {
Daniel Borkmann286aad32014-09-08 08:04:49 +0200350 u16 pages; /* Number of allocated pages */
Daniel Borkmanna91263d2015-09-30 01:41:50 +0200351 kmemcheck_bitfield_begin(meta);
352 u16 jited:1, /* Is our filter JIT'ed? */
Daniel Borkmannc46646d2015-09-30 01:41:51 +0200353 gpl_compatible:1, /* Is filter GPL compatible? */
Alexei Starovoitovff936a02015-10-07 10:55:41 -0700354 cb_access:1, /* Is control block accessed? */
Daniel Borkmannc46646d2015-09-30 01:41:51 +0200355 dst_needed:1; /* Do we need dst entry? */
Daniel Borkmanna91263d2015-09-30 01:41:50 +0200356 kmemcheck_bitfield_end(meta);
Daniel Borkmann286aad32014-09-08 08:04:49 +0200357 u32 len; /* Number of filter blocks */
Daniel Borkmann24701ec2015-03-01 12:31:47 +0100358 enum bpf_prog_type type; /* Type of BPF program */
Alexei Starovoitov09756af2014-09-26 00:17:00 -0700359 struct bpf_prog_aux *aux; /* Auxiliary fields */
Daniel Borkmann24701ec2015-03-01 12:31:47 +0100360 struct sock_fprog_kern *orig_prog; /* Original BPF program */
Eric Dumazet0a148422011-04-20 09:27:32 +0000361 unsigned int (*bpf_func)(const struct sk_buff *skb,
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700362 const struct bpf_insn *filter);
Daniel Borkmann60a3b222014-09-02 22:53:44 +0200363 /* Instructions for interpreter */
Alexei Starovoitovd45ed4a2013-10-04 00:14:06 -0700364 union {
Alexei Starovoitovbd4cf0e2014-03-28 18:58:25 +0100365 struct sock_filter insns[0];
Alexei Starovoitov2695fb52014-07-24 16:38:21 -0700366 struct bpf_insn insnsi[0];
Alexei Starovoitovd45ed4a2013-10-04 00:14:06 -0700367 };
Stephen Hemmingerb7156312008-04-10 01:33:47 -0700368};
369
Alexei Starovoitov7ae457c2014-07-30 20:34:16 -0700370struct sk_filter {
371 atomic_t refcnt;
372 struct rcu_head rcu;
373 struct bpf_prog *prog;
374};
375
376#define BPF_PROG_RUN(filter, ctx) (*filter->bpf_func)(ctx, filter->insnsi)
377
Daniel Borkmann01dd1942016-01-06 22:32:16 +0100378#define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN
379
Alexei Starovoitovdb58ba42016-05-05 19:49:12 -0700380struct bpf_skb_data_end {
381 struct qdisc_skb_cb qdisc_cb;
382 void *data_end;
383};
384
Brenden Blanco6a773a12016-07-19 12:16:47 -0700385struct xdp_buff {
386 void *data;
387 void *data_end;
388};
389
Alexei Starovoitovdb58ba42016-05-05 19:49:12 -0700390/* compute the linear packet data range [data, data_end) which
391 * will be accessed by cls_bpf and act_bpf programs
392 */
393static inline void bpf_compute_data_end(struct sk_buff *skb)
394{
395 struct bpf_skb_data_end *cb = (struct bpf_skb_data_end *)skb->cb;
396
397 BUILD_BUG_ON(sizeof(*cb) > FIELD_SIZEOF(struct sk_buff, cb));
398 cb->data_end = skb->data + skb_headlen(skb);
399}
400
Daniel Borkmann01dd1942016-01-06 22:32:16 +0100401static inline u8 *bpf_skb_cb(struct sk_buff *skb)
402{
403 /* eBPF programs may read/write skb->cb[] area to transfer meta
404 * data between tail calls. Since this also needs to work with
405 * tc, that scratch memory is mapped to qdisc_skb_cb's data area.
406 *
407 * In some socket filter cases, the cb unfortunately needs to be
408 * saved/restored so that protocol specific skb->cb[] data won't
409 * be lost. In any case, due to unpriviledged eBPF programs
410 * attached to sockets, we need to clear the bpf_skb_cb() area
411 * to not leak previous contents to user space.
412 */
413 BUILD_BUG_ON(FIELD_SIZEOF(struct __sk_buff, cb) != BPF_SKB_CB_LEN);
414 BUILD_BUG_ON(FIELD_SIZEOF(struct __sk_buff, cb) !=
415 FIELD_SIZEOF(struct qdisc_skb_cb, data));
416
417 return qdisc_skb_cb(skb)->data;
418}
419
Alexei Starovoitovff936a02015-10-07 10:55:41 -0700420static inline u32 bpf_prog_run_save_cb(const struct bpf_prog *prog,
421 struct sk_buff *skb)
422{
Daniel Borkmann01dd1942016-01-06 22:32:16 +0100423 u8 *cb_data = bpf_skb_cb(skb);
424 u8 cb_saved[BPF_SKB_CB_LEN];
Alexei Starovoitovff936a02015-10-07 10:55:41 -0700425 u32 res;
426
Alexei Starovoitovff936a02015-10-07 10:55:41 -0700427 if (unlikely(prog->cb_access)) {
Daniel Borkmann01dd1942016-01-06 22:32:16 +0100428 memcpy(cb_saved, cb_data, sizeof(cb_saved));
429 memset(cb_data, 0, sizeof(cb_saved));
Alexei Starovoitovff936a02015-10-07 10:55:41 -0700430 }
431
432 res = BPF_PROG_RUN(prog, skb);
433
434 if (unlikely(prog->cb_access))
Daniel Borkmann01dd1942016-01-06 22:32:16 +0100435 memcpy(cb_data, cb_saved, sizeof(cb_saved));
Alexei Starovoitovff936a02015-10-07 10:55:41 -0700436
437 return res;
438}
439
440static inline u32 bpf_prog_run_clear_cb(const struct bpf_prog *prog,
441 struct sk_buff *skb)
442{
Daniel Borkmann01dd1942016-01-06 22:32:16 +0100443 u8 *cb_data = bpf_skb_cb(skb);
Alexei Starovoitovff936a02015-10-07 10:55:41 -0700444
445 if (unlikely(prog->cb_access))
Daniel Borkmann01dd1942016-01-06 22:32:16 +0100446 memset(cb_data, 0, BPF_SKB_CB_LEN);
447
Alexei Starovoitovff936a02015-10-07 10:55:41 -0700448 return BPF_PROG_RUN(prog, skb);
449}
450
Brenden Blanco6a773a12016-07-19 12:16:47 -0700451static inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog,
452 struct xdp_buff *xdp)
453{
454 u32 ret;
455
456 rcu_read_lock();
457 ret = BPF_PROG_RUN(prog, (void *)xdp);
458 rcu_read_unlock();
459
460 return ret;
461}
462
Alexei Starovoitov7ae457c2014-07-30 20:34:16 -0700463static inline unsigned int bpf_prog_size(unsigned int proglen)
Stephen Hemmingerb7156312008-04-10 01:33:47 -0700464{
Alexei Starovoitov7ae457c2014-07-30 20:34:16 -0700465 return max(sizeof(struct bpf_prog),
466 offsetof(struct bpf_prog, insns[proglen]));
Stephen Hemmingerb7156312008-04-10 01:33:47 -0700467}
468
Daniel Borkmann7b36f922015-07-30 12:42:47 +0200469static inline bool bpf_prog_was_classic(const struct bpf_prog *prog)
470{
471 /* When classic BPF programs have been loaded and the arch
472 * does not have a classic BPF JIT (anymore), they have been
473 * converted via bpf_migrate_filter() to eBPF and thus always
474 * have an unspec program type.
475 */
476 return prog->type == BPF_PROG_TYPE_UNSPEC;
477}
478
Alexei Starovoitov009937e2014-07-30 20:34:13 -0700479#define bpf_classic_proglen(fprog) (fprog->len * sizeof(fprog->filter[0]))
Daniel Borkmanna3ea2692014-03-28 18:58:19 +0100480
Daniel Borkmann60a3b222014-09-02 22:53:44 +0200481#ifdef CONFIG_DEBUG_SET_MODULE_RONX
482static inline void bpf_prog_lock_ro(struct bpf_prog *fp)
483{
484 set_memory_ro((unsigned long)fp, fp->pages);
485}
486
487static inline void bpf_prog_unlock_ro(struct bpf_prog *fp)
488{
489 set_memory_rw((unsigned long)fp, fp->pages);
490}
491#else
492static inline void bpf_prog_lock_ro(struct bpf_prog *fp)
493{
494}
495
496static inline void bpf_prog_unlock_ro(struct bpf_prog *fp)
497{
498}
499#endif /* CONFIG_DEBUG_SET_MODULE_RONX */
500
Willem de Bruijnf4979fc2016-07-12 18:18:56 -0400501int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap);
502static inline int sk_filter(struct sock *sk, struct sk_buff *skb)
503{
504 return sk_filter_trim_cap(sk, skb, 1);
505}
Alexei Starovoitovbd4cf0e2014-03-28 18:58:25 +0100506
Daniel Borkmannd1c55ab2016-05-13 19:08:31 +0200507struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err);
Alexei Starovoitov7ae457c2014-07-30 20:34:16 -0700508void bpf_prog_free(struct bpf_prog *fp);
Alexei Starovoitovbd4cf0e2014-03-28 18:58:25 +0100509
Daniel Borkmann60a3b222014-09-02 22:53:44 +0200510struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags);
511struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size,
512 gfp_t gfp_extra_flags);
513void __bpf_prog_free(struct bpf_prog *fp);
514
515static inline void bpf_prog_unlock_free(struct bpf_prog *fp)
516{
517 bpf_prog_unlock_ro(fp);
518 __bpf_prog_free(fp);
519}
520
Daniel Borkmannac67eb22015-05-06 16:12:30 +0200521typedef int (*bpf_aux_classic_check_t)(struct sock_filter *filter,
522 unsigned int flen);
523
Alexei Starovoitov7ae457c2014-07-30 20:34:16 -0700524int bpf_prog_create(struct bpf_prog **pfp, struct sock_fprog_kern *fprog);
Daniel Borkmannac67eb22015-05-06 16:12:30 +0200525int bpf_prog_create_from_user(struct bpf_prog **pfp, struct sock_fprog *fprog,
Daniel Borkmannbab18992015-10-02 15:17:33 +0200526 bpf_aux_classic_check_t trans, bool save_orig);
Alexei Starovoitov7ae457c2014-07-30 20:34:16 -0700527void bpf_prog_destroy(struct bpf_prog *fp);
Daniel Borkmanna3ea2692014-03-28 18:58:19 +0100528
Daniel Borkmannfbc907f2014-03-28 18:58:20 +0100529int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk);
Alexei Starovoitov89aa0752014-12-01 15:06:35 -0800530int sk_attach_bpf(u32 ufd, struct sock *sk);
Craig Gallek538950a2016-01-04 17:41:47 -0500531int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk);
532int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk);
Daniel Borkmannfbc907f2014-03-28 18:58:20 +0100533int sk_detach_filter(struct sock *sk);
Daniel Borkmannfbc907f2014-03-28 18:58:20 +0100534int sk_get_filter(struct sock *sk, struct sock_filter __user *filter,
535 unsigned int len);
Daniel Borkmannfbc907f2014-03-28 18:58:20 +0100536
Alexei Starovoitov278571b2014-07-30 20:34:12 -0700537bool sk_filter_charge(struct sock *sk, struct sk_filter *fp);
Daniel Borkmannfbc907f2014-03-28 18:58:20 +0100538void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp);
Eric Dumazet0a148422011-04-20 09:27:32 +0000539
Alexei Starovoitov62258272014-05-13 19:50:46 -0700540u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
Daniel Borkmannd1c55ab2016-05-13 19:08:31 +0200541
542struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog);
Alexei Starovoitov4e10df92015-07-20 20:34:18 -0700543bool bpf_helper_changes_skb_data(void *func);
Alexei Starovoitov62258272014-05-13 19:50:46 -0700544
Daniel Borkmannc237ee52016-05-13 19:08:30 +0200545struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off,
546 const struct bpf_insn *patch, u32 len);
Brenden Blanco6a773a12016-07-19 12:16:47 -0700547void bpf_warn_invalid_xdp_action(u32 act);
Daniel Borkmannc237ee52016-05-13 19:08:30 +0200548
Daniel Borkmannb954d832014-09-10 15:01:02 +0200549#ifdef CONFIG_BPF_JIT
Daniel Borkmannc94987e2016-05-13 19:08:27 +0200550extern int bpf_jit_enable;
Daniel Borkmann4f3446b2016-05-13 19:08:32 +0200551extern int bpf_jit_harden;
Daniel Borkmannc94987e2016-05-13 19:08:27 +0200552
Daniel Borkmannb954d832014-09-10 15:01:02 +0200553typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size);
554
555struct bpf_binary_header *
556bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
557 unsigned int alignment,
558 bpf_jit_fill_hole_t bpf_fill_ill_insns);
559void bpf_jit_binary_free(struct bpf_binary_header *hdr);
560
561void bpf_jit_compile(struct bpf_prog *fp);
562void bpf_jit_free(struct bpf_prog *fp);
563
Daniel Borkmann4f3446b2016-05-13 19:08:32 +0200564struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *fp);
565void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other);
566
Daniel Borkmannb954d832014-09-10 15:01:02 +0200567static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen,
568 u32 pass, void *image)
569{
Daniel Borkmannb13138e2015-07-30 12:42:49 +0200570 pr_err("flen=%u proglen=%u pass=%u image=%pK from=%s pid=%d\n", flen,
571 proglen, pass, image, current->comm, task_pid_nr(current));
572
Daniel Borkmannb954d832014-09-10 15:01:02 +0200573 if (image)
574 print_hex_dump(KERN_ERR, "JIT code: ", DUMP_PREFIX_OFFSET,
575 16, 1, image, proglen, false);
576}
Daniel Borkmann4f3446b2016-05-13 19:08:32 +0200577
578static inline bool bpf_jit_is_ebpf(void)
579{
580# ifdef CONFIG_HAVE_EBPF_JIT
581 return true;
582# else
583 return false;
584# endif
585}
586
587static inline bool bpf_jit_blinding_enabled(void)
588{
589 /* These are the prerequisites, should someone ever have the
590 * idea to call blinding outside of them, we make sure to
591 * bail out.
592 */
593 if (!bpf_jit_is_ebpf())
594 return false;
595 if (!bpf_jit_enable)
596 return false;
597 if (!bpf_jit_harden)
598 return false;
599 if (bpf_jit_harden == 1 && capable(CAP_SYS_ADMIN))
600 return false;
601
602 return true;
603}
Daniel Borkmannb954d832014-09-10 15:01:02 +0200604#else
605static inline void bpf_jit_compile(struct bpf_prog *fp)
606{
607}
608
609static inline void bpf_jit_free(struct bpf_prog *fp)
610{
611 bpf_prog_unlock_free(fp);
612}
613#endif /* CONFIG_BPF_JIT */
614
Daniel Borkmann34805932014-05-29 10:22:50 +0200615#define BPF_ANC BIT(15)
616
Rabin Vincent55795ef2016-01-05 16:23:07 +0100617static inline bool bpf_needs_clear_a(const struct sock_filter *first)
618{
619 switch (first->code) {
620 case BPF_RET | BPF_K:
621 case BPF_LD | BPF_W | BPF_LEN:
622 return false;
623
624 case BPF_LD | BPF_W | BPF_ABS:
625 case BPF_LD | BPF_H | BPF_ABS:
626 case BPF_LD | BPF_B | BPF_ABS:
627 if (first->k == SKF_AD_OFF + SKF_AD_ALU_XOR_X)
628 return true;
629 return false;
630
631 default:
632 return true;
633 }
634}
635
Daniel Borkmann34805932014-05-29 10:22:50 +0200636static inline u16 bpf_anc_helper(const struct sock_filter *ftest)
637{
638 BUG_ON(ftest->code & BPF_ANC);
639
640 switch (ftest->code) {
641 case BPF_LD | BPF_W | BPF_ABS:
642 case BPF_LD | BPF_H | BPF_ABS:
643 case BPF_LD | BPF_B | BPF_ABS:
644#define BPF_ANCILLARY(CODE) case SKF_AD_OFF + SKF_AD_##CODE: \
645 return BPF_ANC | SKF_AD_##CODE
646 switch (ftest->k) {
647 BPF_ANCILLARY(PROTOCOL);
648 BPF_ANCILLARY(PKTTYPE);
649 BPF_ANCILLARY(IFINDEX);
650 BPF_ANCILLARY(NLATTR);
651 BPF_ANCILLARY(NLATTR_NEST);
652 BPF_ANCILLARY(MARK);
653 BPF_ANCILLARY(QUEUE);
654 BPF_ANCILLARY(HATYPE);
655 BPF_ANCILLARY(RXHASH);
656 BPF_ANCILLARY(CPU);
657 BPF_ANCILLARY(ALU_XOR_X);
658 BPF_ANCILLARY(VLAN_TAG);
659 BPF_ANCILLARY(VLAN_TAG_PRESENT);
660 BPF_ANCILLARY(PAY_OFFSET);
661 BPF_ANCILLARY(RANDOM);
Michal Sekletar27cd5452015-03-24 14:48:41 +0100662 BPF_ANCILLARY(VLAN_TPID);
Daniel Borkmann34805932014-05-29 10:22:50 +0200663 }
664 /* Fallthrough. */
665 default:
666 return ftest->code;
667 }
668}
669
Zi Shen Lim9f12fbe2014-07-03 07:56:54 -0700670void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb,
671 int k, unsigned int size);
672
673static inline void *bpf_load_pointer(const struct sk_buff *skb, int k,
674 unsigned int size, void *buffer)
675{
676 if (k >= 0)
677 return skb_header_pointer(skb, k, size, buffer);
678
679 return bpf_internal_load_pointer_neg_helper(skb, k, size);
680}
681
Michal Sekletarea02f942014-01-17 17:09:45 +0100682static inline int bpf_tell_extensions(void)
683{
Daniel Borkmann37692292014-01-21 00:19:37 +0100684 return SKF_AD_MAX;
Michal Sekletarea02f942014-01-17 17:09:45 +0100685}
686
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687#endif /* __LINUX_FILTER_H__ */