blob: f2a7faf4706eb64debcc65d3c9ad0b0fd9c70599 [file] [log] [blame]
Eric Dumazet0a148422011-04-20 09:27:32 +00001/* bpf_jit.S : BPF JIT helper functions
2 *
3 * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; version 2
8 * of the License.
9 */
10#include <linux/linkage.h>
Josh Poimboeufd21001c2016-01-21 16:49:27 -060011#include <asm/frame.h>
Eric Dumazet0a148422011-04-20 09:27:32 +000012
13/*
14 * Calling convention :
Alexei Starovoitov62258272014-05-13 19:50:46 -070015 * rbx : skb pointer (callee saved)
Eric Dumazet0a148422011-04-20 09:27:32 +000016 * esi : offset of byte(s) to fetch in skb (can be scratched)
Alexei Starovoitov62258272014-05-13 19:50:46 -070017 * r10 : copy of skb->data
Eric Dumazet0a148422011-04-20 09:27:32 +000018 * r9d : hlen = skb->len - skb->data_len
19 */
Alexei Starovoitov62258272014-05-13 19:50:46 -070020#define SKBDATA %r10
Jan Seifferta998d432012-03-30 05:24:05 +000021#define SKF_MAX_NEG_OFF $(-0x200000) /* SKF_LL_OFF from filter.h */
Alexei Starovoitov62258272014-05-13 19:50:46 -070022#define MAX_BPF_STACK (512 /* from filter.h */ + \
23 32 /* space for rbx,r13,r14,r15 */ + \
24 8 /* space for skb_copy_bits */)
Eric Dumazet0a148422011-04-20 09:27:32 +000025
Josh Poimboeuf2d8fe902016-01-21 16:49:26 -060026#define FUNC(name) \
27 .globl name; \
28 .type name, @function; \
29 name:
Eric Dumazet0a148422011-04-20 09:27:32 +000030
Josh Poimboeuf2d8fe902016-01-21 16:49:26 -060031FUNC(sk_load_word)
Jan Seifferta998d432012-03-30 05:24:05 +000032 test %esi,%esi
33 js bpf_slow_path_word_neg
34
Josh Poimboeuf2d8fe902016-01-21 16:49:26 -060035FUNC(sk_load_word_positive_offset)
Eric Dumazet0a148422011-04-20 09:27:32 +000036 mov %r9d,%eax # hlen
37 sub %esi,%eax # hlen - offset
38 cmp $3,%eax
39 jle bpf_slow_path_word
40 mov (SKBDATA,%rsi),%eax
41 bswap %eax /* ntohl() */
42 ret
43
Josh Poimboeuf2d8fe902016-01-21 16:49:26 -060044FUNC(sk_load_half)
Jan Seifferta998d432012-03-30 05:24:05 +000045 test %esi,%esi
46 js bpf_slow_path_half_neg
47
Josh Poimboeuf2d8fe902016-01-21 16:49:26 -060048FUNC(sk_load_half_positive_offset)
Eric Dumazet0a148422011-04-20 09:27:32 +000049 mov %r9d,%eax
50 sub %esi,%eax # hlen - offset
51 cmp $1,%eax
52 jle bpf_slow_path_half
53 movzwl (SKBDATA,%rsi),%eax
54 rol $8,%ax # ntohs()
55 ret
56
Josh Poimboeuf2d8fe902016-01-21 16:49:26 -060057FUNC(sk_load_byte)
Jan Seifferta998d432012-03-30 05:24:05 +000058 test %esi,%esi
59 js bpf_slow_path_byte_neg
60
Josh Poimboeuf2d8fe902016-01-21 16:49:26 -060061FUNC(sk_load_byte_positive_offset)
Eric Dumazet0a148422011-04-20 09:27:32 +000062 cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte */
63 jle bpf_slow_path_byte
64 movzbl (SKBDATA,%rsi),%eax
65 ret
66
Eric Dumazet0a148422011-04-20 09:27:32 +000067/* rsi contains offset and can be scratched */
68#define bpf_slow_path_common(LEN) \
Josh Poimboeufd21001c2016-01-21 16:49:27 -060069 lea -MAX_BPF_STACK + 32(%rbp), %rdx;\
70 FRAME_BEGIN; \
Alexei Starovoitov62258272014-05-13 19:50:46 -070071 mov %rbx, %rdi; /* arg1 == skb */ \
Eric Dumazet0a148422011-04-20 09:27:32 +000072 push %r9; \
73 push SKBDATA; \
74/* rsi already has offset */ \
75 mov $LEN,%ecx; /* len */ \
Eric Dumazet0a148422011-04-20 09:27:32 +000076 call skb_copy_bits; \
77 test %eax,%eax; \
78 pop SKBDATA; \
Josh Poimboeufd21001c2016-01-21 16:49:27 -060079 pop %r9; \
80 FRAME_END
Eric Dumazet0a148422011-04-20 09:27:32 +000081
82
83bpf_slow_path_word:
84 bpf_slow_path_common(4)
85 js bpf_error
Alexei Starovoitov62258272014-05-13 19:50:46 -070086 mov - MAX_BPF_STACK + 32(%rbp),%eax
Eric Dumazet0a148422011-04-20 09:27:32 +000087 bswap %eax
88 ret
89
90bpf_slow_path_half:
91 bpf_slow_path_common(2)
92 js bpf_error
Alexei Starovoitov62258272014-05-13 19:50:46 -070093 mov - MAX_BPF_STACK + 32(%rbp),%ax
Eric Dumazet0a148422011-04-20 09:27:32 +000094 rol $8,%ax
95 movzwl %ax,%eax
96 ret
97
98bpf_slow_path_byte:
99 bpf_slow_path_common(1)
100 js bpf_error
Alexei Starovoitov62258272014-05-13 19:50:46 -0700101 movzbl - MAX_BPF_STACK + 32(%rbp),%eax
Eric Dumazet0a148422011-04-20 09:27:32 +0000102 ret
Jan Seifferta998d432012-03-30 05:24:05 +0000103
104#define sk_negative_common(SIZE) \
Josh Poimboeufd21001c2016-01-21 16:49:27 -0600105 FRAME_BEGIN; \
Alexei Starovoitov62258272014-05-13 19:50:46 -0700106 mov %rbx, %rdi; /* arg1 == skb */ \
Jan Seifferta998d432012-03-30 05:24:05 +0000107 push %r9; \
108 push SKBDATA; \
109/* rsi already has offset */ \
Alexei Starovoitovfdfaf642014-03-10 15:56:51 -0700110 mov $SIZE,%edx; /* size */ \
Jan Seifferta998d432012-03-30 05:24:05 +0000111 call bpf_internal_load_pointer_neg_helper; \
112 test %rax,%rax; \
113 pop SKBDATA; \
114 pop %r9; \
Josh Poimboeufd21001c2016-01-21 16:49:27 -0600115 FRAME_END; \
Jan Seifferta998d432012-03-30 05:24:05 +0000116 jz bpf_error
117
Jan Seifferta998d432012-03-30 05:24:05 +0000118bpf_slow_path_word_neg:
119 cmp SKF_MAX_NEG_OFF, %esi /* test range */
120 jl bpf_error /* offset lower -> error */
Josh Poimboeuf2d8fe902016-01-21 16:49:26 -0600121
122FUNC(sk_load_word_negative_offset)
Jan Seifferta998d432012-03-30 05:24:05 +0000123 sk_negative_common(4)
124 mov (%rax), %eax
125 bswap %eax
126 ret
127
128bpf_slow_path_half_neg:
129 cmp SKF_MAX_NEG_OFF, %esi
130 jl bpf_error
Josh Poimboeuf2d8fe902016-01-21 16:49:26 -0600131
132FUNC(sk_load_half_negative_offset)
Jan Seifferta998d432012-03-30 05:24:05 +0000133 sk_negative_common(2)
134 mov (%rax),%ax
135 rol $8,%ax
136 movzwl %ax,%eax
137 ret
138
139bpf_slow_path_byte_neg:
140 cmp SKF_MAX_NEG_OFF, %esi
141 jl bpf_error
Josh Poimboeuf2d8fe902016-01-21 16:49:26 -0600142
143FUNC(sk_load_byte_negative_offset)
Jan Seifferta998d432012-03-30 05:24:05 +0000144 sk_negative_common(1)
145 movzbl (%rax), %eax
146 ret
147
Jan Seifferta998d432012-03-30 05:24:05 +0000148bpf_error:
149# force a return 0 from jit handler
Alexei Starovoitov62258272014-05-13 19:50:46 -0700150 xor %eax,%eax
151 mov - MAX_BPF_STACK(%rbp),%rbx
152 mov - MAX_BPF_STACK + 8(%rbp),%r13
153 mov - MAX_BPF_STACK + 16(%rbp),%r14
154 mov - MAX_BPF_STACK + 24(%rbp),%r15
Jan Seifferta998d432012-03-30 05:24:05 +0000155 leaveq
156 ret