blob: 01495755701bd3d068db95df291ef71096d9cf33 [file] [log] [blame]
Eric Dumazet0a148422011-04-20 09:27:32 +00001/* bpf_jit.S : BPF JIT helper functions
2 *
3 * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; version 2
8 * of the License.
9 */
10#include <linux/linkage.h>
11#include <asm/dwarf2.h>
12
13/*
14 * Calling convention :
15 * rdi : skb pointer
16 * esi : offset of byte(s) to fetch in skb (can be scratched)
17 * r8 : copy of skb->data
18 * r9d : hlen = skb->len - skb->data_len
19 */
20#define SKBDATA %r8
Jan Seifferta998d432012-03-30 05:24:05 +000021#define SKF_MAX_NEG_OFF $(-0x200000) /* SKF_LL_OFF from filter.h */
Eric Dumazet0a148422011-04-20 09:27:32 +000022
23sk_load_word:
24 .globl sk_load_word
25
Jan Seifferta998d432012-03-30 05:24:05 +000026 test %esi,%esi
27 js bpf_slow_path_word_neg
28
29sk_load_word_positive_offset:
30 .globl sk_load_word_positive_offset
31
Eric Dumazet0a148422011-04-20 09:27:32 +000032 mov %r9d,%eax # hlen
33 sub %esi,%eax # hlen - offset
34 cmp $3,%eax
35 jle bpf_slow_path_word
36 mov (SKBDATA,%rsi),%eax
37 bswap %eax /* ntohl() */
38 ret
39
Eric Dumazet0a148422011-04-20 09:27:32 +000040sk_load_half:
41 .globl sk_load_half
42
Jan Seifferta998d432012-03-30 05:24:05 +000043 test %esi,%esi
44 js bpf_slow_path_half_neg
45
46sk_load_half_positive_offset:
47 .globl sk_load_half_positive_offset
48
Eric Dumazet0a148422011-04-20 09:27:32 +000049 mov %r9d,%eax
50 sub %esi,%eax # hlen - offset
51 cmp $1,%eax
52 jle bpf_slow_path_half
53 movzwl (SKBDATA,%rsi),%eax
54 rol $8,%ax # ntohs()
55 ret
56
Eric Dumazet0a148422011-04-20 09:27:32 +000057sk_load_byte:
58 .globl sk_load_byte
59
Jan Seifferta998d432012-03-30 05:24:05 +000060 test %esi,%esi
61 js bpf_slow_path_byte_neg
62
63sk_load_byte_positive_offset:
64 .globl sk_load_byte_positive_offset
65
Eric Dumazet0a148422011-04-20 09:27:32 +000066 cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte */
67 jle bpf_slow_path_byte
68 movzbl (SKBDATA,%rsi),%eax
69 ret
70
71/**
72 * sk_load_byte_msh - BPF_S_LDX_B_MSH helper
73 *
74 * Implements BPF_S_LDX_B_MSH : ldxb 4*([offset]&0xf)
75 * Must preserve A accumulator (%eax)
Jan Seifferta998d432012-03-30 05:24:05 +000076 * Inputs : %esi is the offset value
Eric Dumazet0a148422011-04-20 09:27:32 +000077 */
Jan Seifferta998d432012-03-30 05:24:05 +000078sk_load_byte_msh:
79 .globl sk_load_byte_msh
80 test %esi,%esi
81 js bpf_slow_path_byte_msh_neg
82
83sk_load_byte_msh_positive_offset:
84 .globl sk_load_byte_msh_positive_offset
Eric Dumazet0a148422011-04-20 09:27:32 +000085 cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte_msh */
86 jle bpf_slow_path_byte_msh
87 movzbl (SKBDATA,%rsi),%ebx
88 and $15,%bl
89 shl $2,%bl
90 ret
Eric Dumazet0a148422011-04-20 09:27:32 +000091
92/* rsi contains offset and can be scratched */
93#define bpf_slow_path_common(LEN) \
94 push %rdi; /* save skb */ \
95 push %r9; \
96 push SKBDATA; \
97/* rsi already has offset */ \
98 mov $LEN,%ecx; /* len */ \
99 lea -12(%rbp),%rdx; \
100 call skb_copy_bits; \
101 test %eax,%eax; \
102 pop SKBDATA; \
103 pop %r9; \
104 pop %rdi
105
106
107bpf_slow_path_word:
108 bpf_slow_path_common(4)
109 js bpf_error
110 mov -12(%rbp),%eax
111 bswap %eax
112 ret
113
114bpf_slow_path_half:
115 bpf_slow_path_common(2)
116 js bpf_error
117 mov -12(%rbp),%ax
118 rol $8,%ax
119 movzwl %ax,%eax
120 ret
121
122bpf_slow_path_byte:
123 bpf_slow_path_common(1)
124 js bpf_error
125 movzbl -12(%rbp),%eax
126 ret
127
128bpf_slow_path_byte_msh:
129 xchg %eax,%ebx /* dont lose A , X is about to be scratched */
130 bpf_slow_path_common(1)
131 js bpf_error
132 movzbl -12(%rbp),%eax
133 and $15,%al
134 shl $2,%al
135 xchg %eax,%ebx
136 ret
Jan Seifferta998d432012-03-30 05:24:05 +0000137
138#define sk_negative_common(SIZE) \
139 push %rdi; /* save skb */ \
140 push %r9; \
141 push SKBDATA; \
142/* rsi already has offset */ \
Alexei Starovoitovfdfaf642014-03-10 15:56:51 -0700143 mov $SIZE,%edx; /* size */ \
Jan Seifferta998d432012-03-30 05:24:05 +0000144 call bpf_internal_load_pointer_neg_helper; \
145 test %rax,%rax; \
146 pop SKBDATA; \
147 pop %r9; \
148 pop %rdi; \
149 jz bpf_error
150
151
152bpf_slow_path_word_neg:
153 cmp SKF_MAX_NEG_OFF, %esi /* test range */
154 jl bpf_error /* offset lower -> error */
155sk_load_word_negative_offset:
156 .globl sk_load_word_negative_offset
157 sk_negative_common(4)
158 mov (%rax), %eax
159 bswap %eax
160 ret
161
162bpf_slow_path_half_neg:
163 cmp SKF_MAX_NEG_OFF, %esi
164 jl bpf_error
165sk_load_half_negative_offset:
166 .globl sk_load_half_negative_offset
167 sk_negative_common(2)
168 mov (%rax),%ax
169 rol $8,%ax
170 movzwl %ax,%eax
171 ret
172
173bpf_slow_path_byte_neg:
174 cmp SKF_MAX_NEG_OFF, %esi
175 jl bpf_error
176sk_load_byte_negative_offset:
177 .globl sk_load_byte_negative_offset
178 sk_negative_common(1)
179 movzbl (%rax), %eax
180 ret
181
182bpf_slow_path_byte_msh_neg:
183 cmp SKF_MAX_NEG_OFF, %esi
184 jl bpf_error
185sk_load_byte_msh_negative_offset:
186 .globl sk_load_byte_msh_negative_offset
187 xchg %eax,%ebx /* dont lose A , X is about to be scratched */
188 sk_negative_common(1)
189 movzbl (%rax),%eax
190 and $15,%al
191 shl $2,%al
192 xchg %eax,%ebx
193 ret
194
195bpf_error:
196# force a return 0 from jit handler
197 xor %eax,%eax
198 mov -8(%rbp),%rbx
199 leaveq
200 ret