blob: 55ba3855a97f58093ec06f343cde1df2dffbb17b [file] [log] [blame]
Matt Evans0ca87f02011-07-20 15:51:00 +00001/* bpf_jit.S: Packet/header access helper functions
2 * for PPC64 BPF compiler.
3 *
4 * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; version 2
9 * of the License.
10 */
11
12#include <asm/ppc_asm.h>
13#include "bpf_jit.h"
14
15/*
16 * All of these routines are called directly from generated code,
17 * whose register usage is:
18 *
19 * r3 skb
20 * r4,r5 A,X
21 * r6 *** address parameter to helper ***
22 * r7-r10 scratch
23 * r14 skb->data
24 * r15 skb headlen
25 * r16-31 M[]
26 */
27
28/*
29 * To consider: These helpers are so small it could be better to just
30 * generate them inline. Inline code can do the simple headlen check
31 * then branch directly to slow_path_XXX if required. (In fact, could
32 * load a spare GPR with the address of slow_path_generic and pass size
33 * as an argument, making the call site a mtlr, li and bllr.)
Matt Evans0ca87f02011-07-20 15:51:00 +000034 */
35 .globl sk_load_word
36sk_load_word:
37 cmpdi r_addr, 0
Jan Seiffert05be1822012-04-29 19:02:19 +000038 blt bpf_slow_path_word_neg
39 .globl sk_load_word_positive_offset
40sk_load_word_positive_offset:
Matt Evans0ca87f02011-07-20 15:51:00 +000041 /* Are we accessing past headlen? */
42 subi r_scratch1, r_HL, 4
43 cmpd r_scratch1, r_addr
44 blt bpf_slow_path_word
45 /* Nope, just hitting the header. cr0 here is eq or gt! */
46 lwzx r_A, r_D, r_addr
47 /* When big endian we don't need to byteswap. */
48 blr /* Return success, cr0 != LT */
49
50 .globl sk_load_half
51sk_load_half:
52 cmpdi r_addr, 0
Jan Seiffert05be1822012-04-29 19:02:19 +000053 blt bpf_slow_path_half_neg
54 .globl sk_load_half_positive_offset
55sk_load_half_positive_offset:
Matt Evans0ca87f02011-07-20 15:51:00 +000056 subi r_scratch1, r_HL, 2
57 cmpd r_scratch1, r_addr
58 blt bpf_slow_path_half
59 lhzx r_A, r_D, r_addr
60 blr
61
62 .globl sk_load_byte
63sk_load_byte:
64 cmpdi r_addr, 0
Jan Seiffert05be1822012-04-29 19:02:19 +000065 blt bpf_slow_path_byte_neg
66 .globl sk_load_byte_positive_offset
67sk_load_byte_positive_offset:
Matt Evans0ca87f02011-07-20 15:51:00 +000068 cmpd r_HL, r_addr
69 ble bpf_slow_path_byte
70 lbzx r_A, r_D, r_addr
71 blr
72
73/*
74 * BPF_S_LDX_B_MSH: ldxb 4*([offset]&0xf)
Jan Seiffert05be1822012-04-29 19:02:19 +000075 * r_addr is the offset value
Matt Evans0ca87f02011-07-20 15:51:00 +000076 */
77 .globl sk_load_byte_msh
78sk_load_byte_msh:
Jan Seiffert05be1822012-04-29 19:02:19 +000079 cmpdi r_addr, 0
80 blt bpf_slow_path_byte_msh_neg
81 .globl sk_load_byte_msh_positive_offset
82sk_load_byte_msh_positive_offset:
Matt Evans0ca87f02011-07-20 15:51:00 +000083 cmpd r_HL, r_addr
84 ble bpf_slow_path_byte_msh
85 lbzx r_X, r_D, r_addr
86 rlwinm r_X, r_X, 2, 32-4-2, 31-2
87 blr
88
Matt Evans0ca87f02011-07-20 15:51:00 +000089/* Call out to skb_copy_bits:
90 * We'll need to back up our volatile regs first; we have
91 * local variable space at r1+(BPF_PPC_STACK_BASIC).
92 * Allocate a new stack frame here to remain ABI-compliant in
93 * stashing LR.
94 */
95#define bpf_slow_path_common(SIZE) \
96 mflr r0; \
97 std r0, 16(r1); \
98 /* R3 goes in parameter space of caller's frame */ \
99 std r_skb, (BPF_PPC_STACKFRAME+48)(r1); \
100 std r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \
101 std r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \
102 addi r5, r1, BPF_PPC_STACK_BASIC+(2*8); \
103 stdu r1, -BPF_PPC_SLOWPATH_FRAME(r1); \
104 /* R3 = r_skb, as passed */ \
105 mr r4, r_addr; \
106 li r6, SIZE; \
107 bl skb_copy_bits; \
108 /* R3 = 0 on success */ \
109 addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \
110 ld r0, 16(r1); \
111 ld r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \
112 ld r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \
113 mtlr r0; \
114 cmpdi r3, 0; \
115 blt bpf_error; /* cr0 = LT */ \
116 ld r_skb, (BPF_PPC_STACKFRAME+48)(r1); \
117 /* Great success! */
118
119bpf_slow_path_word:
120 bpf_slow_path_common(4)
121 /* Data value is on stack, and cr0 != LT */
122 lwz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
123 blr
124
125bpf_slow_path_half:
126 bpf_slow_path_common(2)
127 lhz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
128 blr
129
130bpf_slow_path_byte:
131 bpf_slow_path_common(1)
132 lbz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1)
133 blr
134
135bpf_slow_path_byte_msh:
136 bpf_slow_path_common(1)
137 lbz r_X, BPF_PPC_STACK_BASIC+(2*8)(r1)
138 rlwinm r_X, r_X, 2, 32-4-2, 31-2
139 blr
Jan Seiffert05be1822012-04-29 19:02:19 +0000140
141/* Call out to bpf_internal_load_pointer_neg_helper:
142 * We'll need to back up our volatile regs first; we have
143 * local variable space at r1+(BPF_PPC_STACK_BASIC).
144 * Allocate a new stack frame here to remain ABI-compliant in
145 * stashing LR.
146 */
147#define sk_negative_common(SIZE) \
148 mflr r0; \
149 std r0, 16(r1); \
150 /* R3 goes in parameter space of caller's frame */ \
151 std r_skb, (BPF_PPC_STACKFRAME+48)(r1); \
152 std r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \
153 std r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \
154 stdu r1, -BPF_PPC_SLOWPATH_FRAME(r1); \
155 /* R3 = r_skb, as passed */ \
156 mr r4, r_addr; \
157 li r5, SIZE; \
158 bl bpf_internal_load_pointer_neg_helper; \
159 /* R3 != 0 on success */ \
160 addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \
161 ld r0, 16(r1); \
162 ld r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \
163 ld r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \
164 mtlr r0; \
165 cmpldi r3, 0; \
166 beq bpf_error_slow; /* cr0 = EQ */ \
167 mr r_addr, r3; \
168 ld r_skb, (BPF_PPC_STACKFRAME+48)(r1); \
169 /* Great success! */
170
171bpf_slow_path_word_neg:
172 lis r_scratch1,-32 /* SKF_LL_OFF */
173 cmpd r_addr, r_scratch1 /* addr < SKF_* */
174 blt bpf_error /* cr0 = LT */
175 .globl sk_load_word_negative_offset
176sk_load_word_negative_offset:
177 sk_negative_common(4)
178 lwz r_A, 0(r_addr)
179 blr
180
181bpf_slow_path_half_neg:
182 lis r_scratch1,-32 /* SKF_LL_OFF */
183 cmpd r_addr, r_scratch1 /* addr < SKF_* */
184 blt bpf_error /* cr0 = LT */
185 .globl sk_load_half_negative_offset
186sk_load_half_negative_offset:
187 sk_negative_common(2)
188 lhz r_A, 0(r_addr)
189 blr
190
191bpf_slow_path_byte_neg:
192 lis r_scratch1,-32 /* SKF_LL_OFF */
193 cmpd r_addr, r_scratch1 /* addr < SKF_* */
194 blt bpf_error /* cr0 = LT */
195 .globl sk_load_byte_negative_offset
196sk_load_byte_negative_offset:
197 sk_negative_common(1)
198 lbz r_A, 0(r_addr)
199 blr
200
201bpf_slow_path_byte_msh_neg:
202 lis r_scratch1,-32 /* SKF_LL_OFF */
203 cmpd r_addr, r_scratch1 /* addr < SKF_* */
204 blt bpf_error /* cr0 = LT */
205 .globl sk_load_byte_msh_negative_offset
206sk_load_byte_msh_negative_offset:
207 sk_negative_common(1)
208 lbz r_X, 0(r_addr)
209 rlwinm r_X, r_X, 2, 32-4-2, 31-2
210 blr
211
212bpf_error_slow:
213 /* fabricate a cr0 = lt */
214 li r_scratch1, -1
215 cmpdi r_scratch1, 0
216bpf_error:
217 /* Entered with cr0 = lt */
218 li r3, 0
219 /* Generated code will 'blt epilogue', returning 0. */
220 blr