Matt Evans | 0ca87f0 | 2011-07-20 15:51:00 +0000 | [diff] [blame] | 1 | /* bpf_jit.S: Packet/header access helper functions |
| 2 | * for PPC64 BPF compiler. |
| 3 | * |
| 4 | * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License |
| 8 | * as published by the Free Software Foundation; version 2 |
| 9 | * of the License. |
| 10 | */ |
| 11 | |
| 12 | #include <asm/ppc_asm.h> |
| 13 | #include "bpf_jit.h" |
| 14 | |
| 15 | /* |
| 16 | * All of these routines are called directly from generated code, |
| 17 | * whose register usage is: |
| 18 | * |
| 19 | * r3 skb |
| 20 | * r4,r5 A,X |
| 21 | * r6 *** address parameter to helper *** |
| 22 | * r7-r10 scratch |
| 23 | * r14 skb->data |
| 24 | * r15 skb headlen |
| 25 | * r16-31 M[] |
| 26 | */ |
| 27 | |
| 28 | /* |
| 29 | * To consider: These helpers are so small it could be better to just |
| 30 | * generate them inline. Inline code can do the simple headlen check |
| 31 | * then branch directly to slow_path_XXX if required. (In fact, could |
| 32 | * load a spare GPR with the address of slow_path_generic and pass size |
| 33 | * as an argument, making the call site a mtlr, li and bllr.) |
Matt Evans | 0ca87f0 | 2011-07-20 15:51:00 +0000 | [diff] [blame] | 34 | */ |
| 35 | .globl sk_load_word |
| 36 | sk_load_word: |
| 37 | cmpdi r_addr, 0 |
Jan Seiffert | 05be182 | 2012-04-29 19:02:19 +0000 | [diff] [blame] | 38 | blt bpf_slow_path_word_neg |
| 39 | .globl sk_load_word_positive_offset |
| 40 | sk_load_word_positive_offset: |
Matt Evans | 0ca87f0 | 2011-07-20 15:51:00 +0000 | [diff] [blame] | 41 | /* Are we accessing past headlen? */ |
| 42 | subi r_scratch1, r_HL, 4 |
| 43 | cmpd r_scratch1, r_addr |
| 44 | blt bpf_slow_path_word |
| 45 | /* Nope, just hitting the header. cr0 here is eq or gt! */ |
| 46 | lwzx r_A, r_D, r_addr |
| 47 | /* When big endian we don't need to byteswap. */ |
| 48 | blr /* Return success, cr0 != LT */ |
| 49 | |
| 50 | .globl sk_load_half |
| 51 | sk_load_half: |
| 52 | cmpdi r_addr, 0 |
Jan Seiffert | 05be182 | 2012-04-29 19:02:19 +0000 | [diff] [blame] | 53 | blt bpf_slow_path_half_neg |
| 54 | .globl sk_load_half_positive_offset |
| 55 | sk_load_half_positive_offset: |
Matt Evans | 0ca87f0 | 2011-07-20 15:51:00 +0000 | [diff] [blame] | 56 | subi r_scratch1, r_HL, 2 |
| 57 | cmpd r_scratch1, r_addr |
| 58 | blt bpf_slow_path_half |
| 59 | lhzx r_A, r_D, r_addr |
| 60 | blr |
| 61 | |
| 62 | .globl sk_load_byte |
| 63 | sk_load_byte: |
| 64 | cmpdi r_addr, 0 |
Jan Seiffert | 05be182 | 2012-04-29 19:02:19 +0000 | [diff] [blame] | 65 | blt bpf_slow_path_byte_neg |
| 66 | .globl sk_load_byte_positive_offset |
| 67 | sk_load_byte_positive_offset: |
Matt Evans | 0ca87f0 | 2011-07-20 15:51:00 +0000 | [diff] [blame] | 68 | cmpd r_HL, r_addr |
| 69 | ble bpf_slow_path_byte |
| 70 | lbzx r_A, r_D, r_addr |
| 71 | blr |
| 72 | |
| 73 | /* |
| 74 | * BPF_S_LDX_B_MSH: ldxb 4*([offset]&0xf) |
Jan Seiffert | 05be182 | 2012-04-29 19:02:19 +0000 | [diff] [blame] | 75 | * r_addr is the offset value |
Matt Evans | 0ca87f0 | 2011-07-20 15:51:00 +0000 | [diff] [blame] | 76 | */ |
| 77 | .globl sk_load_byte_msh |
| 78 | sk_load_byte_msh: |
Jan Seiffert | 05be182 | 2012-04-29 19:02:19 +0000 | [diff] [blame] | 79 | cmpdi r_addr, 0 |
| 80 | blt bpf_slow_path_byte_msh_neg |
| 81 | .globl sk_load_byte_msh_positive_offset |
| 82 | sk_load_byte_msh_positive_offset: |
Matt Evans | 0ca87f0 | 2011-07-20 15:51:00 +0000 | [diff] [blame] | 83 | cmpd r_HL, r_addr |
| 84 | ble bpf_slow_path_byte_msh |
| 85 | lbzx r_X, r_D, r_addr |
| 86 | rlwinm r_X, r_X, 2, 32-4-2, 31-2 |
| 87 | blr |
| 88 | |
Matt Evans | 0ca87f0 | 2011-07-20 15:51:00 +0000 | [diff] [blame] | 89 | /* Call out to skb_copy_bits: |
| 90 | * We'll need to back up our volatile regs first; we have |
| 91 | * local variable space at r1+(BPF_PPC_STACK_BASIC). |
| 92 | * Allocate a new stack frame here to remain ABI-compliant in |
| 93 | * stashing LR. |
| 94 | */ |
| 95 | #define bpf_slow_path_common(SIZE) \ |
| 96 | mflr r0; \ |
| 97 | std r0, 16(r1); \ |
| 98 | /* R3 goes in parameter space of caller's frame */ \ |
| 99 | std r_skb, (BPF_PPC_STACKFRAME+48)(r1); \ |
| 100 | std r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \ |
| 101 | std r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \ |
| 102 | addi r5, r1, BPF_PPC_STACK_BASIC+(2*8); \ |
| 103 | stdu r1, -BPF_PPC_SLOWPATH_FRAME(r1); \ |
| 104 | /* R3 = r_skb, as passed */ \ |
| 105 | mr r4, r_addr; \ |
| 106 | li r6, SIZE; \ |
| 107 | bl skb_copy_bits; \ |
| 108 | /* R3 = 0 on success */ \ |
| 109 | addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \ |
| 110 | ld r0, 16(r1); \ |
| 111 | ld r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \ |
| 112 | ld r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \ |
| 113 | mtlr r0; \ |
| 114 | cmpdi r3, 0; \ |
| 115 | blt bpf_error; /* cr0 = LT */ \ |
| 116 | ld r_skb, (BPF_PPC_STACKFRAME+48)(r1); \ |
| 117 | /* Great success! */ |
| 118 | |
| 119 | bpf_slow_path_word: |
| 120 | bpf_slow_path_common(4) |
| 121 | /* Data value is on stack, and cr0 != LT */ |
| 122 | lwz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1) |
| 123 | blr |
| 124 | |
| 125 | bpf_slow_path_half: |
| 126 | bpf_slow_path_common(2) |
| 127 | lhz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1) |
| 128 | blr |
| 129 | |
| 130 | bpf_slow_path_byte: |
| 131 | bpf_slow_path_common(1) |
| 132 | lbz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1) |
| 133 | blr |
| 134 | |
| 135 | bpf_slow_path_byte_msh: |
| 136 | bpf_slow_path_common(1) |
| 137 | lbz r_X, BPF_PPC_STACK_BASIC+(2*8)(r1) |
| 138 | rlwinm r_X, r_X, 2, 32-4-2, 31-2 |
| 139 | blr |
Jan Seiffert | 05be182 | 2012-04-29 19:02:19 +0000 | [diff] [blame] | 140 | |
| 141 | /* Call out to bpf_internal_load_pointer_neg_helper: |
| 142 | * We'll need to back up our volatile regs first; we have |
| 143 | * local variable space at r1+(BPF_PPC_STACK_BASIC). |
| 144 | * Allocate a new stack frame here to remain ABI-compliant in |
| 145 | * stashing LR. |
| 146 | */ |
| 147 | #define sk_negative_common(SIZE) \ |
| 148 | mflr r0; \ |
| 149 | std r0, 16(r1); \ |
| 150 | /* R3 goes in parameter space of caller's frame */ \ |
| 151 | std r_skb, (BPF_PPC_STACKFRAME+48)(r1); \ |
| 152 | std r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \ |
| 153 | std r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \ |
| 154 | stdu r1, -BPF_PPC_SLOWPATH_FRAME(r1); \ |
| 155 | /* R3 = r_skb, as passed */ \ |
| 156 | mr r4, r_addr; \ |
| 157 | li r5, SIZE; \ |
| 158 | bl bpf_internal_load_pointer_neg_helper; \ |
| 159 | /* R3 != 0 on success */ \ |
| 160 | addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \ |
| 161 | ld r0, 16(r1); \ |
| 162 | ld r_A, (BPF_PPC_STACK_BASIC+(0*8))(r1); \ |
| 163 | ld r_X, (BPF_PPC_STACK_BASIC+(1*8))(r1); \ |
| 164 | mtlr r0; \ |
| 165 | cmpldi r3, 0; \ |
| 166 | beq bpf_error_slow; /* cr0 = EQ */ \ |
| 167 | mr r_addr, r3; \ |
| 168 | ld r_skb, (BPF_PPC_STACKFRAME+48)(r1); \ |
| 169 | /* Great success! */ |
| 170 | |
| 171 | bpf_slow_path_word_neg: |
| 172 | lis r_scratch1,-32 /* SKF_LL_OFF */ |
| 173 | cmpd r_addr, r_scratch1 /* addr < SKF_* */ |
| 174 | blt bpf_error /* cr0 = LT */ |
| 175 | .globl sk_load_word_negative_offset |
| 176 | sk_load_word_negative_offset: |
| 177 | sk_negative_common(4) |
| 178 | lwz r_A, 0(r_addr) |
| 179 | blr |
| 180 | |
| 181 | bpf_slow_path_half_neg: |
| 182 | lis r_scratch1,-32 /* SKF_LL_OFF */ |
| 183 | cmpd r_addr, r_scratch1 /* addr < SKF_* */ |
| 184 | blt bpf_error /* cr0 = LT */ |
| 185 | .globl sk_load_half_negative_offset |
| 186 | sk_load_half_negative_offset: |
| 187 | sk_negative_common(2) |
| 188 | lhz r_A, 0(r_addr) |
| 189 | blr |
| 190 | |
| 191 | bpf_slow_path_byte_neg: |
| 192 | lis r_scratch1,-32 /* SKF_LL_OFF */ |
| 193 | cmpd r_addr, r_scratch1 /* addr < SKF_* */ |
| 194 | blt bpf_error /* cr0 = LT */ |
| 195 | .globl sk_load_byte_negative_offset |
| 196 | sk_load_byte_negative_offset: |
| 197 | sk_negative_common(1) |
| 198 | lbz r_A, 0(r_addr) |
| 199 | blr |
| 200 | |
| 201 | bpf_slow_path_byte_msh_neg: |
| 202 | lis r_scratch1,-32 /* SKF_LL_OFF */ |
| 203 | cmpd r_addr, r_scratch1 /* addr < SKF_* */ |
| 204 | blt bpf_error /* cr0 = LT */ |
| 205 | .globl sk_load_byte_msh_negative_offset |
| 206 | sk_load_byte_msh_negative_offset: |
| 207 | sk_negative_common(1) |
| 208 | lbz r_X, 0(r_addr) |
| 209 | rlwinm r_X, r_X, 2, 32-4-2, 31-2 |
| 210 | blr |
| 211 | |
| 212 | bpf_error_slow: |
| 213 | /* fabricate a cr0 = lt */ |
| 214 | li r_scratch1, -1 |
| 215 | cmpdi r_scratch1, 0 |
| 216 | bpf_error: |
| 217 | /* Entered with cr0 = lt */ |
| 218 | li r3, 0 |
| 219 | /* Generated code will 'blt epilogue', returning 0. */ |
| 220 | blr |