| Naveen N. Rao | 156d0e2 | 2016-06-22 21:55:07 +0530 | [diff] [blame] | 1 | /* |
| 2 | * bpf_jit_asm64.S: Packet/header access helper functions |
| 3 | * for PPC64 BPF compiler. |
| 4 | * |
| 5 | * Copyright 2016, Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> |
| 6 | * IBM Corporation |
| 7 | * |
| 8 | * Based on bpf_jit_asm.S by Matt Evans |
| 9 | * |
| 10 | * This program is free software; you can redistribute it and/or |
| 11 | * modify it under the terms of the GNU General Public License |
| 12 | * as published by the Free Software Foundation; version 2 |
| 13 | * of the License. |
| 14 | */ |
| 15 | |
| 16 | #include <asm/ppc_asm.h> |
| 17 | #include <asm/ptrace.h> |
| 18 | #include "bpf_jit64.h" |
| 19 | |
| 20 | /* |
| 21 | * All of these routines are called directly from generated code, |
| 22 | * with the below register usage: |
| 23 | * r27 skb pointer (ctx) |
| 24 | * r25 skb header length |
| 25 | * r26 skb->data pointer |
| 26 | * r4 offset |
| 27 | * |
| 28 | * Result is passed back in: |
| 29 | * r8 data read in host endian format (accumulator) |
| 30 | * |
| 31 | * r9 is used as a temporary register |
| 32 | */ |
| 33 | |
| 34 | #define r_skb r27 |
| 35 | #define r_hlen r25 |
| 36 | #define r_data r26 |
| 37 | #define r_off r4 |
| 38 | #define r_val r8 |
| 39 | #define r_tmp r9 |
| 40 | |
| 41 | _GLOBAL_TOC(sk_load_word) |
| 42 | cmpdi r_off, 0 |
| 43 | blt bpf_slow_path_word_neg |
| 44 | b sk_load_word_positive_offset |
| 45 | |
| 46 | _GLOBAL_TOC(sk_load_word_positive_offset) |
| 47 | /* Are we accessing past headlen? */ |
| 48 | subi r_tmp, r_hlen, 4 |
| 49 | cmpd r_tmp, r_off |
| 50 | blt bpf_slow_path_word |
| 51 | /* Nope, just hitting the header. cr0 here is eq or gt! */ |
| 52 | LWZX_BE r_val, r_data, r_off |
| 53 | blr /* Return success, cr0 != LT */ |
| 54 | |
| 55 | _GLOBAL_TOC(sk_load_half) |
| 56 | cmpdi r_off, 0 |
| 57 | blt bpf_slow_path_half_neg |
| 58 | b sk_load_half_positive_offset |
| 59 | |
| 60 | _GLOBAL_TOC(sk_load_half_positive_offset) |
| 61 | subi r_tmp, r_hlen, 2 |
| 62 | cmpd r_tmp, r_off |
| 63 | blt bpf_slow_path_half |
| 64 | LHZX_BE r_val, r_data, r_off |
| 65 | blr |
| 66 | |
| 67 | _GLOBAL_TOC(sk_load_byte) |
| 68 | cmpdi r_off, 0 |
| 69 | blt bpf_slow_path_byte_neg |
| 70 | b sk_load_byte_positive_offset |
| 71 | |
| 72 | _GLOBAL_TOC(sk_load_byte_positive_offset) |
| 73 | cmpd r_hlen, r_off |
| 74 | ble bpf_slow_path_byte |
| 75 | lbzx r_val, r_data, r_off |
| 76 | blr |
| 77 | |
| 78 | /* |
| 79 | * Call out to skb_copy_bits: |
| 80 | * Allocate a new stack frame here to remain ABI-compliant in |
| 81 | * stashing LR. |
| 82 | */ |
| 83 | #define bpf_slow_path_common(SIZE) \ |
| 84 | mflr r0; \ |
| 85 | std r0, PPC_LR_STKOFF(r1); \ |
| 86 | stdu r1, -(STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_LOCALS)(r1); \ |
| 87 | mr r3, r_skb; \ |
| 88 | /* r4 = r_off as passed */ \ |
| 89 | addi r5, r1, STACK_FRAME_MIN_SIZE; \ |
| 90 | li r6, SIZE; \ |
| 91 | bl skb_copy_bits; \ |
| 92 | nop; \ |
| 93 | /* save r5 */ \ |
| 94 | addi r5, r1, STACK_FRAME_MIN_SIZE; \ |
| 95 | /* r3 = 0 on success */ \ |
| 96 | addi r1, r1, STACK_FRAME_MIN_SIZE + BPF_PPC_STACK_LOCALS; \ |
| 97 | ld r0, PPC_LR_STKOFF(r1); \ |
| 98 | mtlr r0; \ |
| 99 | cmpdi r3, 0; \ |
| 100 | blt bpf_error; /* cr0 = LT */ |
| 101 | |
| 102 | bpf_slow_path_word: |
| 103 | bpf_slow_path_common(4) |
| 104 | /* Data value is on stack, and cr0 != LT */ |
| 105 | LWZX_BE r_val, 0, r5 |
| 106 | blr |
| 107 | |
| 108 | bpf_slow_path_half: |
| 109 | bpf_slow_path_common(2) |
| 110 | LHZX_BE r_val, 0, r5 |
| 111 | blr |
| 112 | |
| 113 | bpf_slow_path_byte: |
| 114 | bpf_slow_path_common(1) |
| 115 | lbzx r_val, 0, r5 |
| 116 | blr |
| 117 | |
| 118 | /* |
| 119 | * Call out to bpf_internal_load_pointer_neg_helper |
| 120 | */ |
| 121 | #define sk_negative_common(SIZE) \ |
| 122 | mflr r0; \ |
| 123 | std r0, PPC_LR_STKOFF(r1); \ |
| 124 | stdu r1, -STACK_FRAME_MIN_SIZE(r1); \ |
| 125 | mr r3, r_skb; \ |
| 126 | /* r4 = r_off, as passed */ \ |
| 127 | li r5, SIZE; \ |
| 128 | bl bpf_internal_load_pointer_neg_helper; \ |
| 129 | nop; \ |
| 130 | addi r1, r1, STACK_FRAME_MIN_SIZE; \ |
| 131 | ld r0, PPC_LR_STKOFF(r1); \ |
| 132 | mtlr r0; \ |
| 133 | /* R3 != 0 on success */ \ |
| 134 | cmpldi r3, 0; \ |
| 135 | beq bpf_error_slow; /* cr0 = EQ */ |
| 136 | |
| 137 | bpf_slow_path_word_neg: |
| 138 | lis r_tmp, -32 /* SKF_LL_OFF */ |
| 139 | cmpd r_off, r_tmp /* addr < SKF_* */ |
| 140 | blt bpf_error /* cr0 = LT */ |
| 141 | b sk_load_word_negative_offset |
| 142 | |
| 143 | _GLOBAL_TOC(sk_load_word_negative_offset) |
| 144 | sk_negative_common(4) |
| 145 | LWZX_BE r_val, 0, r3 |
| 146 | blr |
| 147 | |
| 148 | bpf_slow_path_half_neg: |
| 149 | lis r_tmp, -32 /* SKF_LL_OFF */ |
| 150 | cmpd r_off, r_tmp /* addr < SKF_* */ |
| 151 | blt bpf_error /* cr0 = LT */ |
| 152 | b sk_load_half_negative_offset |
| 153 | |
| 154 | _GLOBAL_TOC(sk_load_half_negative_offset) |
| 155 | sk_negative_common(2) |
| 156 | LHZX_BE r_val, 0, r3 |
| 157 | blr |
| 158 | |
| 159 | bpf_slow_path_byte_neg: |
| 160 | lis r_tmp, -32 /* SKF_LL_OFF */ |
| 161 | cmpd r_off, r_tmp /* addr < SKF_* */ |
| 162 | blt bpf_error /* cr0 = LT */ |
| 163 | b sk_load_byte_negative_offset |
| 164 | |
| 165 | _GLOBAL_TOC(sk_load_byte_negative_offset) |
| 166 | sk_negative_common(1) |
| 167 | lbzx r_val, 0, r3 |
| 168 | blr |
| 169 | |
| 170 | bpf_error_slow: |
| 171 | /* fabricate a cr0 = lt */ |
| 172 | li r_tmp, -1 |
| 173 | cmpdi r_tmp, 0 |
| 174 | bpf_error: |
| 175 | /* |
| 176 | * Entered with cr0 = lt |
| 177 | * Generated code will 'blt epilogue', returning 0. |
| 178 | */ |
| 179 | li r_val, 0 |
| 180 | blr |