| /* bpf_jit.S : BPF JIT helper functions |
| * |
| * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com) |
| * |
| * This program is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU General Public License |
| * as published by the Free Software Foundation; version 2 |
| * of the License. |
| */ |
| #include <linux/linkage.h> |
| |
| /* |
| * Calling convention : |
| * rbx : skb pointer (callee saved) |
| * esi : offset of byte(s) to fetch in skb (can be scratched) |
| * r10 : copy of skb->data |
| * r9d : hlen = skb->len - skb->data_len |
| */ |
| #define SKBDATA %r10 |
| #define SKF_MAX_NEG_OFF $(-0x200000) /* SKF_LL_OFF from filter.h */ |
| #define MAX_BPF_STACK (512 /* from filter.h */ + \ |
| 32 /* space for rbx,r13,r14,r15 */ + \ |
| 8 /* space for skb_copy_bits */) |
| |
| sk_load_word: |
| .globl sk_load_word |
| |
| test %esi,%esi |
| js bpf_slow_path_word_neg |
| |
| sk_load_word_positive_offset: |
| .globl sk_load_word_positive_offset |
| |
| mov %r9d,%eax # hlen |
| sub %esi,%eax # hlen - offset |
| cmp $3,%eax |
| jle bpf_slow_path_word |
| mov (SKBDATA,%rsi),%eax |
| bswap %eax /* ntohl() */ |
| ret |
| |
| sk_load_half: |
| .globl sk_load_half |
| |
| test %esi,%esi |
| js bpf_slow_path_half_neg |
| |
| sk_load_half_positive_offset: |
| .globl sk_load_half_positive_offset |
| |
| mov %r9d,%eax |
| sub %esi,%eax # hlen - offset |
| cmp $1,%eax |
| jle bpf_slow_path_half |
| movzwl (SKBDATA,%rsi),%eax |
| rol $8,%ax # ntohs() |
| ret |
| |
| sk_load_byte: |
| .globl sk_load_byte |
| |
| test %esi,%esi |
| js bpf_slow_path_byte_neg |
| |
| sk_load_byte_positive_offset: |
| .globl sk_load_byte_positive_offset |
| |
| cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte */ |
| jle bpf_slow_path_byte |
| movzbl (SKBDATA,%rsi),%eax |
| ret |
| |
| /* rsi contains offset and can be scratched */ |
| #define bpf_slow_path_common(LEN) \ |
| mov %rbx, %rdi; /* arg1 == skb */ \ |
| push %r9; \ |
| push SKBDATA; \ |
| /* rsi already has offset */ \ |
| mov $LEN,%ecx; /* len */ \ |
| lea - MAX_BPF_STACK + 32(%rbp),%rdx; \ |
| call skb_copy_bits; \ |
| test %eax,%eax; \ |
| pop SKBDATA; \ |
| pop %r9; |
| |
| |
| bpf_slow_path_word: |
| bpf_slow_path_common(4) |
| js bpf_error |
| mov - MAX_BPF_STACK + 32(%rbp),%eax |
| bswap %eax |
| ret |
| |
| bpf_slow_path_half: |
| bpf_slow_path_common(2) |
| js bpf_error |
| mov - MAX_BPF_STACK + 32(%rbp),%ax |
| rol $8,%ax |
| movzwl %ax,%eax |
| ret |
| |
| bpf_slow_path_byte: |
| bpf_slow_path_common(1) |
| js bpf_error |
| movzbl - MAX_BPF_STACK + 32(%rbp),%eax |
| ret |
| |
| #define sk_negative_common(SIZE) \ |
| mov %rbx, %rdi; /* arg1 == skb */ \ |
| push %r9; \ |
| push SKBDATA; \ |
| /* rsi already has offset */ \ |
| mov $SIZE,%edx; /* size */ \ |
| call bpf_internal_load_pointer_neg_helper; \ |
| test %rax,%rax; \ |
| pop SKBDATA; \ |
| pop %r9; \ |
| jz bpf_error |
| |
| bpf_slow_path_word_neg: |
| cmp SKF_MAX_NEG_OFF, %esi /* test range */ |
| jl bpf_error /* offset lower -> error */ |
| sk_load_word_negative_offset: |
| .globl sk_load_word_negative_offset |
| sk_negative_common(4) |
| mov (%rax), %eax |
| bswap %eax |
| ret |
| |
| bpf_slow_path_half_neg: |
| cmp SKF_MAX_NEG_OFF, %esi |
| jl bpf_error |
| sk_load_half_negative_offset: |
| .globl sk_load_half_negative_offset |
| sk_negative_common(2) |
| mov (%rax),%ax |
| rol $8,%ax |
| movzwl %ax,%eax |
| ret |
| |
| bpf_slow_path_byte_neg: |
| cmp SKF_MAX_NEG_OFF, %esi |
| jl bpf_error |
| sk_load_byte_negative_offset: |
| .globl sk_load_byte_negative_offset |
| sk_negative_common(1) |
| movzbl (%rax), %eax |
| ret |
| |
| bpf_error: |
| # force a return 0 from jit handler |
| xor %eax,%eax |
| mov - MAX_BPF_STACK(%rbp),%rbx |
| mov - MAX_BPF_STACK + 8(%rbp),%r13 |
| mov - MAX_BPF_STACK + 16(%rbp),%r14 |
| mov - MAX_BPF_STACK + 24(%rbp),%r15 |
| leaveq |
| ret |