Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* Normally compiler builtins are used, but sometimes the compiler calls out |
| 2 | of line code. Based on asm-i386/string.h. |
| 3 | */ |
| 4 | #define _STRING_C |
| 5 | #include <linux/string.h> |
Andi Kleen | 2ee60e17 | 2006-06-26 13:59:44 +0200 | [diff] [blame] | 6 | #include <linux/module.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 7 | |
| 8 | #undef memmove |
Paolo Ciarrocchi | e940659 | 2008-02-19 20:53:38 +0100 | [diff] [blame] | 9 | void *memmove(void *dest, const void *src, size_t count) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 10 | { |
Ma Ling | 3b4b682 | 2010-09-17 03:12:40 +0800 | [diff] [blame] | 11 | unsigned long d0,d1,d2,d3,d4,d5,d6,d7; |
| 12 | char *ret; |
| 13 | |
| 14 | __asm__ __volatile__( |
| 15 | /* Handle more 32bytes in loop */ |
| 16 | "mov %2, %3\n\t" |
| 17 | "cmp $0x20, %0\n\t" |
| 18 | "jb 1f\n\t" |
| 19 | |
| 20 | /* Decide forward/backward copy mode */ |
| 21 | "cmp %2, %1\n\t" |
| 22 | "jb 2f\n\t" |
| 23 | |
| 24 | /* |
| 25 | * movsq instruction have many startup latency |
| 26 | * so we handle small size by general register. |
| 27 | */ |
| 28 | "cmp $680, %0\n\t" |
| 29 | "jb 3f\n\t" |
| 30 | /* |
| 31 | * movsq instruction is only good for aligned case. |
| 32 | */ |
| 33 | "cmpb %%dil, %%sil\n\t" |
| 34 | "je 4f\n\t" |
| 35 | "3:\n\t" |
| 36 | "sub $0x20, %0\n\t" |
| 37 | /* |
| 38 | * We gobble 32byts forward in each loop. |
| 39 | */ |
| 40 | "5:\n\t" |
| 41 | "sub $0x20, %0\n\t" |
| 42 | "movq 0*8(%1), %4\n\t" |
| 43 | "movq 1*8(%1), %5\n\t" |
| 44 | "movq 2*8(%1), %6\n\t" |
| 45 | "movq 3*8(%1), %7\n\t" |
| 46 | "leaq 4*8(%1), %1\n\t" |
| 47 | |
| 48 | "movq %4, 0*8(%2)\n\t" |
| 49 | "movq %5, 1*8(%2)\n\t" |
| 50 | "movq %6, 2*8(%2)\n\t" |
| 51 | "movq %7, 3*8(%2)\n\t" |
| 52 | "leaq 4*8(%2), %2\n\t" |
| 53 | "jae 5b\n\t" |
| 54 | "addq $0x20, %0\n\t" |
| 55 | "jmp 1f\n\t" |
| 56 | /* |
| 57 | * Handle data forward by movsq. |
| 58 | */ |
| 59 | ".p2align 4\n\t" |
| 60 | "4:\n\t" |
| 61 | "movq %0, %8\n\t" |
| 62 | "movq -8(%1, %0), %4\n\t" |
| 63 | "lea -8(%2, %0), %5\n\t" |
| 64 | "shrq $3, %8\n\t" |
| 65 | "rep movsq\n\t" |
| 66 | "movq %4, (%5)\n\t" |
| 67 | "jmp 13f\n\t" |
| 68 | /* |
| 69 | * Handle data backward by movsq. |
| 70 | */ |
| 71 | ".p2align 4\n\t" |
| 72 | "7:\n\t" |
| 73 | "movq %0, %8\n\t" |
| 74 | "movq (%1), %4\n\t" |
| 75 | "movq %2, %5\n\t" |
| 76 | "leaq -8(%1, %0), %1\n\t" |
| 77 | "leaq -8(%2, %0), %2\n\t" |
| 78 | "shrq $3, %8\n\t" |
| 79 | "std\n\t" |
| 80 | "rep movsq\n\t" |
| 81 | "cld\n\t" |
| 82 | "movq %4, (%5)\n\t" |
| 83 | "jmp 13f\n\t" |
| 84 | |
| 85 | /* |
| 86 | * Start to prepare for backward copy. |
| 87 | */ |
| 88 | ".p2align 4\n\t" |
| 89 | "2:\n\t" |
| 90 | "cmp $680, %0\n\t" |
| 91 | "jb 6f \n\t" |
| 92 | "cmp %%dil, %%sil\n\t" |
| 93 | "je 7b \n\t" |
| 94 | "6:\n\t" |
| 95 | /* |
| 96 | * Calculate copy position to tail. |
| 97 | */ |
| 98 | "addq %0, %1\n\t" |
| 99 | "addq %0, %2\n\t" |
| 100 | "subq $0x20, %0\n\t" |
| 101 | /* |
| 102 | * We gobble 32byts backward in each loop. |
| 103 | */ |
| 104 | "8:\n\t" |
| 105 | "subq $0x20, %0\n\t" |
| 106 | "movq -1*8(%1), %4\n\t" |
| 107 | "movq -2*8(%1), %5\n\t" |
| 108 | "movq -3*8(%1), %6\n\t" |
| 109 | "movq -4*8(%1), %7\n\t" |
| 110 | "leaq -4*8(%1), %1\n\t" |
| 111 | |
| 112 | "movq %4, -1*8(%2)\n\t" |
| 113 | "movq %5, -2*8(%2)\n\t" |
| 114 | "movq %6, -3*8(%2)\n\t" |
| 115 | "movq %7, -4*8(%2)\n\t" |
| 116 | "leaq -4*8(%2), %2\n\t" |
| 117 | "jae 8b\n\t" |
| 118 | /* |
| 119 | * Calculate copy position to head. |
| 120 | */ |
| 121 | "addq $0x20, %0\n\t" |
| 122 | "subq %0, %1\n\t" |
| 123 | "subq %0, %2\n\t" |
| 124 | "1:\n\t" |
| 125 | "cmpq $16, %0\n\t" |
| 126 | "jb 9f\n\t" |
| 127 | /* |
| 128 | * Move data from 16 bytes to 31 bytes. |
| 129 | */ |
| 130 | "movq 0*8(%1), %4\n\t" |
| 131 | "movq 1*8(%1), %5\n\t" |
| 132 | "movq -2*8(%1, %0), %6\n\t" |
| 133 | "movq -1*8(%1, %0), %7\n\t" |
| 134 | "movq %4, 0*8(%2)\n\t" |
| 135 | "movq %5, 1*8(%2)\n\t" |
| 136 | "movq %6, -2*8(%2, %0)\n\t" |
| 137 | "movq %7, -1*8(%2, %0)\n\t" |
| 138 | "jmp 13f\n\t" |
| 139 | ".p2align 4\n\t" |
| 140 | "9:\n\t" |
| 141 | "cmpq $8, %0\n\t" |
| 142 | "jb 10f\n\t" |
| 143 | /* |
| 144 | * Move data from 8 bytes to 15 bytes. |
| 145 | */ |
| 146 | "movq 0*8(%1), %4\n\t" |
| 147 | "movq -1*8(%1, %0), %5\n\t" |
| 148 | "movq %4, 0*8(%2)\n\t" |
| 149 | "movq %5, -1*8(%2, %0)\n\t" |
| 150 | "jmp 13f\n\t" |
| 151 | "10:\n\t" |
| 152 | "cmpq $4, %0\n\t" |
| 153 | "jb 11f\n\t" |
| 154 | /* |
| 155 | * Move data from 4 bytes to 7 bytes. |
| 156 | */ |
| 157 | "movl (%1), %4d\n\t" |
| 158 | "movl -4(%1, %0), %5d\n\t" |
| 159 | "movl %4d, (%2)\n\t" |
| 160 | "movl %5d, -4(%2, %0)\n\t" |
| 161 | "jmp 13f\n\t" |
| 162 | "11:\n\t" |
| 163 | "cmp $2, %0\n\t" |
| 164 | "jb 12f\n\t" |
| 165 | /* |
| 166 | * Move data from 2 bytes to 3 bytes. |
| 167 | */ |
| 168 | "movw (%1), %4w\n\t" |
| 169 | "movw -2(%1, %0), %5w\n\t" |
| 170 | "movw %4w, (%2)\n\t" |
| 171 | "movw %5w, -2(%2, %0)\n\t" |
| 172 | "jmp 13f\n\t" |
| 173 | "12:\n\t" |
| 174 | "cmp $1, %0\n\t" |
| 175 | "jb 13f\n\t" |
| 176 | /* |
| 177 | * Move data for 1 byte. |
| 178 | */ |
| 179 | "movb (%1), %4b\n\t" |
| 180 | "movb %4b, (%2)\n\t" |
| 181 | "13:\n\t" |
| 182 | : "=&d" (d0), "=&S" (d1), "=&D" (d2), "=&a" (ret) , |
| 183 | "=r"(d3), "=r"(d4), "=r"(d5), "=r"(d6), "=&c" (d7) |
| 184 | :"0" (count), |
| 185 | "1" (src), |
| 186 | "2" (dest) |
| 187 | :"memory"); |
| 188 | |
| 189 | return ret; |
| 190 | |
Paolo Ciarrocchi | e940659 | 2008-02-19 20:53:38 +0100 | [diff] [blame] | 191 | } |
Andi Kleen | 2ee60e17 | 2006-06-26 13:59:44 +0200 | [diff] [blame] | 192 | EXPORT_SYMBOL(memmove); |