Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* -*- linux-c -*- ------------------------------------------------------- * |
| 2 | * |
| 3 | * Copyright 2002 H. Peter Anvin - All Rights Reserved |
| 4 | * |
| 5 | * This program is free software; you can redistribute it and/or modify |
| 6 | * it under the terms of the GNU General Public License as published by |
| 7 | * the Free Software Foundation, Inc., 53 Temple Place Ste 330, |
Atsushi SAKAI | 93ed05e | 2009-03-31 14:57:37 +1100 | [diff] [blame^] | 8 | * Boston MA 02111-1307, USA; either version 2 of the License, or |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 9 | * (at your option) any later version; incorporated herein by reference. |
| 10 | * |
| 11 | * ----------------------------------------------------------------------- */ |
| 12 | |
| 13 | /* |
| 14 | * raid6sse2.c |
| 15 | * |
| 16 | * SSE-2 implementation of RAID-6 syndrome functions |
| 17 | * |
| 18 | */ |
| 19 | |
Al Viro | ca5cd87 | 2007-10-29 04:31:16 +0000 | [diff] [blame] | 20 | #if (defined(__i386__) || defined(__x86_64__)) && !defined(__arch_um__) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 21 | |
| 22 | #include "raid6.h" |
| 23 | #include "raid6x86.h" |
| 24 | |
| 25 | static const struct raid6_sse_constants { |
| 26 | u64 x1d[2]; |
| 27 | } raid6_sse_constants __attribute__((aligned(16))) = { |
| 28 | { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL }, |
| 29 | }; |
| 30 | |
| 31 | static int raid6_have_sse2(void) |
| 32 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 33 | /* Not really boot_cpu but "all_cpus" */ |
| 34 | return boot_cpu_has(X86_FEATURE_MMX) && |
| 35 | boot_cpu_has(X86_FEATURE_FXSR) && |
| 36 | boot_cpu_has(X86_FEATURE_XMM) && |
| 37 | boot_cpu_has(X86_FEATURE_XMM2); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 38 | } |
| 39 | |
| 40 | /* |
| 41 | * Plain SSE2 implementation |
| 42 | */ |
| 43 | static void raid6_sse21_gen_syndrome(int disks, size_t bytes, void **ptrs) |
| 44 | { |
| 45 | u8 **dptr = (u8 **)ptrs; |
| 46 | u8 *p, *q; |
| 47 | int d, z, z0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 48 | |
| 49 | z0 = disks - 3; /* Highest data disk */ |
| 50 | p = dptr[z0+1]; /* XOR parity */ |
| 51 | q = dptr[z0+2]; /* RS syndrome */ |
| 52 | |
H. Peter Anvin | a723406 | 2007-02-28 20:11:25 -0800 | [diff] [blame] | 53 | kernel_fpu_begin(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 54 | |
| 55 | asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0])); |
| 56 | asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */ |
| 57 | |
| 58 | for ( d = 0 ; d < bytes ; d += 16 ) { |
| 59 | asm volatile("prefetchnta %0" : : "m" (dptr[z0][d])); |
| 60 | asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d])); /* P[0] */ |
| 61 | asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d])); |
| 62 | asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */ |
| 63 | asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z0-1][d])); |
| 64 | for ( z = z0-2 ; z >= 0 ; z-- ) { |
| 65 | asm volatile("prefetchnta %0" : : "m" (dptr[z][d])); |
| 66 | asm volatile("pcmpgtb %xmm4,%xmm5"); |
| 67 | asm volatile("paddb %xmm4,%xmm4"); |
| 68 | asm volatile("pand %xmm0,%xmm5"); |
| 69 | asm volatile("pxor %xmm5,%xmm4"); |
| 70 | asm volatile("pxor %xmm5,%xmm5"); |
| 71 | asm volatile("pxor %xmm6,%xmm2"); |
| 72 | asm volatile("pxor %xmm6,%xmm4"); |
| 73 | asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z][d])); |
| 74 | } |
| 75 | asm volatile("pcmpgtb %xmm4,%xmm5"); |
| 76 | asm volatile("paddb %xmm4,%xmm4"); |
| 77 | asm volatile("pand %xmm0,%xmm5"); |
| 78 | asm volatile("pxor %xmm5,%xmm4"); |
| 79 | asm volatile("pxor %xmm5,%xmm5"); |
| 80 | asm volatile("pxor %xmm6,%xmm2"); |
| 81 | asm volatile("pxor %xmm6,%xmm4"); |
| 82 | |
| 83 | asm volatile("movntdq %%xmm2,%0" : "=m" (p[d])); |
| 84 | asm volatile("pxor %xmm2,%xmm2"); |
| 85 | asm volatile("movntdq %%xmm4,%0" : "=m" (q[d])); |
| 86 | asm volatile("pxor %xmm4,%xmm4"); |
| 87 | } |
| 88 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 89 | asm volatile("sfence" : : : "memory"); |
H. Peter Anvin | a723406 | 2007-02-28 20:11:25 -0800 | [diff] [blame] | 90 | kernel_fpu_end(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 91 | } |
| 92 | |
| 93 | const struct raid6_calls raid6_sse2x1 = { |
| 94 | raid6_sse21_gen_syndrome, |
| 95 | raid6_have_sse2, |
| 96 | "sse2x1", |
| 97 | 1 /* Has cache hints */ |
| 98 | }; |
| 99 | |
| 100 | /* |
| 101 | * Unrolled-by-2 SSE2 implementation |
| 102 | */ |
| 103 | static void raid6_sse22_gen_syndrome(int disks, size_t bytes, void **ptrs) |
| 104 | { |
| 105 | u8 **dptr = (u8 **)ptrs; |
| 106 | u8 *p, *q; |
| 107 | int d, z, z0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 108 | |
| 109 | z0 = disks - 3; /* Highest data disk */ |
| 110 | p = dptr[z0+1]; /* XOR parity */ |
| 111 | q = dptr[z0+2]; /* RS syndrome */ |
| 112 | |
H. Peter Anvin | a723406 | 2007-02-28 20:11:25 -0800 | [diff] [blame] | 113 | kernel_fpu_begin(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 114 | |
| 115 | asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0])); |
| 116 | asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */ |
| 117 | asm volatile("pxor %xmm7,%xmm7"); /* Zero temp */ |
| 118 | |
| 119 | /* We uniformly assume a single prefetch covers at least 32 bytes */ |
| 120 | for ( d = 0 ; d < bytes ; d += 32 ) { |
| 121 | asm volatile("prefetchnta %0" : : "m" (dptr[z0][d])); |
| 122 | asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d])); /* P[0] */ |
| 123 | asm volatile("movdqa %0,%%xmm3" : : "m" (dptr[z0][d+16])); /* P[1] */ |
| 124 | asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */ |
| 125 | asm volatile("movdqa %xmm3,%xmm6"); /* Q[1] */ |
| 126 | for ( z = z0-1 ; z >= 0 ; z-- ) { |
| 127 | asm volatile("prefetchnta %0" : : "m" (dptr[z][d])); |
| 128 | asm volatile("pcmpgtb %xmm4,%xmm5"); |
| 129 | asm volatile("pcmpgtb %xmm6,%xmm7"); |
| 130 | asm volatile("paddb %xmm4,%xmm4"); |
| 131 | asm volatile("paddb %xmm6,%xmm6"); |
| 132 | asm volatile("pand %xmm0,%xmm5"); |
| 133 | asm volatile("pand %xmm0,%xmm7"); |
| 134 | asm volatile("pxor %xmm5,%xmm4"); |
| 135 | asm volatile("pxor %xmm7,%xmm6"); |
| 136 | asm volatile("movdqa %0,%%xmm5" : : "m" (dptr[z][d])); |
| 137 | asm volatile("movdqa %0,%%xmm7" : : "m" (dptr[z][d+16])); |
| 138 | asm volatile("pxor %xmm5,%xmm2"); |
| 139 | asm volatile("pxor %xmm7,%xmm3"); |
| 140 | asm volatile("pxor %xmm5,%xmm4"); |
| 141 | asm volatile("pxor %xmm7,%xmm6"); |
| 142 | asm volatile("pxor %xmm5,%xmm5"); |
| 143 | asm volatile("pxor %xmm7,%xmm7"); |
| 144 | } |
| 145 | asm volatile("movntdq %%xmm2,%0" : "=m" (p[d])); |
| 146 | asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16])); |
| 147 | asm volatile("movntdq %%xmm4,%0" : "=m" (q[d])); |
| 148 | asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16])); |
| 149 | } |
| 150 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 151 | asm volatile("sfence" : : : "memory"); |
H. Peter Anvin | a723406 | 2007-02-28 20:11:25 -0800 | [diff] [blame] | 152 | kernel_fpu_end(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 153 | } |
| 154 | |
| 155 | const struct raid6_calls raid6_sse2x2 = { |
| 156 | raid6_sse22_gen_syndrome, |
| 157 | raid6_have_sse2, |
| 158 | "sse2x2", |
| 159 | 1 /* Has cache hints */ |
| 160 | }; |
| 161 | |
| 162 | #endif |
| 163 | |
Al Viro | ca5cd87 | 2007-10-29 04:31:16 +0000 | [diff] [blame] | 164 | #if defined(__x86_64__) && !defined(__arch_um__) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 165 | |
| 166 | /* |
| 167 | * Unrolled-by-4 SSE2 implementation |
| 168 | */ |
| 169 | static void raid6_sse24_gen_syndrome(int disks, size_t bytes, void **ptrs) |
| 170 | { |
| 171 | u8 **dptr = (u8 **)ptrs; |
| 172 | u8 *p, *q; |
| 173 | int d, z, z0; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 174 | |
| 175 | z0 = disks - 3; /* Highest data disk */ |
| 176 | p = dptr[z0+1]; /* XOR parity */ |
| 177 | q = dptr[z0+2]; /* RS syndrome */ |
| 178 | |
H. Peter Anvin | a723406 | 2007-02-28 20:11:25 -0800 | [diff] [blame] | 179 | kernel_fpu_begin(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 180 | |
| 181 | asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0])); |
| 182 | asm volatile("pxor %xmm2,%xmm2"); /* P[0] */ |
| 183 | asm volatile("pxor %xmm3,%xmm3"); /* P[1] */ |
| 184 | asm volatile("pxor %xmm4,%xmm4"); /* Q[0] */ |
| 185 | asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */ |
| 186 | asm volatile("pxor %xmm6,%xmm6"); /* Q[1] */ |
| 187 | asm volatile("pxor %xmm7,%xmm7"); /* Zero temp */ |
| 188 | asm volatile("pxor %xmm10,%xmm10"); /* P[2] */ |
| 189 | asm volatile("pxor %xmm11,%xmm11"); /* P[3] */ |
| 190 | asm volatile("pxor %xmm12,%xmm12"); /* Q[2] */ |
| 191 | asm volatile("pxor %xmm13,%xmm13"); /* Zero temp */ |
| 192 | asm volatile("pxor %xmm14,%xmm14"); /* Q[3] */ |
| 193 | asm volatile("pxor %xmm15,%xmm15"); /* Zero temp */ |
| 194 | |
| 195 | for ( d = 0 ; d < bytes ; d += 64 ) { |
| 196 | for ( z = z0 ; z >= 0 ; z-- ) { |
| 197 | /* The second prefetch seems to improve performance... */ |
| 198 | asm volatile("prefetchnta %0" :: "m" (dptr[z][d])); |
| 199 | asm volatile("prefetchnta %0" :: "m" (dptr[z][d+32])); |
| 200 | asm volatile("pcmpgtb %xmm4,%xmm5"); |
| 201 | asm volatile("pcmpgtb %xmm6,%xmm7"); |
| 202 | asm volatile("pcmpgtb %xmm12,%xmm13"); |
| 203 | asm volatile("pcmpgtb %xmm14,%xmm15"); |
| 204 | asm volatile("paddb %xmm4,%xmm4"); |
| 205 | asm volatile("paddb %xmm6,%xmm6"); |
| 206 | asm volatile("paddb %xmm12,%xmm12"); |
| 207 | asm volatile("paddb %xmm14,%xmm14"); |
| 208 | asm volatile("pand %xmm0,%xmm5"); |
| 209 | asm volatile("pand %xmm0,%xmm7"); |
| 210 | asm volatile("pand %xmm0,%xmm13"); |
| 211 | asm volatile("pand %xmm0,%xmm15"); |
| 212 | asm volatile("pxor %xmm5,%xmm4"); |
| 213 | asm volatile("pxor %xmm7,%xmm6"); |
| 214 | asm volatile("pxor %xmm13,%xmm12"); |
| 215 | asm volatile("pxor %xmm15,%xmm14"); |
| 216 | asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d])); |
| 217 | asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16])); |
| 218 | asm volatile("movdqa %0,%%xmm13" :: "m" (dptr[z][d+32])); |
| 219 | asm volatile("movdqa %0,%%xmm15" :: "m" (dptr[z][d+48])); |
| 220 | asm volatile("pxor %xmm5,%xmm2"); |
| 221 | asm volatile("pxor %xmm7,%xmm3"); |
| 222 | asm volatile("pxor %xmm13,%xmm10"); |
| 223 | asm volatile("pxor %xmm15,%xmm11"); |
| 224 | asm volatile("pxor %xmm5,%xmm4"); |
| 225 | asm volatile("pxor %xmm7,%xmm6"); |
| 226 | asm volatile("pxor %xmm13,%xmm12"); |
| 227 | asm volatile("pxor %xmm15,%xmm14"); |
| 228 | asm volatile("pxor %xmm5,%xmm5"); |
| 229 | asm volatile("pxor %xmm7,%xmm7"); |
| 230 | asm volatile("pxor %xmm13,%xmm13"); |
| 231 | asm volatile("pxor %xmm15,%xmm15"); |
| 232 | } |
| 233 | asm volatile("movntdq %%xmm2,%0" : "=m" (p[d])); |
| 234 | asm volatile("pxor %xmm2,%xmm2"); |
| 235 | asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16])); |
| 236 | asm volatile("pxor %xmm3,%xmm3"); |
| 237 | asm volatile("movntdq %%xmm10,%0" : "=m" (p[d+32])); |
| 238 | asm volatile("pxor %xmm10,%xmm10"); |
| 239 | asm volatile("movntdq %%xmm11,%0" : "=m" (p[d+48])); |
| 240 | asm volatile("pxor %xmm11,%xmm11"); |
| 241 | asm volatile("movntdq %%xmm4,%0" : "=m" (q[d])); |
| 242 | asm volatile("pxor %xmm4,%xmm4"); |
| 243 | asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16])); |
| 244 | asm volatile("pxor %xmm6,%xmm6"); |
| 245 | asm volatile("movntdq %%xmm12,%0" : "=m" (q[d+32])); |
| 246 | asm volatile("pxor %xmm12,%xmm12"); |
| 247 | asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48])); |
| 248 | asm volatile("pxor %xmm14,%xmm14"); |
| 249 | } |
H. Peter Anvin | a723406 | 2007-02-28 20:11:25 -0800 | [diff] [blame] | 250 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 251 | asm volatile("sfence" : : : "memory"); |
H. Peter Anvin | a723406 | 2007-02-28 20:11:25 -0800 | [diff] [blame] | 252 | kernel_fpu_end(); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 253 | } |
| 254 | |
| 255 | const struct raid6_calls raid6_sse2x4 = { |
| 256 | raid6_sse24_gen_syndrome, |
| 257 | raid6_have_sse2, |
| 258 | "sse2x4", |
| 259 | 1 /* Has cache hints */ |
| 260 | }; |
| 261 | |
| 262 | #endif |