blob: 85b82c85f28e61699bbf94e19e3f2f7750108d10 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* -*- linux-c -*- ------------------------------------------------------- *
2 *
3 * Copyright 2002 H. Peter Anvin - All Rights Reserved
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation, Inc., 53 Temple Place Ste 330,
Atsushi SAKAI93ed05e2009-03-31 14:57:37 +11008 * Boston MA 02111-1307, USA; either version 2 of the License, or
Linus Torvalds1da177e2005-04-16 15:20:36 -07009 * (at your option) any later version; incorporated herein by reference.
10 *
11 * ----------------------------------------------------------------------- */
12
13/*
NeilBrowna8e026c2010-08-12 06:44:54 +100014 * raid6/sse2.c
Linus Torvalds1da177e2005-04-16 15:20:36 -070015 *
16 * SSE-2 implementation of RAID-6 syndrome functions
17 *
18 */
19
Dan Williamsf701d582009-03-31 15:09:39 +110020#include <linux/raid/pq.h>
NeilBrowna8e026c2010-08-12 06:44:54 +100021#include "x86.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070022
23static const struct raid6_sse_constants {
24 u64 x1d[2];
25} raid6_sse_constants __attribute__((aligned(16))) = {
26 { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL },
27};
28
29static int raid6_have_sse2(void)
30{
Linus Torvalds1da177e2005-04-16 15:20:36 -070031 /* Not really boot_cpu but "all_cpus" */
32 return boot_cpu_has(X86_FEATURE_MMX) &&
33 boot_cpu_has(X86_FEATURE_FXSR) &&
34 boot_cpu_has(X86_FEATURE_XMM) &&
35 boot_cpu_has(X86_FEATURE_XMM2);
Linus Torvalds1da177e2005-04-16 15:20:36 -070036}
37
38/*
39 * Plain SSE2 implementation
40 */
41static void raid6_sse21_gen_syndrome(int disks, size_t bytes, void **ptrs)
42{
43 u8 **dptr = (u8 **)ptrs;
44 u8 *p, *q;
45 int d, z, z0;
Linus Torvalds1da177e2005-04-16 15:20:36 -070046
47 z0 = disks - 3; /* Highest data disk */
48 p = dptr[z0+1]; /* XOR parity */
49 q = dptr[z0+2]; /* RS syndrome */
50
H. Peter Anvina7234062007-02-28 20:11:25 -080051 kernel_fpu_begin();
Linus Torvalds1da177e2005-04-16 15:20:36 -070052
53 asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
54 asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */
55
56 for ( d = 0 ; d < bytes ; d += 16 ) {
57 asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
58 asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d])); /* P[0] */
59 asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
60 asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
61 asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z0-1][d]));
62 for ( z = z0-2 ; z >= 0 ; z-- ) {
63 asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
64 asm volatile("pcmpgtb %xmm4,%xmm5");
65 asm volatile("paddb %xmm4,%xmm4");
66 asm volatile("pand %xmm0,%xmm5");
67 asm volatile("pxor %xmm5,%xmm4");
68 asm volatile("pxor %xmm5,%xmm5");
69 asm volatile("pxor %xmm6,%xmm2");
70 asm volatile("pxor %xmm6,%xmm4");
71 asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z][d]));
72 }
73 asm volatile("pcmpgtb %xmm4,%xmm5");
74 asm volatile("paddb %xmm4,%xmm4");
75 asm volatile("pand %xmm0,%xmm5");
76 asm volatile("pxor %xmm5,%xmm4");
77 asm volatile("pxor %xmm5,%xmm5");
78 asm volatile("pxor %xmm6,%xmm2");
79 asm volatile("pxor %xmm6,%xmm4");
80
81 asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
82 asm volatile("pxor %xmm2,%xmm2");
83 asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
84 asm volatile("pxor %xmm4,%xmm4");
85 }
86
Linus Torvalds1da177e2005-04-16 15:20:36 -070087 asm volatile("sfence" : : : "memory");
H. Peter Anvina7234062007-02-28 20:11:25 -080088 kernel_fpu_end();
Linus Torvalds1da177e2005-04-16 15:20:36 -070089}
90
91const struct raid6_calls raid6_sse2x1 = {
92 raid6_sse21_gen_syndrome,
93 raid6_have_sse2,
94 "sse2x1",
95 1 /* Has cache hints */
96};
97
98/*
99 * Unrolled-by-2 SSE2 implementation
100 */
101static void raid6_sse22_gen_syndrome(int disks, size_t bytes, void **ptrs)
102{
103 u8 **dptr = (u8 **)ptrs;
104 u8 *p, *q;
105 int d, z, z0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106
107 z0 = disks - 3; /* Highest data disk */
108 p = dptr[z0+1]; /* XOR parity */
109 q = dptr[z0+2]; /* RS syndrome */
110
H. Peter Anvina7234062007-02-28 20:11:25 -0800111 kernel_fpu_begin();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112
113 asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
114 asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */
115 asm volatile("pxor %xmm7,%xmm7"); /* Zero temp */
116
117 /* We uniformly assume a single prefetch covers at least 32 bytes */
118 for ( d = 0 ; d < bytes ; d += 32 ) {
119 asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
120 asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d])); /* P[0] */
121 asm volatile("movdqa %0,%%xmm3" : : "m" (dptr[z0][d+16])); /* P[1] */
122 asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
123 asm volatile("movdqa %xmm3,%xmm6"); /* Q[1] */
124 for ( z = z0-1 ; z >= 0 ; z-- ) {
125 asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
126 asm volatile("pcmpgtb %xmm4,%xmm5");
127 asm volatile("pcmpgtb %xmm6,%xmm7");
128 asm volatile("paddb %xmm4,%xmm4");
129 asm volatile("paddb %xmm6,%xmm6");
130 asm volatile("pand %xmm0,%xmm5");
131 asm volatile("pand %xmm0,%xmm7");
132 asm volatile("pxor %xmm5,%xmm4");
133 asm volatile("pxor %xmm7,%xmm6");
134 asm volatile("movdqa %0,%%xmm5" : : "m" (dptr[z][d]));
135 asm volatile("movdqa %0,%%xmm7" : : "m" (dptr[z][d+16]));
136 asm volatile("pxor %xmm5,%xmm2");
137 asm volatile("pxor %xmm7,%xmm3");
138 asm volatile("pxor %xmm5,%xmm4");
139 asm volatile("pxor %xmm7,%xmm6");
140 asm volatile("pxor %xmm5,%xmm5");
141 asm volatile("pxor %xmm7,%xmm7");
142 }
143 asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
144 asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
145 asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
146 asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
147 }
148
Linus Torvalds1da177e2005-04-16 15:20:36 -0700149 asm volatile("sfence" : : : "memory");
H. Peter Anvina7234062007-02-28 20:11:25 -0800150 kernel_fpu_end();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151}
152
153const struct raid6_calls raid6_sse2x2 = {
154 raid6_sse22_gen_syndrome,
155 raid6_have_sse2,
156 "sse2x2",
157 1 /* Has cache hints */
158};
159
Yuanhan Liu4f8c55c2012-11-30 13:10:40 -0800160#ifdef CONFIG_X86_64
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161
162/*
163 * Unrolled-by-4 SSE2 implementation
164 */
165static void raid6_sse24_gen_syndrome(int disks, size_t bytes, void **ptrs)
166{
167 u8 **dptr = (u8 **)ptrs;
168 u8 *p, *q;
169 int d, z, z0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170
171 z0 = disks - 3; /* Highest data disk */
172 p = dptr[z0+1]; /* XOR parity */
173 q = dptr[z0+2]; /* RS syndrome */
174
H. Peter Anvina7234062007-02-28 20:11:25 -0800175 kernel_fpu_begin();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700176
177 asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0]));
178 asm volatile("pxor %xmm2,%xmm2"); /* P[0] */
179 asm volatile("pxor %xmm3,%xmm3"); /* P[1] */
180 asm volatile("pxor %xmm4,%xmm4"); /* Q[0] */
181 asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */
182 asm volatile("pxor %xmm6,%xmm6"); /* Q[1] */
183 asm volatile("pxor %xmm7,%xmm7"); /* Zero temp */
184 asm volatile("pxor %xmm10,%xmm10"); /* P[2] */
185 asm volatile("pxor %xmm11,%xmm11"); /* P[3] */
186 asm volatile("pxor %xmm12,%xmm12"); /* Q[2] */
187 asm volatile("pxor %xmm13,%xmm13"); /* Zero temp */
188 asm volatile("pxor %xmm14,%xmm14"); /* Q[3] */
189 asm volatile("pxor %xmm15,%xmm15"); /* Zero temp */
190
191 for ( d = 0 ; d < bytes ; d += 64 ) {
192 for ( z = z0 ; z >= 0 ; z-- ) {
193 /* The second prefetch seems to improve performance... */
194 asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
195 asm volatile("prefetchnta %0" :: "m" (dptr[z][d+32]));
196 asm volatile("pcmpgtb %xmm4,%xmm5");
197 asm volatile("pcmpgtb %xmm6,%xmm7");
198 asm volatile("pcmpgtb %xmm12,%xmm13");
199 asm volatile("pcmpgtb %xmm14,%xmm15");
200 asm volatile("paddb %xmm4,%xmm4");
201 asm volatile("paddb %xmm6,%xmm6");
202 asm volatile("paddb %xmm12,%xmm12");
203 asm volatile("paddb %xmm14,%xmm14");
204 asm volatile("pand %xmm0,%xmm5");
205 asm volatile("pand %xmm0,%xmm7");
206 asm volatile("pand %xmm0,%xmm13");
207 asm volatile("pand %xmm0,%xmm15");
208 asm volatile("pxor %xmm5,%xmm4");
209 asm volatile("pxor %xmm7,%xmm6");
210 asm volatile("pxor %xmm13,%xmm12");
211 asm volatile("pxor %xmm15,%xmm14");
212 asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
213 asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
214 asm volatile("movdqa %0,%%xmm13" :: "m" (dptr[z][d+32]));
215 asm volatile("movdqa %0,%%xmm15" :: "m" (dptr[z][d+48]));
216 asm volatile("pxor %xmm5,%xmm2");
217 asm volatile("pxor %xmm7,%xmm3");
218 asm volatile("pxor %xmm13,%xmm10");
219 asm volatile("pxor %xmm15,%xmm11");
220 asm volatile("pxor %xmm5,%xmm4");
221 asm volatile("pxor %xmm7,%xmm6");
222 asm volatile("pxor %xmm13,%xmm12");
223 asm volatile("pxor %xmm15,%xmm14");
224 asm volatile("pxor %xmm5,%xmm5");
225 asm volatile("pxor %xmm7,%xmm7");
226 asm volatile("pxor %xmm13,%xmm13");
227 asm volatile("pxor %xmm15,%xmm15");
228 }
229 asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
230 asm volatile("pxor %xmm2,%xmm2");
231 asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
232 asm volatile("pxor %xmm3,%xmm3");
233 asm volatile("movntdq %%xmm10,%0" : "=m" (p[d+32]));
234 asm volatile("pxor %xmm10,%xmm10");
235 asm volatile("movntdq %%xmm11,%0" : "=m" (p[d+48]));
236 asm volatile("pxor %xmm11,%xmm11");
237 asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
238 asm volatile("pxor %xmm4,%xmm4");
239 asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
240 asm volatile("pxor %xmm6,%xmm6");
241 asm volatile("movntdq %%xmm12,%0" : "=m" (q[d+32]));
242 asm volatile("pxor %xmm12,%xmm12");
243 asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48]));
244 asm volatile("pxor %xmm14,%xmm14");
245 }
H. Peter Anvina7234062007-02-28 20:11:25 -0800246
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247 asm volatile("sfence" : : : "memory");
H. Peter Anvina7234062007-02-28 20:11:25 -0800248 kernel_fpu_end();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249}
250
251const struct raid6_calls raid6_sse2x4 = {
252 raid6_sse24_gen_syndrome,
253 raid6_have_sse2,
254 "sse2x4",
255 1 /* Has cache hints */
256};
257
Yuanhan Liu4f8c55c2012-11-30 13:10:40 -0800258#endif /* CONFIG_X86_64 */