blob: 3a0dddca5a1097e473d103a15797547da66e7bf1 [file] [log] [blame]
Dan Williams9bc89cd2007-01-02 11:10:44 -07001/*
2 * xor offload engine api
3 *
4 * Copyright © 2006, Intel Corporation.
5 *
6 * Dan Williams <dan.j.williams@intel.com>
7 *
8 * with architecture considerations by:
9 * Neil Brown <neilb@suse.de>
10 * Jeff Garzik <jeff@garzik.org>
11 *
12 * This program is free software; you can redistribute it and/or modify it
13 * under the terms and conditions of the GNU General Public License,
14 * version 2, as published by the Free Software Foundation.
15 *
16 * This program is distributed in the hope it will be useful, but WITHOUT
17 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
19 * more details.
20 *
21 * You should have received a copy of the GNU General Public License along with
22 * this program; if not, write to the Free Software Foundation, Inc.,
23 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
24 *
25 */
26#include <linux/kernel.h>
27#include <linux/interrupt.h>
28#include <linux/mm.h>
29#include <linux/dma-mapping.h>
30#include <linux/raid/xor.h>
31#include <linux/async_tx.h>
32
Dan Williams1367a3d2008-02-02 18:46:43 -070033/* do_async_xor - dma map the pages and perform the xor with an engine.
34 * This routine is marked __always_inline so it can be compiled away
35 * when CONFIG_DMA_ENGINE=n
36 */
Dan Williams00367312008-02-02 19:49:57 -070037static __always_inline struct dma_async_tx_descriptor *
38do_async_xor(struct dma_device *device,
Dan Williams9bc89cd2007-01-02 11:10:44 -070039 struct dma_chan *chan, struct page *dest, struct page **src_list,
40 unsigned int offset, unsigned int src_cnt, size_t len,
41 enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx,
42 dma_async_tx_callback cb_fn, void *cb_param)
43{
Dan Williams00367312008-02-02 19:49:57 -070044 dma_addr_t dma_dest;
45 dma_addr_t *dma_src = (dma_addr_t *) src_list;
46 struct dma_async_tx_descriptor *tx;
Dan Williams9bc89cd2007-01-02 11:10:44 -070047 int i;
Dan Williamsd4c56f92008-02-02 19:49:58 -070048 unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0;
Dan Williams9bc89cd2007-01-02 11:10:44 -070049
Dan Williams3280ab3e2008-03-13 17:45:28 -070050 pr_debug("%s: len: %zu\n", __func__, len);
Dan Williams9bc89cd2007-01-02 11:10:44 -070051
Dan Williams00367312008-02-02 19:49:57 -070052 dma_dest = dma_map_page(device->dev, dest, offset, len,
Dan Williamsd909b342008-02-02 19:30:14 -070053 DMA_FROM_DEVICE);
Dan Williams9bc89cd2007-01-02 11:10:44 -070054
Dan Williams00367312008-02-02 19:49:57 -070055 for (i = 0; i < src_cnt; i++)
56 dma_src[i] = dma_map_page(device->dev, src_list[i], offset,
57 len, DMA_TO_DEVICE);
58
59 /* Since we have clobbered the src_list we are committed
60 * to doing this asynchronously. Drivers force forward progress
61 * in case they can not provide a descriptor
62 */
63 tx = device->device_prep_dma_xor(chan, dma_dest, dma_src, src_cnt, len,
Dan Williamsd4c56f92008-02-02 19:49:58 -070064 dma_prep_flags);
Dan Williams00367312008-02-02 19:49:57 -070065 if (!tx) {
66 if (depend_tx)
67 dma_wait_for_async_tx(depend_tx);
68
69 while (!tx)
70 tx = device->device_prep_dma_xor(chan, dma_dest,
71 dma_src, src_cnt, len,
Dan Williamsd4c56f92008-02-02 19:49:58 -070072 dma_prep_flags);
Dan Williams9bc89cd2007-01-02 11:10:44 -070073 }
74
75 async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param);
Dan Williams00367312008-02-02 19:49:57 -070076
77 return tx;
Dan Williams9bc89cd2007-01-02 11:10:44 -070078}
79
80static void
81do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset,
82 unsigned int src_cnt, size_t len, enum async_tx_flags flags,
83 struct dma_async_tx_descriptor *depend_tx,
84 dma_async_tx_callback cb_fn, void *cb_param)
85{
86 void *_dest;
87 int i;
88
Dan Williams3280ab3e2008-03-13 17:45:28 -070089 pr_debug("%s: len: %zu\n", __func__, len);
Dan Williams9bc89cd2007-01-02 11:10:44 -070090
91 /* reuse the 'src_list' array to convert to buffer pointers */
92 for (i = 0; i < src_cnt; i++)
93 src_list[i] = (struct page *)
94 (page_address(src_list[i]) + offset);
95
96 /* set destination address */
97 _dest = page_address(dest) + offset;
98
99 if (flags & ASYNC_TX_XOR_ZERO_DST)
100 memset(_dest, 0, len);
101
102 xor_blocks(src_cnt, len, _dest,
103 (void **) src_list);
104
105 async_tx_sync_epilog(flags, depend_tx, cb_fn, cb_param);
106}
107
108/**
109 * async_xor - attempt to xor a set of blocks with a dma engine.
110 * xor_blocks always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST
111 * flag must be set to not include dest data in the calculation. The
112 * assumption with dma eninges is that they only use the destination
113 * buffer as a source when it is explicity specified in the source list.
114 * @dest: destination page
115 * @src_list: array of source pages (if the dest is also a source it must be
116 * at index zero). The contents of this array may be overwritten.
117 * @offset: offset in pages to start transaction
118 * @src_cnt: number of source pages
119 * @len: length in bytes
120 * @flags: ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DEST,
Dan Williamsd909b342008-02-02 19:30:14 -0700121 * ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
Dan Williams9bc89cd2007-01-02 11:10:44 -0700122 * @depend_tx: xor depends on the result of this transaction.
123 * @cb_fn: function to call when the xor completes
124 * @cb_param: parameter to pass to the callback routine
125 */
126struct dma_async_tx_descriptor *
127async_xor(struct page *dest, struct page **src_list, unsigned int offset,
128 int src_cnt, size_t len, enum async_tx_flags flags,
129 struct dma_async_tx_descriptor *depend_tx,
130 dma_async_tx_callback cb_fn, void *cb_param)
131{
Dan Williams47437b22008-02-02 19:49:59 -0700132 struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_XOR,
133 &dest, 1, src_list,
134 src_cnt, len);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700135 struct dma_device *device = chan ? chan->device : NULL;
136 struct dma_async_tx_descriptor *tx = NULL;
137 dma_async_tx_callback _cb_fn;
138 void *_cb_param;
139 unsigned long local_flags;
140 int xor_src_cnt;
Dan Williams00367312008-02-02 19:49:57 -0700141 int i = 0, src_off = 0;
Dan Williams9bc89cd2007-01-02 11:10:44 -0700142
143 BUG_ON(src_cnt <= 1);
144
145 while (src_cnt) {
146 local_flags = flags;
147 if (device) { /* run the xor asynchronously */
148 xor_src_cnt = min(src_cnt, device->max_xor);
149 /* if we are submitting additional xors
150 * only set the callback on the last transaction
151 */
152 if (src_cnt > xor_src_cnt) {
153 local_flags &= ~ASYNC_TX_ACK;
154 _cb_fn = NULL;
155 _cb_param = NULL;
156 } else {
157 _cb_fn = cb_fn;
158 _cb_param = cb_param;
159 }
160
Dan Williams00367312008-02-02 19:49:57 -0700161 tx = do_async_xor(device, chan, dest,
162 &src_list[src_off], offset,
163 xor_src_cnt, len, local_flags,
164 depend_tx, _cb_fn, _cb_param);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700165 } else { /* run the xor synchronously */
Dan Williams9bc89cd2007-01-02 11:10:44 -0700166 /* in the sync case the dest is an implied source
167 * (assumes the dest is at the src_off index)
168 */
169 if (flags & ASYNC_TX_XOR_DROP_DST) {
170 src_cnt--;
171 src_off++;
172 }
173
174 /* process up to 'MAX_XOR_BLOCKS' sources */
175 xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS);
176
177 /* if we are submitting additional xors
178 * only set the callback on the last transaction
179 */
180 if (src_cnt > xor_src_cnt) {
181 local_flags &= ~ASYNC_TX_ACK;
182 _cb_fn = NULL;
183 _cb_param = NULL;
184 } else {
185 _cb_fn = cb_fn;
186 _cb_param = cb_param;
187 }
188
189 /* wait for any prerequisite operations */
190 if (depend_tx) {
191 /* if ack is already set then we cannot be sure
192 * we are referring to the correct operation
193 */
Dan Williams636bdea2008-04-17 20:17:26 -0700194 BUG_ON(async_tx_test_ack(depend_tx));
Dan Williams9bc89cd2007-01-02 11:10:44 -0700195 if (dma_wait_for_async_tx(depend_tx) ==
196 DMA_ERROR)
197 panic("%s: DMA_ERROR waiting for "
198 "depend_tx\n",
Dan Williams3280ab3e2008-03-13 17:45:28 -0700199 __func__);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700200 }
201
202 do_sync_xor(dest, &src_list[src_off], offset,
203 xor_src_cnt, len, local_flags, depend_tx,
204 _cb_fn, _cb_param);
205 }
206
207 /* the previous tx is hidden from the client,
208 * so ack it
209 */
210 if (i && depend_tx)
211 async_tx_ack(depend_tx);
212
213 depend_tx = tx;
214
215 if (src_cnt > xor_src_cnt) {
216 /* drop completed sources */
217 src_cnt -= xor_src_cnt;
218 src_off += xor_src_cnt;
219
220 /* unconditionally preserve the destination */
221 flags &= ~ASYNC_TX_XOR_ZERO_DST;
222
223 /* use the intermediate result a source, but remember
224 * it's dropped, because it's implied, in the sync case
225 */
226 src_list[--src_off] = dest;
227 src_cnt++;
228 flags |= ASYNC_TX_XOR_DROP_DST;
229 } else
230 src_cnt = 0;
231 i++;
232 }
233
234 return tx;
235}
236EXPORT_SYMBOL_GPL(async_xor);
237
238static int page_is_zero(struct page *p, unsigned int offset, size_t len)
239{
240 char *a = page_address(p) + offset;
241 return ((*(u32 *) a) == 0 &&
242 memcmp(a, a + 4, len - 4) == 0);
243}
244
245/**
246 * async_xor_zero_sum - attempt a xor parity check with a dma engine.
247 * @dest: destination page used if the xor is performed synchronously
248 * @src_list: array of source pages. The dest page must be listed as a source
249 * at index zero. The contents of this array may be overwritten.
250 * @offset: offset in pages to start transaction
251 * @src_cnt: number of source pages
252 * @len: length in bytes
253 * @result: 0 if sum == 0 else non-zero
Dan Williamsd909b342008-02-02 19:30:14 -0700254 * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
Dan Williams9bc89cd2007-01-02 11:10:44 -0700255 * @depend_tx: xor depends on the result of this transaction.
256 * @cb_fn: function to call when the xor completes
257 * @cb_param: parameter to pass to the callback routine
258 */
259struct dma_async_tx_descriptor *
260async_xor_zero_sum(struct page *dest, struct page **src_list,
261 unsigned int offset, int src_cnt, size_t len,
262 u32 *result, enum async_tx_flags flags,
263 struct dma_async_tx_descriptor *depend_tx,
264 dma_async_tx_callback cb_fn, void *cb_param)
265{
Dan Williams47437b22008-02-02 19:49:59 -0700266 struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_ZERO_SUM,
267 &dest, 1, src_list,
268 src_cnt, len);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700269 struct dma_device *device = chan ? chan->device : NULL;
Dan Williams00367312008-02-02 19:49:57 -0700270 struct dma_async_tx_descriptor *tx = NULL;
Dan Williams9bc89cd2007-01-02 11:10:44 -0700271
272 BUG_ON(src_cnt <= 1);
273
Dan Williams8d8002f2008-03-18 21:23:59 -0700274 if (device && src_cnt <= device->max_xor) {
Dan Williams00367312008-02-02 19:49:57 -0700275 dma_addr_t *dma_src = (dma_addr_t *) src_list;
Dan Williamsd4c56f92008-02-02 19:49:58 -0700276 unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0;
Dan Williams00367312008-02-02 19:49:57 -0700277 int i;
Dan Williams9bc89cd2007-01-02 11:10:44 -0700278
Dan Williams3280ab3e2008-03-13 17:45:28 -0700279 pr_debug("%s: (async) len: %zu\n", __func__, len);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700280
Dan Williams00367312008-02-02 19:49:57 -0700281 for (i = 0; i < src_cnt; i++)
282 dma_src[i] = dma_map_page(device->dev, src_list[i],
283 offset, len, DMA_TO_DEVICE);
284
285 tx = device->device_prep_dma_zero_sum(chan, dma_src, src_cnt,
286 len, result,
Dan Williamsd4c56f92008-02-02 19:49:58 -0700287 dma_prep_flags);
Dan Williams00367312008-02-02 19:49:57 -0700288 if (!tx) {
289 if (depend_tx)
290 dma_wait_for_async_tx(depend_tx);
291
292 while (!tx)
293 tx = device->device_prep_dma_zero_sum(chan,
294 dma_src, src_cnt, len, result,
Dan Williamsd4c56f92008-02-02 19:49:58 -0700295 dma_prep_flags);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700296 }
297
298 async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param);
299 } else {
300 unsigned long xor_flags = flags;
301
Dan Williams3280ab3e2008-03-13 17:45:28 -0700302 pr_debug("%s: (sync) len: %zu\n", __func__, len);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700303
304 xor_flags |= ASYNC_TX_XOR_DROP_DST;
305 xor_flags &= ~ASYNC_TX_ACK;
306
307 tx = async_xor(dest, src_list, offset, src_cnt, len, xor_flags,
308 depend_tx, NULL, NULL);
309
310 if (tx) {
311 if (dma_wait_for_async_tx(tx) == DMA_ERROR)
312 panic("%s: DMA_ERROR waiting for tx\n",
Dan Williams3280ab3e2008-03-13 17:45:28 -0700313 __func__);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700314 async_tx_ack(tx);
315 }
316
317 *result = page_is_zero(dest, offset, len) ? 0 : 1;
318
319 tx = NULL;
320
321 async_tx_sync_epilog(flags, depend_tx, cb_fn, cb_param);
322 }
323
324 return tx;
325}
326EXPORT_SYMBOL_GPL(async_xor_zero_sum);
327
328static int __init async_xor_init(void)
329{
Dan Williams00367312008-02-02 19:49:57 -0700330 #ifdef CONFIG_DMA_ENGINE
331 /* To conserve stack space the input src_list (array of page pointers)
332 * is reused to hold the array of dma addresses passed to the driver.
333 * This conversion is only possible when dma_addr_t is less than the
334 * the size of a pointer. HIGHMEM64G is known to violate this
335 * assumption.
336 */
337 BUILD_BUG_ON(sizeof(dma_addr_t) > sizeof(struct page *));
338 #endif
339
Dan Williams9bc89cd2007-01-02 11:10:44 -0700340 return 0;
341}
342
343static void __exit async_xor_exit(void)
344{
345 do { } while (0);
346}
347
348module_init(async_xor_init);
349module_exit(async_xor_exit);
350
351MODULE_AUTHOR("Intel Corporation");
352MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api");
353MODULE_LICENSE("GPL");