blob: 595b78672b36ad90e27cf50ed28126bb43970fb5 [file] [log] [blame]
Dan Williams9bc89cd2007-01-02 11:10:44 -07001/*
2 * xor offload engine api
3 *
4 * Copyright © 2006, Intel Corporation.
5 *
6 * Dan Williams <dan.j.williams@intel.com>
7 *
8 * with architecture considerations by:
9 * Neil Brown <neilb@suse.de>
10 * Jeff Garzik <jeff@garzik.org>
11 *
12 * This program is free software; you can redistribute it and/or modify it
13 * under the terms and conditions of the GNU General Public License,
14 * version 2, as published by the Free Software Foundation.
15 *
16 * This program is distributed in the hope it will be useful, but WITHOUT
17 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
19 * more details.
20 *
21 * You should have received a copy of the GNU General Public License along with
22 * this program; if not, write to the Free Software Foundation, Inc.,
23 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
24 *
25 */
26#include <linux/kernel.h>
27#include <linux/interrupt.h>
28#include <linux/mm.h>
29#include <linux/dma-mapping.h>
30#include <linux/raid/xor.h>
31#include <linux/async_tx.h>
32
Dan Williams1367a3d2008-02-02 18:46:43 -070033/* do_async_xor - dma map the pages and perform the xor with an engine.
34 * This routine is marked __always_inline so it can be compiled away
35 * when CONFIG_DMA_ENGINE=n
36 */
Dan Williams00367312008-02-02 19:49:57 -070037static __always_inline struct dma_async_tx_descriptor *
Dan Williams1e55db22008-07-16 19:44:56 -070038do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,
39 unsigned int offset, int src_cnt, size_t len,
40 enum async_tx_flags flags,
41 struct dma_async_tx_descriptor *depend_tx,
42 dma_async_tx_callback cb_fn, void *cb_param)
Dan Williams9bc89cd2007-01-02 11:10:44 -070043{
Dan Williams1e55db22008-07-16 19:44:56 -070044 struct dma_device *dma = chan->device;
Dan Williams00367312008-02-02 19:49:57 -070045 dma_addr_t *dma_src = (dma_addr_t *) src_list;
Dan Williams1e55db22008-07-16 19:44:56 -070046 struct dma_async_tx_descriptor *tx = NULL;
47 int src_off = 0;
Dan Williams9bc89cd2007-01-02 11:10:44 -070048 int i;
Dan Williams1e55db22008-07-16 19:44:56 -070049 dma_async_tx_callback _cb_fn;
50 void *_cb_param;
51 enum async_tx_flags async_flags;
52 enum dma_ctrl_flags dma_flags;
53 int xor_src_cnt;
54 dma_addr_t dma_dest;
Dan Williams9bc89cd2007-01-02 11:10:44 -070055
Dan Williamsa06d5682008-12-08 13:46:00 -070056 /* map the dest bidrectional in case it is re-used as a source */
57 dma_dest = dma_map_page(dma->dev, dest, offset, len, DMA_BIDIRECTIONAL);
58 for (i = 0; i < src_cnt; i++) {
59 /* only map the dest once */
60 if (unlikely(src_list[i] == dest)) {
61 dma_src[i] = dma_dest;
62 continue;
63 }
Dan Williams1e55db22008-07-16 19:44:56 -070064 dma_src[i] = dma_map_page(dma->dev, src_list[i], offset,
Dan Williams00367312008-02-02 19:49:57 -070065 len, DMA_TO_DEVICE);
Dan Williamsa06d5682008-12-08 13:46:00 -070066 }
Dan Williams00367312008-02-02 19:49:57 -070067
Dan Williams1e55db22008-07-16 19:44:56 -070068 while (src_cnt) {
69 async_flags = flags;
70 dma_flags = 0;
71 xor_src_cnt = min(src_cnt, dma->max_xor);
72 /* if we are submitting additional xors, leave the chain open,
73 * clear the callback parameters, and leave the destination
74 * buffer mapped
75 */
76 if (src_cnt > xor_src_cnt) {
77 async_flags &= ~ASYNC_TX_ACK;
78 dma_flags = DMA_COMPL_SKIP_DEST_UNMAP;
79 _cb_fn = NULL;
80 _cb_param = NULL;
81 } else {
82 _cb_fn = cb_fn;
83 _cb_param = cb_param;
84 }
85 if (_cb_fn)
86 dma_flags |= DMA_PREP_INTERRUPT;
87
88 /* Since we have clobbered the src_list we are committed
89 * to doing this asynchronously. Drivers force forward progress
90 * in case they can not provide a descriptor
91 */
92 tx = dma->device_prep_dma_xor(chan, dma_dest, &dma_src[src_off],
93 xor_src_cnt, len, dma_flags);
94
Dan Williams669ab0b2008-07-17 17:59:55 -070095 if (unlikely(!tx))
96 async_tx_quiesce(&depend_tx);
Dan Williams00367312008-02-02 19:49:57 -070097
Dan Williams1e55db22008-07-16 19:44:56 -070098 /* spin wait for the preceeding transactions to complete */
Dan Williams669ab0b2008-07-17 17:59:55 -070099 while (unlikely(!tx)) {
100 dma_async_issue_pending(chan);
Dan Williams1e55db22008-07-16 19:44:56 -0700101 tx = dma->device_prep_dma_xor(chan, dma_dest,
102 &dma_src[src_off],
103 xor_src_cnt, len,
104 dma_flags);
Dan Williams669ab0b2008-07-17 17:59:55 -0700105 }
Dan Williams9bc89cd2007-01-02 11:10:44 -0700106
Dan Williams1e55db22008-07-16 19:44:56 -0700107 async_tx_submit(chan, tx, async_flags, depend_tx, _cb_fn,
108 _cb_param);
109
110 depend_tx = tx;
111 flags |= ASYNC_TX_DEP_ACK;
112
113 if (src_cnt > xor_src_cnt) {
114 /* drop completed sources */
115 src_cnt -= xor_src_cnt;
116 src_off += xor_src_cnt;
117
118 /* use the intermediate result a source */
119 dma_src[--src_off] = dma_dest;
120 src_cnt++;
121 } else
122 break;
123 }
Dan Williams00367312008-02-02 19:49:57 -0700124
125 return tx;
Dan Williams9bc89cd2007-01-02 11:10:44 -0700126}
127
128static void
129do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset,
Dan Williams1e55db22008-07-16 19:44:56 -0700130 int src_cnt, size_t len, enum async_tx_flags flags,
Dan Williams1e55db22008-07-16 19:44:56 -0700131 dma_async_tx_callback cb_fn, void *cb_param)
Dan Williams9bc89cd2007-01-02 11:10:44 -0700132{
Dan Williams9bc89cd2007-01-02 11:10:44 -0700133 int i;
Dan Williams1e55db22008-07-16 19:44:56 -0700134 int xor_src_cnt;
135 int src_off = 0;
136 void *dest_buf;
137 void **srcs = (void **) src_list;
Dan Williams9bc89cd2007-01-02 11:10:44 -0700138
139 /* reuse the 'src_list' array to convert to buffer pointers */
140 for (i = 0; i < src_cnt; i++)
Dan Williams1e55db22008-07-16 19:44:56 -0700141 srcs[i] = page_address(src_list[i]) + offset;
Dan Williams9bc89cd2007-01-02 11:10:44 -0700142
143 /* set destination address */
Dan Williams1e55db22008-07-16 19:44:56 -0700144 dest_buf = page_address(dest) + offset;
Dan Williams9bc89cd2007-01-02 11:10:44 -0700145
146 if (flags & ASYNC_TX_XOR_ZERO_DST)
Dan Williams1e55db22008-07-16 19:44:56 -0700147 memset(dest_buf, 0, len);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700148
Dan Williams1e55db22008-07-16 19:44:56 -0700149 while (src_cnt > 0) {
150 /* process up to 'MAX_XOR_BLOCKS' sources */
151 xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS);
152 xor_blocks(xor_src_cnt, len, dest_buf, &srcs[src_off]);
153
154 /* drop completed sources */
155 src_cnt -= xor_src_cnt;
156 src_off += xor_src_cnt;
157 }
Dan Williams9bc89cd2007-01-02 11:10:44 -0700158
Dan Williams3dce0172008-07-17 17:59:55 -0700159 async_tx_sync_epilog(cb_fn, cb_param);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700160}
161
162/**
163 * async_xor - attempt to xor a set of blocks with a dma engine.
164 * xor_blocks always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST
165 * flag must be set to not include dest data in the calculation. The
166 * assumption with dma eninges is that they only use the destination
167 * buffer as a source when it is explicity specified in the source list.
168 * @dest: destination page
169 * @src_list: array of source pages (if the dest is also a source it must be
170 * at index zero). The contents of this array may be overwritten.
171 * @offset: offset in pages to start transaction
172 * @src_cnt: number of source pages
173 * @len: length in bytes
174 * @flags: ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DEST,
Dan Williamsd909b342008-02-02 19:30:14 -0700175 * ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
Dan Williams9bc89cd2007-01-02 11:10:44 -0700176 * @depend_tx: xor depends on the result of this transaction.
177 * @cb_fn: function to call when the xor completes
178 * @cb_param: parameter to pass to the callback routine
179 */
180struct dma_async_tx_descriptor *
181async_xor(struct page *dest, struct page **src_list, unsigned int offset,
182 int src_cnt, size_t len, enum async_tx_flags flags,
183 struct dma_async_tx_descriptor *depend_tx,
184 dma_async_tx_callback cb_fn, void *cb_param)
185{
Dan Williams47437b22008-02-02 19:49:59 -0700186 struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_XOR,
187 &dest, 1, src_list,
188 src_cnt, len);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700189 BUG_ON(src_cnt <= 1);
190
Dan Williams1e55db22008-07-16 19:44:56 -0700191 if (chan) {
192 /* run the xor asynchronously */
193 pr_debug("%s (async): len: %zu\n", __func__, len);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700194
Dan Williams1e55db22008-07-16 19:44:56 -0700195 return do_async_xor(chan, dest, src_list, offset, src_cnt, len,
196 flags, depend_tx, cb_fn, cb_param);
197 } else {
198 /* run the xor synchronously */
199 pr_debug("%s (sync): len: %zu\n", __func__, len);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700200
Dan Williams1e55db22008-07-16 19:44:56 -0700201 /* in the sync case the dest is an implied source
202 * (assumes the dest is the first source)
203 */
204 if (flags & ASYNC_TX_XOR_DROP_DST) {
205 src_cnt--;
206 src_list++;
Dan Williams9bc89cd2007-01-02 11:10:44 -0700207 }
208
Dan Williams1e55db22008-07-16 19:44:56 -0700209 /* wait for any prerequisite operations */
Dan Williamsd2c52b72008-07-17 17:59:55 -0700210 async_tx_quiesce(&depend_tx);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700211
Dan Williams1e55db22008-07-16 19:44:56 -0700212 do_sync_xor(dest, src_list, offset, src_cnt, len,
Dan Williams3dce0172008-07-17 17:59:55 -0700213 flags, cb_fn, cb_param);
Dan Williams1e55db22008-07-16 19:44:56 -0700214
215 return NULL;
216 }
Dan Williams9bc89cd2007-01-02 11:10:44 -0700217}
218EXPORT_SYMBOL_GPL(async_xor);
219
220static int page_is_zero(struct page *p, unsigned int offset, size_t len)
221{
222 char *a = page_address(p) + offset;
223 return ((*(u32 *) a) == 0 &&
224 memcmp(a, a + 4, len - 4) == 0);
225}
226
227/**
228 * async_xor_zero_sum - attempt a xor parity check with a dma engine.
229 * @dest: destination page used if the xor is performed synchronously
230 * @src_list: array of source pages. The dest page must be listed as a source
231 * at index zero. The contents of this array may be overwritten.
232 * @offset: offset in pages to start transaction
233 * @src_cnt: number of source pages
234 * @len: length in bytes
235 * @result: 0 if sum == 0 else non-zero
Dan Williamsd909b342008-02-02 19:30:14 -0700236 * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
Dan Williams9bc89cd2007-01-02 11:10:44 -0700237 * @depend_tx: xor depends on the result of this transaction.
238 * @cb_fn: function to call when the xor completes
239 * @cb_param: parameter to pass to the callback routine
240 */
241struct dma_async_tx_descriptor *
242async_xor_zero_sum(struct page *dest, struct page **src_list,
243 unsigned int offset, int src_cnt, size_t len,
244 u32 *result, enum async_tx_flags flags,
245 struct dma_async_tx_descriptor *depend_tx,
246 dma_async_tx_callback cb_fn, void *cb_param)
247{
Dan Williams47437b22008-02-02 19:49:59 -0700248 struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_ZERO_SUM,
249 &dest, 1, src_list,
250 src_cnt, len);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700251 struct dma_device *device = chan ? chan->device : NULL;
Dan Williams00367312008-02-02 19:49:57 -0700252 struct dma_async_tx_descriptor *tx = NULL;
Dan Williams9bc89cd2007-01-02 11:10:44 -0700253
254 BUG_ON(src_cnt <= 1);
255
Dan Williams8d8002f2008-03-18 21:23:59 -0700256 if (device && src_cnt <= device->max_xor) {
Dan Williams00367312008-02-02 19:49:57 -0700257 dma_addr_t *dma_src = (dma_addr_t *) src_list;
Dan Williamsd4c56f92008-02-02 19:49:58 -0700258 unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0;
Dan Williams00367312008-02-02 19:49:57 -0700259 int i;
Dan Williams9bc89cd2007-01-02 11:10:44 -0700260
Dan Williams3280ab3e2008-03-13 17:45:28 -0700261 pr_debug("%s: (async) len: %zu\n", __func__, len);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700262
Dan Williams00367312008-02-02 19:49:57 -0700263 for (i = 0; i < src_cnt; i++)
264 dma_src[i] = dma_map_page(device->dev, src_list[i],
265 offset, len, DMA_TO_DEVICE);
266
267 tx = device->device_prep_dma_zero_sum(chan, dma_src, src_cnt,
268 len, result,
Dan Williamsd4c56f92008-02-02 19:49:58 -0700269 dma_prep_flags);
Dan Williams669ab0b2008-07-17 17:59:55 -0700270 if (unlikely(!tx)) {
271 async_tx_quiesce(&depend_tx);
Dan Williams00367312008-02-02 19:49:57 -0700272
Dan Williamse34a8ae2008-08-05 10:22:05 -0700273 while (!tx) {
Dan Williams669ab0b2008-07-17 17:59:55 -0700274 dma_async_issue_pending(chan);
Dan Williams00367312008-02-02 19:49:57 -0700275 tx = device->device_prep_dma_zero_sum(chan,
276 dma_src, src_cnt, len, result,
Dan Williamsd4c56f92008-02-02 19:49:58 -0700277 dma_prep_flags);
Dan Williamse34a8ae2008-08-05 10:22:05 -0700278 }
Dan Williams9bc89cd2007-01-02 11:10:44 -0700279 }
280
281 async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param);
282 } else {
283 unsigned long xor_flags = flags;
284
Dan Williams3280ab3e2008-03-13 17:45:28 -0700285 pr_debug("%s: (sync) len: %zu\n", __func__, len);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700286
287 xor_flags |= ASYNC_TX_XOR_DROP_DST;
288 xor_flags &= ~ASYNC_TX_ACK;
289
290 tx = async_xor(dest, src_list, offset, src_cnt, len, xor_flags,
291 depend_tx, NULL, NULL);
292
Dan Williamsd2c52b72008-07-17 17:59:55 -0700293 async_tx_quiesce(&tx);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700294
295 *result = page_is_zero(dest, offset, len) ? 0 : 1;
296
Dan Williams3dce0172008-07-17 17:59:55 -0700297 async_tx_sync_epilog(cb_fn, cb_param);
Dan Williams9bc89cd2007-01-02 11:10:44 -0700298 }
299
300 return tx;
301}
302EXPORT_SYMBOL_GPL(async_xor_zero_sum);
303
304static int __init async_xor_init(void)
305{
Dan Williams00367312008-02-02 19:49:57 -0700306 #ifdef CONFIG_DMA_ENGINE
307 /* To conserve stack space the input src_list (array of page pointers)
308 * is reused to hold the array of dma addresses passed to the driver.
309 * This conversion is only possible when dma_addr_t is less than the
310 * the size of a pointer. HIGHMEM64G is known to violate this
311 * assumption.
312 */
313 BUILD_BUG_ON(sizeof(dma_addr_t) > sizeof(struct page *));
314 #endif
315
Dan Williams9bc89cd2007-01-02 11:10:44 -0700316 return 0;
317}
318
319static void __exit async_xor_exit(void)
320{
321 do { } while (0);
322}
323
324module_init(async_xor_init);
325module_exit(async_xor_exit);
326
327MODULE_AUTHOR("Intel Corporation");
328MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api");
329MODULE_LICENSE("GPL");