blob: b4f5c32b6a470f1a0d12c746fdbf3a7fcc9596a7 [file] [log] [blame]
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -07001/*
2 * DMA Engine test module
3 *
4 * Copyright (C) 2007 Atmel Corporation
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10#include <linux/delay.h>
11#include <linux/dmaengine.h>
12#include <linux/init.h>
13#include <linux/kthread.h>
14#include <linux/module.h>
15#include <linux/moduleparam.h>
16#include <linux/random.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090017#include <linux/slab.h>
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070018#include <linux/wait.h>
19
20static unsigned int test_buf_size = 16384;
21module_param(test_buf_size, uint, S_IRUGO);
22MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
23
Kay Sievers06190d82008-11-11 13:12:33 -070024static char test_channel[20];
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070025module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO);
26MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
27
Kay Sievers06190d82008-11-11 13:12:33 -070028static char test_device[20];
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070029module_param_string(device, test_device, sizeof(test_device), S_IRUGO);
30MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
31
32static unsigned int threads_per_chan = 1;
33module_param(threads_per_chan, uint, S_IRUGO);
34MODULE_PARM_DESC(threads_per_chan,
35 "Number of threads to start per channel (default: 1)");
36
37static unsigned int max_channels;
38module_param(max_channels, uint, S_IRUGO);
Dan Williams33df8ca2009-01-06 11:38:15 -070039MODULE_PARM_DESC(max_channels,
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070040 "Maximum number of channels to use (default: all)");
41
Nicolas Ferre0a2ff57d2009-07-03 19:26:51 +020042static unsigned int iterations;
43module_param(iterations, uint, S_IRUGO);
44MODULE_PARM_DESC(iterations,
45 "Iterations before stopping test (default: infinite)");
46
Dan Williamsb54d5cb2009-03-25 09:13:25 -070047static unsigned int xor_sources = 3;
48module_param(xor_sources, uint, S_IRUGO);
49MODULE_PARM_DESC(xor_sources,
50 "Number of xor source buffers (default: 3)");
51
Dan Williams58691d62009-08-29 19:09:27 -070052static unsigned int pq_sources = 3;
53module_param(pq_sources, uint, S_IRUGO);
54MODULE_PARM_DESC(pq_sources,
55 "Number of p+q source buffers (default: 3)");
56
Viresh Kumard42efe62011-03-22 17:27:25 +053057static int timeout = 3000;
58module_param(timeout, uint, S_IRUGO);
Joe Perches85ee7a12011-04-23 20:38:19 -070059MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
60 "Pass -1 for infinite timeout");
Viresh Kumard42efe62011-03-22 17:27:25 +053061
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070062/*
63 * Initialization patterns. All bytes in the source buffer has bit 7
64 * set, all bytes in the destination buffer has bit 7 cleared.
65 *
66 * Bit 6 is set for all bytes which are to be copied by the DMA
67 * engine. Bit 5 is set for all bytes which are to be overwritten by
68 * the DMA engine.
69 *
70 * The remaining bits are the inverse of a counter which increments by
71 * one for each byte address.
72 */
73#define PATTERN_SRC 0x80
74#define PATTERN_DST 0x00
75#define PATTERN_COPY 0x40
76#define PATTERN_OVERWRITE 0x20
77#define PATTERN_COUNT_MASK 0x1f
78
79struct dmatest_thread {
80 struct list_head node;
81 struct task_struct *task;
82 struct dma_chan *chan;
Dan Williamsb54d5cb2009-03-25 09:13:25 -070083 u8 **srcs;
84 u8 **dsts;
85 enum dma_transaction_type type;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070086};
87
88struct dmatest_chan {
89 struct list_head node;
90 struct dma_chan *chan;
91 struct list_head threads;
92};
93
94/*
95 * These are protected by dma_list_mutex since they're only used by
Dan Williams33df8ca2009-01-06 11:38:15 -070096 * the DMA filter function callback
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070097 */
98static LIST_HEAD(dmatest_channels);
99static unsigned int nr_channels;
100
101static bool dmatest_match_channel(struct dma_chan *chan)
102{
103 if (test_channel[0] == '\0')
104 return true;
Dan Williams41d5e592009-01-06 11:38:21 -0700105 return strcmp(dma_chan_name(chan), test_channel) == 0;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700106}
107
108static bool dmatest_match_device(struct dma_device *device)
109{
110 if (test_device[0] == '\0')
111 return true;
Kay Sievers06190d82008-11-11 13:12:33 -0700112 return strcmp(dev_name(device->dev), test_device) == 0;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700113}
114
115static unsigned long dmatest_random(void)
116{
117 unsigned long buf;
118
119 get_random_bytes(&buf, sizeof(buf));
120 return buf;
121}
122
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700123static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700124{
125 unsigned int i;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700126 u8 *buf;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700127
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700128 for (; (buf = *bufs); bufs++) {
129 for (i = 0; i < start; i++)
130 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
131 for ( ; i < start + len; i++)
132 buf[i] = PATTERN_SRC | PATTERN_COPY
Joe Perchesc0198942009-06-28 09:26:21 -0700133 | (~i & PATTERN_COUNT_MASK);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700134 for ( ; i < test_buf_size; i++)
135 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
136 buf++;
137 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700138}
139
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700140static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700141{
142 unsigned int i;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700143 u8 *buf;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700144
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700145 for (; (buf = *bufs); bufs++) {
146 for (i = 0; i < start; i++)
147 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
148 for ( ; i < start + len; i++)
149 buf[i] = PATTERN_DST | PATTERN_OVERWRITE
150 | (~i & PATTERN_COUNT_MASK);
151 for ( ; i < test_buf_size; i++)
152 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
153 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700154}
155
156static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
157 unsigned int counter, bool is_srcbuf)
158{
159 u8 diff = actual ^ pattern;
160 u8 expected = pattern | (~counter & PATTERN_COUNT_MASK);
161 const char *thread_name = current->comm;
162
163 if (is_srcbuf)
164 pr_warning("%s: srcbuf[0x%x] overwritten!"
165 " Expected %02x, got %02x\n",
166 thread_name, index, expected, actual);
167 else if ((pattern & PATTERN_COPY)
168 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
169 pr_warning("%s: dstbuf[0x%x] not copied!"
170 " Expected %02x, got %02x\n",
171 thread_name, index, expected, actual);
172 else if (diff & PATTERN_SRC)
173 pr_warning("%s: dstbuf[0x%x] was copied!"
174 " Expected %02x, got %02x\n",
175 thread_name, index, expected, actual);
176 else
177 pr_warning("%s: dstbuf[0x%x] mismatch!"
178 " Expected %02x, got %02x\n",
179 thread_name, index, expected, actual);
180}
181
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700182static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700183 unsigned int end, unsigned int counter, u8 pattern,
184 bool is_srcbuf)
185{
186 unsigned int i;
187 unsigned int error_count = 0;
188 u8 actual;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700189 u8 expected;
190 u8 *buf;
191 unsigned int counter_orig = counter;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700192
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700193 for (; (buf = *bufs); bufs++) {
194 counter = counter_orig;
195 for (i = start; i < end; i++) {
196 actual = buf[i];
197 expected = pattern | (~counter & PATTERN_COUNT_MASK);
198 if (actual != expected) {
199 if (error_count < 32)
200 dmatest_mismatch(actual, pattern, i,
201 counter, is_srcbuf);
202 error_count++;
203 }
204 counter++;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700205 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700206 }
207
208 if (error_count > 32)
209 pr_warning("%s: %u errors suppressed\n",
210 current->comm, error_count - 32);
211
212 return error_count;
213}
214
Dan Williamse44e0aa2009-03-25 09:13:25 -0700215static void dmatest_callback(void *completion)
216{
217 complete(completion);
218}
219
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700220/*
221 * This function repeatedly tests DMA transfers of various lengths and
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700222 * offsets for a given operation type until it is told to exit by
223 * kthread_stop(). There may be multiple threads running this function
224 * in parallel for a single channel, and there may be multiple channels
225 * being tested in parallel.
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700226 *
227 * Before each test, the source and destination buffer is initialized
228 * with a known pattern. This pattern is different depending on
229 * whether it's in an area which is supposed to be copied or
230 * overwritten, and different in the source and destination buffers.
231 * So if the DMA engine doesn't copy exactly what we tell it to copy,
232 * we'll notice.
233 */
234static int dmatest_func(void *data)
235{
236 struct dmatest_thread *thread = data;
237 struct dma_chan *chan;
238 const char *thread_name;
239 unsigned int src_off, dst_off, len;
240 unsigned int error_count;
241 unsigned int failed_tests = 0;
242 unsigned int total_tests = 0;
243 dma_cookie_t cookie;
244 enum dma_status status;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700245 enum dma_ctrl_flags flags;
Anatolij Gustschin94de6482010-02-15 22:35:23 +0100246 u8 pq_coefs[pq_sources + 1];
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700247 int ret;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700248 int src_cnt;
249 int dst_cnt;
250 int i;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700251
252 thread_name = current->comm;
253
254 ret = -ENOMEM;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700255
256 smp_rmb();
257 chan = thread->chan;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700258 if (thread->type == DMA_MEMCPY)
259 src_cnt = dst_cnt = 1;
260 else if (thread->type == DMA_XOR) {
261 src_cnt = xor_sources | 1; /* force odd to ensure dst = src */
262 dst_cnt = 1;
Dan Williams58691d62009-08-29 19:09:27 -0700263 } else if (thread->type == DMA_PQ) {
264 src_cnt = pq_sources | 1; /* force odd to ensure dst = src */
265 dst_cnt = 2;
Anatolij Gustschin94de6482010-02-15 22:35:23 +0100266 for (i = 0; i < src_cnt; i++)
Dan Williams58691d62009-08-29 19:09:27 -0700267 pq_coefs[i] = 1;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700268 } else
269 goto err_srcs;
270
271 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
272 if (!thread->srcs)
273 goto err_srcs;
274 for (i = 0; i < src_cnt; i++) {
275 thread->srcs[i] = kmalloc(test_buf_size, GFP_KERNEL);
276 if (!thread->srcs[i])
277 goto err_srcbuf;
278 }
279 thread->srcs[i] = NULL;
280
281 thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
282 if (!thread->dsts)
283 goto err_dsts;
284 for (i = 0; i < dst_cnt; i++) {
285 thread->dsts[i] = kmalloc(test_buf_size, GFP_KERNEL);
286 if (!thread->dsts[i])
287 goto err_dstbuf;
288 }
289 thread->dsts[i] = NULL;
290
Dan Williamse44e0aa2009-03-25 09:13:25 -0700291 set_user_nice(current, 10);
292
Ira Snyderb203bd32011-03-03 07:54:53 +0000293 /*
294 * src buffers are freed by the DMAEngine code with dma_unmap_single()
295 * dst buffers are freed by ourselves below
296 */
297 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT
298 | DMA_COMPL_SKIP_DEST_UNMAP | DMA_COMPL_SRC_UNMAP_SINGLE;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700299
Nicolas Ferre0a2ff57d2009-07-03 19:26:51 +0200300 while (!kthread_should_stop()
301 && !(iterations && total_tests >= iterations)) {
Atsushi Nemotod86be862009-01-13 09:22:20 -0700302 struct dma_device *dev = chan->device;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700303 struct dma_async_tx_descriptor *tx = NULL;
304 dma_addr_t dma_srcs[src_cnt];
305 dma_addr_t dma_dsts[dst_cnt];
Dan Williamse44e0aa2009-03-25 09:13:25 -0700306 struct completion cmp;
Viresh Kumard42efe62011-03-22 17:27:25 +0530307 unsigned long tmo = msecs_to_jiffies(timeout);
Dan Williams83544ae2009-09-08 17:42:53 -0700308 u8 align = 0;
Atsushi Nemotod86be862009-01-13 09:22:20 -0700309
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700310 total_tests++;
311
Dan Williams83544ae2009-09-08 17:42:53 -0700312 /* honor alignment restrictions */
313 if (thread->type == DMA_MEMCPY)
314 align = dev->copy_align;
315 else if (thread->type == DMA_XOR)
316 align = dev->xor_align;
317 else if (thread->type == DMA_PQ)
318 align = dev->pq_align;
319
Guennadi Liakhovetskicfe4f272009-12-04 19:44:48 +0100320 if (1 << align > test_buf_size) {
321 pr_err("%u-byte buffer too small for %d-byte alignment\n",
322 test_buf_size, 1 << align);
323 break;
324 }
325
326 len = dmatest_random() % test_buf_size + 1;
Dan Williams83544ae2009-09-08 17:42:53 -0700327 len = (len >> align) << align;
Guennadi Liakhovetskicfe4f272009-12-04 19:44:48 +0100328 if (!len)
329 len = 1 << align;
330 src_off = dmatest_random() % (test_buf_size - len + 1);
331 dst_off = dmatest_random() % (test_buf_size - len + 1);
332
Dan Williams83544ae2009-09-08 17:42:53 -0700333 src_off = (src_off >> align) << align;
334 dst_off = (dst_off >> align) << align;
335
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700336 dmatest_init_srcs(thread->srcs, src_off, len);
337 dmatest_init_dsts(thread->dsts, dst_off, len);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700338
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700339 for (i = 0; i < src_cnt; i++) {
340 u8 *buf = thread->srcs[i] + src_off;
341
342 dma_srcs[i] = dma_map_single(dev->dev, buf, len,
343 DMA_TO_DEVICE);
344 }
Atsushi Nemotod86be862009-01-13 09:22:20 -0700345 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700346 for (i = 0; i < dst_cnt; i++) {
347 dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i],
348 test_buf_size,
349 DMA_BIDIRECTIONAL);
350 }
Atsushi Nemotod86be862009-01-13 09:22:20 -0700351
Dan Williams83544ae2009-09-08 17:42:53 -0700352
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700353 if (thread->type == DMA_MEMCPY)
354 tx = dev->device_prep_dma_memcpy(chan,
355 dma_dsts[0] + dst_off,
356 dma_srcs[0], len,
357 flags);
358 else if (thread->type == DMA_XOR)
359 tx = dev->device_prep_dma_xor(chan,
360 dma_dsts[0] + dst_off,
Dan Williams67b91242010-02-28 22:20:18 -0700361 dma_srcs, src_cnt,
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700362 len, flags);
Dan Williams58691d62009-08-29 19:09:27 -0700363 else if (thread->type == DMA_PQ) {
364 dma_addr_t dma_pq[dst_cnt];
365
366 for (i = 0; i < dst_cnt; i++)
367 dma_pq[i] = dma_dsts[i] + dst_off;
368 tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs,
Anatolij Gustschin94de6482010-02-15 22:35:23 +0100369 src_cnt, pq_coefs,
Dan Williams58691d62009-08-29 19:09:27 -0700370 len, flags);
371 }
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700372
Atsushi Nemotod86be862009-01-13 09:22:20 -0700373 if (!tx) {
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700374 for (i = 0; i < src_cnt; i++)
375 dma_unmap_single(dev->dev, dma_srcs[i], len,
376 DMA_TO_DEVICE);
377 for (i = 0; i < dst_cnt; i++)
378 dma_unmap_single(dev->dev, dma_dsts[i],
379 test_buf_size,
380 DMA_BIDIRECTIONAL);
Atsushi Nemotod86be862009-01-13 09:22:20 -0700381 pr_warning("%s: #%u: prep error with src_off=0x%x "
382 "dst_off=0x%x len=0x%x\n",
383 thread_name, total_tests - 1,
384 src_off, dst_off, len);
385 msleep(100);
386 failed_tests++;
387 continue;
388 }
Dan Williamse44e0aa2009-03-25 09:13:25 -0700389
390 init_completion(&cmp);
391 tx->callback = dmatest_callback;
392 tx->callback_param = &cmp;
Atsushi Nemotod86be862009-01-13 09:22:20 -0700393 cookie = tx->tx_submit(tx);
394
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700395 if (dma_submit_error(cookie)) {
396 pr_warning("%s: #%u: submit error %d with src_off=0x%x "
397 "dst_off=0x%x len=0x%x\n",
398 thread_name, total_tests - 1, cookie,
399 src_off, dst_off, len);
400 msleep(100);
401 failed_tests++;
402 continue;
403 }
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700404 dma_async_issue_pending(chan);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700405
Dan Williamse44e0aa2009-03-25 09:13:25 -0700406 tmo = wait_for_completion_timeout(&cmp, tmo);
407 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700408
Dan Williamse44e0aa2009-03-25 09:13:25 -0700409 if (tmo == 0) {
410 pr_warning("%s: #%u: test timed out\n",
411 thread_name, total_tests - 1);
412 failed_tests++;
413 continue;
414 } else if (status != DMA_SUCCESS) {
415 pr_warning("%s: #%u: got completion callback,"
416 " but status is \'%s\'\n",
417 thread_name, total_tests - 1,
418 status == DMA_ERROR ? "error" : "in progress");
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700419 failed_tests++;
420 continue;
421 }
Dan Williamse44e0aa2009-03-25 09:13:25 -0700422
Atsushi Nemotod86be862009-01-13 09:22:20 -0700423 /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700424 for (i = 0; i < dst_cnt; i++)
425 dma_unmap_single(dev->dev, dma_dsts[i], test_buf_size,
426 DMA_BIDIRECTIONAL);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700427
428 error_count = 0;
429
430 pr_debug("%s: verifying source buffer...\n", thread_name);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700431 error_count += dmatest_verify(thread->srcs, 0, src_off,
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700432 0, PATTERN_SRC, true);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700433 error_count += dmatest_verify(thread->srcs, src_off,
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700434 src_off + len, src_off,
435 PATTERN_SRC | PATTERN_COPY, true);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700436 error_count += dmatest_verify(thread->srcs, src_off + len,
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700437 test_buf_size, src_off + len,
438 PATTERN_SRC, true);
439
440 pr_debug("%s: verifying dest buffer...\n",
441 thread->task->comm);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700442 error_count += dmatest_verify(thread->dsts, 0, dst_off,
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700443 0, PATTERN_DST, false);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700444 error_count += dmatest_verify(thread->dsts, dst_off,
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700445 dst_off + len, src_off,
446 PATTERN_SRC | PATTERN_COPY, false);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700447 error_count += dmatest_verify(thread->dsts, dst_off + len,
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700448 test_buf_size, dst_off + len,
449 PATTERN_DST, false);
450
451 if (error_count) {
452 pr_warning("%s: #%u: %u errors with "
453 "src_off=0x%x dst_off=0x%x len=0x%x\n",
454 thread_name, total_tests - 1, error_count,
455 src_off, dst_off, len);
456 failed_tests++;
457 } else {
458 pr_debug("%s: #%u: No errors with "
459 "src_off=0x%x dst_off=0x%x len=0x%x\n",
460 thread_name, total_tests - 1,
461 src_off, dst_off, len);
462 }
463 }
464
465 ret = 0;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700466 for (i = 0; thread->dsts[i]; i++)
467 kfree(thread->dsts[i]);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700468err_dstbuf:
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700469 kfree(thread->dsts);
470err_dsts:
471 for (i = 0; thread->srcs[i]; i++)
472 kfree(thread->srcs[i]);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700473err_srcbuf:
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700474 kfree(thread->srcs);
475err_srcs:
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700476 pr_notice("%s: terminating after %u tests, %u failures (status %d)\n",
477 thread_name, total_tests, failed_tests, ret);
Nicolas Ferre0a2ff57d2009-07-03 19:26:51 +0200478
479 if (iterations > 0)
480 while (!kthread_should_stop()) {
Yong Zhangb953df72010-02-05 21:52:37 +0800481 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit);
Nicolas Ferre0a2ff57d2009-07-03 19:26:51 +0200482 interruptible_sleep_on(&wait_dmatest_exit);
483 }
484
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700485 return ret;
486}
487
488static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
489{
490 struct dmatest_thread *thread;
491 struct dmatest_thread *_thread;
492 int ret;
493
494 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
495 ret = kthread_stop(thread->task);
496 pr_debug("dmatest: thread %s exited with status %d\n",
497 thread->task->comm, ret);
498 list_del(&thread->node);
499 kfree(thread);
500 }
501 kfree(dtc);
502}
503
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700504static int dmatest_add_threads(struct dmatest_chan *dtc, enum dma_transaction_type type)
505{
506 struct dmatest_thread *thread;
507 struct dma_chan *chan = dtc->chan;
508 char *op;
509 unsigned int i;
510
511 if (type == DMA_MEMCPY)
512 op = "copy";
513 else if (type == DMA_XOR)
514 op = "xor";
Dan Williams58691d62009-08-29 19:09:27 -0700515 else if (type == DMA_PQ)
516 op = "pq";
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700517 else
518 return -EINVAL;
519
520 for (i = 0; i < threads_per_chan; i++) {
521 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
522 if (!thread) {
523 pr_warning("dmatest: No memory for %s-%s%u\n",
524 dma_chan_name(chan), op, i);
525
526 break;
527 }
528 thread->chan = dtc->chan;
529 thread->type = type;
530 smp_wmb();
531 thread->task = kthread_run(dmatest_func, thread, "%s-%s%u",
532 dma_chan_name(chan), op, i);
533 if (IS_ERR(thread->task)) {
534 pr_warning("dmatest: Failed to run thread %s-%s%u\n",
535 dma_chan_name(chan), op, i);
536 kfree(thread);
537 break;
538 }
539
540 /* srcbuf and dstbuf are allocated by the thread itself */
541
542 list_add_tail(&thread->node, &dtc->threads);
543 }
544
545 return i;
546}
547
Dan Williams33df8ca2009-01-06 11:38:15 -0700548static int dmatest_add_channel(struct dma_chan *chan)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700549{
550 struct dmatest_chan *dtc;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700551 struct dma_device *dma_dev = chan->device;
552 unsigned int thread_count = 0;
Kulikov Vasiliyb9033e62010-07-17 19:19:48 +0400553 int cnt;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700554
Andrew Morton6fdb8bd2008-09-19 04:16:23 -0700555 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700556 if (!dtc) {
Dan Williams41d5e592009-01-06 11:38:21 -0700557 pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan));
Dan Williams33df8ca2009-01-06 11:38:15 -0700558 return -ENOMEM;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700559 }
560
561 dtc->chan = chan;
562 INIT_LIST_HEAD(&dtc->threads);
563
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700564 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
565 cnt = dmatest_add_threads(dtc, DMA_MEMCPY);
Nicolas Ferref1aef8b2009-07-06 18:19:44 +0200566 thread_count += cnt > 0 ? cnt : 0;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700567 }
568 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
569 cnt = dmatest_add_threads(dtc, DMA_XOR);
Nicolas Ferref1aef8b2009-07-06 18:19:44 +0200570 thread_count += cnt > 0 ? cnt : 0;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700571 }
Dan Williams58691d62009-08-29 19:09:27 -0700572 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
573 cnt = dmatest_add_threads(dtc, DMA_PQ);
574 thread_count += cnt > 0 ?: 0;
575 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700576
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700577 pr_info("dmatest: Started %u threads using %s\n",
578 thread_count, dma_chan_name(chan));
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700579
580 list_add_tail(&dtc->node, &dmatest_channels);
581 nr_channels++;
582
Dan Williams33df8ca2009-01-06 11:38:15 -0700583 return 0;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700584}
585
Dan Williams7dd60252009-01-06 11:38:19 -0700586static bool filter(struct dma_chan *chan, void *param)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700587{
Dan Williams33df8ca2009-01-06 11:38:15 -0700588 if (!dmatest_match_channel(chan) || !dmatest_match_device(chan->device))
Dan Williams7dd60252009-01-06 11:38:19 -0700589 return false;
Dan Williams33df8ca2009-01-06 11:38:15 -0700590 else
Dan Williams7dd60252009-01-06 11:38:19 -0700591 return true;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700592}
593
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700594static int __init dmatest_init(void)
595{
Dan Williams33df8ca2009-01-06 11:38:15 -0700596 dma_cap_mask_t mask;
597 struct dma_chan *chan;
598 int err = 0;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700599
Dan Williams33df8ca2009-01-06 11:38:15 -0700600 dma_cap_zero(mask);
601 dma_cap_set(DMA_MEMCPY, mask);
602 for (;;) {
603 chan = dma_request_channel(mask, filter, NULL);
604 if (chan) {
605 err = dmatest_add_channel(chan);
Dan Williamsc56c81a2009-04-08 15:08:23 -0700606 if (err) {
Dan Williams33df8ca2009-01-06 11:38:15 -0700607 dma_release_channel(chan);
608 break; /* add_channel failed, punt */
609 }
610 } else
611 break; /* no more channels available */
612 if (max_channels && nr_channels >= max_channels)
613 break; /* we have all we need */
614 }
615
616 return err;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700617}
Dan Williams33df8ca2009-01-06 11:38:15 -0700618/* when compiled-in wait for drivers to load first */
619late_initcall(dmatest_init);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700620
621static void __exit dmatest_exit(void)
622{
Dan Williams33df8ca2009-01-06 11:38:15 -0700623 struct dmatest_chan *dtc, *_dtc;
Dan Williams7cbd4872009-03-04 16:06:03 -0700624 struct dma_chan *chan;
Dan Williams33df8ca2009-01-06 11:38:15 -0700625
626 list_for_each_entry_safe(dtc, _dtc, &dmatest_channels, node) {
627 list_del(&dtc->node);
Dan Williams7cbd4872009-03-04 16:06:03 -0700628 chan = dtc->chan;
Dan Williams33df8ca2009-01-06 11:38:15 -0700629 dmatest_cleanup_channel(dtc);
630 pr_debug("dmatest: dropped channel %s\n",
Dan Williams7cbd4872009-03-04 16:06:03 -0700631 dma_chan_name(chan));
632 dma_release_channel(chan);
Dan Williams33df8ca2009-01-06 11:38:15 -0700633 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700634}
635module_exit(dmatest_exit);
636
Jean Delvaree05503e2011-05-18 16:49:24 +0200637MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700638MODULE_LICENSE("GPL v2");