blob: e0bd578a253afe624678499cc87ffa5d33741b77 [file] [log] [blame]
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -07001/*
2 * DMA Engine test module
3 *
4 * Copyright (C) 2007 Atmel Corporation
Andy Shevchenko851b7e12013-03-04 11:09:30 +02005 * Copyright (C) 2013 Intel Corporation
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -07006 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
Dan Williams872f05c2013-11-06 16:29:58 -080011#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070013#include <linux/delay.h>
Alexey Dobriyanb7f080c2011-06-16 11:01:34 +000014#include <linux/dma-mapping.h>
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070015#include <linux/dmaengine.h>
Guennadi Liakhovetski981ed702011-08-18 16:50:51 +020016#include <linux/freezer.h>
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070017#include <linux/init.h>
18#include <linux/kthread.h>
19#include <linux/module.h>
20#include <linux/moduleparam.h>
21#include <linux/random.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090022#include <linux/slab.h>
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070023#include <linux/wait.h>
24
25static unsigned int test_buf_size = 16384;
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030026module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070027MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
28
Kay Sievers06190d82008-11-11 13:12:33 -070029static char test_channel[20];
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030030module_param_string(channel, test_channel, sizeof(test_channel),
31 S_IRUGO | S_IWUSR);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070032MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
33
Guennadi Liakhovetskia85159f2013-12-30 14:58:04 +010034static char test_device[32];
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030035module_param_string(device, test_device, sizeof(test_device),
36 S_IRUGO | S_IWUSR);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070037MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
38
39static unsigned int threads_per_chan = 1;
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030040module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070041MODULE_PARM_DESC(threads_per_chan,
42 "Number of threads to start per channel (default: 1)");
43
44static unsigned int max_channels;
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030045module_param(max_channels, uint, S_IRUGO | S_IWUSR);
Dan Williams33df8ca2009-01-06 11:38:15 -070046MODULE_PARM_DESC(max_channels,
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070047 "Maximum number of channels to use (default: all)");
48
Nicolas Ferre0a2ff57d2009-07-03 19:26:51 +020049static unsigned int iterations;
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030050module_param(iterations, uint, S_IRUGO | S_IWUSR);
Nicolas Ferre0a2ff57d2009-07-03 19:26:51 +020051MODULE_PARM_DESC(iterations,
52 "Iterations before stopping test (default: infinite)");
53
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +053054static unsigned int sg_buffers = 1;
55module_param(sg_buffers, uint, S_IRUGO | S_IWUSR);
56MODULE_PARM_DESC(sg_buffers,
57 "Number of scatter gather buffers (default: 1)");
58
Eugeniy Paltsevd8646722016-09-14 20:40:38 +030059static unsigned int dmatest;
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +053060module_param(dmatest, uint, S_IRUGO | S_IWUSR);
61MODULE_PARM_DESC(dmatest,
Eugeniy Paltsevd8646722016-09-14 20:40:38 +030062 "dmatest 0-memcpy 1-slave_sg (default: 0)");
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +053063
Dan Williamsb54d5cb2009-03-25 09:13:25 -070064static unsigned int xor_sources = 3;
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030065module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
Dan Williamsb54d5cb2009-03-25 09:13:25 -070066MODULE_PARM_DESC(xor_sources,
67 "Number of xor source buffers (default: 3)");
68
Dan Williams58691d62009-08-29 19:09:27 -070069static unsigned int pq_sources = 3;
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030070module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
Dan Williams58691d62009-08-29 19:09:27 -070071MODULE_PARM_DESC(pq_sources,
72 "Number of p+q source buffers (default: 3)");
73
Viresh Kumard42efe62011-03-22 17:27:25 +053074static int timeout = 3000;
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030075module_param(timeout, uint, S_IRUGO | S_IWUSR);
Joe Perches85ee7a12011-04-23 20:38:19 -070076MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
77 "Pass -1 for infinite timeout");
Viresh Kumard42efe62011-03-22 17:27:25 +053078
Dan Williamse3b9c342013-11-06 16:30:05 -080079static bool noverify;
80module_param(noverify, bool, S_IRUGO | S_IWUSR);
81MODULE_PARM_DESC(noverify, "Disable random data setup and verification");
Andy Shevchenko74b5c072013-03-04 11:09:32 +020082
Dan Williams50137a72013-11-08 12:26:26 -080083static bool verbose;
84module_param(verbose, bool, S_IRUGO | S_IWUSR);
85MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070086
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +020087/**
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +020088 * struct dmatest_params - test parameters.
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +020089 * @buf_size: size of the memcpy test buffer
90 * @channel: bus ID of the channel to test
91 * @device: bus ID of the DMA Engine to test
92 * @threads_per_chan: number of threads to start per channel
93 * @max_channels: maximum number of channels to use
94 * @iterations: iterations before stopping test
95 * @xor_sources: number of xor source buffers
96 * @pq_sources: number of p+q source buffers
97 * @timeout: transfer timeout in msec, -1 for infinite timeout
98 */
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +020099struct dmatest_params {
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200100 unsigned int buf_size;
101 char channel[20];
Guennadi Liakhovetskia85159f2013-12-30 14:58:04 +0100102 char device[32];
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200103 unsigned int threads_per_chan;
104 unsigned int max_channels;
105 unsigned int iterations;
106 unsigned int xor_sources;
107 unsigned int pq_sources;
108 int timeout;
Dan Williamse3b9c342013-11-06 16:30:05 -0800109 bool noverify;
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200110};
111
112/**
113 * struct dmatest_info - test information.
114 * @params: test parameters
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200115 * @lock: access protection to the fields of this structure
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200116 */
Dan Williamsa310d032013-11-06 16:30:01 -0800117static struct dmatest_info {
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200118 /* Test parameters */
119 struct dmatest_params params;
Andy Shevchenko838cc702013-03-04 11:09:28 +0200120
121 /* Internal state */
122 struct list_head channels;
123 unsigned int nr_channels;
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200124 struct mutex lock;
Dan Williamsa310d032013-11-06 16:30:01 -0800125 bool did_init;
126} test_info = {
127 .channels = LIST_HEAD_INIT(test_info.channels),
128 .lock = __MUTEX_INITIALIZER(test_info.lock),
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200129};
130
Dan Williamsa310d032013-11-06 16:30:01 -0800131static int dmatest_run_set(const char *val, const struct kernel_param *kp);
132static int dmatest_run_get(char *val, const struct kernel_param *kp);
Luis R. Rodriguez9c278472015-05-27 11:09:38 +0930133static const struct kernel_param_ops run_ops = {
Dan Williamsa310d032013-11-06 16:30:01 -0800134 .set = dmatest_run_set,
135 .get = dmatest_run_get,
136};
137static bool dmatest_run;
138module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
139MODULE_PARM_DESC(run, "Run the test (default: false)");
140
141/* Maximum amount of mismatched bytes in buffer to print */
142#define MAX_ERROR_COUNT 32
143
144/*
145 * Initialization patterns. All bytes in the source buffer has bit 7
146 * set, all bytes in the destination buffer has bit 7 cleared.
147 *
148 * Bit 6 is set for all bytes which are to be copied by the DMA
149 * engine. Bit 5 is set for all bytes which are to be overwritten by
150 * the DMA engine.
151 *
152 * The remaining bits are the inverse of a counter which increments by
153 * one for each byte address.
154 */
155#define PATTERN_SRC 0x80
156#define PATTERN_DST 0x00
157#define PATTERN_COPY 0x40
158#define PATTERN_OVERWRITE 0x20
159#define PATTERN_COUNT_MASK 0x1f
160
Adam Wallis679dbea2017-11-27 10:45:01 -0500161/* poor man's completion - we want to use wait_event_freezable() on it */
162struct dmatest_done {
163 bool done;
164 wait_queue_head_t *wait;
165};
166
Dan Williamsa310d032013-11-06 16:30:01 -0800167struct dmatest_thread {
168 struct list_head node;
169 struct dmatest_info *info;
170 struct task_struct *task;
171 struct dma_chan *chan;
172 u8 **srcs;
173 u8 **dsts;
174 enum dma_transaction_type type;
Adam Wallis679dbea2017-11-27 10:45:01 -0500175 wait_queue_head_t done_wait;
176 struct dmatest_done test_done;
Dan Williamsa310d032013-11-06 16:30:01 -0800177 bool done;
178};
179
180struct dmatest_chan {
181 struct list_head node;
182 struct dma_chan *chan;
183 struct list_head threads;
184};
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200185
Dan Williams2d88ce72013-11-06 16:30:09 -0800186static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
187static bool wait;
188
189static bool is_threaded_test_run(struct dmatest_info *info)
190{
191 struct dmatest_chan *dtc;
192
193 list_for_each_entry(dtc, &info->channels, node) {
194 struct dmatest_thread *thread;
195
196 list_for_each_entry(thread, &dtc->threads, node) {
197 if (!thread->done)
198 return true;
199 }
200 }
201
202 return false;
203}
204
205static int dmatest_wait_get(char *val, const struct kernel_param *kp)
206{
207 struct dmatest_info *info = &test_info;
208 struct dmatest_params *params = &info->params;
209
210 if (params->iterations)
211 wait_event(thread_wait, !is_threaded_test_run(info));
212 wait = true;
213 return param_get_bool(val, kp);
214}
215
Luis R. Rodriguez9c278472015-05-27 11:09:38 +0930216static const struct kernel_param_ops wait_ops = {
Dan Williams2d88ce72013-11-06 16:30:09 -0800217 .get = dmatest_wait_get,
218 .set = param_set_bool,
219};
220module_param_cb(wait, &wait_ops, &wait, S_IRUGO);
221MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700222
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200223static bool dmatest_match_channel(struct dmatest_params *params,
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200224 struct dma_chan *chan)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700225{
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200226 if (params->channel[0] == '\0')
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700227 return true;
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200228 return strcmp(dma_chan_name(chan), params->channel) == 0;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700229}
230
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200231static bool dmatest_match_device(struct dmatest_params *params,
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200232 struct dma_device *device)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700233{
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200234 if (params->device[0] == '\0')
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700235 return true;
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200236 return strcmp(dev_name(device->dev), params->device) == 0;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700237}
238
239static unsigned long dmatest_random(void)
240{
241 unsigned long buf;
242
Dan Williamsbe9fa5a2013-11-06 16:30:03 -0800243 prandom_bytes(&buf, sizeof(buf));
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700244 return buf;
245}
246
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200247static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
248 unsigned int buf_size)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700249{
250 unsigned int i;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700251 u8 *buf;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700252
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700253 for (; (buf = *bufs); bufs++) {
254 for (i = 0; i < start; i++)
255 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
256 for ( ; i < start + len; i++)
257 buf[i] = PATTERN_SRC | PATTERN_COPY
Joe Perchesc0198942009-06-28 09:26:21 -0700258 | (~i & PATTERN_COUNT_MASK);
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200259 for ( ; i < buf_size; i++)
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700260 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
261 buf++;
262 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700263}
264
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200265static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
266 unsigned int buf_size)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700267{
268 unsigned int i;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700269 u8 *buf;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700270
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700271 for (; (buf = *bufs); bufs++) {
272 for (i = 0; i < start; i++)
273 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
274 for ( ; i < start + len; i++)
275 buf[i] = PATTERN_DST | PATTERN_OVERWRITE
276 | (~i & PATTERN_COUNT_MASK);
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200277 for ( ; i < buf_size; i++)
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700278 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
279 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700280}
281
Dan Williams7b610172013-11-06 16:29:57 -0800282static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
283 unsigned int counter, bool is_srcbuf)
284{
285 u8 diff = actual ^ pattern;
286 u8 expected = pattern | (~counter & PATTERN_COUNT_MASK);
287 const char *thread_name = current->comm;
288
289 if (is_srcbuf)
290 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
291 thread_name, index, expected, actual);
292 else if ((pattern & PATTERN_COPY)
293 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
294 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
295 thread_name, index, expected, actual);
296 else if (diff & PATTERN_SRC)
297 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
298 thread_name, index, expected, actual);
299 else
300 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
301 thread_name, index, expected, actual);
302}
303
304static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
305 unsigned int end, unsigned int counter, u8 pattern,
306 bool is_srcbuf)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700307{
308 unsigned int i;
309 unsigned int error_count = 0;
310 u8 actual;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700311 u8 expected;
312 u8 *buf;
313 unsigned int counter_orig = counter;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700314
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700315 for (; (buf = *bufs); bufs++) {
316 counter = counter_orig;
317 for (i = start; i < end; i++) {
318 actual = buf[i];
319 expected = pattern | (~counter & PATTERN_COUNT_MASK);
320 if (actual != expected) {
Dan Williams7b610172013-11-06 16:29:57 -0800321 if (error_count < MAX_ERROR_COUNT)
322 dmatest_mismatch(actual, pattern, i,
323 counter, is_srcbuf);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700324 error_count++;
325 }
326 counter++;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700327 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700328 }
329
Andy Shevchenko74b5c072013-03-04 11:09:32 +0200330 if (error_count > MAX_ERROR_COUNT)
Dan Williams7b610172013-11-06 16:29:57 -0800331 pr_warn("%s: %u errors suppressed\n",
Andy Shevchenko74b5c072013-03-04 11:09:32 +0200332 current->comm, error_count - MAX_ERROR_COUNT);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700333
334 return error_count;
335}
336
Tejun Heoadfa5432011-11-23 09:28:16 -0800337
338static void dmatest_callback(void *arg)
Dan Williamse44e0aa2009-03-25 09:13:25 -0700339{
Tejun Heoadfa5432011-11-23 09:28:16 -0800340 struct dmatest_done *done = arg;
Adam Wallis679dbea2017-11-27 10:45:01 -0500341 struct dmatest_thread *thread =
342 container_of(arg, struct dmatest_thread, done_wait);
343 if (!thread->done) {
344 done->done = true;
345 wake_up_all(done->wait);
346 } else {
347 /*
348 * If thread->done, it means that this callback occurred
349 * after the parent thread has cleaned up. This can
350 * happen in the case that driver doesn't implement
351 * the terminate_all() functionality and a dma operation
352 * did not occur within the timeout period
353 */
354 WARN(1, "dmatest: Kernel memory may be corrupted!!\n");
355 }
Dan Williamse44e0aa2009-03-25 09:13:25 -0700356}
357
Akinobu Mita8be9e32b2012-10-28 00:49:32 +0900358static unsigned int min_odd(unsigned int x, unsigned int y)
359{
360 unsigned int val = min(x, y);
361
362 return val % 2 ? val : val - 1;
363}
364
Dan Williams872f05c2013-11-06 16:29:58 -0800365static void result(const char *err, unsigned int n, unsigned int src_off,
366 unsigned int dst_off, unsigned int len, unsigned long data)
Andy Shevchenkod86b2f22013-03-04 11:09:34 +0200367{
Jerome Blin2acec152014-03-04 10:38:55 +0100368 pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
Dan Williams872f05c2013-11-06 16:29:58 -0800369 current->comm, n, err, src_off, dst_off, len, data);
Andy Shevchenkod86b2f22013-03-04 11:09:34 +0200370}
371
Dan Williams872f05c2013-11-06 16:29:58 -0800372static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
373 unsigned int dst_off, unsigned int len,
374 unsigned long data)
Andy Shevchenko95019c82013-03-04 11:09:33 +0200375{
Jerome Blin2acec152014-03-04 10:38:55 +0100376 pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
Andy Shevchenkoa835bb82014-10-22 16:16:42 +0300377 current->comm, n, err, src_off, dst_off, len, data);
Andy Shevchenko95019c82013-03-04 11:09:33 +0200378}
379
Andy Shevchenkoa835bb82014-10-22 16:16:42 +0300380#define verbose_result(err, n, src_off, dst_off, len, data) ({ \
381 if (verbose) \
382 result(err, n, src_off, dst_off, len, data); \
383 else \
384 dbg_result(err, n, src_off, dst_off, len, data);\
Dan Williams50137a72013-11-08 12:26:26 -0800385})
386
Dan Williams86727442013-11-06 16:30:07 -0800387static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
Andy Shevchenko95019c82013-03-04 11:09:33 +0200388{
Dan Williams86727442013-11-06 16:30:07 -0800389 unsigned long long per_sec = 1000000;
Andy Shevchenko95019c82013-03-04 11:09:33 +0200390
Dan Williams86727442013-11-06 16:30:07 -0800391 if (runtime <= 0)
392 return 0;
Andy Shevchenko95019c82013-03-04 11:09:33 +0200393
Dan Williams86727442013-11-06 16:30:07 -0800394 /* drop precision until runtime is 32-bits */
395 while (runtime > UINT_MAX) {
396 runtime >>= 1;
397 per_sec <<= 1;
398 }
Andy Shevchenko95019c82013-03-04 11:09:33 +0200399
Dan Williams86727442013-11-06 16:30:07 -0800400 per_sec *= val;
401 do_div(per_sec, runtime);
402 return per_sec;
Andy Shevchenko95019c82013-03-04 11:09:33 +0200403}
404
Dan Williams86727442013-11-06 16:30:07 -0800405static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
Andy Shevchenkod86b2f22013-03-04 11:09:34 +0200406{
Dan Williams86727442013-11-06 16:30:07 -0800407 return dmatest_persec(runtime, len >> 10);
Andy Shevchenko95019c82013-03-04 11:09:33 +0200408}
409
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700410/*
411 * This function repeatedly tests DMA transfers of various lengths and
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700412 * offsets for a given operation type until it is told to exit by
413 * kthread_stop(). There may be multiple threads running this function
414 * in parallel for a single channel, and there may be multiple channels
415 * being tested in parallel.
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700416 *
417 * Before each test, the source and destination buffer is initialized
418 * with a known pattern. This pattern is different depending on
419 * whether it's in an area which is supposed to be copied or
420 * overwritten, and different in the source and destination buffers.
421 * So if the DMA engine doesn't copy exactly what we tell it to copy,
422 * we'll notice.
423 */
424static int dmatest_func(void *data)
425{
426 struct dmatest_thread *thread = data;
Adam Wallis679dbea2017-11-27 10:45:01 -0500427 struct dmatest_done *done = &thread->test_done;
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200428 struct dmatest_info *info;
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200429 struct dmatest_params *params;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700430 struct dma_chan *chan;
Akinobu Mita8be9e32b2012-10-28 00:49:32 +0900431 struct dma_device *dev;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700432 unsigned int error_count;
433 unsigned int failed_tests = 0;
434 unsigned int total_tests = 0;
435 dma_cookie_t cookie;
436 enum dma_status status;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700437 enum dma_ctrl_flags flags;
Andy Shevchenko945b5af2013-03-04 11:09:26 +0200438 u8 *pq_coefs = NULL;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700439 int ret;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700440 int src_cnt;
441 int dst_cnt;
442 int i;
Sinan Kayae9405ef2016-09-01 10:02:55 -0400443 ktime_t ktime, start, diff;
444 ktime_t filltime = ktime_set(0, 0);
445 ktime_t comparetime = ktime_set(0, 0);
Dan Williams86727442013-11-06 16:30:07 -0800446 s64 runtime = 0;
447 unsigned long long total_len = 0;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700448
Tejun Heoadfa5432011-11-23 09:28:16 -0800449 set_freezable();
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700450
451 ret = -ENOMEM;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700452
453 smp_rmb();
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200454 info = thread->info;
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200455 params = &info->params;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700456 chan = thread->chan;
Akinobu Mita8be9e32b2012-10-28 00:49:32 +0900457 dev = chan->device;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700458 if (thread->type == DMA_MEMCPY)
459 src_cnt = dst_cnt = 1;
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530460 else if (thread->type == DMA_SG)
461 src_cnt = dst_cnt = sg_buffers;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700462 else if (thread->type == DMA_XOR) {
Akinobu Mita8be9e32b2012-10-28 00:49:32 +0900463 /* force odd to ensure dst = src */
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200464 src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700465 dst_cnt = 1;
Dan Williams58691d62009-08-29 19:09:27 -0700466 } else if (thread->type == DMA_PQ) {
Akinobu Mita8be9e32b2012-10-28 00:49:32 +0900467 /* force odd to ensure dst = src */
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200468 src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
Dan Williams58691d62009-08-29 19:09:27 -0700469 dst_cnt = 2;
Andy Shevchenko945b5af2013-03-04 11:09:26 +0200470
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200471 pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL);
Andy Shevchenko945b5af2013-03-04 11:09:26 +0200472 if (!pq_coefs)
473 goto err_thread_type;
474
Anatolij Gustschin94de6482010-02-15 22:35:23 +0100475 for (i = 0; i < src_cnt; i++)
Dan Williams58691d62009-08-29 19:09:27 -0700476 pq_coefs[i] = 1;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700477 } else
Andy Shevchenko945b5af2013-03-04 11:09:26 +0200478 goto err_thread_type;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700479
480 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
481 if (!thread->srcs)
482 goto err_srcs;
483 for (i = 0; i < src_cnt; i++) {
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200484 thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700485 if (!thread->srcs[i])
486 goto err_srcbuf;
487 }
488 thread->srcs[i] = NULL;
489
490 thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
491 if (!thread->dsts)
492 goto err_dsts;
493 for (i = 0; i < dst_cnt; i++) {
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200494 thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700495 if (!thread->dsts[i])
496 goto err_dstbuf;
497 }
498 thread->dsts[i] = NULL;
499
Dan Williamse44e0aa2009-03-25 09:13:25 -0700500 set_user_nice(current, 10);
501
Ira Snyderb203bd32011-03-03 07:54:53 +0000502 /*
Bartlomiej Zolnierkiewiczd1cab342013-10-18 19:35:21 +0200503 * src and dst buffers are freed by ourselves below
Ira Snyderb203bd32011-03-03 07:54:53 +0000504 */
Bartlomiej Zolnierkiewicz0776ae72013-10-18 19:35:33 +0200505 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700506
Dan Williams86727442013-11-06 16:30:07 -0800507 ktime = ktime_get();
Nicolas Ferre0a2ff57d2009-07-03 19:26:51 +0200508 while (!kthread_should_stop()
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200509 && !(params->iterations && total_tests >= params->iterations)) {
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700510 struct dma_async_tx_descriptor *tx = NULL;
Dan Williams4076e752013-11-06 16:30:10 -0800511 struct dmaengine_unmap_data *um;
512 dma_addr_t srcs[src_cnt];
513 dma_addr_t *dsts;
Andy Shevchenkoede23a52014-10-22 16:16:43 +0300514 unsigned int src_off, dst_off, len;
Dan Williams83544ae2009-09-08 17:42:53 -0700515 u8 align = 0;
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530516 struct scatterlist tx_sg[src_cnt];
517 struct scatterlist rx_sg[src_cnt];
Atsushi Nemotod86be862009-01-13 09:22:20 -0700518
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700519 total_tests++;
520
Dan Williams83544ae2009-09-08 17:42:53 -0700521 /* honor alignment restrictions */
Nicolin Chenc8a2c192016-09-07 18:24:28 -0700522 if (thread->type == DMA_MEMCPY || thread->type == DMA_SG)
Dan Williams83544ae2009-09-08 17:42:53 -0700523 align = dev->copy_align;
524 else if (thread->type == DMA_XOR)
525 align = dev->xor_align;
526 else if (thread->type == DMA_PQ)
527 align = dev->pq_align;
528
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200529 if (1 << align > params->buf_size) {
Guennadi Liakhovetskicfe4f272009-12-04 19:44:48 +0100530 pr_err("%u-byte buffer too small for %d-byte alignment\n",
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200531 params->buf_size, 1 << align);
Guennadi Liakhovetskicfe4f272009-12-04 19:44:48 +0100532 break;
533 }
534
Andy Shevchenkoede23a52014-10-22 16:16:43 +0300535 if (params->noverify)
Dan Williamse3b9c342013-11-06 16:30:05 -0800536 len = params->buf_size;
Andy Shevchenkoede23a52014-10-22 16:16:43 +0300537 else
538 len = dmatest_random() % params->buf_size + 1;
539
540 len = (len >> align) << align;
541 if (!len)
542 len = 1 << align;
543
544 total_len += len;
545
546 if (params->noverify) {
Dan Williamse3b9c342013-11-06 16:30:05 -0800547 src_off = 0;
548 dst_off = 0;
549 } else {
Sinan Kayae9405ef2016-09-01 10:02:55 -0400550 start = ktime_get();
Dan Williamse3b9c342013-11-06 16:30:05 -0800551 src_off = dmatest_random() % (params->buf_size - len + 1);
552 dst_off = dmatest_random() % (params->buf_size - len + 1);
553
554 src_off = (src_off >> align) << align;
555 dst_off = (dst_off >> align) << align;
556
557 dmatest_init_srcs(thread->srcs, src_off, len,
558 params->buf_size);
559 dmatest_init_dsts(thread->dsts, dst_off, len,
560 params->buf_size);
Sinan Kayae9405ef2016-09-01 10:02:55 -0400561
562 diff = ktime_sub(ktime_get(), start);
563 filltime = ktime_add(filltime, diff);
Dan Williamse3b9c342013-11-06 16:30:05 -0800564 }
565
Dan Williams4076e752013-11-06 16:30:10 -0800566 um = dmaengine_get_unmap_data(dev->dev, src_cnt+dst_cnt,
567 GFP_KERNEL);
568 if (!um) {
569 failed_tests++;
570 result("unmap data NULL", total_tests,
571 src_off, dst_off, len, ret);
572 continue;
573 }
Dan Williams83544ae2009-09-08 17:42:53 -0700574
Dan Williams4076e752013-11-06 16:30:10 -0800575 um->len = params->buf_size;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700576 for (i = 0; i < src_cnt; i++) {
Dan Williams745c00d2013-12-09 11:16:01 -0800577 void *buf = thread->srcs[i];
Dan Williams4076e752013-11-06 16:30:10 -0800578 struct page *pg = virt_to_page(buf);
Dan Williams745c00d2013-12-09 11:16:01 -0800579 unsigned pg_off = (unsigned long) buf & ~PAGE_MASK;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700580
Dan Williams4076e752013-11-06 16:30:10 -0800581 um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
582 um->len, DMA_TO_DEVICE);
583 srcs[i] = um->addr[i] + src_off;
584 ret = dma_mapping_error(dev->dev, um->addr[i]);
Andy Shevchenkoafde3be2012-12-17 15:59:53 -0800585 if (ret) {
Dan Williams4076e752013-11-06 16:30:10 -0800586 dmaengine_unmap_put(um);
Dan Williams872f05c2013-11-06 16:29:58 -0800587 result("src mapping error", total_tests,
588 src_off, dst_off, len, ret);
Andy Shevchenkoafde3be2012-12-17 15:59:53 -0800589 failed_tests++;
590 continue;
591 }
Dan Williams4076e752013-11-06 16:30:10 -0800592 um->to_cnt++;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700593 }
Atsushi Nemotod86be862009-01-13 09:22:20 -0700594 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
Dan Williams4076e752013-11-06 16:30:10 -0800595 dsts = &um->addr[src_cnt];
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700596 for (i = 0; i < dst_cnt; i++) {
Dan Williams745c00d2013-12-09 11:16:01 -0800597 void *buf = thread->dsts[i];
Dan Williams4076e752013-11-06 16:30:10 -0800598 struct page *pg = virt_to_page(buf);
Dan Williams745c00d2013-12-09 11:16:01 -0800599 unsigned pg_off = (unsigned long) buf & ~PAGE_MASK;
Dan Williams4076e752013-11-06 16:30:10 -0800600
601 dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
602 DMA_BIDIRECTIONAL);
603 ret = dma_mapping_error(dev->dev, dsts[i]);
Andy Shevchenkoafde3be2012-12-17 15:59:53 -0800604 if (ret) {
Dan Williams4076e752013-11-06 16:30:10 -0800605 dmaengine_unmap_put(um);
Dan Williams872f05c2013-11-06 16:29:58 -0800606 result("dst mapping error", total_tests,
607 src_off, dst_off, len, ret);
Andy Shevchenkoafde3be2012-12-17 15:59:53 -0800608 failed_tests++;
609 continue;
610 }
Dan Williams4076e752013-11-06 16:30:10 -0800611 um->bidi_cnt++;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700612 }
Atsushi Nemotod86be862009-01-13 09:22:20 -0700613
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530614 sg_init_table(tx_sg, src_cnt);
615 sg_init_table(rx_sg, src_cnt);
616 for (i = 0; i < src_cnt; i++) {
617 sg_dma_address(&rx_sg[i]) = srcs[i];
618 sg_dma_address(&tx_sg[i]) = dsts[i] + dst_off;
619 sg_dma_len(&tx_sg[i]) = len;
620 sg_dma_len(&rx_sg[i]) = len;
621 }
622
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700623 if (thread->type == DMA_MEMCPY)
624 tx = dev->device_prep_dma_memcpy(chan,
Dan Williams4076e752013-11-06 16:30:10 -0800625 dsts[0] + dst_off,
626 srcs[0], len, flags);
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530627 else if (thread->type == DMA_SG)
628 tx = dev->device_prep_dma_sg(chan, tx_sg, src_cnt,
629 rx_sg, src_cnt, flags);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700630 else if (thread->type == DMA_XOR)
631 tx = dev->device_prep_dma_xor(chan,
Dan Williams4076e752013-11-06 16:30:10 -0800632 dsts[0] + dst_off,
633 srcs, src_cnt,
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700634 len, flags);
Dan Williams58691d62009-08-29 19:09:27 -0700635 else if (thread->type == DMA_PQ) {
636 dma_addr_t dma_pq[dst_cnt];
637
638 for (i = 0; i < dst_cnt; i++)
Dan Williams4076e752013-11-06 16:30:10 -0800639 dma_pq[i] = dsts[i] + dst_off;
640 tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
Anatolij Gustschin94de6482010-02-15 22:35:23 +0100641 src_cnt, pq_coefs,
Dan Williams58691d62009-08-29 19:09:27 -0700642 len, flags);
643 }
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700644
Atsushi Nemotod86be862009-01-13 09:22:20 -0700645 if (!tx) {
Dan Williams4076e752013-11-06 16:30:10 -0800646 dmaengine_unmap_put(um);
Dan Williams872f05c2013-11-06 16:29:58 -0800647 result("prep error", total_tests, src_off,
648 dst_off, len, ret);
Atsushi Nemotod86be862009-01-13 09:22:20 -0700649 msleep(100);
650 failed_tests++;
651 continue;
652 }
Dan Williamse44e0aa2009-03-25 09:13:25 -0700653
Adam Wallis679dbea2017-11-27 10:45:01 -0500654 done->done = false;
Dan Williamse44e0aa2009-03-25 09:13:25 -0700655 tx->callback = dmatest_callback;
Adam Wallis679dbea2017-11-27 10:45:01 -0500656 tx->callback_param = done;
Atsushi Nemotod86be862009-01-13 09:22:20 -0700657 cookie = tx->tx_submit(tx);
658
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700659 if (dma_submit_error(cookie)) {
Dan Williams4076e752013-11-06 16:30:10 -0800660 dmaengine_unmap_put(um);
Dan Williams872f05c2013-11-06 16:29:58 -0800661 result("submit error", total_tests, src_off,
662 dst_off, len, ret);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700663 msleep(100);
664 failed_tests++;
665 continue;
666 }
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700667 dma_async_issue_pending(chan);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700668
Adam Wallis679dbea2017-11-27 10:45:01 -0500669 wait_event_freezable_timeout(thread->done_wait, done->done,
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200670 msecs_to_jiffies(params->timeout));
Guennadi Liakhovetski981ed702011-08-18 16:50:51 +0200671
Dan Williamse44e0aa2009-03-25 09:13:25 -0700672 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700673
Adam Wallis679dbea2017-11-27 10:45:01 -0500674 if (!done->done) {
Dan Williams4076e752013-11-06 16:30:10 -0800675 dmaengine_unmap_put(um);
Dan Williams872f05c2013-11-06 16:29:58 -0800676 result("test timed out", total_tests, src_off, dst_off,
677 len, 0);
Dan Williamse44e0aa2009-03-25 09:13:25 -0700678 failed_tests++;
679 continue;
Vinod Koul19e9f992013-10-16 13:37:27 +0530680 } else if (status != DMA_COMPLETE) {
Dan Williams4076e752013-11-06 16:30:10 -0800681 dmaengine_unmap_put(um);
Dan Williams872f05c2013-11-06 16:29:58 -0800682 result(status == DMA_ERROR ?
683 "completion error status" :
684 "completion busy status", total_tests, src_off,
685 dst_off, len, ret);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700686 failed_tests++;
687 continue;
688 }
Dan Williamse44e0aa2009-03-25 09:13:25 -0700689
Dan Williams4076e752013-11-06 16:30:10 -0800690 dmaengine_unmap_put(um);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700691
Dan Williamse3b9c342013-11-06 16:30:05 -0800692 if (params->noverify) {
Dan Williams50137a72013-11-08 12:26:26 -0800693 verbose_result("test passed", total_tests, src_off,
694 dst_off, len, 0);
Dan Williamse3b9c342013-11-06 16:30:05 -0800695 continue;
696 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700697
Sinan Kayae9405ef2016-09-01 10:02:55 -0400698 start = ktime_get();
Dan Williams872f05c2013-11-06 16:29:58 -0800699 pr_debug("%s: verifying source buffer...\n", current->comm);
Dan Williamse3b9c342013-11-06 16:30:05 -0800700 error_count = dmatest_verify(thread->srcs, 0, src_off,
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700701 0, PATTERN_SRC, true);
Dan Williams7b610172013-11-06 16:29:57 -0800702 error_count += dmatest_verify(thread->srcs, src_off,
703 src_off + len, src_off,
704 PATTERN_SRC | PATTERN_COPY, true);
705 error_count += dmatest_verify(thread->srcs, src_off + len,
706 params->buf_size, src_off + len,
707 PATTERN_SRC, true);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700708
Dan Williams872f05c2013-11-06 16:29:58 -0800709 pr_debug("%s: verifying dest buffer...\n", current->comm);
Dan Williams7b610172013-11-06 16:29:57 -0800710 error_count += dmatest_verify(thread->dsts, 0, dst_off,
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700711 0, PATTERN_DST, false);
Dan Williams7b610172013-11-06 16:29:57 -0800712 error_count += dmatest_verify(thread->dsts, dst_off,
713 dst_off + len, src_off,
714 PATTERN_SRC | PATTERN_COPY, false);
715 error_count += dmatest_verify(thread->dsts, dst_off + len,
716 params->buf_size, dst_off + len,
717 PATTERN_DST, false);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700718
Sinan Kayae9405ef2016-09-01 10:02:55 -0400719 diff = ktime_sub(ktime_get(), start);
720 comparetime = ktime_add(comparetime, diff);
721
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700722 if (error_count) {
Dan Williams872f05c2013-11-06 16:29:58 -0800723 result("data error", total_tests, src_off, dst_off,
724 len, error_count);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700725 failed_tests++;
726 } else {
Dan Williams50137a72013-11-08 12:26:26 -0800727 verbose_result("test passed", total_tests, src_off,
728 dst_off, len, 0);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700729 }
730 }
Sinan Kayae9405ef2016-09-01 10:02:55 -0400731 ktime = ktime_sub(ktime_get(), ktime);
732 ktime = ktime_sub(ktime, comparetime);
733 ktime = ktime_sub(ktime, filltime);
734 runtime = ktime_to_us(ktime);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700735
736 ret = 0;
Andy Shevchenko8e1f50d2014-08-22 15:19:44 +0300737err_dstbuf:
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700738 for (i = 0; thread->dsts[i]; i++)
739 kfree(thread->dsts[i]);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700740 kfree(thread->dsts);
741err_dsts:
Andy Shevchenko8e1f50d2014-08-22 15:19:44 +0300742err_srcbuf:
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700743 for (i = 0; thread->srcs[i]; i++)
744 kfree(thread->srcs[i]);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700745 kfree(thread->srcs);
746err_srcs:
Andy Shevchenko945b5af2013-03-04 11:09:26 +0200747 kfree(pq_coefs);
748err_thread_type:
Dan Williams86727442013-11-06 16:30:07 -0800749 pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n",
750 current->comm, total_tests, failed_tests,
751 dmatest_persec(runtime, total_tests),
752 dmatest_KBs(runtime, total_len), ret);
Nicolas Ferre0a2ff57d2009-07-03 19:26:51 +0200753
Viresh Kumar9704efa2011-07-29 16:21:57 +0530754 /* terminate all transfers on specified channels */
Adam Wallis679dbea2017-11-27 10:45:01 -0500755 if (ret || failed_tests)
Shiraz Hashim5e034f72012-11-09 15:26:29 +0000756 dmaengine_terminate_all(chan);
757
Andy Shevchenko3e5ccd82013-03-04 11:09:31 +0200758 thread->done = true;
Dan Williams2d88ce72013-11-06 16:30:09 -0800759 wake_up(&thread_wait);
Nicolas Ferre0a2ff57d2009-07-03 19:26:51 +0200760
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700761 return ret;
762}
763
764static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
765{
766 struct dmatest_thread *thread;
767 struct dmatest_thread *_thread;
768 int ret;
769
770 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
771 ret = kthread_stop(thread->task);
Dan Williams0adff802013-11-06 16:30:00 -0800772 pr_debug("thread %s exited with status %d\n",
773 thread->task->comm, ret);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700774 list_del(&thread->node);
Dan Williams2d88ce72013-11-06 16:30:09 -0800775 put_task_struct(thread->task);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700776 kfree(thread);
777 }
Viresh Kumar9704efa2011-07-29 16:21:57 +0530778
779 /* terminate all transfers on specified channels */
Jon Mason944ea4d2012-11-11 23:03:20 +0000780 dmaengine_terminate_all(dtc->chan);
Viresh Kumar9704efa2011-07-29 16:21:57 +0530781
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700782 kfree(dtc);
783}
784
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200785static int dmatest_add_threads(struct dmatest_info *info,
786 struct dmatest_chan *dtc, enum dma_transaction_type type)
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700787{
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200788 struct dmatest_params *params = &info->params;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700789 struct dmatest_thread *thread;
790 struct dma_chan *chan = dtc->chan;
791 char *op;
792 unsigned int i;
793
794 if (type == DMA_MEMCPY)
795 op = "copy";
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530796 else if (type == DMA_SG)
797 op = "sg";
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700798 else if (type == DMA_XOR)
799 op = "xor";
Dan Williams58691d62009-08-29 19:09:27 -0700800 else if (type == DMA_PQ)
801 op = "pq";
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700802 else
803 return -EINVAL;
804
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200805 for (i = 0; i < params->threads_per_chan; i++) {
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700806 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
807 if (!thread) {
Dan Williams0adff802013-11-06 16:30:00 -0800808 pr_warn("No memory for %s-%s%u\n",
809 dma_chan_name(chan), op, i);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700810 break;
811 }
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200812 thread->info = info;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700813 thread->chan = dtc->chan;
814 thread->type = type;
Adam Wallis679dbea2017-11-27 10:45:01 -0500815 thread->test_done.wait = &thread->done_wait;
816 init_waitqueue_head(&thread->done_wait);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700817 smp_wmb();
Dan Williams2d88ce72013-11-06 16:30:09 -0800818 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700819 dma_chan_name(chan), op, i);
820 if (IS_ERR(thread->task)) {
Dan Williams2d88ce72013-11-06 16:30:09 -0800821 pr_warn("Failed to create thread %s-%s%u\n",
Dan Williams0adff802013-11-06 16:30:00 -0800822 dma_chan_name(chan), op, i);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700823 kfree(thread);
824 break;
825 }
826
827 /* srcbuf and dstbuf are allocated by the thread itself */
Dan Williams2d88ce72013-11-06 16:30:09 -0800828 get_task_struct(thread->task);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700829 list_add_tail(&thread->node, &dtc->threads);
Dan Williams2d88ce72013-11-06 16:30:09 -0800830 wake_up_process(thread->task);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700831 }
832
833 return i;
834}
835
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200836static int dmatest_add_channel(struct dmatest_info *info,
837 struct dma_chan *chan)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700838{
839 struct dmatest_chan *dtc;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700840 struct dma_device *dma_dev = chan->device;
841 unsigned int thread_count = 0;
Kulikov Vasiliyb9033e62010-07-17 19:19:48 +0400842 int cnt;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700843
Andrew Morton6fdb8bd2008-09-19 04:16:23 -0700844 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700845 if (!dtc) {
Dan Williams0adff802013-11-06 16:30:00 -0800846 pr_warn("No memory for %s\n", dma_chan_name(chan));
Dan Williams33df8ca2009-01-06 11:38:15 -0700847 return -ENOMEM;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700848 }
849
850 dtc->chan = chan;
851 INIT_LIST_HEAD(&dtc->threads);
852
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700853 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530854 if (dmatest == 0) {
855 cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
856 thread_count += cnt > 0 ? cnt : 0;
857 }
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700858 }
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530859
860 if (dma_has_cap(DMA_SG, dma_dev->cap_mask)) {
861 if (dmatest == 1) {
862 cnt = dmatest_add_threads(info, dtc, DMA_SG);
863 thread_count += cnt > 0 ? cnt : 0;
864 }
865 }
866
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700867 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200868 cnt = dmatest_add_threads(info, dtc, DMA_XOR);
Nicolas Ferref1aef8b2009-07-06 18:19:44 +0200869 thread_count += cnt > 0 ? cnt : 0;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700870 }
Dan Williams58691d62009-08-29 19:09:27 -0700871 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200872 cnt = dmatest_add_threads(info, dtc, DMA_PQ);
Dr. David Alan Gilbertd07a74a2011-08-25 16:13:55 -0700873 thread_count += cnt > 0 ? cnt : 0;
Dan Williams58691d62009-08-29 19:09:27 -0700874 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700875
Dan Williams0adff802013-11-06 16:30:00 -0800876 pr_info("Started %u threads using %s\n",
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700877 thread_count, dma_chan_name(chan));
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700878
Andy Shevchenko838cc702013-03-04 11:09:28 +0200879 list_add_tail(&dtc->node, &info->channels);
880 info->nr_channels++;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700881
Dan Williams33df8ca2009-01-06 11:38:15 -0700882 return 0;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700883}
884
Dan Williams7dd60252009-01-06 11:38:19 -0700885static bool filter(struct dma_chan *chan, void *param)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700886{
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200887 struct dmatest_params *params = param;
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200888
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200889 if (!dmatest_match_channel(params, chan) ||
890 !dmatest_match_device(params, chan->device))
Dan Williams7dd60252009-01-06 11:38:19 -0700891 return false;
Dan Williams33df8ca2009-01-06 11:38:15 -0700892 else
Dan Williams7dd60252009-01-06 11:38:19 -0700893 return true;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700894}
895
Dan Williamsa9e55492013-11-06 16:30:02 -0800896static void request_channels(struct dmatest_info *info,
897 enum dma_transaction_type type)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700898{
Dan Williams33df8ca2009-01-06 11:38:15 -0700899 dma_cap_mask_t mask;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700900
Dan Williams33df8ca2009-01-06 11:38:15 -0700901 dma_cap_zero(mask);
Dan Williamsa9e55492013-11-06 16:30:02 -0800902 dma_cap_set(type, mask);
Dan Williams33df8ca2009-01-06 11:38:15 -0700903 for (;;) {
Dan Williamsa9e55492013-11-06 16:30:02 -0800904 struct dmatest_params *params = &info->params;
905 struct dma_chan *chan;
906
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200907 chan = dma_request_channel(mask, filter, params);
Dan Williams33df8ca2009-01-06 11:38:15 -0700908 if (chan) {
Dan Williamsa9e55492013-11-06 16:30:02 -0800909 if (dmatest_add_channel(info, chan)) {
Dan Williams33df8ca2009-01-06 11:38:15 -0700910 dma_release_channel(chan);
911 break; /* add_channel failed, punt */
912 }
913 } else
914 break; /* no more channels available */
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200915 if (params->max_channels &&
916 info->nr_channels >= params->max_channels)
Dan Williams33df8ca2009-01-06 11:38:15 -0700917 break; /* we have all we need */
918 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700919}
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700920
Dan Williamsa9e55492013-11-06 16:30:02 -0800921static void run_threaded_test(struct dmatest_info *info)
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200922{
923 struct dmatest_params *params = &info->params;
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200924
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200925 /* Copy test parameters */
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +0300926 params->buf_size = test_buf_size;
927 strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
928 strlcpy(params->device, strim(test_device), sizeof(params->device));
929 params->threads_per_chan = threads_per_chan;
930 params->max_channels = max_channels;
931 params->iterations = iterations;
932 params->xor_sources = xor_sources;
933 params->pq_sources = pq_sources;
934 params->timeout = timeout;
Dan Williamse3b9c342013-11-06 16:30:05 -0800935 params->noverify = noverify;
Dan Williamsa310d032013-11-06 16:30:01 -0800936
Dan Williamsa9e55492013-11-06 16:30:02 -0800937 request_channels(info, DMA_MEMCPY);
938 request_channels(info, DMA_XOR);
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530939 request_channels(info, DMA_SG);
Dan Williamsa9e55492013-11-06 16:30:02 -0800940 request_channels(info, DMA_PQ);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700941}
942
Dan Williamsa310d032013-11-06 16:30:01 -0800943static void stop_threaded_test(struct dmatest_info *info)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700944{
945 struct dmatest_chan *dtc, *_dtc;
946 struct dma_chan *chan;
947
948 list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
949 list_del(&dtc->node);
950 chan = dtc->chan;
951 dmatest_cleanup_channel(dtc);
Dan Williams0adff802013-11-06 16:30:00 -0800952 pr_debug("dropped channel %s\n", dma_chan_name(chan));
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200953 dma_release_channel(chan);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700954 }
Dan Williams33df8ca2009-01-06 11:38:15 -0700955
Dan Williams7cbd4872009-03-04 16:06:03 -0700956 info->nr_channels = 0;
Dan Williams33df8ca2009-01-06 11:38:15 -0700957}
Andy Shevchenko838cc702013-03-04 11:09:28 +0200958
Dan Williamsa9e55492013-11-06 16:30:02 -0800959static void restart_threaded_test(struct dmatest_info *info, bool run)
Dan Williams7cbd4872009-03-04 16:06:03 -0700960{
Dan Williamsa310d032013-11-06 16:30:01 -0800961 /* we might be called early to set run=, defer running until all
962 * parameters have been evaluated
963 */
964 if (!info->did_init)
Dan Williamsa9e55492013-11-06 16:30:02 -0800965 return;
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200966
Dan Williamsa310d032013-11-06 16:30:01 -0800967 /* Stop any running test first */
968 stop_threaded_test(info);
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200969
970 /* Run test with new parameters */
Dan Williamsa9e55492013-11-06 16:30:02 -0800971 run_threaded_test(info);
Andy Shevchenkobcc567e2013-05-23 14:29:53 +0300972}
973
Dan Williamsa310d032013-11-06 16:30:01 -0800974static int dmatest_run_get(char *val, const struct kernel_param *kp)
Andy Shevchenkobcc567e2013-05-23 14:29:53 +0300975{
Dan Williamsa310d032013-11-06 16:30:01 -0800976 struct dmatest_info *info = &test_info;
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200977
978 mutex_lock(&info->lock);
Dan Williamsa310d032013-11-06 16:30:01 -0800979 if (is_threaded_test_run(info)) {
980 dmatest_run = true;
Andy Shevchenko3e5ccd82013-03-04 11:09:31 +0200981 } else {
Dan Williamsa310d032013-11-06 16:30:01 -0800982 stop_threaded_test(info);
983 dmatest_run = false;
Andy Shevchenko3e5ccd82013-03-04 11:09:31 +0200984 }
Dan Williamsa310d032013-11-06 16:30:01 -0800985 mutex_unlock(&info->lock);
986
987 return param_get_bool(val, kp);
988}
989
990static int dmatest_run_set(const char *val, const struct kernel_param *kp)
991{
992 struct dmatest_info *info = &test_info;
993 int ret;
994
995 mutex_lock(&info->lock);
996 ret = param_set_bool(val, kp);
997 if (ret) {
998 mutex_unlock(&info->lock);
999 return ret;
1000 }
1001
1002 if (is_threaded_test_run(info))
1003 ret = -EBUSY;
1004 else if (dmatest_run)
Dan Williamsa9e55492013-11-06 16:30:02 -08001005 restart_threaded_test(info, dmatest_run);
Andy Shevchenko3e5ccd82013-03-04 11:09:31 +02001006
Andy Shevchenko851b7e12013-03-04 11:09:30 +02001007 mutex_unlock(&info->lock);
Andy Shevchenko851b7e12013-03-04 11:09:30 +02001008
Dan Williamsa310d032013-11-06 16:30:01 -08001009 return ret;
Andy Shevchenko851b7e12013-03-04 11:09:30 +02001010}
1011
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +02001012static int __init dmatest_init(void)
1013{
1014 struct dmatest_info *info = &test_info;
Dan Williams2d88ce72013-11-06 16:30:09 -08001015 struct dmatest_params *params = &info->params;
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +02001016
Dan Williamsa310d032013-11-06 16:30:01 -08001017 if (dmatest_run) {
1018 mutex_lock(&info->lock);
Dan Williamsa9e55492013-11-06 16:30:02 -08001019 run_threaded_test(info);
Dan Williamsa310d032013-11-06 16:30:01 -08001020 mutex_unlock(&info->lock);
1021 }
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +02001022
Dan Williams2d88ce72013-11-06 16:30:09 -08001023 if (params->iterations && wait)
1024 wait_event(thread_wait, !is_threaded_test_run(info));
Andy Shevchenko838cc702013-03-04 11:09:28 +02001025
Dan Williamsa310d032013-11-06 16:30:01 -08001026 /* module parameters are stable, inittime tests are started,
1027 * let userspace take over 'run' control
1028 */
1029 info->did_init = true;
Andy Shevchenko95019c82013-03-04 11:09:33 +02001030
Andy Shevchenko851b7e12013-03-04 11:09:30 +02001031 return 0;
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +02001032}
1033/* when compiled-in wait for drivers to load first */
1034late_initcall(dmatest_init);
1035
1036static void __exit dmatest_exit(void)
1037{
1038 struct dmatest_info *info = &test_info;
1039
Dan Williamsa310d032013-11-06 16:30:01 -08001040 mutex_lock(&info->lock);
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +02001041 stop_threaded_test(info);
Dan Williamsa310d032013-11-06 16:30:01 -08001042 mutex_unlock(&info->lock);
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +02001043}
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -07001044module_exit(dmatest_exit);
1045
Jean Delvaree05503e2011-05-18 16:49:24 +02001046MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -07001047MODULE_LICENSE("GPL v2");