blob: 7dd46cf5ed845a764a893eb28a5cd794dcb49948 [file] [log] [blame]
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -07001/*
2 * DMA Engine test module
3 *
4 * Copyright (C) 2007 Atmel Corporation
Andy Shevchenko851b7e12013-03-04 11:09:30 +02005 * Copyright (C) 2013 Intel Corporation
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -07006 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
Dan Williams872f05c2013-11-06 16:29:58 -080011#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070013#include <linux/delay.h>
Alexey Dobriyanb7f080c2011-06-16 11:01:34 +000014#include <linux/dma-mapping.h>
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070015#include <linux/dmaengine.h>
Guennadi Liakhovetski981ed702011-08-18 16:50:51 +020016#include <linux/freezer.h>
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070017#include <linux/init.h>
18#include <linux/kthread.h>
19#include <linux/module.h>
20#include <linux/moduleparam.h>
21#include <linux/random.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090022#include <linux/slab.h>
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070023#include <linux/wait.h>
24
25static unsigned int test_buf_size = 16384;
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030026module_param(test_buf_size, uint, S_IRUGO | S_IWUSR);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070027MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
28
Kay Sievers06190d82008-11-11 13:12:33 -070029static char test_channel[20];
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030030module_param_string(channel, test_channel, sizeof(test_channel),
31 S_IRUGO | S_IWUSR);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070032MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
33
Guennadi Liakhovetskia85159f2013-12-30 14:58:04 +010034static char test_device[32];
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030035module_param_string(device, test_device, sizeof(test_device),
36 S_IRUGO | S_IWUSR);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070037MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
38
39static unsigned int threads_per_chan = 1;
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030040module_param(threads_per_chan, uint, S_IRUGO | S_IWUSR);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070041MODULE_PARM_DESC(threads_per_chan,
42 "Number of threads to start per channel (default: 1)");
43
44static unsigned int max_channels;
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030045module_param(max_channels, uint, S_IRUGO | S_IWUSR);
Dan Williams33df8ca2009-01-06 11:38:15 -070046MODULE_PARM_DESC(max_channels,
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070047 "Maximum number of channels to use (default: all)");
48
Nicolas Ferre0a2ff57d2009-07-03 19:26:51 +020049static unsigned int iterations;
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030050module_param(iterations, uint, S_IRUGO | S_IWUSR);
Nicolas Ferre0a2ff57d2009-07-03 19:26:51 +020051MODULE_PARM_DESC(iterations,
52 "Iterations before stopping test (default: infinite)");
53
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +053054static unsigned int sg_buffers = 1;
55module_param(sg_buffers, uint, S_IRUGO | S_IWUSR);
56MODULE_PARM_DESC(sg_buffers,
57 "Number of scatter gather buffers (default: 1)");
58
Eugeniy Paltsevd8646722016-09-14 20:40:38 +030059static unsigned int dmatest;
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +053060module_param(dmatest, uint, S_IRUGO | S_IWUSR);
61MODULE_PARM_DESC(dmatest,
Eugeniy Paltsevd8646722016-09-14 20:40:38 +030062 "dmatest 0-memcpy 1-slave_sg (default: 0)");
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +053063
Dan Williamsb54d5cb2009-03-25 09:13:25 -070064static unsigned int xor_sources = 3;
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030065module_param(xor_sources, uint, S_IRUGO | S_IWUSR);
Dan Williamsb54d5cb2009-03-25 09:13:25 -070066MODULE_PARM_DESC(xor_sources,
67 "Number of xor source buffers (default: 3)");
68
Dan Williams58691d62009-08-29 19:09:27 -070069static unsigned int pq_sources = 3;
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030070module_param(pq_sources, uint, S_IRUGO | S_IWUSR);
Dan Williams58691d62009-08-29 19:09:27 -070071MODULE_PARM_DESC(pq_sources,
72 "Number of p+q source buffers (default: 3)");
73
Viresh Kumard42efe62011-03-22 17:27:25 +053074static int timeout = 3000;
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +030075module_param(timeout, uint, S_IRUGO | S_IWUSR);
Joe Perches85ee7a12011-04-23 20:38:19 -070076MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), "
77 "Pass -1 for infinite timeout");
Viresh Kumard42efe62011-03-22 17:27:25 +053078
Dan Williamse3b9c342013-11-06 16:30:05 -080079static bool noverify;
80module_param(noverify, bool, S_IRUGO | S_IWUSR);
81MODULE_PARM_DESC(noverify, "Disable random data setup and verification");
Andy Shevchenko74b5c072013-03-04 11:09:32 +020082
Dan Williams50137a72013-11-08 12:26:26 -080083static bool verbose;
84module_param(verbose, bool, S_IRUGO | S_IWUSR);
85MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)");
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -070086
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +020087/**
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +020088 * struct dmatest_params - test parameters.
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +020089 * @buf_size: size of the memcpy test buffer
90 * @channel: bus ID of the channel to test
91 * @device: bus ID of the DMA Engine to test
92 * @threads_per_chan: number of threads to start per channel
93 * @max_channels: maximum number of channels to use
94 * @iterations: iterations before stopping test
95 * @xor_sources: number of xor source buffers
96 * @pq_sources: number of p+q source buffers
97 * @timeout: transfer timeout in msec, -1 for infinite timeout
98 */
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +020099struct dmatest_params {
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200100 unsigned int buf_size;
101 char channel[20];
Guennadi Liakhovetskia85159f2013-12-30 14:58:04 +0100102 char device[32];
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200103 unsigned int threads_per_chan;
104 unsigned int max_channels;
105 unsigned int iterations;
106 unsigned int xor_sources;
107 unsigned int pq_sources;
108 int timeout;
Dan Williamse3b9c342013-11-06 16:30:05 -0800109 bool noverify;
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200110};
111
112/**
113 * struct dmatest_info - test information.
114 * @params: test parameters
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200115 * @lock: access protection to the fields of this structure
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200116 */
Dan Williamsa310d032013-11-06 16:30:01 -0800117static struct dmatest_info {
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200118 /* Test parameters */
119 struct dmatest_params params;
Andy Shevchenko838cc702013-03-04 11:09:28 +0200120
121 /* Internal state */
122 struct list_head channels;
123 unsigned int nr_channels;
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200124 struct mutex lock;
Dan Williamsa310d032013-11-06 16:30:01 -0800125 bool did_init;
126} test_info = {
127 .channels = LIST_HEAD_INIT(test_info.channels),
128 .lock = __MUTEX_INITIALIZER(test_info.lock),
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200129};
130
Dan Williamsa310d032013-11-06 16:30:01 -0800131static int dmatest_run_set(const char *val, const struct kernel_param *kp);
132static int dmatest_run_get(char *val, const struct kernel_param *kp);
Luis R. Rodriguez9c278472015-05-27 11:09:38 +0930133static const struct kernel_param_ops run_ops = {
Dan Williamsa310d032013-11-06 16:30:01 -0800134 .set = dmatest_run_set,
135 .get = dmatest_run_get,
136};
137static bool dmatest_run;
138module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR);
139MODULE_PARM_DESC(run, "Run the test (default: false)");
140
141/* Maximum amount of mismatched bytes in buffer to print */
142#define MAX_ERROR_COUNT 32
143
144/*
145 * Initialization patterns. All bytes in the source buffer has bit 7
146 * set, all bytes in the destination buffer has bit 7 cleared.
147 *
148 * Bit 6 is set for all bytes which are to be copied by the DMA
149 * engine. Bit 5 is set for all bytes which are to be overwritten by
150 * the DMA engine.
151 *
152 * The remaining bits are the inverse of a counter which increments by
153 * one for each byte address.
154 */
155#define PATTERN_SRC 0x80
156#define PATTERN_DST 0x00
157#define PATTERN_COPY 0x40
158#define PATTERN_OVERWRITE 0x20
159#define PATTERN_COUNT_MASK 0x1f
160
Adam Wallis679dbea2017-11-27 10:45:01 -0500161/* poor man's completion - we want to use wait_event_freezable() on it */
162struct dmatest_done {
163 bool done;
164 wait_queue_head_t *wait;
165};
166
Dan Williamsa310d032013-11-06 16:30:01 -0800167struct dmatest_thread {
168 struct list_head node;
169 struct dmatest_info *info;
170 struct task_struct *task;
171 struct dma_chan *chan;
172 u8 **srcs;
173 u8 **dsts;
174 enum dma_transaction_type type;
Adam Wallis679dbea2017-11-27 10:45:01 -0500175 wait_queue_head_t done_wait;
176 struct dmatest_done test_done;
Dan Williamsa310d032013-11-06 16:30:01 -0800177 bool done;
178};
179
180struct dmatest_chan {
181 struct list_head node;
182 struct dma_chan *chan;
183 struct list_head threads;
184};
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200185
Dan Williams2d88ce72013-11-06 16:30:09 -0800186static DECLARE_WAIT_QUEUE_HEAD(thread_wait);
187static bool wait;
188
189static bool is_threaded_test_run(struct dmatest_info *info)
190{
191 struct dmatest_chan *dtc;
192
193 list_for_each_entry(dtc, &info->channels, node) {
194 struct dmatest_thread *thread;
195
196 list_for_each_entry(thread, &dtc->threads, node) {
197 if (!thread->done)
198 return true;
199 }
200 }
201
202 return false;
203}
204
205static int dmatest_wait_get(char *val, const struct kernel_param *kp)
206{
207 struct dmatest_info *info = &test_info;
208 struct dmatest_params *params = &info->params;
209
210 if (params->iterations)
211 wait_event(thread_wait, !is_threaded_test_run(info));
212 wait = true;
213 return param_get_bool(val, kp);
214}
215
Luis R. Rodriguez9c278472015-05-27 11:09:38 +0930216static const struct kernel_param_ops wait_ops = {
Dan Williams2d88ce72013-11-06 16:30:09 -0800217 .get = dmatest_wait_get,
218 .set = param_set_bool,
219};
220module_param_cb(wait, &wait_ops, &wait, S_IRUGO);
221MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)");
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700222
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200223static bool dmatest_match_channel(struct dmatest_params *params,
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200224 struct dma_chan *chan)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700225{
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200226 if (params->channel[0] == '\0')
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700227 return true;
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200228 return strcmp(dma_chan_name(chan), params->channel) == 0;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700229}
230
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200231static bool dmatest_match_device(struct dmatest_params *params,
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200232 struct dma_device *device)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700233{
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200234 if (params->device[0] == '\0')
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700235 return true;
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200236 return strcmp(dev_name(device->dev), params->device) == 0;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700237}
238
239static unsigned long dmatest_random(void)
240{
241 unsigned long buf;
242
Dan Williamsbe9fa5a2013-11-06 16:30:03 -0800243 prandom_bytes(&buf, sizeof(buf));
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700244 return buf;
245}
246
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200247static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len,
248 unsigned int buf_size)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700249{
250 unsigned int i;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700251 u8 *buf;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700252
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700253 for (; (buf = *bufs); bufs++) {
254 for (i = 0; i < start; i++)
255 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
256 for ( ; i < start + len; i++)
257 buf[i] = PATTERN_SRC | PATTERN_COPY
Joe Perchesc0198942009-06-28 09:26:21 -0700258 | (~i & PATTERN_COUNT_MASK);
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200259 for ( ; i < buf_size; i++)
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700260 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
261 buf++;
262 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700263}
264
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200265static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len,
266 unsigned int buf_size)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700267{
268 unsigned int i;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700269 u8 *buf;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700270
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700271 for (; (buf = *bufs); bufs++) {
272 for (i = 0; i < start; i++)
273 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
274 for ( ; i < start + len; i++)
275 buf[i] = PATTERN_DST | PATTERN_OVERWRITE
276 | (~i & PATTERN_COUNT_MASK);
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200277 for ( ; i < buf_size; i++)
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700278 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
279 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700280}
281
Dan Williams7b610172013-11-06 16:29:57 -0800282static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
283 unsigned int counter, bool is_srcbuf)
284{
285 u8 diff = actual ^ pattern;
286 u8 expected = pattern | (~counter & PATTERN_COUNT_MASK);
287 const char *thread_name = current->comm;
288
289 if (is_srcbuf)
290 pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n",
291 thread_name, index, expected, actual);
292 else if ((pattern & PATTERN_COPY)
293 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
294 pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n",
295 thread_name, index, expected, actual);
296 else if (diff & PATTERN_SRC)
297 pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n",
298 thread_name, index, expected, actual);
299 else
300 pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n",
301 thread_name, index, expected, actual);
302}
303
304static unsigned int dmatest_verify(u8 **bufs, unsigned int start,
305 unsigned int end, unsigned int counter, u8 pattern,
306 bool is_srcbuf)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700307{
308 unsigned int i;
309 unsigned int error_count = 0;
310 u8 actual;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700311 u8 expected;
312 u8 *buf;
313 unsigned int counter_orig = counter;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700314
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700315 for (; (buf = *bufs); bufs++) {
316 counter = counter_orig;
317 for (i = start; i < end; i++) {
318 actual = buf[i];
319 expected = pattern | (~counter & PATTERN_COUNT_MASK);
320 if (actual != expected) {
Dan Williams7b610172013-11-06 16:29:57 -0800321 if (error_count < MAX_ERROR_COUNT)
322 dmatest_mismatch(actual, pattern, i,
323 counter, is_srcbuf);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700324 error_count++;
325 }
326 counter++;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700327 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700328 }
329
Andy Shevchenko74b5c072013-03-04 11:09:32 +0200330 if (error_count > MAX_ERROR_COUNT)
Dan Williams7b610172013-11-06 16:29:57 -0800331 pr_warn("%s: %u errors suppressed\n",
Andy Shevchenko74b5c072013-03-04 11:09:32 +0200332 current->comm, error_count - MAX_ERROR_COUNT);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700333
334 return error_count;
335}
336
Tejun Heoadfa5432011-11-23 09:28:16 -0800337
338static void dmatest_callback(void *arg)
Dan Williamse44e0aa2009-03-25 09:13:25 -0700339{
Tejun Heoadfa5432011-11-23 09:28:16 -0800340 struct dmatest_done *done = arg;
Adam Wallis679dbea2017-11-27 10:45:01 -0500341 struct dmatest_thread *thread =
Yang Shunyong297c7cc2018-01-29 14:40:11 +0800342 container_of(done, struct dmatest_thread, test_done);
Adam Wallis679dbea2017-11-27 10:45:01 -0500343 if (!thread->done) {
344 done->done = true;
345 wake_up_all(done->wait);
346 } else {
347 /*
348 * If thread->done, it means that this callback occurred
349 * after the parent thread has cleaned up. This can
350 * happen in the case that driver doesn't implement
351 * the terminate_all() functionality and a dma operation
352 * did not occur within the timeout period
353 */
354 WARN(1, "dmatest: Kernel memory may be corrupted!!\n");
355 }
Dan Williamse44e0aa2009-03-25 09:13:25 -0700356}
357
Akinobu Mita8be9e32b2012-10-28 00:49:32 +0900358static unsigned int min_odd(unsigned int x, unsigned int y)
359{
360 unsigned int val = min(x, y);
361
362 return val % 2 ? val : val - 1;
363}
364
Dan Williams872f05c2013-11-06 16:29:58 -0800365static void result(const char *err, unsigned int n, unsigned int src_off,
366 unsigned int dst_off, unsigned int len, unsigned long data)
Andy Shevchenkod86b2f22013-03-04 11:09:34 +0200367{
Jerome Blin2acec152014-03-04 10:38:55 +0100368 pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
Dan Williams872f05c2013-11-06 16:29:58 -0800369 current->comm, n, err, src_off, dst_off, len, data);
Andy Shevchenkod86b2f22013-03-04 11:09:34 +0200370}
371
Dan Williams872f05c2013-11-06 16:29:58 -0800372static void dbg_result(const char *err, unsigned int n, unsigned int src_off,
373 unsigned int dst_off, unsigned int len,
374 unsigned long data)
Andy Shevchenko95019c82013-03-04 11:09:33 +0200375{
Jerome Blin2acec152014-03-04 10:38:55 +0100376 pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n",
Andy Shevchenkoa835bb82014-10-22 16:16:42 +0300377 current->comm, n, err, src_off, dst_off, len, data);
Andy Shevchenko95019c82013-03-04 11:09:33 +0200378}
379
Andy Shevchenkoa835bb82014-10-22 16:16:42 +0300380#define verbose_result(err, n, src_off, dst_off, len, data) ({ \
381 if (verbose) \
382 result(err, n, src_off, dst_off, len, data); \
383 else \
384 dbg_result(err, n, src_off, dst_off, len, data);\
Dan Williams50137a72013-11-08 12:26:26 -0800385})
386
Dan Williams86727442013-11-06 16:30:07 -0800387static unsigned long long dmatest_persec(s64 runtime, unsigned int val)
Andy Shevchenko95019c82013-03-04 11:09:33 +0200388{
Dan Williams86727442013-11-06 16:30:07 -0800389 unsigned long long per_sec = 1000000;
Andy Shevchenko95019c82013-03-04 11:09:33 +0200390
Dan Williams86727442013-11-06 16:30:07 -0800391 if (runtime <= 0)
392 return 0;
Andy Shevchenko95019c82013-03-04 11:09:33 +0200393
Dan Williams86727442013-11-06 16:30:07 -0800394 /* drop precision until runtime is 32-bits */
395 while (runtime > UINT_MAX) {
396 runtime >>= 1;
397 per_sec <<= 1;
398 }
Andy Shevchenko95019c82013-03-04 11:09:33 +0200399
Dan Williams86727442013-11-06 16:30:07 -0800400 per_sec *= val;
401 do_div(per_sec, runtime);
402 return per_sec;
Andy Shevchenko95019c82013-03-04 11:09:33 +0200403}
404
Dan Williams86727442013-11-06 16:30:07 -0800405static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len)
Andy Shevchenkod86b2f22013-03-04 11:09:34 +0200406{
Dan Williams86727442013-11-06 16:30:07 -0800407 return dmatest_persec(runtime, len >> 10);
Andy Shevchenko95019c82013-03-04 11:09:33 +0200408}
409
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700410/*
411 * This function repeatedly tests DMA transfers of various lengths and
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700412 * offsets for a given operation type until it is told to exit by
413 * kthread_stop(). There may be multiple threads running this function
414 * in parallel for a single channel, and there may be multiple channels
415 * being tested in parallel.
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700416 *
417 * Before each test, the source and destination buffer is initialized
418 * with a known pattern. This pattern is different depending on
419 * whether it's in an area which is supposed to be copied or
420 * overwritten, and different in the source and destination buffers.
421 * So if the DMA engine doesn't copy exactly what we tell it to copy,
422 * we'll notice.
423 */
424static int dmatest_func(void *data)
425{
426 struct dmatest_thread *thread = data;
Adam Wallis679dbea2017-11-27 10:45:01 -0500427 struct dmatest_done *done = &thread->test_done;
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200428 struct dmatest_info *info;
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200429 struct dmatest_params *params;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700430 struct dma_chan *chan;
Akinobu Mita8be9e32b2012-10-28 00:49:32 +0900431 struct dma_device *dev;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700432 unsigned int error_count;
433 unsigned int failed_tests = 0;
434 unsigned int total_tests = 0;
435 dma_cookie_t cookie;
436 enum dma_status status;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700437 enum dma_ctrl_flags flags;
Andy Shevchenko945b5af2013-03-04 11:09:26 +0200438 u8 *pq_coefs = NULL;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700439 int ret;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700440 int src_cnt;
441 int dst_cnt;
442 int i;
Sinan Kayae9405ef2016-09-01 10:02:55 -0400443 ktime_t ktime, start, diff;
444 ktime_t filltime = ktime_set(0, 0);
445 ktime_t comparetime = ktime_set(0, 0);
Dan Williams86727442013-11-06 16:30:07 -0800446 s64 runtime = 0;
447 unsigned long long total_len = 0;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700448
Tejun Heoadfa5432011-11-23 09:28:16 -0800449 set_freezable();
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700450
451 ret = -ENOMEM;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700452
453 smp_rmb();
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200454 info = thread->info;
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200455 params = &info->params;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700456 chan = thread->chan;
Akinobu Mita8be9e32b2012-10-28 00:49:32 +0900457 dev = chan->device;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700458 if (thread->type == DMA_MEMCPY)
459 src_cnt = dst_cnt = 1;
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530460 else if (thread->type == DMA_SG)
461 src_cnt = dst_cnt = sg_buffers;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700462 else if (thread->type == DMA_XOR) {
Akinobu Mita8be9e32b2012-10-28 00:49:32 +0900463 /* force odd to ensure dst = src */
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200464 src_cnt = min_odd(params->xor_sources | 1, dev->max_xor);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700465 dst_cnt = 1;
Dan Williams58691d62009-08-29 19:09:27 -0700466 } else if (thread->type == DMA_PQ) {
Akinobu Mita8be9e32b2012-10-28 00:49:32 +0900467 /* force odd to ensure dst = src */
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200468 src_cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0));
Dan Williams58691d62009-08-29 19:09:27 -0700469 dst_cnt = 2;
Andy Shevchenko945b5af2013-03-04 11:09:26 +0200470
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200471 pq_coefs = kmalloc(params->pq_sources+1, GFP_KERNEL);
Andy Shevchenko945b5af2013-03-04 11:09:26 +0200472 if (!pq_coefs)
473 goto err_thread_type;
474
Anatolij Gustschin94de6482010-02-15 22:35:23 +0100475 for (i = 0; i < src_cnt; i++)
Dan Williams58691d62009-08-29 19:09:27 -0700476 pq_coefs[i] = 1;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700477 } else
Andy Shevchenko945b5af2013-03-04 11:09:26 +0200478 goto err_thread_type;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700479
480 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL);
481 if (!thread->srcs)
482 goto err_srcs;
483 for (i = 0; i < src_cnt; i++) {
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200484 thread->srcs[i] = kmalloc(params->buf_size, GFP_KERNEL);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700485 if (!thread->srcs[i])
486 goto err_srcbuf;
487 }
488 thread->srcs[i] = NULL;
489
490 thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL);
491 if (!thread->dsts)
492 goto err_dsts;
493 for (i = 0; i < dst_cnt; i++) {
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200494 thread->dsts[i] = kmalloc(params->buf_size, GFP_KERNEL);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700495 if (!thread->dsts[i])
496 goto err_dstbuf;
497 }
498 thread->dsts[i] = NULL;
499
Dan Williamse44e0aa2009-03-25 09:13:25 -0700500 set_user_nice(current, 10);
501
Ira Snyderb203bd32011-03-03 07:54:53 +0000502 /*
Bartlomiej Zolnierkiewiczd1cab342013-10-18 19:35:21 +0200503 * src and dst buffers are freed by ourselves below
Ira Snyderb203bd32011-03-03 07:54:53 +0000504 */
Bartlomiej Zolnierkiewicz0776ae72013-10-18 19:35:33 +0200505 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700506
Dan Williams86727442013-11-06 16:30:07 -0800507 ktime = ktime_get();
Nicolas Ferre0a2ff57d2009-07-03 19:26:51 +0200508 while (!kthread_should_stop()
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200509 && !(params->iterations && total_tests >= params->iterations)) {
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700510 struct dma_async_tx_descriptor *tx = NULL;
Dan Williams4076e752013-11-06 16:30:10 -0800511 struct dmaengine_unmap_data *um;
512 dma_addr_t srcs[src_cnt];
513 dma_addr_t *dsts;
Andy Shevchenkoede23a52014-10-22 16:16:43 +0300514 unsigned int src_off, dst_off, len;
Dan Williams83544ae2009-09-08 17:42:53 -0700515 u8 align = 0;
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530516 struct scatterlist tx_sg[src_cnt];
517 struct scatterlist rx_sg[src_cnt];
Atsushi Nemotod86be862009-01-13 09:22:20 -0700518
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700519 total_tests++;
520
Dan Williams83544ae2009-09-08 17:42:53 -0700521 /* honor alignment restrictions */
Nicolin Chenc8a2c192016-09-07 18:24:28 -0700522 if (thread->type == DMA_MEMCPY || thread->type == DMA_SG)
Dan Williams83544ae2009-09-08 17:42:53 -0700523 align = dev->copy_align;
524 else if (thread->type == DMA_XOR)
525 align = dev->xor_align;
526 else if (thread->type == DMA_PQ)
527 align = dev->pq_align;
528
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200529 if (1 << align > params->buf_size) {
Guennadi Liakhovetskicfe4f272009-12-04 19:44:48 +0100530 pr_err("%u-byte buffer too small for %d-byte alignment\n",
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200531 params->buf_size, 1 << align);
Guennadi Liakhovetskicfe4f272009-12-04 19:44:48 +0100532 break;
533 }
534
Andy Shevchenkoede23a52014-10-22 16:16:43 +0300535 if (params->noverify)
Dan Williamse3b9c342013-11-06 16:30:05 -0800536 len = params->buf_size;
Andy Shevchenkoede23a52014-10-22 16:16:43 +0300537 else
538 len = dmatest_random() % params->buf_size + 1;
539
540 len = (len >> align) << align;
541 if (!len)
542 len = 1 << align;
543
544 total_len += len;
545
546 if (params->noverify) {
Dan Williamse3b9c342013-11-06 16:30:05 -0800547 src_off = 0;
548 dst_off = 0;
549 } else {
Sinan Kayae9405ef2016-09-01 10:02:55 -0400550 start = ktime_get();
Dan Williamse3b9c342013-11-06 16:30:05 -0800551 src_off = dmatest_random() % (params->buf_size - len + 1);
552 dst_off = dmatest_random() % (params->buf_size - len + 1);
553
554 src_off = (src_off >> align) << align;
555 dst_off = (dst_off >> align) << align;
556
557 dmatest_init_srcs(thread->srcs, src_off, len,
558 params->buf_size);
559 dmatest_init_dsts(thread->dsts, dst_off, len,
560 params->buf_size);
Sinan Kayae9405ef2016-09-01 10:02:55 -0400561
562 diff = ktime_sub(ktime_get(), start);
563 filltime = ktime_add(filltime, diff);
Dan Williamse3b9c342013-11-06 16:30:05 -0800564 }
565
Dan Williams4076e752013-11-06 16:30:10 -0800566 um = dmaengine_get_unmap_data(dev->dev, src_cnt+dst_cnt,
567 GFP_KERNEL);
568 if (!um) {
569 failed_tests++;
570 result("unmap data NULL", total_tests,
571 src_off, dst_off, len, ret);
572 continue;
573 }
Dan Williams83544ae2009-09-08 17:42:53 -0700574
Dan Williams4076e752013-11-06 16:30:10 -0800575 um->len = params->buf_size;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700576 for (i = 0; i < src_cnt; i++) {
Dan Williams745c00d2013-12-09 11:16:01 -0800577 void *buf = thread->srcs[i];
Dan Williams4076e752013-11-06 16:30:10 -0800578 struct page *pg = virt_to_page(buf);
Dan Williams745c00d2013-12-09 11:16:01 -0800579 unsigned pg_off = (unsigned long) buf & ~PAGE_MASK;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700580
Dan Williams4076e752013-11-06 16:30:10 -0800581 um->addr[i] = dma_map_page(dev->dev, pg, pg_off,
582 um->len, DMA_TO_DEVICE);
583 srcs[i] = um->addr[i] + src_off;
584 ret = dma_mapping_error(dev->dev, um->addr[i]);
Andy Shevchenkoafde3be2012-12-17 15:59:53 -0800585 if (ret) {
Dan Williams872f05c2013-11-06 16:29:58 -0800586 result("src mapping error", total_tests,
587 src_off, dst_off, len, ret);
Andy Shevchenkoad303932019-01-30 21:48:44 +0200588 goto error_unmap_continue;
Andy Shevchenkoafde3be2012-12-17 15:59:53 -0800589 }
Dan Williams4076e752013-11-06 16:30:10 -0800590 um->to_cnt++;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700591 }
Atsushi Nemotod86be862009-01-13 09:22:20 -0700592 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */
Dan Williams4076e752013-11-06 16:30:10 -0800593 dsts = &um->addr[src_cnt];
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700594 for (i = 0; i < dst_cnt; i++) {
Dan Williams745c00d2013-12-09 11:16:01 -0800595 void *buf = thread->dsts[i];
Dan Williams4076e752013-11-06 16:30:10 -0800596 struct page *pg = virt_to_page(buf);
Dan Williams745c00d2013-12-09 11:16:01 -0800597 unsigned pg_off = (unsigned long) buf & ~PAGE_MASK;
Dan Williams4076e752013-11-06 16:30:10 -0800598
599 dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len,
600 DMA_BIDIRECTIONAL);
601 ret = dma_mapping_error(dev->dev, dsts[i]);
Andy Shevchenkoafde3be2012-12-17 15:59:53 -0800602 if (ret) {
Dan Williams872f05c2013-11-06 16:29:58 -0800603 result("dst mapping error", total_tests,
604 src_off, dst_off, len, ret);
Andy Shevchenkoad303932019-01-30 21:48:44 +0200605 goto error_unmap_continue;
Andy Shevchenkoafde3be2012-12-17 15:59:53 -0800606 }
Dan Williams4076e752013-11-06 16:30:10 -0800607 um->bidi_cnt++;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700608 }
Atsushi Nemotod86be862009-01-13 09:22:20 -0700609
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530610 sg_init_table(tx_sg, src_cnt);
611 sg_init_table(rx_sg, src_cnt);
612 for (i = 0; i < src_cnt; i++) {
613 sg_dma_address(&rx_sg[i]) = srcs[i];
614 sg_dma_address(&tx_sg[i]) = dsts[i] + dst_off;
615 sg_dma_len(&tx_sg[i]) = len;
616 sg_dma_len(&rx_sg[i]) = len;
617 }
618
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700619 if (thread->type == DMA_MEMCPY)
620 tx = dev->device_prep_dma_memcpy(chan,
Dan Williams4076e752013-11-06 16:30:10 -0800621 dsts[0] + dst_off,
622 srcs[0], len, flags);
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530623 else if (thread->type == DMA_SG)
624 tx = dev->device_prep_dma_sg(chan, tx_sg, src_cnt,
625 rx_sg, src_cnt, flags);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700626 else if (thread->type == DMA_XOR)
627 tx = dev->device_prep_dma_xor(chan,
Dan Williams4076e752013-11-06 16:30:10 -0800628 dsts[0] + dst_off,
629 srcs, src_cnt,
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700630 len, flags);
Dan Williams58691d62009-08-29 19:09:27 -0700631 else if (thread->type == DMA_PQ) {
632 dma_addr_t dma_pq[dst_cnt];
633
634 for (i = 0; i < dst_cnt; i++)
Dan Williams4076e752013-11-06 16:30:10 -0800635 dma_pq[i] = dsts[i] + dst_off;
636 tx = dev->device_prep_dma_pq(chan, dma_pq, srcs,
Anatolij Gustschin94de6482010-02-15 22:35:23 +0100637 src_cnt, pq_coefs,
Dan Williams58691d62009-08-29 19:09:27 -0700638 len, flags);
639 }
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700640
Atsushi Nemotod86be862009-01-13 09:22:20 -0700641 if (!tx) {
Dan Williams872f05c2013-11-06 16:29:58 -0800642 result("prep error", total_tests, src_off,
643 dst_off, len, ret);
Atsushi Nemotod86be862009-01-13 09:22:20 -0700644 msleep(100);
Andy Shevchenkoad303932019-01-30 21:48:44 +0200645 goto error_unmap_continue;
Atsushi Nemotod86be862009-01-13 09:22:20 -0700646 }
Dan Williamse44e0aa2009-03-25 09:13:25 -0700647
Adam Wallis679dbea2017-11-27 10:45:01 -0500648 done->done = false;
Dan Williamse44e0aa2009-03-25 09:13:25 -0700649 tx->callback = dmatest_callback;
Adam Wallis679dbea2017-11-27 10:45:01 -0500650 tx->callback_param = done;
Atsushi Nemotod86be862009-01-13 09:22:20 -0700651 cookie = tx->tx_submit(tx);
652
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700653 if (dma_submit_error(cookie)) {
Dan Williams872f05c2013-11-06 16:29:58 -0800654 result("submit error", total_tests, src_off,
655 dst_off, len, ret);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700656 msleep(100);
Andy Shevchenkoad303932019-01-30 21:48:44 +0200657 goto error_unmap_continue;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700658 }
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700659 dma_async_issue_pending(chan);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700660
Adam Wallis679dbea2017-11-27 10:45:01 -0500661 wait_event_freezable_timeout(thread->done_wait, done->done,
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200662 msecs_to_jiffies(params->timeout));
Guennadi Liakhovetski981ed702011-08-18 16:50:51 +0200663
Dan Williamse44e0aa2009-03-25 09:13:25 -0700664 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700665
Adam Wallis679dbea2017-11-27 10:45:01 -0500666 if (!done->done) {
Dan Williams4076e752013-11-06 16:30:10 -0800667 dmaengine_unmap_put(um);
Dan Williams872f05c2013-11-06 16:29:58 -0800668 result("test timed out", total_tests, src_off, dst_off,
669 len, 0);
Andy Shevchenkoad303932019-01-30 21:48:44 +0200670 goto error_unmap_continue;
Vinod Koul19e9f992013-10-16 13:37:27 +0530671 } else if (status != DMA_COMPLETE) {
Dan Williams4076e752013-11-06 16:30:10 -0800672 dmaengine_unmap_put(um);
Dan Williams872f05c2013-11-06 16:29:58 -0800673 result(status == DMA_ERROR ?
674 "completion error status" :
675 "completion busy status", total_tests, src_off,
676 dst_off, len, ret);
Andy Shevchenkoad303932019-01-30 21:48:44 +0200677 goto error_unmap_continue;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700678 }
Dan Williamse44e0aa2009-03-25 09:13:25 -0700679
Dan Williams4076e752013-11-06 16:30:10 -0800680 dmaengine_unmap_put(um);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700681
Dan Williamse3b9c342013-11-06 16:30:05 -0800682 if (params->noverify) {
Dan Williams50137a72013-11-08 12:26:26 -0800683 verbose_result("test passed", total_tests, src_off,
684 dst_off, len, 0);
Dan Williamse3b9c342013-11-06 16:30:05 -0800685 continue;
686 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700687
Sinan Kayae9405ef2016-09-01 10:02:55 -0400688 start = ktime_get();
Dan Williams872f05c2013-11-06 16:29:58 -0800689 pr_debug("%s: verifying source buffer...\n", current->comm);
Dan Williamse3b9c342013-11-06 16:30:05 -0800690 error_count = dmatest_verify(thread->srcs, 0, src_off,
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700691 0, PATTERN_SRC, true);
Dan Williams7b610172013-11-06 16:29:57 -0800692 error_count += dmatest_verify(thread->srcs, src_off,
693 src_off + len, src_off,
694 PATTERN_SRC | PATTERN_COPY, true);
695 error_count += dmatest_verify(thread->srcs, src_off + len,
696 params->buf_size, src_off + len,
697 PATTERN_SRC, true);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700698
Dan Williams872f05c2013-11-06 16:29:58 -0800699 pr_debug("%s: verifying dest buffer...\n", current->comm);
Dan Williams7b610172013-11-06 16:29:57 -0800700 error_count += dmatest_verify(thread->dsts, 0, dst_off,
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700701 0, PATTERN_DST, false);
Dan Williams7b610172013-11-06 16:29:57 -0800702 error_count += dmatest_verify(thread->dsts, dst_off,
703 dst_off + len, src_off,
704 PATTERN_SRC | PATTERN_COPY, false);
705 error_count += dmatest_verify(thread->dsts, dst_off + len,
706 params->buf_size, dst_off + len,
707 PATTERN_DST, false);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700708
Sinan Kayae9405ef2016-09-01 10:02:55 -0400709 diff = ktime_sub(ktime_get(), start);
710 comparetime = ktime_add(comparetime, diff);
711
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700712 if (error_count) {
Dan Williams872f05c2013-11-06 16:29:58 -0800713 result("data error", total_tests, src_off, dst_off,
714 len, error_count);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700715 failed_tests++;
716 } else {
Dan Williams50137a72013-11-08 12:26:26 -0800717 verbose_result("test passed", total_tests, src_off,
718 dst_off, len, 0);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700719 }
Andy Shevchenkoad303932019-01-30 21:48:44 +0200720
721 continue;
722
723error_unmap_continue:
724 dmaengine_unmap_put(um);
725 failed_tests++;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700726 }
Sinan Kayae9405ef2016-09-01 10:02:55 -0400727 ktime = ktime_sub(ktime_get(), ktime);
728 ktime = ktime_sub(ktime, comparetime);
729 ktime = ktime_sub(ktime, filltime);
730 runtime = ktime_to_us(ktime);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700731
732 ret = 0;
Andy Shevchenko8e1f50d2014-08-22 15:19:44 +0300733err_dstbuf:
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700734 for (i = 0; thread->dsts[i]; i++)
735 kfree(thread->dsts[i]);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700736 kfree(thread->dsts);
737err_dsts:
Andy Shevchenko8e1f50d2014-08-22 15:19:44 +0300738err_srcbuf:
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700739 for (i = 0; thread->srcs[i]; i++)
740 kfree(thread->srcs[i]);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700741 kfree(thread->srcs);
742err_srcs:
Andy Shevchenko945b5af2013-03-04 11:09:26 +0200743 kfree(pq_coefs);
744err_thread_type:
Dan Williams86727442013-11-06 16:30:07 -0800745 pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n",
746 current->comm, total_tests, failed_tests,
747 dmatest_persec(runtime, total_tests),
748 dmatest_KBs(runtime, total_len), ret);
Nicolas Ferre0a2ff57d2009-07-03 19:26:51 +0200749
Viresh Kumar9704efa2011-07-29 16:21:57 +0530750 /* terminate all transfers on specified channels */
Adam Wallis679dbea2017-11-27 10:45:01 -0500751 if (ret || failed_tests)
Shiraz Hashim5e034f72012-11-09 15:26:29 +0000752 dmaengine_terminate_all(chan);
753
Andy Shevchenko3e5ccd82013-03-04 11:09:31 +0200754 thread->done = true;
Dan Williams2d88ce72013-11-06 16:30:09 -0800755 wake_up(&thread_wait);
Nicolas Ferre0a2ff57d2009-07-03 19:26:51 +0200756
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700757 return ret;
758}
759
760static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
761{
762 struct dmatest_thread *thread;
763 struct dmatest_thread *_thread;
764 int ret;
765
766 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
767 ret = kthread_stop(thread->task);
Dan Williams0adff802013-11-06 16:30:00 -0800768 pr_debug("thread %s exited with status %d\n",
769 thread->task->comm, ret);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700770 list_del(&thread->node);
Dan Williams2d88ce72013-11-06 16:30:09 -0800771 put_task_struct(thread->task);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700772 kfree(thread);
773 }
Viresh Kumar9704efa2011-07-29 16:21:57 +0530774
775 /* terminate all transfers on specified channels */
Jon Mason944ea4d2012-11-11 23:03:20 +0000776 dmaengine_terminate_all(dtc->chan);
Viresh Kumar9704efa2011-07-29 16:21:57 +0530777
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700778 kfree(dtc);
779}
780
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200781static int dmatest_add_threads(struct dmatest_info *info,
782 struct dmatest_chan *dtc, enum dma_transaction_type type)
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700783{
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200784 struct dmatest_params *params = &info->params;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700785 struct dmatest_thread *thread;
786 struct dma_chan *chan = dtc->chan;
787 char *op;
788 unsigned int i;
789
790 if (type == DMA_MEMCPY)
791 op = "copy";
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530792 else if (type == DMA_SG)
793 op = "sg";
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700794 else if (type == DMA_XOR)
795 op = "xor";
Dan Williams58691d62009-08-29 19:09:27 -0700796 else if (type == DMA_PQ)
797 op = "pq";
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700798 else
799 return -EINVAL;
800
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200801 for (i = 0; i < params->threads_per_chan; i++) {
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700802 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
803 if (!thread) {
Dan Williams0adff802013-11-06 16:30:00 -0800804 pr_warn("No memory for %s-%s%u\n",
805 dma_chan_name(chan), op, i);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700806 break;
807 }
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200808 thread->info = info;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700809 thread->chan = dtc->chan;
810 thread->type = type;
Adam Wallis679dbea2017-11-27 10:45:01 -0500811 thread->test_done.wait = &thread->done_wait;
812 init_waitqueue_head(&thread->done_wait);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700813 smp_wmb();
Dan Williams2d88ce72013-11-06 16:30:09 -0800814 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u",
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700815 dma_chan_name(chan), op, i);
816 if (IS_ERR(thread->task)) {
Dan Williams2d88ce72013-11-06 16:30:09 -0800817 pr_warn("Failed to create thread %s-%s%u\n",
Dan Williams0adff802013-11-06 16:30:00 -0800818 dma_chan_name(chan), op, i);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700819 kfree(thread);
820 break;
821 }
822
823 /* srcbuf and dstbuf are allocated by the thread itself */
Dan Williams2d88ce72013-11-06 16:30:09 -0800824 get_task_struct(thread->task);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700825 list_add_tail(&thread->node, &dtc->threads);
Dan Williams2d88ce72013-11-06 16:30:09 -0800826 wake_up_process(thread->task);
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700827 }
828
829 return i;
830}
831
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200832static int dmatest_add_channel(struct dmatest_info *info,
833 struct dma_chan *chan)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700834{
835 struct dmatest_chan *dtc;
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700836 struct dma_device *dma_dev = chan->device;
837 unsigned int thread_count = 0;
Kulikov Vasiliyb9033e62010-07-17 19:19:48 +0400838 int cnt;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700839
Andrew Morton6fdb8bd2008-09-19 04:16:23 -0700840 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700841 if (!dtc) {
Dan Williams0adff802013-11-06 16:30:00 -0800842 pr_warn("No memory for %s\n", dma_chan_name(chan));
Dan Williams33df8ca2009-01-06 11:38:15 -0700843 return -ENOMEM;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700844 }
845
846 dtc->chan = chan;
847 INIT_LIST_HEAD(&dtc->threads);
848
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700849 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530850 if (dmatest == 0) {
851 cnt = dmatest_add_threads(info, dtc, DMA_MEMCPY);
852 thread_count += cnt > 0 ? cnt : 0;
853 }
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700854 }
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530855
856 if (dma_has_cap(DMA_SG, dma_dev->cap_mask)) {
857 if (dmatest == 1) {
858 cnt = dmatest_add_threads(info, dtc, DMA_SG);
859 thread_count += cnt > 0 ? cnt : 0;
860 }
861 }
862
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700863 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200864 cnt = dmatest_add_threads(info, dtc, DMA_XOR);
Nicolas Ferref1aef8b2009-07-06 18:19:44 +0200865 thread_count += cnt > 0 ? cnt : 0;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700866 }
Dan Williams58691d62009-08-29 19:09:27 -0700867 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200868 cnt = dmatest_add_threads(info, dtc, DMA_PQ);
Dr. David Alan Gilbertd07a74a2011-08-25 16:13:55 -0700869 thread_count += cnt > 0 ? cnt : 0;
Dan Williams58691d62009-08-29 19:09:27 -0700870 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700871
Dan Williams0adff802013-11-06 16:30:00 -0800872 pr_info("Started %u threads using %s\n",
Dan Williamsb54d5cb2009-03-25 09:13:25 -0700873 thread_count, dma_chan_name(chan));
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700874
Andy Shevchenko838cc702013-03-04 11:09:28 +0200875 list_add_tail(&dtc->node, &info->channels);
876 info->nr_channels++;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700877
Dan Williams33df8ca2009-01-06 11:38:15 -0700878 return 0;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700879}
880
Dan Williams7dd60252009-01-06 11:38:19 -0700881static bool filter(struct dma_chan *chan, void *param)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700882{
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200883 struct dmatest_params *params = param;
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200884
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200885 if (!dmatest_match_channel(params, chan) ||
886 !dmatest_match_device(params, chan->device))
Dan Williams7dd60252009-01-06 11:38:19 -0700887 return false;
Dan Williams33df8ca2009-01-06 11:38:15 -0700888 else
Dan Williams7dd60252009-01-06 11:38:19 -0700889 return true;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700890}
891
Dan Williamsa9e55492013-11-06 16:30:02 -0800892static void request_channels(struct dmatest_info *info,
893 enum dma_transaction_type type)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700894{
Dan Williams33df8ca2009-01-06 11:38:15 -0700895 dma_cap_mask_t mask;
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700896
Dan Williams33df8ca2009-01-06 11:38:15 -0700897 dma_cap_zero(mask);
Dan Williamsa9e55492013-11-06 16:30:02 -0800898 dma_cap_set(type, mask);
Dan Williams33df8ca2009-01-06 11:38:15 -0700899 for (;;) {
Dan Williamsa9e55492013-11-06 16:30:02 -0800900 struct dmatest_params *params = &info->params;
901 struct dma_chan *chan;
902
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200903 chan = dma_request_channel(mask, filter, params);
Dan Williams33df8ca2009-01-06 11:38:15 -0700904 if (chan) {
Dan Williamsa9e55492013-11-06 16:30:02 -0800905 if (dmatest_add_channel(info, chan)) {
Dan Williams33df8ca2009-01-06 11:38:15 -0700906 dma_release_channel(chan);
907 break; /* add_channel failed, punt */
908 }
909 } else
910 break; /* no more channels available */
Andy Shevchenko15b8a8e2013-03-04 11:09:29 +0200911 if (params->max_channels &&
912 info->nr_channels >= params->max_channels)
Dan Williams33df8ca2009-01-06 11:38:15 -0700913 break; /* we have all we need */
914 }
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700915}
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700916
Dan Williamsa9e55492013-11-06 16:30:02 -0800917static void run_threaded_test(struct dmatest_info *info)
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200918{
919 struct dmatest_params *params = &info->params;
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200920
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200921 /* Copy test parameters */
Andy Shevchenkoa6c268d2013-07-23 18:36:46 +0300922 params->buf_size = test_buf_size;
923 strlcpy(params->channel, strim(test_channel), sizeof(params->channel));
924 strlcpy(params->device, strim(test_device), sizeof(params->device));
925 params->threads_per_chan = threads_per_chan;
926 params->max_channels = max_channels;
927 params->iterations = iterations;
928 params->xor_sources = xor_sources;
929 params->pq_sources = pq_sources;
930 params->timeout = timeout;
Dan Williamse3b9c342013-11-06 16:30:05 -0800931 params->noverify = noverify;
Dan Williamsa310d032013-11-06 16:30:01 -0800932
Dan Williamsa9e55492013-11-06 16:30:02 -0800933 request_channels(info, DMA_MEMCPY);
934 request_channels(info, DMA_XOR);
Kedareswara rao Appanaa0d4cb42016-06-09 21:10:14 +0530935 request_channels(info, DMA_SG);
Dan Williamsa9e55492013-11-06 16:30:02 -0800936 request_channels(info, DMA_PQ);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700937}
938
Dan Williamsa310d032013-11-06 16:30:01 -0800939static void stop_threaded_test(struct dmatest_info *info)
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700940{
941 struct dmatest_chan *dtc, *_dtc;
942 struct dma_chan *chan;
943
944 list_for_each_entry_safe(dtc, _dtc, &info->channels, node) {
945 list_del(&dtc->node);
946 chan = dtc->chan;
947 dmatest_cleanup_channel(dtc);
Dan Williams0adff802013-11-06 16:30:00 -0800948 pr_debug("dropped channel %s\n", dma_chan_name(chan));
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +0200949 dma_release_channel(chan);
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -0700950 }
Dan Williams33df8ca2009-01-06 11:38:15 -0700951
Dan Williams7cbd4872009-03-04 16:06:03 -0700952 info->nr_channels = 0;
Dan Williams33df8ca2009-01-06 11:38:15 -0700953}
Andy Shevchenko838cc702013-03-04 11:09:28 +0200954
Dan Williamsa9e55492013-11-06 16:30:02 -0800955static void restart_threaded_test(struct dmatest_info *info, bool run)
Dan Williams7cbd4872009-03-04 16:06:03 -0700956{
Dan Williamsa310d032013-11-06 16:30:01 -0800957 /* we might be called early to set run=, defer running until all
958 * parameters have been evaluated
959 */
960 if (!info->did_init)
Dan Williamsa9e55492013-11-06 16:30:02 -0800961 return;
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200962
Dan Williamsa310d032013-11-06 16:30:01 -0800963 /* Stop any running test first */
964 stop_threaded_test(info);
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200965
966 /* Run test with new parameters */
Dan Williamsa9e55492013-11-06 16:30:02 -0800967 run_threaded_test(info);
Andy Shevchenkobcc567e2013-05-23 14:29:53 +0300968}
969
Dan Williamsa310d032013-11-06 16:30:01 -0800970static int dmatest_run_get(char *val, const struct kernel_param *kp)
Andy Shevchenkobcc567e2013-05-23 14:29:53 +0300971{
Dan Williamsa310d032013-11-06 16:30:01 -0800972 struct dmatest_info *info = &test_info;
Andy Shevchenko851b7e12013-03-04 11:09:30 +0200973
974 mutex_lock(&info->lock);
Dan Williamsa310d032013-11-06 16:30:01 -0800975 if (is_threaded_test_run(info)) {
976 dmatest_run = true;
Andy Shevchenko3e5ccd82013-03-04 11:09:31 +0200977 } else {
Dan Williamsa310d032013-11-06 16:30:01 -0800978 stop_threaded_test(info);
979 dmatest_run = false;
Andy Shevchenko3e5ccd82013-03-04 11:09:31 +0200980 }
Dan Williamsa310d032013-11-06 16:30:01 -0800981 mutex_unlock(&info->lock);
982
983 return param_get_bool(val, kp);
984}
985
986static int dmatest_run_set(const char *val, const struct kernel_param *kp)
987{
988 struct dmatest_info *info = &test_info;
989 int ret;
990
991 mutex_lock(&info->lock);
992 ret = param_set_bool(val, kp);
993 if (ret) {
994 mutex_unlock(&info->lock);
995 return ret;
996 }
997
998 if (is_threaded_test_run(info))
999 ret = -EBUSY;
1000 else if (dmatest_run)
Dan Williamsa9e55492013-11-06 16:30:02 -08001001 restart_threaded_test(info, dmatest_run);
Andy Shevchenko3e5ccd82013-03-04 11:09:31 +02001002
Andy Shevchenko851b7e12013-03-04 11:09:30 +02001003 mutex_unlock(&info->lock);
Andy Shevchenko851b7e12013-03-04 11:09:30 +02001004
Dan Williamsa310d032013-11-06 16:30:01 -08001005 return ret;
Andy Shevchenko851b7e12013-03-04 11:09:30 +02001006}
1007
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +02001008static int __init dmatest_init(void)
1009{
1010 struct dmatest_info *info = &test_info;
Dan Williams2d88ce72013-11-06 16:30:09 -08001011 struct dmatest_params *params = &info->params;
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +02001012
Dan Williamsa310d032013-11-06 16:30:01 -08001013 if (dmatest_run) {
1014 mutex_lock(&info->lock);
Dan Williamsa9e55492013-11-06 16:30:02 -08001015 run_threaded_test(info);
Dan Williamsa310d032013-11-06 16:30:01 -08001016 mutex_unlock(&info->lock);
1017 }
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +02001018
Dan Williams2d88ce72013-11-06 16:30:09 -08001019 if (params->iterations && wait)
1020 wait_event(thread_wait, !is_threaded_test_run(info));
Andy Shevchenko838cc702013-03-04 11:09:28 +02001021
Dan Williamsa310d032013-11-06 16:30:01 -08001022 /* module parameters are stable, inittime tests are started,
1023 * let userspace take over 'run' control
1024 */
1025 info->did_init = true;
Andy Shevchenko95019c82013-03-04 11:09:33 +02001026
Andy Shevchenko851b7e12013-03-04 11:09:30 +02001027 return 0;
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +02001028}
1029/* when compiled-in wait for drivers to load first */
1030late_initcall(dmatest_init);
1031
1032static void __exit dmatest_exit(void)
1033{
1034 struct dmatest_info *info = &test_info;
1035
Dan Williamsa310d032013-11-06 16:30:01 -08001036 mutex_lock(&info->lock);
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +02001037 stop_threaded_test(info);
Dan Williamsa310d032013-11-06 16:30:01 -08001038 mutex_unlock(&info->lock);
Andy Shevchenkoe03e93a2013-03-04 11:09:27 +02001039}
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -07001040module_exit(dmatest_exit);
1041
Jean Delvaree05503e2011-05-18 16:49:24 +02001042MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
Haavard Skinnemoen4a776f02008-07-08 11:58:45 -07001043MODULE_LICENSE("GPL v2");