blob: 8d9169b7f20813374ba19058663c7367935b8e6d [file] [log] [blame]
Matt Porterc2dde5f2012-08-22 21:09:34 -04001/*
2 * TI EDMA DMA engine driver
3 *
4 * Copyright 2012 Texas Instruments
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation version 2.
9 *
10 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
11 * kind, whether express or implied; without even the implied warranty
12 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 */
15
16#include <linux/dmaengine.h>
17#include <linux/dma-mapping.h>
Lad, Prabhakarb7a4fd52015-02-04 13:03:27 +000018#include <linux/edma.h>
Matt Porterc2dde5f2012-08-22 21:09:34 -040019#include <linux/err.h>
20#include <linux/init.h>
21#include <linux/interrupt.h>
22#include <linux/list.h>
23#include <linux/module.h>
24#include <linux/platform_device.h>
25#include <linux/slab.h>
26#include <linux/spinlock.h>
Peter Ujfalusied646102014-07-31 13:12:38 +030027#include <linux/of.h>
Peter Ujfalusidc9b60552015-10-14 14:42:47 +030028#include <linux/of_dma.h>
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +030029#include <linux/of_irq.h>
30#include <linux/of_address.h>
31#include <linux/of_device.h>
32#include <linux/pm_runtime.h>
Matt Porterc2dde5f2012-08-22 21:09:34 -040033
Matt Porter3ad7a422013-03-06 11:15:31 -050034#include <linux/platform_data/edma.h>
Matt Porterc2dde5f2012-08-22 21:09:34 -040035
36#include "dmaengine.h"
37#include "virt-dma.h"
38
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +030039/* Offsets matching "struct edmacc_param" */
40#define PARM_OPT 0x00
41#define PARM_SRC 0x04
42#define PARM_A_B_CNT 0x08
43#define PARM_DST 0x0c
44#define PARM_SRC_DST_BIDX 0x10
45#define PARM_LINK_BCNTRLD 0x14
46#define PARM_SRC_DST_CIDX 0x18
47#define PARM_CCNT 0x1c
48
49#define PARM_SIZE 0x20
50
51/* Offsets for EDMA CC global channel registers and their shadows */
52#define SH_ER 0x00 /* 64 bits */
53#define SH_ECR 0x08 /* 64 bits */
54#define SH_ESR 0x10 /* 64 bits */
55#define SH_CER 0x18 /* 64 bits */
56#define SH_EER 0x20 /* 64 bits */
57#define SH_EECR 0x28 /* 64 bits */
58#define SH_EESR 0x30 /* 64 bits */
59#define SH_SER 0x38 /* 64 bits */
60#define SH_SECR 0x40 /* 64 bits */
61#define SH_IER 0x50 /* 64 bits */
62#define SH_IECR 0x58 /* 64 bits */
63#define SH_IESR 0x60 /* 64 bits */
64#define SH_IPR 0x68 /* 64 bits */
65#define SH_ICR 0x70 /* 64 bits */
66#define SH_IEVAL 0x78
67#define SH_QER 0x80
68#define SH_QEER 0x84
69#define SH_QEECR 0x88
70#define SH_QEESR 0x8c
71#define SH_QSER 0x90
72#define SH_QSECR 0x94
73#define SH_SIZE 0x200
74
75/* Offsets for EDMA CC global registers */
76#define EDMA_REV 0x0000
77#define EDMA_CCCFG 0x0004
78#define EDMA_QCHMAP 0x0200 /* 8 registers */
79#define EDMA_DMAQNUM 0x0240 /* 8 registers (4 on OMAP-L1xx) */
80#define EDMA_QDMAQNUM 0x0260
81#define EDMA_QUETCMAP 0x0280
82#define EDMA_QUEPRI 0x0284
83#define EDMA_EMR 0x0300 /* 64 bits */
84#define EDMA_EMCR 0x0308 /* 64 bits */
85#define EDMA_QEMR 0x0310
86#define EDMA_QEMCR 0x0314
87#define EDMA_CCERR 0x0318
88#define EDMA_CCERRCLR 0x031c
89#define EDMA_EEVAL 0x0320
90#define EDMA_DRAE 0x0340 /* 4 x 64 bits*/
91#define EDMA_QRAE 0x0380 /* 4 registers */
92#define EDMA_QUEEVTENTRY 0x0400 /* 2 x 16 registers */
93#define EDMA_QSTAT 0x0600 /* 2 registers */
94#define EDMA_QWMTHRA 0x0620
95#define EDMA_QWMTHRB 0x0624
96#define EDMA_CCSTAT 0x0640
97
98#define EDMA_M 0x1000 /* global channel registers */
99#define EDMA_ECR 0x1008
100#define EDMA_ECRH 0x100C
101#define EDMA_SHADOW0 0x2000 /* 4 shadow regions */
102#define EDMA_PARM 0x4000 /* PaRAM entries */
103
104#define PARM_OFFSET(param_no) (EDMA_PARM + ((param_no) << 5))
105
106#define EDMA_DCHMAP 0x0100 /* 64 registers */
107
108/* CCCFG register */
109#define GET_NUM_DMACH(x) (x & 0x7) /* bits 0-2 */
110#define GET_NUM_PAENTRY(x) ((x & 0x7000) >> 12) /* bits 12-14 */
111#define GET_NUM_EVQUE(x) ((x & 0x70000) >> 16) /* bits 16-18 */
112#define GET_NUM_REGN(x) ((x & 0x300000) >> 20) /* bits 20-21 */
113#define CHMAP_EXIST BIT(24)
114
Matt Porterc2dde5f2012-08-22 21:09:34 -0400115/*
Joel Fernandes2abd5f12013-09-23 18:05:15 -0500116 * Max of 20 segments per channel to conserve PaRAM slots
117 * Also note that MAX_NR_SG should be atleast the no.of periods
118 * that are required for ASoC, otherwise DMA prep calls will
119 * fail. Today davinci-pcm is the only user of this driver and
120 * requires atleast 17 slots, so we setup the default to 20.
121 */
122#define MAX_NR_SG 20
Matt Porterc2dde5f2012-08-22 21:09:34 -0400123#define EDMA_MAX_SLOTS MAX_NR_SG
124#define EDMA_DESCRIPTORS 16
125
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300126#define EDMA_CHANNEL_ANY -1 /* for edma_alloc_channel() */
127#define EDMA_SLOT_ANY -1 /* for edma_alloc_slot() */
128#define EDMA_CONT_PARAMS_ANY 1001
129#define EDMA_CONT_PARAMS_FIXED_EXACT 1002
130#define EDMA_CONT_PARAMS_FIXED_NOT_EXACT 1003
131
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300132/* PaRAM slots are laid out like this */
133struct edmacc_param {
134 u32 opt;
135 u32 src;
136 u32 a_b_cnt;
137 u32 dst;
138 u32 src_dst_bidx;
139 u32 link_bcntrld;
140 u32 src_dst_cidx;
141 u32 ccnt;
142} __packed;
143
144/* fields in edmacc_param.opt */
145#define SAM BIT(0)
146#define DAM BIT(1)
147#define SYNCDIM BIT(2)
148#define STATIC BIT(3)
149#define EDMA_FWID (0x07 << 8)
150#define TCCMODE BIT(11)
151#define EDMA_TCC(t) ((t) << 12)
152#define TCINTEN BIT(20)
153#define ITCINTEN BIT(21)
154#define TCCHEN BIT(22)
155#define ITCCHEN BIT(23)
156
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500157struct edma_pset {
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500158 u32 len;
159 dma_addr_t addr;
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500160 struct edmacc_param param;
161};
162
Matt Porterc2dde5f2012-08-22 21:09:34 -0400163struct edma_desc {
164 struct virt_dma_desc vdesc;
165 struct list_head node;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500166 enum dma_transfer_direction direction;
Joel Fernandes50a9c702013-10-31 16:31:23 -0500167 int cyclic;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400168 int absync;
169 int pset_nr;
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500170 struct edma_chan *echan;
Joel Fernandes04361d82014-04-28 15:19:31 -0500171 int processed;
172
173 /*
174 * The following 4 elements are used for residue accounting.
175 *
176 * - processed_stat: the number of SG elements we have traversed
177 * so far to cover accounting. This is updated directly to processed
178 * during edma_callback and is always <= processed, because processed
179 * refers to the number of pending transfer (programmed to EDMA
180 * controller), where as processed_stat tracks number of transfers
181 * accounted for so far.
182 *
183 * - residue: The amount of bytes we have left to transfer for this desc
184 *
185 * - residue_stat: The residue in bytes of data we have covered
186 * so far for accounting. This is updated directly to residue
187 * during callbacks to keep it current.
188 *
189 * - sg_len: Tracks the length of the current intermediate transfer,
190 * this is required to update the residue during intermediate transfer
191 * completion callback.
192 */
193 int processed_stat;
194 u32 sg_len;
195 u32 residue;
196 u32 residue_stat;
197
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500198 struct edma_pset pset[0];
Matt Porterc2dde5f2012-08-22 21:09:34 -0400199};
200
201struct edma_cc;
202
203struct edma_chan {
204 struct virt_dma_chan vchan;
205 struct list_head node;
206 struct edma_desc *edesc;
207 struct edma_cc *ecc;
208 int ch_num;
209 bool alloced;
210 int slot[EDMA_MAX_SLOTS];
Joel Fernandesc5f47992013-08-29 18:05:43 -0500211 int missed;
Matt Porter661f7cb2013-01-10 13:41:04 -0500212 struct dma_slave_config cfg;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400213};
214
215struct edma_cc {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300216 struct device *dev;
217 struct edma_soc_info *info;
218 void __iomem *base;
219 int id;
220
221 /* eDMA3 resource information */
222 unsigned num_channels;
223 unsigned num_region;
224 unsigned num_slots;
225 unsigned num_tc;
Peter Ujfalusi4ab54f62015-10-14 14:43:04 +0300226 bool chmap_exist;
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300227 enum dma_event_q default_queue;
228
229 bool unused_chan_list_done;
Peter Ujfalusi7a73b132015-10-14 14:43:05 +0300230 /* The slot_inuse bit for each PaRAM slot is clear unless the
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300231 * channel is in use ... by ARM or DSP, for QDMA, or whatever.
232 */
Peter Ujfalusi7a73b132015-10-14 14:43:05 +0300233 unsigned long *slot_inuse;
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300234
Peter Ujfalusi7a73b132015-10-14 14:43:05 +0300235 /* The channel_unused bit for each channel is clear unless
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300236 * it is not being used on this platform. It uses a bit
237 * of SOC-specific initialization code.
238 */
Peter Ujfalusi7a73b132015-10-14 14:43:05 +0300239 unsigned long *channel_unused;
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300240
Matt Porterc2dde5f2012-08-22 21:09:34 -0400241 struct dma_device dma_slave;
Peter Ujfalusicb782052015-10-14 14:42:54 +0300242 struct edma_chan *slave_chans;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400243 int dummy_slot;
244};
245
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300246/* dummy param set used to (re)initialize parameter RAM slots */
247static const struct edmacc_param dummy_paramset = {
248 .link_bcntrld = 0xffff,
249 .ccnt = 1,
250};
251
252static const struct of_device_id edma_of_ids[] = {
253 { .compatible = "ti,edma3", },
254 {}
255};
256
257static inline unsigned int edma_read(struct edma_cc *ecc, int offset)
258{
259 return (unsigned int)__raw_readl(ecc->base + offset);
260}
261
262static inline void edma_write(struct edma_cc *ecc, int offset, int val)
263{
264 __raw_writel(val, ecc->base + offset);
265}
266
267static inline void edma_modify(struct edma_cc *ecc, int offset, unsigned and,
268 unsigned or)
269{
270 unsigned val = edma_read(ecc, offset);
271
272 val &= and;
273 val |= or;
274 edma_write(ecc, offset, val);
275}
276
277static inline void edma_and(struct edma_cc *ecc, int offset, unsigned and)
278{
279 unsigned val = edma_read(ecc, offset);
280
281 val &= and;
282 edma_write(ecc, offset, val);
283}
284
285static inline void edma_or(struct edma_cc *ecc, int offset, unsigned or)
286{
287 unsigned val = edma_read(ecc, offset);
288
289 val |= or;
290 edma_write(ecc, offset, val);
291}
292
293static inline unsigned int edma_read_array(struct edma_cc *ecc, int offset,
294 int i)
295{
296 return edma_read(ecc, offset + (i << 2));
297}
298
299static inline void edma_write_array(struct edma_cc *ecc, int offset, int i,
300 unsigned val)
301{
302 edma_write(ecc, offset + (i << 2), val);
303}
304
305static inline void edma_modify_array(struct edma_cc *ecc, int offset, int i,
306 unsigned and, unsigned or)
307{
308 edma_modify(ecc, offset + (i << 2), and, or);
309}
310
311static inline void edma_or_array(struct edma_cc *ecc, int offset, int i,
312 unsigned or)
313{
314 edma_or(ecc, offset + (i << 2), or);
315}
316
317static inline void edma_or_array2(struct edma_cc *ecc, int offset, int i, int j,
318 unsigned or)
319{
320 edma_or(ecc, offset + ((i * 2 + j) << 2), or);
321}
322
323static inline void edma_write_array2(struct edma_cc *ecc, int offset, int i,
324 int j, unsigned val)
325{
326 edma_write(ecc, offset + ((i * 2 + j) << 2), val);
327}
328
329static inline unsigned int edma_shadow0_read(struct edma_cc *ecc, int offset)
330{
331 return edma_read(ecc, EDMA_SHADOW0 + offset);
332}
333
334static inline unsigned int edma_shadow0_read_array(struct edma_cc *ecc,
335 int offset, int i)
336{
337 return edma_read(ecc, EDMA_SHADOW0 + offset + (i << 2));
338}
339
340static inline void edma_shadow0_write(struct edma_cc *ecc, int offset,
341 unsigned val)
342{
343 edma_write(ecc, EDMA_SHADOW0 + offset, val);
344}
345
346static inline void edma_shadow0_write_array(struct edma_cc *ecc, int offset,
347 int i, unsigned val)
348{
349 edma_write(ecc, EDMA_SHADOW0 + offset + (i << 2), val);
350}
351
352static inline unsigned int edma_parm_read(struct edma_cc *ecc, int offset,
353 int param_no)
354{
355 return edma_read(ecc, EDMA_PARM + offset + (param_no << 5));
356}
357
358static inline void edma_parm_write(struct edma_cc *ecc, int offset,
359 int param_no, unsigned val)
360{
361 edma_write(ecc, EDMA_PARM + offset + (param_no << 5), val);
362}
363
364static inline void edma_parm_modify(struct edma_cc *ecc, int offset,
365 int param_no, unsigned and, unsigned or)
366{
367 edma_modify(ecc, EDMA_PARM + offset + (param_no << 5), and, or);
368}
369
370static inline void edma_parm_and(struct edma_cc *ecc, int offset, int param_no,
371 unsigned and)
372{
373 edma_and(ecc, EDMA_PARM + offset + (param_no << 5), and);
374}
375
376static inline void edma_parm_or(struct edma_cc *ecc, int offset, int param_no,
377 unsigned or)
378{
379 edma_or(ecc, EDMA_PARM + offset + (param_no << 5), or);
380}
381
382static inline void set_bits(int offset, int len, unsigned long *p)
383{
384 for (; len > 0; len--)
385 set_bit(offset + (len - 1), p);
386}
387
388static inline void clear_bits(int offset, int len, unsigned long *p)
389{
390 for (; len > 0; len--)
391 clear_bit(offset + (len - 1), p);
392}
393
394static void edma_map_dmach_to_queue(struct edma_cc *ecc, unsigned ch_no,
395 enum dma_event_q queue_no)
396{
397 int bit = (ch_no & 0x7) * 4;
398
399 /* default to low priority queue */
400 if (queue_no == EVENTQ_DEFAULT)
401 queue_no = ecc->default_queue;
402
403 queue_no &= 7;
404 edma_modify_array(ecc, EDMA_DMAQNUM, (ch_no >> 3), ~(0x7 << bit),
405 queue_no << bit);
406}
407
408static void edma_assign_priority_to_queue(struct edma_cc *ecc, int queue_no,
409 int priority)
410{
411 int bit = queue_no * 4;
412
413 edma_modify(ecc, EDMA_QUEPRI, ~(0x7 << bit), ((priority & 0x7) << bit));
414}
415
416static void edma_direct_dmach_to_param_mapping(struct edma_cc *ecc)
417{
418 int i;
419
420 for (i = 0; i < ecc->num_channels; i++)
421 edma_write_array(ecc, EDMA_DCHMAP, i, (i << 5));
422}
423
424static int prepare_unused_channel_list(struct device *dev, void *data)
425{
426 struct platform_device *pdev = to_platform_device(dev);
427 struct edma_cc *ecc = data;
Peter Ujfalusicb782052015-10-14 14:42:54 +0300428 int dma_req_min = EDMA_CTLR_CHAN(ecc->id, 0);
429 int dma_req_max = dma_req_min + ecc->num_channels;
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300430 int i, count;
431 struct of_phandle_args dma_spec;
432
433 if (dev->of_node) {
434 struct platform_device *dma_pdev;
435
436 count = of_property_count_strings(dev->of_node, "dma-names");
437 if (count < 0)
438 return 0;
439 for (i = 0; i < count; i++) {
440 if (of_parse_phandle_with_args(dev->of_node, "dmas",
441 "#dma-cells", i,
442 &dma_spec))
443 continue;
444
445 if (!of_match_node(edma_of_ids, dma_spec.np)) {
446 of_node_put(dma_spec.np);
447 continue;
448 }
449
450 dma_pdev = of_find_device_by_node(dma_spec.np);
451 if (&dma_pdev->dev != ecc->dev)
452 continue;
453
454 clear_bit(EDMA_CHAN_SLOT(dma_spec.args[0]),
Peter Ujfalusi7a73b132015-10-14 14:43:05 +0300455 ecc->channel_unused);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300456 of_node_put(dma_spec.np);
457 }
458 return 0;
459 }
460
461 /* For non-OF case */
462 for (i = 0; i < pdev->num_resources; i++) {
463 struct resource *res = &pdev->resource[i];
Peter Ujfalusicb782052015-10-14 14:42:54 +0300464 int dma_req;
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300465
Peter Ujfalusicb782052015-10-14 14:42:54 +0300466 if (!(res->flags & IORESOURCE_DMA))
467 continue;
468
469 dma_req = (int)res->start;
470 if (dma_req >= dma_req_min && dma_req < dma_req_max)
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300471 clear_bit(EDMA_CHAN_SLOT(pdev->resource[i].start),
Peter Ujfalusi7a73b132015-10-14 14:43:05 +0300472 ecc->channel_unused);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300473 }
474
475 return 0;
476}
477
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +0300478static void edma_setup_interrupt(struct edma_cc *ecc, unsigned lch, bool enable)
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300479{
480 lch = EDMA_CHAN_SLOT(lch);
481
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +0300482 if (enable) {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300483 edma_shadow0_write_array(ecc, SH_ICR, lch >> 5,
484 BIT(lch & 0x1f));
485 edma_shadow0_write_array(ecc, SH_IESR, lch >> 5,
486 BIT(lch & 0x1f));
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +0300487 } else {
488 edma_shadow0_write_array(ecc, SH_IECR, lch >> 5,
489 BIT(lch & 0x1f));
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300490 }
491}
492
493/*
Peter Ujfalusi11c15732015-10-14 14:43:00 +0300494 * paRAM slot management functions
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300495 */
496static void edma_write_slot(struct edma_cc *ecc, unsigned slot,
497 const struct edmacc_param *param)
498{
499 slot = EDMA_CHAN_SLOT(slot);
500 if (slot >= ecc->num_slots)
501 return;
502 memcpy_toio(ecc->base + PARM_OFFSET(slot), param, PARM_SIZE);
503}
504
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300505static void edma_read_slot(struct edma_cc *ecc, unsigned slot,
506 struct edmacc_param *param)
507{
508 slot = EDMA_CHAN_SLOT(slot);
509 if (slot >= ecc->num_slots)
510 return;
511 memcpy_fromio(param, ecc->base + PARM_OFFSET(slot), PARM_SIZE);
512}
513
514/**
515 * edma_alloc_slot - allocate DMA parameter RAM
516 * @ecc: pointer to edma_cc struct
517 * @slot: specific slot to allocate; negative for "any unused slot"
518 *
519 * This allocates a parameter RAM slot, initializing it to hold a
520 * dummy transfer. Slots allocated using this routine have not been
521 * mapped to a hardware DMA channel, and will normally be used by
522 * linking to them from a slot associated with a DMA channel.
523 *
524 * Normal use is to pass EDMA_SLOT_ANY as the @slot, but specific
525 * slots may be allocated on behalf of DSP firmware.
526 *
527 * Returns the number of the slot, else negative errno.
528 */
529static int edma_alloc_slot(struct edma_cc *ecc, int slot)
530{
531 if (slot > 0)
532 slot = EDMA_CHAN_SLOT(slot);
533 if (slot < 0) {
534 slot = ecc->num_channels;
535 for (;;) {
Peter Ujfalusi7a73b132015-10-14 14:43:05 +0300536 slot = find_next_zero_bit(ecc->slot_inuse,
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300537 ecc->num_slots,
538 slot);
539 if (slot == ecc->num_slots)
540 return -ENOMEM;
Peter Ujfalusi7a73b132015-10-14 14:43:05 +0300541 if (!test_and_set_bit(slot, ecc->slot_inuse))
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300542 break;
543 }
544 } else if (slot < ecc->num_channels || slot >= ecc->num_slots) {
545 return -EINVAL;
Peter Ujfalusi7a73b132015-10-14 14:43:05 +0300546 } else if (test_and_set_bit(slot, ecc->slot_inuse)) {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300547 return -EBUSY;
548 }
549
550 edma_write_slot(ecc, slot, &dummy_paramset);
551
552 return EDMA_CTLR_CHAN(ecc->id, slot);
553}
554
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300555static void edma_free_slot(struct edma_cc *ecc, unsigned slot)
556{
557 slot = EDMA_CHAN_SLOT(slot);
558 if (slot < ecc->num_channels || slot >= ecc->num_slots)
559 return;
560
561 edma_write_slot(ecc, slot, &dummy_paramset);
Peter Ujfalusi7a73b132015-10-14 14:43:05 +0300562 clear_bit(slot, ecc->slot_inuse);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300563}
564
565/**
566 * edma_link - link one parameter RAM slot to another
567 * @ecc: pointer to edma_cc struct
568 * @from: parameter RAM slot originating the link
569 * @to: parameter RAM slot which is the link target
570 *
571 * The originating slot should not be part of any active DMA transfer.
572 */
573static void edma_link(struct edma_cc *ecc, unsigned from, unsigned to)
574{
Peter Ujfalusifc014092015-10-14 14:42:59 +0300575 if (unlikely(EDMA_CTLR(from) != EDMA_CTLR(to)))
576 dev_warn(ecc->dev, "Ignoring eDMA instance for linking\n");
577
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300578 from = EDMA_CHAN_SLOT(from);
579 to = EDMA_CHAN_SLOT(to);
580 if (from >= ecc->num_slots || to >= ecc->num_slots)
581 return;
582
583 edma_parm_modify(ecc, PARM_LINK_BCNTRLD, from, 0xffff0000,
584 PARM_OFFSET(to));
585}
586
587/**
588 * edma_get_position - returns the current transfer point
589 * @ecc: pointer to edma_cc struct
590 * @slot: parameter RAM slot being examined
591 * @dst: true selects the dest position, false the source
592 *
593 * Returns the position of the current active slot
594 */
595static dma_addr_t edma_get_position(struct edma_cc *ecc, unsigned slot,
596 bool dst)
597{
598 u32 offs;
599
600 slot = EDMA_CHAN_SLOT(slot);
601 offs = PARM_OFFSET(slot);
602 offs += dst ? PARM_DST : PARM_SRC;
603
604 return edma_read(ecc, offs);
605}
606
607/*-----------------------------------------------------------------------*/
608/**
609 * edma_start - start dma on a channel
610 * @ecc: pointer to edma_cc struct
611 * @channel: channel being activated
612 *
613 * Channels with event associations will be triggered by their hardware
614 * events, and channels without such associations will be triggered by
615 * software. (At this writing there is no interface for using software
616 * triggers except with channels that don't support hardware triggers.)
617 *
618 * Returns zero on success, else negative errno.
619 */
620static int edma_start(struct edma_cc *ecc, unsigned channel)
621{
622 if (ecc->id != EDMA_CTLR(channel)) {
623 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
624 ecc->id, EDMA_CTLR(channel));
625 return -EINVAL;
626 }
627 channel = EDMA_CHAN_SLOT(channel);
628
629 if (channel < ecc->num_channels) {
630 int j = channel >> 5;
631 unsigned int mask = BIT(channel & 0x1f);
632
633 /* EDMA channels without event association */
Peter Ujfalusi7a73b132015-10-14 14:43:05 +0300634 if (test_bit(channel, ecc->channel_unused)) {
Peter Ujfalusi3287fb42015-10-14 14:42:57 +0300635 dev_dbg(ecc->dev, "ESR%d %08x\n", j,
636 edma_shadow0_read_array(ecc, SH_ESR, j));
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300637 edma_shadow0_write_array(ecc, SH_ESR, j, mask);
638 return 0;
639 }
640
641 /* EDMA channel with event association */
Peter Ujfalusi3287fb42015-10-14 14:42:57 +0300642 dev_dbg(ecc->dev, "ER%d %08x\n", j,
643 edma_shadow0_read_array(ecc, SH_ER, j));
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300644 /* Clear any pending event or error */
645 edma_write_array(ecc, EDMA_ECR, j, mask);
646 edma_write_array(ecc, EDMA_EMCR, j, mask);
647 /* Clear any SER */
648 edma_shadow0_write_array(ecc, SH_SECR, j, mask);
649 edma_shadow0_write_array(ecc, SH_EESR, j, mask);
Peter Ujfalusi3287fb42015-10-14 14:42:57 +0300650 dev_dbg(ecc->dev, "EER%d %08x\n", j,
651 edma_shadow0_read_array(ecc, SH_EER, j));
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300652 return 0;
653 }
654
655 return -EINVAL;
656}
657
658/**
659 * edma_stop - stops dma on the channel passed
660 * @ecc: pointer to edma_cc struct
661 * @channel: channel being deactivated
662 *
Peter Ujfalusi11c15732015-10-14 14:43:00 +0300663 * Any active transfer is paused and all pending hardware events are cleared.
664 * The current transfer may not be resumed, and the channel's Parameter RAM
665 * should be reinitialized before being reused.
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300666 */
667static void edma_stop(struct edma_cc *ecc, unsigned channel)
668{
669 if (ecc->id != EDMA_CTLR(channel)) {
670 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
671 ecc->id, EDMA_CTLR(channel));
672 return;
673 }
674 channel = EDMA_CHAN_SLOT(channel);
675
676 if (channel < ecc->num_channels) {
677 int j = channel >> 5;
678 unsigned int mask = BIT(channel & 0x1f);
679
680 edma_shadow0_write_array(ecc, SH_EECR, j, mask);
681 edma_shadow0_write_array(ecc, SH_ECR, j, mask);
682 edma_shadow0_write_array(ecc, SH_SECR, j, mask);
683 edma_write_array(ecc, EDMA_EMCR, j, mask);
684
685 /* clear possibly pending completion interrupt */
686 edma_shadow0_write_array(ecc, SH_ICR, j, mask);
687
Peter Ujfalusi3287fb42015-10-14 14:42:57 +0300688 dev_dbg(ecc->dev, "EER%d %08x\n", j,
689 edma_shadow0_read_array(ecc, SH_EER, j));
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300690
691 /* REVISIT: consider guarding against inappropriate event
692 * chaining by overwriting with dummy_paramset.
693 */
694 }
695}
696
Peter Ujfalusi11c15732015-10-14 14:43:00 +0300697/*
698 * Temporarily disable EDMA hardware events on the specified channel,
699 * preventing them from triggering new transfers
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300700 */
701static void edma_pause(struct edma_cc *ecc, unsigned channel)
702{
703 if (ecc->id != EDMA_CTLR(channel)) {
704 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
705 ecc->id, EDMA_CTLR(channel));
706 return;
707 }
708 channel = EDMA_CHAN_SLOT(channel);
709
710 if (channel < ecc->num_channels) {
711 unsigned int mask = BIT(channel & 0x1f);
712
713 edma_shadow0_write_array(ecc, SH_EECR, channel >> 5, mask);
714 }
715}
716
Peter Ujfalusi11c15732015-10-14 14:43:00 +0300717/* Re-enable EDMA hardware events on the specified channel. */
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300718static void edma_resume(struct edma_cc *ecc, unsigned channel)
719{
720 if (ecc->id != EDMA_CTLR(channel)) {
721 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
722 ecc->id, EDMA_CTLR(channel));
723 return;
724 }
725 channel = EDMA_CHAN_SLOT(channel);
726
727 if (channel < ecc->num_channels) {
728 unsigned int mask = BIT(channel & 0x1f);
729
730 edma_shadow0_write_array(ecc, SH_EESR, channel >> 5, mask);
731 }
732}
733
734static int edma_trigger_channel(struct edma_cc *ecc, unsigned channel)
735{
736 unsigned int mask;
737
738 if (ecc->id != EDMA_CTLR(channel)) {
739 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
740 ecc->id, EDMA_CTLR(channel));
741 return -EINVAL;
742 }
743 channel = EDMA_CHAN_SLOT(channel);
744 mask = BIT(channel & 0x1f);
745
746 edma_shadow0_write_array(ecc, SH_ESR, (channel >> 5), mask);
747
Peter Ujfalusi3287fb42015-10-14 14:42:57 +0300748 dev_dbg(ecc->dev, "ESR%d %08x\n", (channel >> 5),
749 edma_shadow0_read_array(ecc, SH_ESR, (channel >> 5)));
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300750 return 0;
751}
752
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300753static void edma_clean_channel(struct edma_cc *ecc, unsigned channel)
754{
755 if (ecc->id != EDMA_CTLR(channel)) {
756 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
757 ecc->id, EDMA_CTLR(channel));
758 return;
759 }
760 channel = EDMA_CHAN_SLOT(channel);
761
762 if (channel < ecc->num_channels) {
763 int j = (channel >> 5);
764 unsigned int mask = BIT(channel & 0x1f);
765
Peter Ujfalusi3287fb42015-10-14 14:42:57 +0300766 dev_dbg(ecc->dev, "EMR%d %08x\n", j,
767 edma_read_array(ecc, EDMA_EMR, j));
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300768 edma_shadow0_write_array(ecc, SH_ECR, j, mask);
769 /* Clear the corresponding EMR bits */
770 edma_write_array(ecc, EDMA_EMCR, j, mask);
771 /* Clear any SER */
772 edma_shadow0_write_array(ecc, SH_SECR, j, mask);
773 edma_write(ecc, EDMA_CCERRCLR, BIT(16) | BIT(1) | BIT(0));
774 }
775}
776
777/**
778 * edma_alloc_channel - allocate DMA channel and paired parameter RAM
779 * @ecc: pointer to edma_cc struct
780 * @channel: specific channel to allocate; negative for "any unmapped channel"
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300781 * @eventq_no: an EVENTQ_* constant, used to choose which Transfer
782 * Controller (TC) executes requests using this channel. Use
783 * EVENTQ_DEFAULT unless you really need a high priority queue.
784 *
785 * This allocates a DMA channel and its associated parameter RAM slot.
786 * The parameter RAM is initialized to hold a dummy transfer.
787 *
788 * Normal use is to pass a specific channel number as @channel, to make
789 * use of hardware events mapped to that channel. When the channel will
790 * be used only for software triggering or event chaining, channels not
791 * mapped to hardware events (or mapped to unused events) are preferable.
792 *
793 * DMA transfers start from a channel using edma_start(), or by
794 * chaining. When the transfer described in that channel's parameter RAM
795 * slot completes, that slot's data may be reloaded through a link.
796 *
797 * DMA errors are only reported to the @callback associated with the
798 * channel driving that transfer, but transfer completion callbacks can
799 * be sent to another channel under control of the TCC field in
800 * the option word of the transfer's parameter RAM set. Drivers must not
801 * use DMA transfer completion callbacks for channels they did not allocate.
802 * (The same applies to TCC codes used in transfer chaining.)
803 *
804 * Returns the number of the channel, else negative errno.
805 */
806static int edma_alloc_channel(struct edma_cc *ecc, int channel,
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +0300807 enum dma_event_q eventq_no)
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300808{
809 unsigned done = 0;
810 int ret = 0;
811
812 if (!ecc->unused_chan_list_done) {
813 /*
814 * Scan all the platform devices to find out the EDMA channels
815 * used and clear them in the unused list, making the rest
816 * available for ARM usage.
817 */
818 ret = bus_for_each_dev(&platform_bus_type, NULL, ecc,
819 prepare_unused_channel_list);
820 if (ret < 0)
821 return ret;
822
823 ecc->unused_chan_list_done = true;
824 }
825
826 if (channel >= 0) {
827 if (ecc->id != EDMA_CTLR(channel)) {
828 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n",
829 __func__, ecc->id, EDMA_CTLR(channel));
830 return -EINVAL;
831 }
832 channel = EDMA_CHAN_SLOT(channel);
833 }
834
835 if (channel < 0) {
836 channel = 0;
837 for (;;) {
Peter Ujfalusi7a73b132015-10-14 14:43:05 +0300838 channel = find_next_bit(ecc->channel_unused,
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300839 ecc->num_channels, channel);
840 if (channel == ecc->num_channels)
841 break;
Peter Ujfalusi7a73b132015-10-14 14:43:05 +0300842 if (!test_and_set_bit(channel, ecc->slot_inuse)) {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300843 done = 1;
844 break;
845 }
846 channel++;
847 }
848 if (!done)
849 return -ENOMEM;
850 } else if (channel >= ecc->num_channels) {
851 return -EINVAL;
Peter Ujfalusi7a73b132015-10-14 14:43:05 +0300852 } else if (test_and_set_bit(channel, ecc->slot_inuse)) {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300853 return -EBUSY;
854 }
855
856 /* ensure access through shadow region 0 */
857 edma_or_array2(ecc, EDMA_DRAE, 0, channel >> 5, BIT(channel & 0x1f));
858
859 /* ensure no events are pending */
860 edma_stop(ecc, EDMA_CTLR_CHAN(ecc->id, channel));
861 edma_write_slot(ecc, channel, &dummy_paramset);
862
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +0300863 edma_setup_interrupt(ecc, EDMA_CTLR_CHAN(ecc->id, channel), true);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300864
865 edma_map_dmach_to_queue(ecc, channel, eventq_no);
866
867 return EDMA_CTLR_CHAN(ecc->id, channel);
868}
869
870/**
871 * edma_free_channel - deallocate DMA channel
872 * @ecc: pointer to edma_cc struct
873 * @channel: dma channel returned from edma_alloc_channel()
874 *
875 * This deallocates the DMA channel and associated parameter RAM slot
876 * allocated by edma_alloc_channel().
877 *
878 * Callers are responsible for ensuring the channel is inactive, and
879 * will not be reactivated by linking, chaining, or software calls to
880 * edma_start().
881 */
882static void edma_free_channel(struct edma_cc *ecc, unsigned channel)
883{
884 if (ecc->id != EDMA_CTLR(channel)) {
885 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
886 ecc->id, EDMA_CTLR(channel));
887 return;
888 }
889 channel = EDMA_CHAN_SLOT(channel);
890
891 if (channel >= ecc->num_channels)
892 return;
893
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +0300894 edma_setup_interrupt(ecc, channel, false);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300895 /* REVISIT should probably take out of shadow region 0 */
896
Peter Ujfalusi96f5ff02015-10-14 14:42:58 +0300897 edma_write_slot(ecc, channel, &dummy_paramset);
Peter Ujfalusi7a73b132015-10-14 14:43:05 +0300898 clear_bit(channel, ecc->slot_inuse);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300899}
900
Peter Ujfalusi11c15732015-10-14 14:43:00 +0300901/* Move channel to a specific event queue */
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300902static void edma_assign_channel_eventq(struct edma_cc *ecc, unsigned channel,
903 enum dma_event_q eventq_no)
904{
905 if (ecc->id != EDMA_CTLR(channel)) {
906 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
907 ecc->id, EDMA_CTLR(channel));
908 return;
909 }
910 channel = EDMA_CHAN_SLOT(channel);
911
912 if (channel >= ecc->num_channels)
913 return;
914
915 /* default to low priority queue */
916 if (eventq_no == EVENTQ_DEFAULT)
917 eventq_no = ecc->default_queue;
918 if (eventq_no >= ecc->num_tc)
919 return;
920
921 edma_map_dmach_to_queue(ecc, channel, eventq_no);
922}
923
Matt Porterc2dde5f2012-08-22 21:09:34 -0400924static inline struct edma_cc *to_edma_cc(struct dma_device *d)
925{
926 return container_of(d, struct edma_cc, dma_slave);
927}
928
929static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
930{
931 return container_of(c, struct edma_chan, vchan.chan);
932}
933
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300934static inline struct edma_desc *to_edma_desc(struct dma_async_tx_descriptor *tx)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400935{
936 return container_of(tx, struct edma_desc, vdesc.tx);
937}
938
939static void edma_desc_free(struct virt_dma_desc *vdesc)
940{
941 kfree(container_of(vdesc, struct edma_desc, vdesc));
942}
943
944/* Dispatch a queued descriptor to the controller (caller holds lock) */
945static void edma_execute(struct edma_chan *echan)
946{
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300947 struct edma_cc *ecc = echan->ecc;
Joel Fernandes53407062013-09-03 10:02:46 -0500948 struct virt_dma_desc *vdesc;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400949 struct edma_desc *edesc;
Joel Fernandes53407062013-09-03 10:02:46 -0500950 struct device *dev = echan->vchan.chan.device->dev;
951 int i, j, left, nslots;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400952
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300953 if (!echan->edesc) {
954 /* Setup is needed for the first transfer */
Joel Fernandes53407062013-09-03 10:02:46 -0500955 vdesc = vchan_next_desc(&echan->vchan);
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300956 if (!vdesc)
Joel Fernandes53407062013-09-03 10:02:46 -0500957 return;
Joel Fernandes53407062013-09-03 10:02:46 -0500958 list_del(&vdesc->node);
959 echan->edesc = to_edma_desc(&vdesc->tx);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400960 }
961
Joel Fernandes53407062013-09-03 10:02:46 -0500962 edesc = echan->edesc;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400963
Joel Fernandes53407062013-09-03 10:02:46 -0500964 /* Find out how many left */
965 left = edesc->pset_nr - edesc->processed;
966 nslots = min(MAX_NR_SG, left);
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500967 edesc->sg_len = 0;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400968
969 /* Write descriptor PaRAM set(s) */
Joel Fernandes53407062013-09-03 10:02:46 -0500970 for (i = 0; i < nslots; i++) {
971 j = i + edesc->processed;
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300972 edma_write_slot(ecc, echan->slot[i], &edesc->pset[j].param);
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500973 edesc->sg_len += edesc->pset[j].len;
Peter Ujfalusi907f74a2015-10-14 14:42:56 +0300974 dev_vdbg(dev,
975 "\n pset[%d]:\n"
976 " chnum\t%d\n"
977 " slot\t%d\n"
978 " opt\t%08x\n"
979 " src\t%08x\n"
980 " dst\t%08x\n"
981 " abcnt\t%08x\n"
982 " ccnt\t%08x\n"
983 " bidx\t%08x\n"
984 " cidx\t%08x\n"
985 " lkrld\t%08x\n",
986 j, echan->ch_num, echan->slot[i],
987 edesc->pset[j].param.opt,
988 edesc->pset[j].param.src,
989 edesc->pset[j].param.dst,
990 edesc->pset[j].param.a_b_cnt,
991 edesc->pset[j].param.ccnt,
992 edesc->pset[j].param.src_dst_bidx,
993 edesc->pset[j].param.src_dst_cidx,
994 edesc->pset[j].param.link_bcntrld);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400995 /* Link to the previous slot if not the last set */
Joel Fernandes53407062013-09-03 10:02:46 -0500996 if (i != (nslots - 1))
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300997 edma_link(ecc, echan->slot[i], echan->slot[i + 1]);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400998 }
999
Joel Fernandes53407062013-09-03 10:02:46 -05001000 edesc->processed += nslots;
1001
Joel Fernandesb267b3b2013-08-29 18:05:44 -05001002 /*
1003 * If this is either the last set in a set of SG-list transactions
1004 * then setup a link to the dummy slot, this results in all future
1005 * events being absorbed and that's OK because we're done
1006 */
Joel Fernandes50a9c702013-10-31 16:31:23 -05001007 if (edesc->processed == edesc->pset_nr) {
1008 if (edesc->cyclic)
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001009 edma_link(ecc, echan->slot[nslots - 1], echan->slot[1]);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001010 else
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001011 edma_link(ecc, echan->slot[nslots - 1],
Joel Fernandes50a9c702013-10-31 16:31:23 -05001012 echan->ecc->dummy_slot);
1013 }
Joel Fernandesb267b3b2013-08-29 18:05:44 -05001014
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +03001015 if (echan->missed) {
1016 /*
1017 * This happens due to setup times between intermediate
1018 * transfers in long SG lists which have to be broken up into
1019 * transfers of MAX_NR_SG
1020 */
1021 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001022 edma_clean_channel(ecc, echan->ch_num);
1023 edma_stop(ecc, echan->ch_num);
1024 edma_start(ecc, echan->ch_num);
1025 edma_trigger_channel(ecc, echan->ch_num);
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +03001026 echan->missed = 0;
1027 } else if (edesc->processed <= MAX_NR_SG) {
Peter Ujfalusi9aac9092014-04-24 10:29:50 +03001028 dev_dbg(dev, "first transfer starting on channel %d\n",
1029 echan->ch_num);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001030 edma_start(ecc, echan->ch_num);
Sekhar Nori5fc68a62014-03-19 11:25:50 +05301031 } else {
1032 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
1033 echan->ch_num, edesc->processed);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001034 edma_resume(ecc, echan->ch_num);
Joel Fernandes53407062013-09-03 10:02:46 -05001035 }
Matt Porterc2dde5f2012-08-22 21:09:34 -04001036}
1037
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001038static int edma_terminate_all(struct dma_chan *chan)
Matt Porterc2dde5f2012-08-22 21:09:34 -04001039{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001040 struct edma_chan *echan = to_edma_chan(chan);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001041 unsigned long flags;
1042 LIST_HEAD(head);
1043
1044 spin_lock_irqsave(&echan->vchan.lock, flags);
1045
1046 /*
1047 * Stop DMA activity: we assume the callback will not be called
1048 * after edma_dma() returns (even if it does, it will see
1049 * echan->edesc is NULL and exit.)
1050 */
1051 if (echan->edesc) {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001052 edma_stop(echan->ecc, echan->ch_num);
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +03001053 /* Move the cyclic channel back to default queue */
1054 if (echan->edesc->cyclic)
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001055 edma_assign_channel_eventq(echan->ecc, echan->ch_num,
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +03001056 EVENTQ_DEFAULT);
Petr Kulhavy5ca9e7c2015-03-27 13:35:51 +02001057 /*
1058 * free the running request descriptor
1059 * since it is not in any of the vdesc lists
1060 */
1061 edma_desc_free(&echan->edesc->vdesc);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001062 echan->edesc = NULL;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001063 }
1064
1065 vchan_get_all_descriptors(&echan->vchan, &head);
1066 spin_unlock_irqrestore(&echan->vchan.lock, flags);
1067 vchan_dma_desc_free_list(&echan->vchan, &head);
1068
1069 return 0;
1070}
1071
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001072static int edma_slave_config(struct dma_chan *chan,
Matt Porter661f7cb2013-01-10 13:41:04 -05001073 struct dma_slave_config *cfg)
Matt Porterc2dde5f2012-08-22 21:09:34 -04001074{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001075 struct edma_chan *echan = to_edma_chan(chan);
1076
Matt Porter661f7cb2013-01-10 13:41:04 -05001077 if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1078 cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
Matt Porterc2dde5f2012-08-22 21:09:34 -04001079 return -EINVAL;
1080
Matt Porter661f7cb2013-01-10 13:41:04 -05001081 memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
Matt Porterc2dde5f2012-08-22 21:09:34 -04001082
1083 return 0;
1084}
1085
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001086static int edma_dma_pause(struct dma_chan *chan)
Peter Ujfalusi72c7b672014-04-14 14:41:59 +03001087{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001088 struct edma_chan *echan = to_edma_chan(chan);
1089
John Ogness02ec6042015-04-27 13:52:25 +02001090 if (!echan->edesc)
Peter Ujfalusi72c7b672014-04-14 14:41:59 +03001091 return -EINVAL;
1092
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001093 edma_pause(echan->ecc, echan->ch_num);
Peter Ujfalusi72c7b672014-04-14 14:41:59 +03001094 return 0;
1095}
1096
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001097static int edma_dma_resume(struct dma_chan *chan)
Peter Ujfalusi72c7b672014-04-14 14:41:59 +03001098{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001099 struct edma_chan *echan = to_edma_chan(chan);
1100
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001101 edma_resume(echan->ecc, echan->ch_num);
Peter Ujfalusi72c7b672014-04-14 14:41:59 +03001102 return 0;
1103}
1104
Joel Fernandesfd009032013-09-23 18:05:13 -05001105/*
1106 * A PaRAM set configuration abstraction used by other modes
1107 * @chan: Channel who's PaRAM set we're configuring
1108 * @pset: PaRAM set to initialize and setup.
1109 * @src_addr: Source address of the DMA
1110 * @dst_addr: Destination address of the DMA
1111 * @burst: In units of dev_width, how much to send
1112 * @dev_width: How much is the dev_width
1113 * @dma_length: Total length of the DMA transfer
1114 * @direction: Direction of the transfer
1115 */
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001116static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001117 dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
1118 enum dma_slave_buswidth dev_width,
1119 unsigned int dma_length,
1120 enum dma_transfer_direction direction)
Joel Fernandesfd009032013-09-23 18:05:13 -05001121{
1122 struct edma_chan *echan = to_edma_chan(chan);
1123 struct device *dev = chan->device->dev;
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001124 struct edmacc_param *param = &epset->param;
Joel Fernandesfd009032013-09-23 18:05:13 -05001125 int acnt, bcnt, ccnt, cidx;
1126 int src_bidx, dst_bidx, src_cidx, dst_cidx;
1127 int absync;
1128
1129 acnt = dev_width;
Peter Ujfalusib2b617d2014-04-14 14:41:58 +03001130
1131 /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
1132 if (!burst)
1133 burst = 1;
Joel Fernandesfd009032013-09-23 18:05:13 -05001134 /*
1135 * If the maxburst is equal to the fifo width, use
1136 * A-synced transfers. This allows for large contiguous
1137 * buffer transfers using only one PaRAM set.
1138 */
1139 if (burst == 1) {
1140 /*
1141 * For the A-sync case, bcnt and ccnt are the remainder
1142 * and quotient respectively of the division of:
1143 * (dma_length / acnt) by (SZ_64K -1). This is so
1144 * that in case bcnt over flows, we have ccnt to use.
1145 * Note: In A-sync tranfer only, bcntrld is used, but it
1146 * only applies for sg_dma_len(sg) >= SZ_64K.
1147 * In this case, the best way adopted is- bccnt for the
1148 * first frame will be the remainder below. Then for
1149 * every successive frame, bcnt will be SZ_64K-1. This
1150 * is assured as bcntrld = 0xffff in end of function.
1151 */
1152 absync = false;
1153 ccnt = dma_length / acnt / (SZ_64K - 1);
1154 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
1155 /*
1156 * If bcnt is non-zero, we have a remainder and hence an
1157 * extra frame to transfer, so increment ccnt.
1158 */
1159 if (bcnt)
1160 ccnt++;
1161 else
1162 bcnt = SZ_64K - 1;
1163 cidx = acnt;
1164 } else {
1165 /*
1166 * If maxburst is greater than the fifo address_width,
1167 * use AB-synced transfers where A count is the fifo
1168 * address_width and B count is the maxburst. In this
1169 * case, we are limited to transfers of C count frames
1170 * of (address_width * maxburst) where C count is limited
1171 * to SZ_64K-1. This places an upper bound on the length
1172 * of an SG segment that can be handled.
1173 */
1174 absync = true;
1175 bcnt = burst;
1176 ccnt = dma_length / (acnt * bcnt);
1177 if (ccnt > (SZ_64K - 1)) {
1178 dev_err(dev, "Exceeded max SG segment size\n");
1179 return -EINVAL;
1180 }
1181 cidx = acnt * bcnt;
1182 }
1183
Thomas Gleixnerc2da2342014-04-28 14:29:57 -05001184 epset->len = dma_length;
1185
Joel Fernandesfd009032013-09-23 18:05:13 -05001186 if (direction == DMA_MEM_TO_DEV) {
1187 src_bidx = acnt;
1188 src_cidx = cidx;
1189 dst_bidx = 0;
1190 dst_cidx = 0;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -05001191 epset->addr = src_addr;
Joel Fernandesfd009032013-09-23 18:05:13 -05001192 } else if (direction == DMA_DEV_TO_MEM) {
1193 src_bidx = 0;
1194 src_cidx = 0;
1195 dst_bidx = acnt;
1196 dst_cidx = cidx;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -05001197 epset->addr = dst_addr;
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001198 } else if (direction == DMA_MEM_TO_MEM) {
1199 src_bidx = acnt;
1200 src_cidx = cidx;
1201 dst_bidx = acnt;
1202 dst_cidx = cidx;
Joel Fernandesfd009032013-09-23 18:05:13 -05001203 } else {
1204 dev_err(dev, "%s: direction not implemented yet\n", __func__);
1205 return -EINVAL;
1206 }
1207
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001208 param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
Joel Fernandesfd009032013-09-23 18:05:13 -05001209 /* Configure A or AB synchronized transfers */
1210 if (absync)
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001211 param->opt |= SYNCDIM;
Joel Fernandesfd009032013-09-23 18:05:13 -05001212
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001213 param->src = src_addr;
1214 param->dst = dst_addr;
Joel Fernandesfd009032013-09-23 18:05:13 -05001215
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001216 param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
1217 param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
Joel Fernandesfd009032013-09-23 18:05:13 -05001218
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001219 param->a_b_cnt = bcnt << 16 | acnt;
1220 param->ccnt = ccnt;
Joel Fernandesfd009032013-09-23 18:05:13 -05001221 /*
1222 * Only time when (bcntrld) auto reload is required is for
1223 * A-sync case, and in this case, a requirement of reload value
1224 * of SZ_64K-1 only is assured. 'link' is initially set to NULL
1225 * and then later will be populated by edma_execute.
1226 */
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001227 param->link_bcntrld = 0xffffffff;
Joel Fernandesfd009032013-09-23 18:05:13 -05001228 return absync;
1229}
1230
Matt Porterc2dde5f2012-08-22 21:09:34 -04001231static struct dma_async_tx_descriptor *edma_prep_slave_sg(
1232 struct dma_chan *chan, struct scatterlist *sgl,
1233 unsigned int sg_len, enum dma_transfer_direction direction,
1234 unsigned long tx_flags, void *context)
1235{
1236 struct edma_chan *echan = to_edma_chan(chan);
1237 struct device *dev = chan->device->dev;
1238 struct edma_desc *edesc;
Joel Fernandesfd009032013-09-23 18:05:13 -05001239 dma_addr_t src_addr = 0, dst_addr = 0;
Matt Porter661f7cb2013-01-10 13:41:04 -05001240 enum dma_slave_buswidth dev_width;
1241 u32 burst;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001242 struct scatterlist *sg;
Joel Fernandesfd009032013-09-23 18:05:13 -05001243 int i, nslots, ret;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001244
1245 if (unlikely(!echan || !sgl || !sg_len))
1246 return NULL;
1247
Matt Porter661f7cb2013-01-10 13:41:04 -05001248 if (direction == DMA_DEV_TO_MEM) {
Joel Fernandesfd009032013-09-23 18:05:13 -05001249 src_addr = echan->cfg.src_addr;
Matt Porter661f7cb2013-01-10 13:41:04 -05001250 dev_width = echan->cfg.src_addr_width;
1251 burst = echan->cfg.src_maxburst;
1252 } else if (direction == DMA_MEM_TO_DEV) {
Joel Fernandesfd009032013-09-23 18:05:13 -05001253 dst_addr = echan->cfg.dst_addr;
Matt Porter661f7cb2013-01-10 13:41:04 -05001254 dev_width = echan->cfg.dst_addr_width;
1255 burst = echan->cfg.dst_maxburst;
1256 } else {
Peter Ujfalusie6fad592014-04-14 14:42:05 +03001257 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
Matt Porter661f7cb2013-01-10 13:41:04 -05001258 return NULL;
1259 }
1260
1261 if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
Peter Ujfalusic594c892014-04-14 14:42:03 +03001262 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001263 return NULL;
1264 }
1265
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001266 edesc = kzalloc(sizeof(*edesc) + sg_len * sizeof(edesc->pset[0]),
1267 GFP_ATOMIC);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001268 if (!edesc) {
Peter Ujfalusic594c892014-04-14 14:42:03 +03001269 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001270 return NULL;
1271 }
1272
1273 edesc->pset_nr = sg_len;
Thomas Gleixnerb6205c32014-04-28 14:18:45 -05001274 edesc->residue = 0;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -05001275 edesc->direction = direction;
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001276 edesc->echan = echan;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001277
Joel Fernandes6fbe24d2013-08-29 18:05:40 -05001278 /* Allocate a PaRAM slot, if needed */
1279 nslots = min_t(unsigned, MAX_NR_SG, sg_len);
1280
1281 for (i = 0; i < nslots; i++) {
Matt Porterc2dde5f2012-08-22 21:09:34 -04001282 if (echan->slot[i] < 0) {
1283 echan->slot[i] =
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001284 edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001285 if (echan->slot[i] < 0) {
Valentin Ilie4b6271a2013-10-24 16:14:22 +03001286 kfree(edesc);
Peter Ujfalusic594c892014-04-14 14:42:03 +03001287 dev_err(dev, "%s: Failed to allocate slot\n",
1288 __func__);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001289 return NULL;
1290 }
1291 }
Joel Fernandes6fbe24d2013-08-29 18:05:40 -05001292 }
1293
1294 /* Configure PaRAM sets for each SG */
1295 for_each_sg(sgl, sg, sg_len, i) {
Joel Fernandesfd009032013-09-23 18:05:13 -05001296 /* Get address for each SG */
1297 if (direction == DMA_DEV_TO_MEM)
1298 dst_addr = sg_dma_address(sg);
1299 else
1300 src_addr = sg_dma_address(sg);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001301
Joel Fernandesfd009032013-09-23 18:05:13 -05001302 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
1303 dst_addr, burst, dev_width,
1304 sg_dma_len(sg), direction);
Vinod Koulb967aec2013-10-30 13:07:18 +05301305 if (ret < 0) {
1306 kfree(edesc);
Joel Fernandesfd009032013-09-23 18:05:13 -05001307 return NULL;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001308 }
1309
Joel Fernandesfd009032013-09-23 18:05:13 -05001310 edesc->absync = ret;
Thomas Gleixnerb6205c32014-04-28 14:18:45 -05001311 edesc->residue += sg_dma_len(sg);
Joel Fernandes6fbe24d2013-08-29 18:05:40 -05001312
1313 /* If this is the last in a current SG set of transactions,
1314 enable interrupts so that next set is processed */
1315 if (!((i+1) % MAX_NR_SG))
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001316 edesc->pset[i].param.opt |= TCINTEN;
Joel Fernandes6fbe24d2013-08-29 18:05:40 -05001317
Matt Porterc2dde5f2012-08-22 21:09:34 -04001318 /* If this is the last set, enable completion interrupt flag */
1319 if (i == sg_len - 1)
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001320 edesc->pset[i].param.opt |= TCINTEN;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001321 }
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001322 edesc->residue_stat = edesc->residue;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001323
Matt Porterc2dde5f2012-08-22 21:09:34 -04001324 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1325}
Matt Porterc2dde5f2012-08-22 21:09:34 -04001326
Lad, Prabhakarb7a4fd52015-02-04 13:03:27 +00001327static struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001328 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1329 size_t len, unsigned long tx_flags)
1330{
1331 int ret;
1332 struct edma_desc *edesc;
1333 struct device *dev = chan->device->dev;
1334 struct edma_chan *echan = to_edma_chan(chan);
1335
1336 if (unlikely(!echan || !len))
1337 return NULL;
1338
1339 edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC);
1340 if (!edesc) {
1341 dev_dbg(dev, "Failed to allocate a descriptor\n");
1342 return NULL;
1343 }
1344
1345 edesc->pset_nr = 1;
1346
1347 ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
1348 DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM);
1349 if (ret < 0)
1350 return NULL;
1351
1352 edesc->absync = ret;
1353
1354 /*
1355 * Enable intermediate transfer chaining to re-trigger channel
1356 * on completion of every TR, and enable transfer-completion
1357 * interrupt on completion of the whole transfer.
1358 */
Joel Fernandesb0cce4c2014-04-28 15:30:32 -05001359 edesc->pset[0].param.opt |= ITCCHEN;
1360 edesc->pset[0].param.opt |= TCINTEN;
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001361
1362 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1363}
1364
Joel Fernandes50a9c702013-10-31 16:31:23 -05001365static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
1366 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1367 size_t period_len, enum dma_transfer_direction direction,
Laurent Pinchart31c1e5a2014-08-01 12:20:10 +02001368 unsigned long tx_flags)
Joel Fernandes50a9c702013-10-31 16:31:23 -05001369{
1370 struct edma_chan *echan = to_edma_chan(chan);
1371 struct device *dev = chan->device->dev;
1372 struct edma_desc *edesc;
1373 dma_addr_t src_addr, dst_addr;
1374 enum dma_slave_buswidth dev_width;
1375 u32 burst;
1376 int i, ret, nslots;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001377
Joel Fernandes50a9c702013-10-31 16:31:23 -05001378 if (unlikely(!echan || !buf_len || !period_len))
1379 return NULL;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001380
Joel Fernandes50a9c702013-10-31 16:31:23 -05001381 if (direction == DMA_DEV_TO_MEM) {
1382 src_addr = echan->cfg.src_addr;
1383 dst_addr = buf_addr;
1384 dev_width = echan->cfg.src_addr_width;
1385 burst = echan->cfg.src_maxburst;
1386 } else if (direction == DMA_MEM_TO_DEV) {
1387 src_addr = buf_addr;
1388 dst_addr = echan->cfg.dst_addr;
1389 dev_width = echan->cfg.dst_addr_width;
1390 burst = echan->cfg.dst_maxburst;
1391 } else {
Peter Ujfalusie6fad592014-04-14 14:42:05 +03001392 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001393 return NULL;
1394 }
1395
1396 if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
Peter Ujfalusic594c892014-04-14 14:42:03 +03001397 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001398 return NULL;
1399 }
1400
1401 if (unlikely(buf_len % period_len)) {
1402 dev_err(dev, "Period should be multiple of Buffer length\n");
1403 return NULL;
1404 }
1405
1406 nslots = (buf_len / period_len) + 1;
1407
1408 /*
1409 * Cyclic DMA users such as audio cannot tolerate delays introduced
1410 * by cases where the number of periods is more than the maximum
1411 * number of SGs the EDMA driver can handle at a time. For DMA types
1412 * such as Slave SGs, such delays are tolerable and synchronized,
1413 * but the synchronization is difficult to achieve with Cyclic and
1414 * cannot be guaranteed, so we error out early.
1415 */
1416 if (nslots > MAX_NR_SG)
1417 return NULL;
1418
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001419 edesc = kzalloc(sizeof(*edesc) + nslots * sizeof(edesc->pset[0]),
1420 GFP_ATOMIC);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001421 if (!edesc) {
Peter Ujfalusic594c892014-04-14 14:42:03 +03001422 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001423 return NULL;
1424 }
1425
1426 edesc->cyclic = 1;
1427 edesc->pset_nr = nslots;
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001428 edesc->residue = edesc->residue_stat = buf_len;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -05001429 edesc->direction = direction;
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001430 edesc->echan = echan;
Joel Fernandes50a9c702013-10-31 16:31:23 -05001431
Peter Ujfalusi83bb3122014-04-14 14:42:02 +03001432 dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
1433 __func__, echan->ch_num, nslots, period_len, buf_len);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001434
1435 for (i = 0; i < nslots; i++) {
1436 /* Allocate a PaRAM slot, if needed */
1437 if (echan->slot[i] < 0) {
1438 echan->slot[i] =
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001439 edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001440 if (echan->slot[i] < 0) {
Christian Engelmayere3ddc972013-12-30 20:48:39 +01001441 kfree(edesc);
Peter Ujfalusic594c892014-04-14 14:42:03 +03001442 dev_err(dev, "%s: Failed to allocate slot\n",
1443 __func__);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001444 return NULL;
1445 }
1446 }
1447
1448 if (i == nslots - 1) {
1449 memcpy(&edesc->pset[i], &edesc->pset[0],
1450 sizeof(edesc->pset[0]));
1451 break;
1452 }
1453
1454 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
1455 dst_addr, burst, dev_width, period_len,
1456 direction);
Christian Engelmayere3ddc972013-12-30 20:48:39 +01001457 if (ret < 0) {
1458 kfree(edesc);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001459 return NULL;
Christian Engelmayere3ddc972013-12-30 20:48:39 +01001460 }
Joel Fernandes50a9c702013-10-31 16:31:23 -05001461
1462 if (direction == DMA_DEV_TO_MEM)
1463 dst_addr += period_len;
1464 else
1465 src_addr += period_len;
1466
Peter Ujfalusi83bb3122014-04-14 14:42:02 +03001467 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
1468 dev_vdbg(dev,
Joel Fernandes50a9c702013-10-31 16:31:23 -05001469 "\n pset[%d]:\n"
1470 " chnum\t%d\n"
1471 " slot\t%d\n"
1472 " opt\t%08x\n"
1473 " src\t%08x\n"
1474 " dst\t%08x\n"
1475 " abcnt\t%08x\n"
1476 " ccnt\t%08x\n"
1477 " bidx\t%08x\n"
1478 " cidx\t%08x\n"
1479 " lkrld\t%08x\n",
1480 i, echan->ch_num, echan->slot[i],
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001481 edesc->pset[i].param.opt,
1482 edesc->pset[i].param.src,
1483 edesc->pset[i].param.dst,
1484 edesc->pset[i].param.a_b_cnt,
1485 edesc->pset[i].param.ccnt,
1486 edesc->pset[i].param.src_dst_bidx,
1487 edesc->pset[i].param.src_dst_cidx,
1488 edesc->pset[i].param.link_bcntrld);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001489
1490 edesc->absync = ret;
1491
1492 /*
Peter Ujfalusia1f146f2014-07-16 15:29:21 +03001493 * Enable period interrupt only if it is requested
Joel Fernandes50a9c702013-10-31 16:31:23 -05001494 */
Peter Ujfalusia1f146f2014-07-16 15:29:21 +03001495 if (tx_flags & DMA_PREP_INTERRUPT)
1496 edesc->pset[i].param.opt |= TCINTEN;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001497 }
1498
Peter Ujfalusi8e8805d2014-07-08 13:46:38 +03001499 /* Place the cyclic channel to highest priority queue */
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001500 edma_assign_channel_eventq(echan->ecc, echan->ch_num, EVENTQ_0);
Peter Ujfalusi8e8805d2014-07-08 13:46:38 +03001501
Matt Porterc2dde5f2012-08-22 21:09:34 -04001502 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1503}
1504
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001505static void edma_completion_handler(struct edma_chan *echan)
Matt Porterc2dde5f2012-08-22 21:09:34 -04001506{
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001507 struct edma_cc *ecc = echan->ecc;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001508 struct device *dev = echan->vchan.chan.device->dev;
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001509 struct edma_desc *edesc = echan->edesc;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001510
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001511 if (!edesc)
1512 return;
Joel Fernandes50a9c702013-10-31 16:31:23 -05001513
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +03001514 spin_lock(&echan->vchan.lock);
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001515 if (edesc->cyclic) {
1516 vchan_cyclic_callback(&edesc->vdesc);
1517 spin_unlock(&echan->vchan.lock);
1518 return;
1519 } else if (edesc->processed == edesc->pset_nr) {
1520 edesc->residue = 0;
1521 edma_stop(ecc, echan->ch_num);
1522 vchan_cookie_complete(&edesc->vdesc);
1523 echan->edesc = NULL;
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001524
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001525 dev_dbg(dev, "Transfer completed on channel %d\n",
1526 echan->ch_num);
1527 } else {
1528 dev_dbg(dev, "Sub transfer completed on channel %d\n",
1529 echan->ch_num);
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +03001530
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001531 edma_pause(ecc, echan->ch_num);
Joel Fernandesc5f47992013-08-29 18:05:43 -05001532
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001533 /* Update statistics for tx_status */
1534 edesc->residue -= edesc->sg_len;
1535 edesc->residue_stat = edesc->residue;
1536 edesc->processed_stat = edesc->processed;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001537 }
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001538 edma_execute(echan);
1539
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +03001540 spin_unlock(&echan->vchan.lock);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001541}
1542
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001543/* eDMA interrupt handler */
1544static irqreturn_t dma_irq_handler(int irq, void *data)
1545{
1546 struct edma_cc *ecc = data;
1547 int ctlr;
1548 u32 sh_ier;
1549 u32 sh_ipr;
1550 u32 bank;
1551
1552 ctlr = ecc->id;
1553 if (ctlr < 0)
1554 return IRQ_NONE;
1555
1556 dev_vdbg(ecc->dev, "dma_irq_handler\n");
1557
1558 sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 0);
1559 if (!sh_ipr) {
1560 sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 1);
1561 if (!sh_ipr)
1562 return IRQ_NONE;
1563 sh_ier = edma_shadow0_read_array(ecc, SH_IER, 1);
1564 bank = 1;
1565 } else {
1566 sh_ier = edma_shadow0_read_array(ecc, SH_IER, 0);
1567 bank = 0;
1568 }
1569
1570 do {
1571 u32 slot;
1572 u32 channel;
1573
1574 slot = __ffs(sh_ipr);
1575 sh_ipr &= ~(BIT(slot));
1576
1577 if (sh_ier & BIT(slot)) {
1578 channel = (bank << 5) | slot;
1579 /* Clear the corresponding IPR bits */
1580 edma_shadow0_write_array(ecc, SH_ICR, bank, BIT(slot));
1581 edma_completion_handler(&ecc->slave_chans[channel]);
1582 }
1583 } while (sh_ipr);
1584
1585 edma_shadow0_write(ecc, SH_IEVAL, 1);
1586 return IRQ_HANDLED;
1587}
1588
1589static void edma_error_handler(struct edma_chan *echan)
1590{
1591 struct edma_cc *ecc = echan->ecc;
1592 struct device *dev = echan->vchan.chan.device->dev;
1593 struct edmacc_param p;
1594
1595 if (!echan->edesc)
1596 return;
1597
1598 spin_lock(&echan->vchan.lock);
1599
1600 edma_read_slot(ecc, echan->slot[0], &p);
1601 /*
1602 * Issue later based on missed flag which will be sure
1603 * to happen as:
1604 * (1) we finished transmitting an intermediate slot and
1605 * edma_execute is coming up.
1606 * (2) or we finished current transfer and issue will
1607 * call edma_execute.
1608 *
1609 * Important note: issuing can be dangerous here and
1610 * lead to some nasty recursion when we are in a NULL
1611 * slot. So we avoid doing so and set the missed flag.
1612 */
1613 if (p.a_b_cnt == 0 && p.ccnt == 0) {
1614 dev_dbg(dev, "Error on null slot, setting miss\n");
1615 echan->missed = 1;
1616 } else {
1617 /*
1618 * The slot is already programmed but the event got
1619 * missed, so its safe to issue it here.
1620 */
1621 dev_dbg(dev, "Missed event, TRIGGERING\n");
1622 edma_clean_channel(ecc, echan->ch_num);
1623 edma_stop(ecc, echan->ch_num);
1624 edma_start(ecc, echan->ch_num);
1625 edma_trigger_channel(ecc, echan->ch_num);
1626 }
1627 spin_unlock(&echan->vchan.lock);
1628}
1629
Peter Ujfalusi7c3b8b32015-10-14 14:43:02 +03001630static inline bool edma_error_pending(struct edma_cc *ecc)
1631{
1632 if (edma_read_array(ecc, EDMA_EMR, 0) ||
1633 edma_read_array(ecc, EDMA_EMR, 1) ||
1634 edma_read(ecc, EDMA_QEMR) || edma_read(ecc, EDMA_CCERR))
1635 return true;
1636
1637 return false;
1638}
1639
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001640/* eDMA error interrupt handler */
1641static irqreturn_t dma_ccerr_handler(int irq, void *data)
1642{
1643 struct edma_cc *ecc = data;
Peter Ujfalusie4402a12015-10-14 14:43:03 +03001644 int i, j;
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001645 int ctlr;
1646 unsigned int cnt = 0;
Peter Ujfalusie4402a12015-10-14 14:43:03 +03001647 unsigned int val;
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001648
1649 ctlr = ecc->id;
1650 if (ctlr < 0)
1651 return IRQ_NONE;
1652
1653 dev_vdbg(ecc->dev, "dma_ccerr_handler\n");
1654
Peter Ujfalusi7c3b8b32015-10-14 14:43:02 +03001655 if (!edma_error_pending(ecc))
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001656 return IRQ_NONE;
1657
1658 while (1) {
Peter Ujfalusie4402a12015-10-14 14:43:03 +03001659 /* Event missed register(s) */
1660 for (j = 0; j < 2; j++) {
1661 unsigned long emr;
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001662
Peter Ujfalusie4402a12015-10-14 14:43:03 +03001663 val = edma_read_array(ecc, EDMA_EMR, j);
1664 if (!val)
1665 continue;
1666
1667 dev_dbg(ecc->dev, "EMR%d 0x%08x\n", j, val);
1668 emr = val;
1669 for (i = find_next_bit(&emr, 32, 0); i < 32;
1670 i = find_next_bit(&emr, 32, i + 1)) {
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001671 int k = (j << 5) + i;
1672
Peter Ujfalusie4402a12015-10-14 14:43:03 +03001673 /* Clear the corresponding EMR bits */
1674 edma_write_array(ecc, EDMA_EMCR, j, BIT(i));
1675 /* Clear any SER */
1676 edma_shadow0_write_array(ecc, SH_SECR, j,
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001677 BIT(i));
Peter Ujfalusie4402a12015-10-14 14:43:03 +03001678 edma_error_handler(&ecc->slave_chans[k]);
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001679 }
1680 }
Peter Ujfalusie4402a12015-10-14 14:43:03 +03001681
1682 val = edma_read(ecc, EDMA_QEMR);
1683 if (val) {
1684 dev_dbg(ecc->dev, "QEMR 0x%02x\n", val);
1685 /* Not reported, just clear the interrupt reason. */
1686 edma_write(ecc, EDMA_QEMCR, val);
1687 edma_shadow0_write(ecc, SH_QSECR, val);
1688 }
1689
1690 val = edma_read(ecc, EDMA_CCERR);
1691 if (val) {
1692 dev_warn(ecc->dev, "CCERR 0x%08x\n", val);
1693 /* Not reported, just clear the interrupt reason. */
1694 edma_write(ecc, EDMA_CCERRCLR, val);
1695 }
1696
Peter Ujfalusi7c3b8b32015-10-14 14:43:02 +03001697 if (!edma_error_pending(ecc))
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001698 break;
1699 cnt++;
1700 if (cnt > 10)
1701 break;
1702 }
1703 edma_write(ecc, EDMA_EEVAL, 1);
1704 return IRQ_HANDLED;
1705}
1706
Matt Porterc2dde5f2012-08-22 21:09:34 -04001707/* Alloc channel resources */
1708static int edma_alloc_chan_resources(struct dma_chan *chan)
1709{
1710 struct edma_chan *echan = to_edma_chan(chan);
1711 struct device *dev = chan->device->dev;
1712 int ret;
1713 int a_ch_num;
1714 LIST_HEAD(descs);
1715
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001716 a_ch_num = edma_alloc_channel(echan->ecc, echan->ch_num, EVENTQ_DEFAULT);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001717
1718 if (a_ch_num < 0) {
1719 ret = -ENODEV;
1720 goto err_no_chan;
1721 }
1722
1723 if (a_ch_num != echan->ch_num) {
1724 dev_err(dev, "failed to allocate requested channel %u:%u\n",
1725 EDMA_CTLR(echan->ch_num),
1726 EDMA_CHAN_SLOT(echan->ch_num));
1727 ret = -ENODEV;
1728 goto err_wrong_chan;
1729 }
1730
1731 echan->alloced = true;
1732 echan->slot[0] = echan->ch_num;
1733
Peter Ujfalusi9aac9092014-04-24 10:29:50 +03001734 dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num,
Ezequiel Garcia0e772c62013-12-13 11:06:18 -03001735 EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num));
Matt Porterc2dde5f2012-08-22 21:09:34 -04001736
1737 return 0;
1738
1739err_wrong_chan:
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001740 edma_free_channel(echan->ecc, a_ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001741err_no_chan:
1742 return ret;
1743}
1744
1745/* Free channel resources */
1746static void edma_free_chan_resources(struct dma_chan *chan)
1747{
1748 struct edma_chan *echan = to_edma_chan(chan);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001749 int i;
1750
1751 /* Terminate transfers */
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001752 edma_stop(echan->ecc, echan->ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001753
1754 vchan_free_chan_resources(&echan->vchan);
1755
1756 /* Free EDMA PaRAM slots */
1757 for (i = 1; i < EDMA_MAX_SLOTS; i++) {
1758 if (echan->slot[i] >= 0) {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001759 edma_free_slot(echan->ecc, echan->slot[i]);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001760 echan->slot[i] = -1;
1761 }
1762 }
1763
1764 /* Free EDMA channel */
1765 if (echan->alloced) {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001766 edma_free_channel(echan->ecc, echan->ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001767 echan->alloced = false;
1768 }
1769
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03001770 dev_dbg(chan->device->dev, "freeing channel for %u\n", echan->ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001771}
1772
1773/* Send pending descriptor to hardware */
1774static void edma_issue_pending(struct dma_chan *chan)
1775{
1776 struct edma_chan *echan = to_edma_chan(chan);
1777 unsigned long flags;
1778
1779 spin_lock_irqsave(&echan->vchan.lock, flags);
1780 if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
1781 edma_execute(echan);
1782 spin_unlock_irqrestore(&echan->vchan.lock, flags);
1783}
1784
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001785static u32 edma_residue(struct edma_desc *edesc)
1786{
1787 bool dst = edesc->direction == DMA_DEV_TO_MEM;
1788 struct edma_pset *pset = edesc->pset;
1789 dma_addr_t done, pos;
1790 int i;
1791
1792 /*
1793 * We always read the dst/src position from the first RamPar
1794 * pset. That's the one which is active now.
1795 */
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001796 pos = edma_get_position(edesc->echan->ecc, edesc->echan->slot[0], dst);
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001797
1798 /*
1799 * Cyclic is simple. Just subtract pset[0].addr from pos.
1800 *
1801 * We never update edesc->residue in the cyclic case, so we
1802 * can tell the remaining room to the end of the circular
1803 * buffer.
1804 */
1805 if (edesc->cyclic) {
1806 done = pos - pset->addr;
1807 edesc->residue_stat = edesc->residue - done;
1808 return edesc->residue_stat;
1809 }
1810
1811 /*
1812 * For SG operation we catch up with the last processed
1813 * status.
1814 */
1815 pset += edesc->processed_stat;
1816
1817 for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
1818 /*
1819 * If we are inside this pset address range, we know
1820 * this is the active one. Get the current delta and
1821 * stop walking the psets.
1822 */
1823 if (pos >= pset->addr && pos < pset->addr + pset->len)
1824 return edesc->residue_stat - (pos - pset->addr);
1825
1826 /* Otherwise mark it done and update residue_stat. */
1827 edesc->processed_stat++;
1828 edesc->residue_stat -= pset->len;
1829 }
1830 return edesc->residue_stat;
1831}
1832
Matt Porterc2dde5f2012-08-22 21:09:34 -04001833/* Check request completion status */
1834static enum dma_status edma_tx_status(struct dma_chan *chan,
1835 dma_cookie_t cookie,
1836 struct dma_tx_state *txstate)
1837{
1838 struct edma_chan *echan = to_edma_chan(chan);
1839 struct virt_dma_desc *vdesc;
1840 enum dma_status ret;
1841 unsigned long flags;
1842
1843 ret = dma_cookie_status(chan, cookie, txstate);
Vinod Koul9d386ec2013-10-16 13:42:15 +05301844 if (ret == DMA_COMPLETE || !txstate)
Matt Porterc2dde5f2012-08-22 21:09:34 -04001845 return ret;
1846
1847 spin_lock_irqsave(&echan->vchan.lock, flags);
Thomas Gleixnerde135932014-04-28 14:19:51 -05001848 if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001849 txstate->residue = edma_residue(echan->edesc);
Thomas Gleixnerde135932014-04-28 14:19:51 -05001850 else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
1851 txstate->residue = to_edma_desc(&vdesc->tx)->residue;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001852 spin_unlock_irqrestore(&echan->vchan.lock, flags);
1853
1854 return ret;
1855}
1856
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001857static void __init edma_chan_init(struct edma_cc *ecc, struct dma_device *dma,
Matt Porterc2dde5f2012-08-22 21:09:34 -04001858 struct edma_chan *echans)
1859{
1860 int i, j;
1861
Peter Ujfalusicb782052015-10-14 14:42:54 +03001862 for (i = 0; i < ecc->num_channels; i++) {
Matt Porterc2dde5f2012-08-22 21:09:34 -04001863 struct edma_chan *echan = &echans[i];
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001864 echan->ch_num = EDMA_CTLR_CHAN(ecc->id, i);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001865 echan->ecc = ecc;
1866 echan->vchan.desc_free = edma_desc_free;
1867
1868 vchan_init(&echan->vchan, dma);
1869
1870 INIT_LIST_HEAD(&echan->node);
1871 for (j = 0; j < EDMA_MAX_SLOTS; j++)
1872 echan->slot[j] = -1;
1873 }
1874}
1875
Peter Ujfalusi2c88ee62014-04-14 14:42:01 +03001876#define EDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1877 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
Peter Ujfalusie4a899d2014-07-03 07:51:56 +03001878 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
Peter Ujfalusi2c88ee62014-04-14 14:42:01 +03001879 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1880
Matt Porterc2dde5f2012-08-22 21:09:34 -04001881static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
1882 struct device *dev)
1883{
1884 dma->device_prep_slave_sg = edma_prep_slave_sg;
Joel Fernandes50a9c702013-10-31 16:31:23 -05001885 dma->device_prep_dma_cyclic = edma_prep_dma_cyclic;
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001886 dma->device_prep_dma_memcpy = edma_prep_dma_memcpy;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001887 dma->device_alloc_chan_resources = edma_alloc_chan_resources;
1888 dma->device_free_chan_resources = edma_free_chan_resources;
1889 dma->device_issue_pending = edma_issue_pending;
1890 dma->device_tx_status = edma_tx_status;
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001891 dma->device_config = edma_slave_config;
1892 dma->device_pause = edma_dma_pause;
1893 dma->device_resume = edma_dma_resume;
1894 dma->device_terminate_all = edma_terminate_all;
Maxime Ripard9f59cd02014-11-17 14:42:47 +01001895
1896 dma->src_addr_widths = EDMA_DMA_BUSWIDTHS;
1897 dma->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
1898 dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1899 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1900
Matt Porterc2dde5f2012-08-22 21:09:34 -04001901 dma->dev = dev;
1902
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001903 /*
1904 * code using dma memcpy must make sure alignment of
1905 * length is at dma->copy_align boundary.
1906 */
Maxime Ripard77a68e52015-07-20 10:41:32 +02001907 dma->copy_align = DMAENGINE_ALIGN_4_BYTES;
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001908
Matt Porterc2dde5f2012-08-22 21:09:34 -04001909 INIT_LIST_HEAD(&dma->channels);
1910}
1911
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001912static int edma_setup_from_hw(struct device *dev, struct edma_soc_info *pdata,
1913 struct edma_cc *ecc)
1914{
1915 int i;
1916 u32 value, cccfg;
1917 s8 (*queue_priority_map)[2];
1918
1919 /* Decode the eDMA3 configuration from CCCFG register */
1920 cccfg = edma_read(ecc, EDMA_CCCFG);
1921
1922 value = GET_NUM_REGN(cccfg);
1923 ecc->num_region = BIT(value);
1924
1925 value = GET_NUM_DMACH(cccfg);
1926 ecc->num_channels = BIT(value + 1);
1927
1928 value = GET_NUM_PAENTRY(cccfg);
1929 ecc->num_slots = BIT(value + 4);
1930
1931 value = GET_NUM_EVQUE(cccfg);
1932 ecc->num_tc = value + 1;
1933
Peter Ujfalusi4ab54f62015-10-14 14:43:04 +03001934 ecc->chmap_exist = (cccfg & CHMAP_EXIST) ? true : false;
1935
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001936 dev_dbg(dev, "eDMA3 CC HW configuration (cccfg: 0x%08x):\n", cccfg);
1937 dev_dbg(dev, "num_region: %u\n", ecc->num_region);
1938 dev_dbg(dev, "num_channels: %u\n", ecc->num_channels);
1939 dev_dbg(dev, "num_slots: %u\n", ecc->num_slots);
1940 dev_dbg(dev, "num_tc: %u\n", ecc->num_tc);
Peter Ujfalusi4ab54f62015-10-14 14:43:04 +03001941 dev_dbg(dev, "chmap_exist: %s\n", ecc->chmap_exist ? "yes" : "no");
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001942
1943 /* Nothing need to be done if queue priority is provided */
1944 if (pdata->queue_priority_mapping)
1945 return 0;
1946
1947 /*
1948 * Configure TC/queue priority as follows:
1949 * Q0 - priority 0
1950 * Q1 - priority 1
1951 * Q2 - priority 2
1952 * ...
1953 * The meaning of priority numbers: 0 highest priority, 7 lowest
1954 * priority. So Q0 is the highest priority queue and the last queue has
1955 * the lowest priority.
1956 */
Peter Ujfalusi547c6e22015-10-14 14:42:55 +03001957 queue_priority_map = devm_kcalloc(dev, ecc->num_tc + 1, sizeof(s8),
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001958 GFP_KERNEL);
1959 if (!queue_priority_map)
1960 return -ENOMEM;
1961
1962 for (i = 0; i < ecc->num_tc; i++) {
1963 queue_priority_map[i][0] = i;
1964 queue_priority_map[i][1] = i;
1965 }
1966 queue_priority_map[i][0] = -1;
1967 queue_priority_map[i][1] = -1;
1968
1969 pdata->queue_priority_mapping = queue_priority_map;
1970 /* Default queue has the lowest priority */
1971 pdata->default_queue = i - 1;
1972
1973 return 0;
1974}
1975
1976#if IS_ENABLED(CONFIG_OF)
1977static int edma_xbar_event_map(struct device *dev, struct edma_soc_info *pdata,
1978 size_t sz)
1979{
1980 const char pname[] = "ti,edma-xbar-event-map";
1981 struct resource res;
1982 void __iomem *xbar;
1983 s16 (*xbar_chans)[2];
1984 size_t nelm = sz / sizeof(s16);
1985 u32 shift, offset, mux;
1986 int ret, i;
1987
Peter Ujfalusi547c6e22015-10-14 14:42:55 +03001988 xbar_chans = devm_kcalloc(dev, nelm + 2, sizeof(s16), GFP_KERNEL);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001989 if (!xbar_chans)
1990 return -ENOMEM;
1991
1992 ret = of_address_to_resource(dev->of_node, 1, &res);
1993 if (ret)
1994 return -ENOMEM;
1995
1996 xbar = devm_ioremap(dev, res.start, resource_size(&res));
1997 if (!xbar)
1998 return -ENOMEM;
1999
2000 ret = of_property_read_u16_array(dev->of_node, pname, (u16 *)xbar_chans,
2001 nelm);
2002 if (ret)
2003 return -EIO;
2004
2005 /* Invalidate last entry for the other user of this mess */
2006 nelm >>= 1;
2007 xbar_chans[nelm][0] = -1;
2008 xbar_chans[nelm][1] = -1;
2009
2010 for (i = 0; i < nelm; i++) {
2011 shift = (xbar_chans[i][1] & 0x03) << 3;
2012 offset = xbar_chans[i][1] & 0xfffffffc;
2013 mux = readl(xbar + offset);
2014 mux &= ~(0xff << shift);
2015 mux |= xbar_chans[i][0] << shift;
2016 writel(mux, (xbar + offset));
2017 }
2018
2019 pdata->xbar_chans = (const s16 (*)[2]) xbar_chans;
2020 return 0;
2021}
2022
2023static int edma_of_parse_dt(struct device *dev, struct edma_soc_info *pdata)
2024{
2025 int ret = 0;
2026 struct property *prop;
2027 size_t sz;
2028 struct edma_rsv_info *rsv_info;
2029
2030 rsv_info = devm_kzalloc(dev, sizeof(struct edma_rsv_info), GFP_KERNEL);
2031 if (!rsv_info)
2032 return -ENOMEM;
2033 pdata->rsv = rsv_info;
2034
2035 prop = of_find_property(dev->of_node, "ti,edma-xbar-event-map", &sz);
2036 if (prop)
2037 ret = edma_xbar_event_map(dev, pdata, sz);
2038
2039 return ret;
2040}
2041
2042static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev)
2043{
2044 struct edma_soc_info *info;
2045 int ret;
2046
2047 info = devm_kzalloc(dev, sizeof(struct edma_soc_info), GFP_KERNEL);
2048 if (!info)
2049 return ERR_PTR(-ENOMEM);
2050
2051 ret = edma_of_parse_dt(dev, info);
2052 if (ret)
2053 return ERR_PTR(ret);
2054
2055 return info;
2056}
2057#else
2058static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev)
2059{
2060 return ERR_PTR(-EINVAL);
2061}
2062#endif
2063
Bill Pemberton463a1f82012-11-19 13:22:55 -05002064static int edma_probe(struct platform_device *pdev)
Matt Porterc2dde5f2012-08-22 21:09:34 -04002065{
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002066 struct edma_soc_info *info = pdev->dev.platform_data;
2067 s8 (*queue_priority_mapping)[2];
2068 int i, off, ln;
2069 const s16 (*rsv_chans)[2];
2070 const s16 (*rsv_slots)[2];
2071 const s16 (*xbar_chans)[2];
2072 int irq;
2073 char *irq_name;
2074 struct resource *mem;
2075 struct device_node *node = pdev->dev.of_node;
2076 struct device *dev = &pdev->dev;
2077 struct edma_cc *ecc;
Matt Porterc2dde5f2012-08-22 21:09:34 -04002078 int ret;
2079
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002080 if (node) {
2081 info = edma_setup_info_from_dt(dev);
2082 if (IS_ERR(info)) {
2083 dev_err(dev, "failed to get DT data\n");
2084 return PTR_ERR(info);
2085 }
2086 }
2087
2088 if (!info)
2089 return -ENODEV;
2090
2091 pm_runtime_enable(dev);
2092 ret = pm_runtime_get_sync(dev);
2093 if (ret < 0) {
2094 dev_err(dev, "pm_runtime_get_sync() failed\n");
2095 return ret;
2096 }
2097
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03002098 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
Russell King94cb0e72013-06-27 13:45:16 +01002099 if (ret)
2100 return ret;
2101
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03002102 ecc = devm_kzalloc(dev, sizeof(*ecc), GFP_KERNEL);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002103 if (!ecc) {
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03002104 dev_err(dev, "Can't allocate controller\n");
Matt Porterc2dde5f2012-08-22 21:09:34 -04002105 return -ENOMEM;
2106 }
2107
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002108 ecc->dev = dev;
2109 ecc->id = pdev->id;
2110 /* When booting with DT the pdev->id is -1 */
2111 if (ecc->id < 0)
2112 ecc->id = 0;
Peter Ujfalusica304fa2015-10-14 14:42:49 +03002113
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002114 mem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "edma3_cc");
2115 if (!mem) {
2116 dev_dbg(dev, "mem resource not found, using index 0\n");
2117 mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2118 if (!mem) {
2119 dev_err(dev, "no mem resource?\n");
2120 return -ENODEV;
2121 }
2122 }
2123 ecc->base = devm_ioremap_resource(dev, mem);
2124 if (IS_ERR(ecc->base))
2125 return PTR_ERR(ecc->base);
Peter Ujfalusib2c843a2015-10-14 14:42:50 +03002126
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002127 platform_set_drvdata(pdev, ecc);
2128
2129 /* Get eDMA3 configuration from IP */
2130 ret = edma_setup_from_hw(dev, info, ecc);
2131 if (ret)
2132 return ret;
2133
Peter Ujfalusicb782052015-10-14 14:42:54 +03002134 /* Allocate memory based on the information we got from the IP */
2135 ecc->slave_chans = devm_kcalloc(dev, ecc->num_channels,
2136 sizeof(*ecc->slave_chans), GFP_KERNEL);
2137 if (!ecc->slave_chans)
2138 return -ENOMEM;
2139
Peter Ujfalusi7a73b132015-10-14 14:43:05 +03002140 ecc->channel_unused = devm_kcalloc(dev,
2141 BITS_TO_LONGS(ecc->num_channels),
2142 sizeof(unsigned long), GFP_KERNEL);
2143 if (!ecc->channel_unused)
Peter Ujfalusicb782052015-10-14 14:42:54 +03002144 return -ENOMEM;
2145
Peter Ujfalusi7a73b132015-10-14 14:43:05 +03002146 ecc->slot_inuse = devm_kcalloc(dev, BITS_TO_LONGS(ecc->num_slots),
Peter Ujfalusicb782052015-10-14 14:42:54 +03002147 sizeof(unsigned long), GFP_KERNEL);
Peter Ujfalusi7a73b132015-10-14 14:43:05 +03002148 if (!ecc->slot_inuse)
Peter Ujfalusicb782052015-10-14 14:42:54 +03002149 return -ENOMEM;
2150
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002151 ecc->default_queue = info->default_queue;
2152
2153 for (i = 0; i < ecc->num_slots; i++)
2154 edma_write_slot(ecc, i, &dummy_paramset);
2155
2156 /* Mark all channels as unused */
Peter Ujfalusi7a73b132015-10-14 14:43:05 +03002157 memset(ecc->channel_unused, 0xff, sizeof(ecc->channel_unused));
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002158
2159 if (info->rsv) {
2160 /* Clear the reserved channels in unused list */
2161 rsv_chans = info->rsv->rsv_chans;
2162 if (rsv_chans) {
2163 for (i = 0; rsv_chans[i][0] != -1; i++) {
2164 off = rsv_chans[i][0];
2165 ln = rsv_chans[i][1];
Peter Ujfalusi7a73b132015-10-14 14:43:05 +03002166 clear_bits(off, ln, ecc->channel_unused);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002167 }
2168 }
2169
2170 /* Set the reserved slots in inuse list */
2171 rsv_slots = info->rsv->rsv_slots;
2172 if (rsv_slots) {
2173 for (i = 0; rsv_slots[i][0] != -1; i++) {
2174 off = rsv_slots[i][0];
2175 ln = rsv_slots[i][1];
Peter Ujfalusi7a73b132015-10-14 14:43:05 +03002176 set_bits(off, ln, ecc->slot_inuse);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002177 }
2178 }
2179 }
2180
2181 /* Clear the xbar mapped channels in unused list */
2182 xbar_chans = info->xbar_chans;
2183 if (xbar_chans) {
2184 for (i = 0; xbar_chans[i][1] != -1; i++) {
2185 off = xbar_chans[i][1];
Peter Ujfalusi7a73b132015-10-14 14:43:05 +03002186 clear_bits(off, 1, ecc->channel_unused);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002187 }
2188 }
2189
2190 irq = platform_get_irq_byname(pdev, "edma3_ccint");
2191 if (irq < 0 && node)
2192 irq = irq_of_parse_and_map(node, 0);
2193
2194 if (irq >= 0) {
2195 irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccint",
2196 dev_name(dev));
2197 ret = devm_request_irq(dev, irq, dma_irq_handler, 0, irq_name,
2198 ecc);
2199 if (ret) {
2200 dev_err(dev, "CCINT (%d) failed --> %d\n", irq, ret);
2201 return ret;
2202 }
2203 }
2204
2205 irq = platform_get_irq_byname(pdev, "edma3_ccerrint");
2206 if (irq < 0 && node)
2207 irq = irq_of_parse_and_map(node, 2);
2208
2209 if (irq >= 0) {
2210 irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccerrint",
2211 dev_name(dev));
2212 ret = devm_request_irq(dev, irq, dma_ccerr_handler, 0, irq_name,
2213 ecc);
2214 if (ret) {
2215 dev_err(dev, "CCERRINT (%d) failed --> %d\n", irq, ret);
2216 return ret;
2217 }
2218 }
2219
2220 for (i = 0; i < ecc->num_channels; i++)
2221 edma_map_dmach_to_queue(ecc, i, info->default_queue);
2222
2223 queue_priority_mapping = info->queue_priority_mapping;
2224
2225 /* Event queue priority mapping */
2226 for (i = 0; queue_priority_mapping[i][0] != -1; i++)
2227 edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
2228 queue_priority_mapping[i][1]);
2229
2230 /* Map the channel to param entry if channel mapping logic exist */
Peter Ujfalusi4ab54f62015-10-14 14:43:04 +03002231 if (ecc->chmap_exist)
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002232 edma_direct_dmach_to_param_mapping(ecc);
2233
2234 for (i = 0; i < ecc->num_region; i++) {
2235 edma_write_array2(ecc, EDMA_DRAE, i, 0, 0x0);
2236 edma_write_array2(ecc, EDMA_DRAE, i, 1, 0x0);
2237 edma_write_array(ecc, EDMA_QRAE, i, 0x0);
2238 }
2239 ecc->info = info;
2240
2241 ecc->dummy_slot = edma_alloc_slot(ecc, EDMA_SLOT_ANY);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002242 if (ecc->dummy_slot < 0) {
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03002243 dev_err(dev, "Can't allocate PaRAM dummy slot\n");
Peter Ujfalusi04d537d2014-07-31 13:12:37 +03002244 return ecc->dummy_slot;
Matt Porterc2dde5f2012-08-22 21:09:34 -04002245 }
2246
2247 dma_cap_zero(ecc->dma_slave.cap_mask);
2248 dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask);
Peter Ujfalusi232b223d2014-04-14 14:42:00 +03002249 dma_cap_set(DMA_CYCLIC, ecc->dma_slave.cap_mask);
Joel Fernandes8cc3e302014-04-18 21:50:33 -05002250 dma_cap_set(DMA_MEMCPY, ecc->dma_slave.cap_mask);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002251
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03002252 edma_dma_init(ecc, &ecc->dma_slave, dev);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002253
2254 edma_chan_init(ecc, &ecc->dma_slave, ecc->slave_chans);
2255
2256 ret = dma_async_device_register(&ecc->dma_slave);
2257 if (ret)
2258 goto err_reg1;
2259
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002260 if (node)
2261 of_dma_controller_register(node, of_dma_xlate_by_chan_id,
Peter Ujfalusib2c843a2015-10-14 14:42:50 +03002262 &ecc->dma_slave);
Peter Ujfalusidc9b60552015-10-14 14:42:47 +03002263
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03002264 dev_info(dev, "TI EDMA DMA engine driver\n");
Matt Porterc2dde5f2012-08-22 21:09:34 -04002265
2266 return 0;
2267
2268err_reg1:
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002269 edma_free_slot(ecc, ecc->dummy_slot);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002270 return ret;
2271}
2272
Greg Kroah-Hartman4bf27b82012-12-21 15:09:59 -08002273static int edma_remove(struct platform_device *pdev)
Matt Porterc2dde5f2012-08-22 21:09:34 -04002274{
2275 struct device *dev = &pdev->dev;
2276 struct edma_cc *ecc = dev_get_drvdata(dev);
2277
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03002278 if (dev->of_node)
2279 of_dma_controller_free(dev->of_node);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002280 dma_async_device_unregister(&ecc->dma_slave);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002281 edma_free_slot(ecc, ecc->dummy_slot);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002282
2283 return 0;
2284}
2285
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002286#ifdef CONFIG_PM_SLEEP
2287static int edma_pm_resume(struct device *dev)
2288{
2289 struct edma_cc *ecc = dev_get_drvdata(dev);
2290 int i;
2291 s8 (*queue_priority_mapping)[2];
2292
2293 queue_priority_mapping = ecc->info->queue_priority_mapping;
2294
2295 /* Event queue priority mapping */
2296 for (i = 0; queue_priority_mapping[i][0] != -1; i++)
2297 edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
2298 queue_priority_mapping[i][1]);
2299
2300 /* Map the channel to param entry if channel mapping logic */
Peter Ujfalusi4ab54f62015-10-14 14:43:04 +03002301 if (ecc->chmap_exist)
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002302 edma_direct_dmach_to_param_mapping(ecc);
2303
2304 for (i = 0; i < ecc->num_channels; i++) {
Peter Ujfalusi7a73b132015-10-14 14:43:05 +03002305 if (test_bit(i, ecc->slot_inuse)) {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002306 /* ensure access through shadow region 0 */
2307 edma_or_array2(ecc, EDMA_DRAE, 0, i >> 5,
2308 BIT(i & 0x1f));
2309
2310 edma_setup_interrupt(ecc, EDMA_CTLR_CHAN(ecc->id, i),
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03002311 true);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002312 }
2313 }
2314
2315 return 0;
2316}
2317#endif
2318
2319static const struct dev_pm_ops edma_pm_ops = {
2320 SET_LATE_SYSTEM_SLEEP_PM_OPS(NULL, edma_pm_resume)
2321};
2322
Matt Porterc2dde5f2012-08-22 21:09:34 -04002323static struct platform_driver edma_driver = {
2324 .probe = edma_probe,
Bill Pembertona7d6e3e2012-11-19 13:20:04 -05002325 .remove = edma_remove,
Matt Porterc2dde5f2012-08-22 21:09:34 -04002326 .driver = {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002327 .name = "edma",
2328 .pm = &edma_pm_ops,
2329 .of_match_table = edma_of_ids,
Matt Porterc2dde5f2012-08-22 21:09:34 -04002330 },
2331};
2332
2333bool edma_filter_fn(struct dma_chan *chan, void *param)
2334{
2335 if (chan->device->dev->driver == &edma_driver.driver) {
2336 struct edma_chan *echan = to_edma_chan(chan);
2337 unsigned ch_req = *(unsigned *)param;
2338 return ch_req == echan->ch_num;
2339 }
2340 return false;
2341}
2342EXPORT_SYMBOL(edma_filter_fn);
2343
Matt Porterc2dde5f2012-08-22 21:09:34 -04002344static int edma_init(void)
2345{
Arnd Bergmann5305e4d2014-10-24 18:14:01 +02002346 return platform_driver_register(&edma_driver);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002347}
2348subsys_initcall(edma_init);
2349
2350static void __exit edma_exit(void)
2351{
Matt Porterc2dde5f2012-08-22 21:09:34 -04002352 platform_driver_unregister(&edma_driver);
2353}
2354module_exit(edma_exit);
2355
Josh Boyerd71505b2013-09-04 10:32:50 -04002356MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
Matt Porterc2dde5f2012-08-22 21:09:34 -04002357MODULE_DESCRIPTION("TI EDMA DMA engine driver");
2358MODULE_LICENSE("GPL v2");