blob: 84b98a01993aa3a9bf02527e8147b930cf7859b7 [file] [log] [blame]
Matt Porterc2dde5f2012-08-22 21:09:34 -04001/*
2 * TI EDMA DMA engine driver
3 *
4 * Copyright 2012 Texas Instruments
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation version 2.
9 *
10 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
11 * kind, whether express or implied; without even the implied warranty
12 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 */
15
16#include <linux/dmaengine.h>
17#include <linux/dma-mapping.h>
Lad, Prabhakarb7a4fd52015-02-04 13:03:27 +000018#include <linux/edma.h>
Matt Porterc2dde5f2012-08-22 21:09:34 -040019#include <linux/err.h>
20#include <linux/init.h>
21#include <linux/interrupt.h>
22#include <linux/list.h>
23#include <linux/module.h>
24#include <linux/platform_device.h>
25#include <linux/slab.h>
26#include <linux/spinlock.h>
Peter Ujfalusied646102014-07-31 13:12:38 +030027#include <linux/of.h>
Peter Ujfalusidc9b60552015-10-14 14:42:47 +030028#include <linux/of_dma.h>
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +030029#include <linux/of_irq.h>
30#include <linux/of_address.h>
31#include <linux/of_device.h>
32#include <linux/pm_runtime.h>
Matt Porterc2dde5f2012-08-22 21:09:34 -040033
Matt Porter3ad7a422013-03-06 11:15:31 -050034#include <linux/platform_data/edma.h>
Matt Porterc2dde5f2012-08-22 21:09:34 -040035
36#include "dmaengine.h"
37#include "virt-dma.h"
38
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +030039/* Offsets matching "struct edmacc_param" */
40#define PARM_OPT 0x00
41#define PARM_SRC 0x04
42#define PARM_A_B_CNT 0x08
43#define PARM_DST 0x0c
44#define PARM_SRC_DST_BIDX 0x10
45#define PARM_LINK_BCNTRLD 0x14
46#define PARM_SRC_DST_CIDX 0x18
47#define PARM_CCNT 0x1c
48
49#define PARM_SIZE 0x20
50
51/* Offsets for EDMA CC global channel registers and their shadows */
52#define SH_ER 0x00 /* 64 bits */
53#define SH_ECR 0x08 /* 64 bits */
54#define SH_ESR 0x10 /* 64 bits */
55#define SH_CER 0x18 /* 64 bits */
56#define SH_EER 0x20 /* 64 bits */
57#define SH_EECR 0x28 /* 64 bits */
58#define SH_EESR 0x30 /* 64 bits */
59#define SH_SER 0x38 /* 64 bits */
60#define SH_SECR 0x40 /* 64 bits */
61#define SH_IER 0x50 /* 64 bits */
62#define SH_IECR 0x58 /* 64 bits */
63#define SH_IESR 0x60 /* 64 bits */
64#define SH_IPR 0x68 /* 64 bits */
65#define SH_ICR 0x70 /* 64 bits */
66#define SH_IEVAL 0x78
67#define SH_QER 0x80
68#define SH_QEER 0x84
69#define SH_QEECR 0x88
70#define SH_QEESR 0x8c
71#define SH_QSER 0x90
72#define SH_QSECR 0x94
73#define SH_SIZE 0x200
74
75/* Offsets for EDMA CC global registers */
76#define EDMA_REV 0x0000
77#define EDMA_CCCFG 0x0004
78#define EDMA_QCHMAP 0x0200 /* 8 registers */
79#define EDMA_DMAQNUM 0x0240 /* 8 registers (4 on OMAP-L1xx) */
80#define EDMA_QDMAQNUM 0x0260
81#define EDMA_QUETCMAP 0x0280
82#define EDMA_QUEPRI 0x0284
83#define EDMA_EMR 0x0300 /* 64 bits */
84#define EDMA_EMCR 0x0308 /* 64 bits */
85#define EDMA_QEMR 0x0310
86#define EDMA_QEMCR 0x0314
87#define EDMA_CCERR 0x0318
88#define EDMA_CCERRCLR 0x031c
89#define EDMA_EEVAL 0x0320
90#define EDMA_DRAE 0x0340 /* 4 x 64 bits*/
91#define EDMA_QRAE 0x0380 /* 4 registers */
92#define EDMA_QUEEVTENTRY 0x0400 /* 2 x 16 registers */
93#define EDMA_QSTAT 0x0600 /* 2 registers */
94#define EDMA_QWMTHRA 0x0620
95#define EDMA_QWMTHRB 0x0624
96#define EDMA_CCSTAT 0x0640
97
98#define EDMA_M 0x1000 /* global channel registers */
99#define EDMA_ECR 0x1008
100#define EDMA_ECRH 0x100C
101#define EDMA_SHADOW0 0x2000 /* 4 shadow regions */
102#define EDMA_PARM 0x4000 /* PaRAM entries */
103
104#define PARM_OFFSET(param_no) (EDMA_PARM + ((param_no) << 5))
105
106#define EDMA_DCHMAP 0x0100 /* 64 registers */
107
108/* CCCFG register */
109#define GET_NUM_DMACH(x) (x & 0x7) /* bits 0-2 */
110#define GET_NUM_PAENTRY(x) ((x & 0x7000) >> 12) /* bits 12-14 */
111#define GET_NUM_EVQUE(x) ((x & 0x70000) >> 16) /* bits 16-18 */
112#define GET_NUM_REGN(x) ((x & 0x300000) >> 20) /* bits 20-21 */
113#define CHMAP_EXIST BIT(24)
114
Matt Porterc2dde5f2012-08-22 21:09:34 -0400115/*
Joel Fernandes2abd5f12013-09-23 18:05:15 -0500116 * Max of 20 segments per channel to conserve PaRAM slots
117 * Also note that MAX_NR_SG should be atleast the no.of periods
118 * that are required for ASoC, otherwise DMA prep calls will
119 * fail. Today davinci-pcm is the only user of this driver and
120 * requires atleast 17 slots, so we setup the default to 20.
121 */
122#define MAX_NR_SG 20
Matt Porterc2dde5f2012-08-22 21:09:34 -0400123#define EDMA_MAX_SLOTS MAX_NR_SG
124#define EDMA_DESCRIPTORS 16
125
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300126#define EDMA_CHANNEL_ANY -1 /* for edma_alloc_channel() */
127#define EDMA_SLOT_ANY -1 /* for edma_alloc_slot() */
128#define EDMA_CONT_PARAMS_ANY 1001
129#define EDMA_CONT_PARAMS_FIXED_EXACT 1002
130#define EDMA_CONT_PARAMS_FIXED_NOT_EXACT 1003
131
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300132/* PaRAM slots are laid out like this */
133struct edmacc_param {
134 u32 opt;
135 u32 src;
136 u32 a_b_cnt;
137 u32 dst;
138 u32 src_dst_bidx;
139 u32 link_bcntrld;
140 u32 src_dst_cidx;
141 u32 ccnt;
142} __packed;
143
144/* fields in edmacc_param.opt */
145#define SAM BIT(0)
146#define DAM BIT(1)
147#define SYNCDIM BIT(2)
148#define STATIC BIT(3)
149#define EDMA_FWID (0x07 << 8)
150#define TCCMODE BIT(11)
151#define EDMA_TCC(t) ((t) << 12)
152#define TCINTEN BIT(20)
153#define ITCINTEN BIT(21)
154#define TCCHEN BIT(22)
155#define ITCCHEN BIT(23)
156
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500157struct edma_pset {
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500158 u32 len;
159 dma_addr_t addr;
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500160 struct edmacc_param param;
161};
162
Matt Porterc2dde5f2012-08-22 21:09:34 -0400163struct edma_desc {
164 struct virt_dma_desc vdesc;
165 struct list_head node;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500166 enum dma_transfer_direction direction;
Joel Fernandes50a9c702013-10-31 16:31:23 -0500167 int cyclic;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400168 int absync;
169 int pset_nr;
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500170 struct edma_chan *echan;
Joel Fernandes04361d82014-04-28 15:19:31 -0500171 int processed;
172
173 /*
174 * The following 4 elements are used for residue accounting.
175 *
176 * - processed_stat: the number of SG elements we have traversed
177 * so far to cover accounting. This is updated directly to processed
178 * during edma_callback and is always <= processed, because processed
179 * refers to the number of pending transfer (programmed to EDMA
180 * controller), where as processed_stat tracks number of transfers
181 * accounted for so far.
182 *
183 * - residue: The amount of bytes we have left to transfer for this desc
184 *
185 * - residue_stat: The residue in bytes of data we have covered
186 * so far for accounting. This is updated directly to residue
187 * during callbacks to keep it current.
188 *
189 * - sg_len: Tracks the length of the current intermediate transfer,
190 * this is required to update the residue during intermediate transfer
191 * completion callback.
192 */
193 int processed_stat;
194 u32 sg_len;
195 u32 residue;
196 u32 residue_stat;
197
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500198 struct edma_pset pset[0];
Matt Porterc2dde5f2012-08-22 21:09:34 -0400199};
200
201struct edma_cc;
202
203struct edma_chan {
204 struct virt_dma_chan vchan;
205 struct list_head node;
206 struct edma_desc *edesc;
207 struct edma_cc *ecc;
208 int ch_num;
209 bool alloced;
210 int slot[EDMA_MAX_SLOTS];
Joel Fernandesc5f47992013-08-29 18:05:43 -0500211 int missed;
Matt Porter661f7cb2013-01-10 13:41:04 -0500212 struct dma_slave_config cfg;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400213};
214
215struct edma_cc {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300216 struct device *dev;
217 struct edma_soc_info *info;
218 void __iomem *base;
219 int id;
220
221 /* eDMA3 resource information */
222 unsigned num_channels;
223 unsigned num_region;
224 unsigned num_slots;
225 unsigned num_tc;
226 enum dma_event_q default_queue;
227
228 bool unused_chan_list_done;
229 /* The edma_inuse bit for each PaRAM slot is clear unless the
230 * channel is in use ... by ARM or DSP, for QDMA, or whatever.
231 */
Peter Ujfalusicb782052015-10-14 14:42:54 +0300232 unsigned long *edma_inuse;
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300233
234 /* The edma_unused bit for each channel is clear unless
235 * it is not being used on this platform. It uses a bit
236 * of SOC-specific initialization code.
237 */
Peter Ujfalusicb782052015-10-14 14:42:54 +0300238 unsigned long *edma_unused;
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300239
Matt Porterc2dde5f2012-08-22 21:09:34 -0400240 struct dma_device dma_slave;
Peter Ujfalusicb782052015-10-14 14:42:54 +0300241 struct edma_chan *slave_chans;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400242 int dummy_slot;
243};
244
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300245/* dummy param set used to (re)initialize parameter RAM slots */
246static const struct edmacc_param dummy_paramset = {
247 .link_bcntrld = 0xffff,
248 .ccnt = 1,
249};
250
251static const struct of_device_id edma_of_ids[] = {
252 { .compatible = "ti,edma3", },
253 {}
254};
255
256static inline unsigned int edma_read(struct edma_cc *ecc, int offset)
257{
258 return (unsigned int)__raw_readl(ecc->base + offset);
259}
260
261static inline void edma_write(struct edma_cc *ecc, int offset, int val)
262{
263 __raw_writel(val, ecc->base + offset);
264}
265
266static inline void edma_modify(struct edma_cc *ecc, int offset, unsigned and,
267 unsigned or)
268{
269 unsigned val = edma_read(ecc, offset);
270
271 val &= and;
272 val |= or;
273 edma_write(ecc, offset, val);
274}
275
276static inline void edma_and(struct edma_cc *ecc, int offset, unsigned and)
277{
278 unsigned val = edma_read(ecc, offset);
279
280 val &= and;
281 edma_write(ecc, offset, val);
282}
283
284static inline void edma_or(struct edma_cc *ecc, int offset, unsigned or)
285{
286 unsigned val = edma_read(ecc, offset);
287
288 val |= or;
289 edma_write(ecc, offset, val);
290}
291
292static inline unsigned int edma_read_array(struct edma_cc *ecc, int offset,
293 int i)
294{
295 return edma_read(ecc, offset + (i << 2));
296}
297
298static inline void edma_write_array(struct edma_cc *ecc, int offset, int i,
299 unsigned val)
300{
301 edma_write(ecc, offset + (i << 2), val);
302}
303
304static inline void edma_modify_array(struct edma_cc *ecc, int offset, int i,
305 unsigned and, unsigned or)
306{
307 edma_modify(ecc, offset + (i << 2), and, or);
308}
309
310static inline void edma_or_array(struct edma_cc *ecc, int offset, int i,
311 unsigned or)
312{
313 edma_or(ecc, offset + (i << 2), or);
314}
315
316static inline void edma_or_array2(struct edma_cc *ecc, int offset, int i, int j,
317 unsigned or)
318{
319 edma_or(ecc, offset + ((i * 2 + j) << 2), or);
320}
321
322static inline void edma_write_array2(struct edma_cc *ecc, int offset, int i,
323 int j, unsigned val)
324{
325 edma_write(ecc, offset + ((i * 2 + j) << 2), val);
326}
327
328static inline unsigned int edma_shadow0_read(struct edma_cc *ecc, int offset)
329{
330 return edma_read(ecc, EDMA_SHADOW0 + offset);
331}
332
333static inline unsigned int edma_shadow0_read_array(struct edma_cc *ecc,
334 int offset, int i)
335{
336 return edma_read(ecc, EDMA_SHADOW0 + offset + (i << 2));
337}
338
339static inline void edma_shadow0_write(struct edma_cc *ecc, int offset,
340 unsigned val)
341{
342 edma_write(ecc, EDMA_SHADOW0 + offset, val);
343}
344
345static inline void edma_shadow0_write_array(struct edma_cc *ecc, int offset,
346 int i, unsigned val)
347{
348 edma_write(ecc, EDMA_SHADOW0 + offset + (i << 2), val);
349}
350
351static inline unsigned int edma_parm_read(struct edma_cc *ecc, int offset,
352 int param_no)
353{
354 return edma_read(ecc, EDMA_PARM + offset + (param_no << 5));
355}
356
357static inline void edma_parm_write(struct edma_cc *ecc, int offset,
358 int param_no, unsigned val)
359{
360 edma_write(ecc, EDMA_PARM + offset + (param_no << 5), val);
361}
362
363static inline void edma_parm_modify(struct edma_cc *ecc, int offset,
364 int param_no, unsigned and, unsigned or)
365{
366 edma_modify(ecc, EDMA_PARM + offset + (param_no << 5), and, or);
367}
368
369static inline void edma_parm_and(struct edma_cc *ecc, int offset, int param_no,
370 unsigned and)
371{
372 edma_and(ecc, EDMA_PARM + offset + (param_no << 5), and);
373}
374
375static inline void edma_parm_or(struct edma_cc *ecc, int offset, int param_no,
376 unsigned or)
377{
378 edma_or(ecc, EDMA_PARM + offset + (param_no << 5), or);
379}
380
381static inline void set_bits(int offset, int len, unsigned long *p)
382{
383 for (; len > 0; len--)
384 set_bit(offset + (len - 1), p);
385}
386
387static inline void clear_bits(int offset, int len, unsigned long *p)
388{
389 for (; len > 0; len--)
390 clear_bit(offset + (len - 1), p);
391}
392
393static void edma_map_dmach_to_queue(struct edma_cc *ecc, unsigned ch_no,
394 enum dma_event_q queue_no)
395{
396 int bit = (ch_no & 0x7) * 4;
397
398 /* default to low priority queue */
399 if (queue_no == EVENTQ_DEFAULT)
400 queue_no = ecc->default_queue;
401
402 queue_no &= 7;
403 edma_modify_array(ecc, EDMA_DMAQNUM, (ch_no >> 3), ~(0x7 << bit),
404 queue_no << bit);
405}
406
407static void edma_assign_priority_to_queue(struct edma_cc *ecc, int queue_no,
408 int priority)
409{
410 int bit = queue_no * 4;
411
412 edma_modify(ecc, EDMA_QUEPRI, ~(0x7 << bit), ((priority & 0x7) << bit));
413}
414
415static void edma_direct_dmach_to_param_mapping(struct edma_cc *ecc)
416{
417 int i;
418
419 for (i = 0; i < ecc->num_channels; i++)
420 edma_write_array(ecc, EDMA_DCHMAP, i, (i << 5));
421}
422
423static int prepare_unused_channel_list(struct device *dev, void *data)
424{
425 struct platform_device *pdev = to_platform_device(dev);
426 struct edma_cc *ecc = data;
Peter Ujfalusicb782052015-10-14 14:42:54 +0300427 int dma_req_min = EDMA_CTLR_CHAN(ecc->id, 0);
428 int dma_req_max = dma_req_min + ecc->num_channels;
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300429 int i, count;
430 struct of_phandle_args dma_spec;
431
432 if (dev->of_node) {
433 struct platform_device *dma_pdev;
434
435 count = of_property_count_strings(dev->of_node, "dma-names");
436 if (count < 0)
437 return 0;
438 for (i = 0; i < count; i++) {
439 if (of_parse_phandle_with_args(dev->of_node, "dmas",
440 "#dma-cells", i,
441 &dma_spec))
442 continue;
443
444 if (!of_match_node(edma_of_ids, dma_spec.np)) {
445 of_node_put(dma_spec.np);
446 continue;
447 }
448
449 dma_pdev = of_find_device_by_node(dma_spec.np);
450 if (&dma_pdev->dev != ecc->dev)
451 continue;
452
453 clear_bit(EDMA_CHAN_SLOT(dma_spec.args[0]),
454 ecc->edma_unused);
455 of_node_put(dma_spec.np);
456 }
457 return 0;
458 }
459
460 /* For non-OF case */
461 for (i = 0; i < pdev->num_resources; i++) {
462 struct resource *res = &pdev->resource[i];
Peter Ujfalusicb782052015-10-14 14:42:54 +0300463 int dma_req;
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300464
Peter Ujfalusicb782052015-10-14 14:42:54 +0300465 if (!(res->flags & IORESOURCE_DMA))
466 continue;
467
468 dma_req = (int)res->start;
469 if (dma_req >= dma_req_min && dma_req < dma_req_max)
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300470 clear_bit(EDMA_CHAN_SLOT(pdev->resource[i].start),
471 ecc->edma_unused);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300472 }
473
474 return 0;
475}
476
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +0300477static void edma_setup_interrupt(struct edma_cc *ecc, unsigned lch, bool enable)
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300478{
479 lch = EDMA_CHAN_SLOT(lch);
480
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +0300481 if (enable) {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300482 edma_shadow0_write_array(ecc, SH_ICR, lch >> 5,
483 BIT(lch & 0x1f));
484 edma_shadow0_write_array(ecc, SH_IESR, lch >> 5,
485 BIT(lch & 0x1f));
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +0300486 } else {
487 edma_shadow0_write_array(ecc, SH_IECR, lch >> 5,
488 BIT(lch & 0x1f));
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300489 }
490}
491
492/*
Peter Ujfalusi11c15732015-10-14 14:43:00 +0300493 * paRAM slot management functions
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300494 */
495static void edma_write_slot(struct edma_cc *ecc, unsigned slot,
496 const struct edmacc_param *param)
497{
498 slot = EDMA_CHAN_SLOT(slot);
499 if (slot >= ecc->num_slots)
500 return;
501 memcpy_toio(ecc->base + PARM_OFFSET(slot), param, PARM_SIZE);
502}
503
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300504static void edma_read_slot(struct edma_cc *ecc, unsigned slot,
505 struct edmacc_param *param)
506{
507 slot = EDMA_CHAN_SLOT(slot);
508 if (slot >= ecc->num_slots)
509 return;
510 memcpy_fromio(param, ecc->base + PARM_OFFSET(slot), PARM_SIZE);
511}
512
513/**
514 * edma_alloc_slot - allocate DMA parameter RAM
515 * @ecc: pointer to edma_cc struct
516 * @slot: specific slot to allocate; negative for "any unused slot"
517 *
518 * This allocates a parameter RAM slot, initializing it to hold a
519 * dummy transfer. Slots allocated using this routine have not been
520 * mapped to a hardware DMA channel, and will normally be used by
521 * linking to them from a slot associated with a DMA channel.
522 *
523 * Normal use is to pass EDMA_SLOT_ANY as the @slot, but specific
524 * slots may be allocated on behalf of DSP firmware.
525 *
526 * Returns the number of the slot, else negative errno.
527 */
528static int edma_alloc_slot(struct edma_cc *ecc, int slot)
529{
530 if (slot > 0)
531 slot = EDMA_CHAN_SLOT(slot);
532 if (slot < 0) {
533 slot = ecc->num_channels;
534 for (;;) {
535 slot = find_next_zero_bit(ecc->edma_inuse,
536 ecc->num_slots,
537 slot);
538 if (slot == ecc->num_slots)
539 return -ENOMEM;
540 if (!test_and_set_bit(slot, ecc->edma_inuse))
541 break;
542 }
543 } else if (slot < ecc->num_channels || slot >= ecc->num_slots) {
544 return -EINVAL;
545 } else if (test_and_set_bit(slot, ecc->edma_inuse)) {
546 return -EBUSY;
547 }
548
549 edma_write_slot(ecc, slot, &dummy_paramset);
550
551 return EDMA_CTLR_CHAN(ecc->id, slot);
552}
553
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300554static void edma_free_slot(struct edma_cc *ecc, unsigned slot)
555{
556 slot = EDMA_CHAN_SLOT(slot);
557 if (slot < ecc->num_channels || slot >= ecc->num_slots)
558 return;
559
560 edma_write_slot(ecc, slot, &dummy_paramset);
561 clear_bit(slot, ecc->edma_inuse);
562}
563
564/**
565 * edma_link - link one parameter RAM slot to another
566 * @ecc: pointer to edma_cc struct
567 * @from: parameter RAM slot originating the link
568 * @to: parameter RAM slot which is the link target
569 *
570 * The originating slot should not be part of any active DMA transfer.
571 */
572static void edma_link(struct edma_cc *ecc, unsigned from, unsigned to)
573{
Peter Ujfalusifc014092015-10-14 14:42:59 +0300574 if (unlikely(EDMA_CTLR(from) != EDMA_CTLR(to)))
575 dev_warn(ecc->dev, "Ignoring eDMA instance for linking\n");
576
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300577 from = EDMA_CHAN_SLOT(from);
578 to = EDMA_CHAN_SLOT(to);
579 if (from >= ecc->num_slots || to >= ecc->num_slots)
580 return;
581
582 edma_parm_modify(ecc, PARM_LINK_BCNTRLD, from, 0xffff0000,
583 PARM_OFFSET(to));
584}
585
586/**
587 * edma_get_position - returns the current transfer point
588 * @ecc: pointer to edma_cc struct
589 * @slot: parameter RAM slot being examined
590 * @dst: true selects the dest position, false the source
591 *
592 * Returns the position of the current active slot
593 */
594static dma_addr_t edma_get_position(struct edma_cc *ecc, unsigned slot,
595 bool dst)
596{
597 u32 offs;
598
599 slot = EDMA_CHAN_SLOT(slot);
600 offs = PARM_OFFSET(slot);
601 offs += dst ? PARM_DST : PARM_SRC;
602
603 return edma_read(ecc, offs);
604}
605
606/*-----------------------------------------------------------------------*/
607/**
608 * edma_start - start dma on a channel
609 * @ecc: pointer to edma_cc struct
610 * @channel: channel being activated
611 *
612 * Channels with event associations will be triggered by their hardware
613 * events, and channels without such associations will be triggered by
614 * software. (At this writing there is no interface for using software
615 * triggers except with channels that don't support hardware triggers.)
616 *
617 * Returns zero on success, else negative errno.
618 */
619static int edma_start(struct edma_cc *ecc, unsigned channel)
620{
621 if (ecc->id != EDMA_CTLR(channel)) {
622 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
623 ecc->id, EDMA_CTLR(channel));
624 return -EINVAL;
625 }
626 channel = EDMA_CHAN_SLOT(channel);
627
628 if (channel < ecc->num_channels) {
629 int j = channel >> 5;
630 unsigned int mask = BIT(channel & 0x1f);
631
632 /* EDMA channels without event association */
633 if (test_bit(channel, ecc->edma_unused)) {
Peter Ujfalusi3287fb42015-10-14 14:42:57 +0300634 dev_dbg(ecc->dev, "ESR%d %08x\n", j,
635 edma_shadow0_read_array(ecc, SH_ESR, j));
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300636 edma_shadow0_write_array(ecc, SH_ESR, j, mask);
637 return 0;
638 }
639
640 /* EDMA channel with event association */
Peter Ujfalusi3287fb42015-10-14 14:42:57 +0300641 dev_dbg(ecc->dev, "ER%d %08x\n", j,
642 edma_shadow0_read_array(ecc, SH_ER, j));
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300643 /* Clear any pending event or error */
644 edma_write_array(ecc, EDMA_ECR, j, mask);
645 edma_write_array(ecc, EDMA_EMCR, j, mask);
646 /* Clear any SER */
647 edma_shadow0_write_array(ecc, SH_SECR, j, mask);
648 edma_shadow0_write_array(ecc, SH_EESR, j, mask);
Peter Ujfalusi3287fb42015-10-14 14:42:57 +0300649 dev_dbg(ecc->dev, "EER%d %08x\n", j,
650 edma_shadow0_read_array(ecc, SH_EER, j));
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300651 return 0;
652 }
653
654 return -EINVAL;
655}
656
657/**
658 * edma_stop - stops dma on the channel passed
659 * @ecc: pointer to edma_cc struct
660 * @channel: channel being deactivated
661 *
Peter Ujfalusi11c15732015-10-14 14:43:00 +0300662 * Any active transfer is paused and all pending hardware events are cleared.
663 * The current transfer may not be resumed, and the channel's Parameter RAM
664 * should be reinitialized before being reused.
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300665 */
666static void edma_stop(struct edma_cc *ecc, unsigned channel)
667{
668 if (ecc->id != EDMA_CTLR(channel)) {
669 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
670 ecc->id, EDMA_CTLR(channel));
671 return;
672 }
673 channel = EDMA_CHAN_SLOT(channel);
674
675 if (channel < ecc->num_channels) {
676 int j = channel >> 5;
677 unsigned int mask = BIT(channel & 0x1f);
678
679 edma_shadow0_write_array(ecc, SH_EECR, j, mask);
680 edma_shadow0_write_array(ecc, SH_ECR, j, mask);
681 edma_shadow0_write_array(ecc, SH_SECR, j, mask);
682 edma_write_array(ecc, EDMA_EMCR, j, mask);
683
684 /* clear possibly pending completion interrupt */
685 edma_shadow0_write_array(ecc, SH_ICR, j, mask);
686
Peter Ujfalusi3287fb42015-10-14 14:42:57 +0300687 dev_dbg(ecc->dev, "EER%d %08x\n", j,
688 edma_shadow0_read_array(ecc, SH_EER, j));
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300689
690 /* REVISIT: consider guarding against inappropriate event
691 * chaining by overwriting with dummy_paramset.
692 */
693 }
694}
695
Peter Ujfalusi11c15732015-10-14 14:43:00 +0300696/*
697 * Temporarily disable EDMA hardware events on the specified channel,
698 * preventing them from triggering new transfers
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300699 */
700static void edma_pause(struct edma_cc *ecc, unsigned channel)
701{
702 if (ecc->id != EDMA_CTLR(channel)) {
703 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
704 ecc->id, EDMA_CTLR(channel));
705 return;
706 }
707 channel = EDMA_CHAN_SLOT(channel);
708
709 if (channel < ecc->num_channels) {
710 unsigned int mask = BIT(channel & 0x1f);
711
712 edma_shadow0_write_array(ecc, SH_EECR, channel >> 5, mask);
713 }
714}
715
Peter Ujfalusi11c15732015-10-14 14:43:00 +0300716/* Re-enable EDMA hardware events on the specified channel. */
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300717static void edma_resume(struct edma_cc *ecc, unsigned channel)
718{
719 if (ecc->id != EDMA_CTLR(channel)) {
720 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
721 ecc->id, EDMA_CTLR(channel));
722 return;
723 }
724 channel = EDMA_CHAN_SLOT(channel);
725
726 if (channel < ecc->num_channels) {
727 unsigned int mask = BIT(channel & 0x1f);
728
729 edma_shadow0_write_array(ecc, SH_EESR, channel >> 5, mask);
730 }
731}
732
733static int edma_trigger_channel(struct edma_cc *ecc, unsigned channel)
734{
735 unsigned int mask;
736
737 if (ecc->id != EDMA_CTLR(channel)) {
738 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
739 ecc->id, EDMA_CTLR(channel));
740 return -EINVAL;
741 }
742 channel = EDMA_CHAN_SLOT(channel);
743 mask = BIT(channel & 0x1f);
744
745 edma_shadow0_write_array(ecc, SH_ESR, (channel >> 5), mask);
746
Peter Ujfalusi3287fb42015-10-14 14:42:57 +0300747 dev_dbg(ecc->dev, "ESR%d %08x\n", (channel >> 5),
748 edma_shadow0_read_array(ecc, SH_ESR, (channel >> 5)));
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300749 return 0;
750}
751
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300752static void edma_clean_channel(struct edma_cc *ecc, unsigned channel)
753{
754 if (ecc->id != EDMA_CTLR(channel)) {
755 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
756 ecc->id, EDMA_CTLR(channel));
757 return;
758 }
759 channel = EDMA_CHAN_SLOT(channel);
760
761 if (channel < ecc->num_channels) {
762 int j = (channel >> 5);
763 unsigned int mask = BIT(channel & 0x1f);
764
Peter Ujfalusi3287fb42015-10-14 14:42:57 +0300765 dev_dbg(ecc->dev, "EMR%d %08x\n", j,
766 edma_read_array(ecc, EDMA_EMR, j));
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300767 edma_shadow0_write_array(ecc, SH_ECR, j, mask);
768 /* Clear the corresponding EMR bits */
769 edma_write_array(ecc, EDMA_EMCR, j, mask);
770 /* Clear any SER */
771 edma_shadow0_write_array(ecc, SH_SECR, j, mask);
772 edma_write(ecc, EDMA_CCERRCLR, BIT(16) | BIT(1) | BIT(0));
773 }
774}
775
776/**
777 * edma_alloc_channel - allocate DMA channel and paired parameter RAM
778 * @ecc: pointer to edma_cc struct
779 * @channel: specific channel to allocate; negative for "any unmapped channel"
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300780 * @eventq_no: an EVENTQ_* constant, used to choose which Transfer
781 * Controller (TC) executes requests using this channel. Use
782 * EVENTQ_DEFAULT unless you really need a high priority queue.
783 *
784 * This allocates a DMA channel and its associated parameter RAM slot.
785 * The parameter RAM is initialized to hold a dummy transfer.
786 *
787 * Normal use is to pass a specific channel number as @channel, to make
788 * use of hardware events mapped to that channel. When the channel will
789 * be used only for software triggering or event chaining, channels not
790 * mapped to hardware events (or mapped to unused events) are preferable.
791 *
792 * DMA transfers start from a channel using edma_start(), or by
793 * chaining. When the transfer described in that channel's parameter RAM
794 * slot completes, that slot's data may be reloaded through a link.
795 *
796 * DMA errors are only reported to the @callback associated with the
797 * channel driving that transfer, but transfer completion callbacks can
798 * be sent to another channel under control of the TCC field in
799 * the option word of the transfer's parameter RAM set. Drivers must not
800 * use DMA transfer completion callbacks for channels they did not allocate.
801 * (The same applies to TCC codes used in transfer chaining.)
802 *
803 * Returns the number of the channel, else negative errno.
804 */
805static int edma_alloc_channel(struct edma_cc *ecc, int channel,
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +0300806 enum dma_event_q eventq_no)
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300807{
808 unsigned done = 0;
809 int ret = 0;
810
811 if (!ecc->unused_chan_list_done) {
812 /*
813 * Scan all the platform devices to find out the EDMA channels
814 * used and clear them in the unused list, making the rest
815 * available for ARM usage.
816 */
817 ret = bus_for_each_dev(&platform_bus_type, NULL, ecc,
818 prepare_unused_channel_list);
819 if (ret < 0)
820 return ret;
821
822 ecc->unused_chan_list_done = true;
823 }
824
825 if (channel >= 0) {
826 if (ecc->id != EDMA_CTLR(channel)) {
827 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n",
828 __func__, ecc->id, EDMA_CTLR(channel));
829 return -EINVAL;
830 }
831 channel = EDMA_CHAN_SLOT(channel);
832 }
833
834 if (channel < 0) {
835 channel = 0;
836 for (;;) {
837 channel = find_next_bit(ecc->edma_unused,
838 ecc->num_channels, channel);
839 if (channel == ecc->num_channels)
840 break;
841 if (!test_and_set_bit(channel, ecc->edma_inuse)) {
842 done = 1;
843 break;
844 }
845 channel++;
846 }
847 if (!done)
848 return -ENOMEM;
849 } else if (channel >= ecc->num_channels) {
850 return -EINVAL;
851 } else if (test_and_set_bit(channel, ecc->edma_inuse)) {
852 return -EBUSY;
853 }
854
855 /* ensure access through shadow region 0 */
856 edma_or_array2(ecc, EDMA_DRAE, 0, channel >> 5, BIT(channel & 0x1f));
857
858 /* ensure no events are pending */
859 edma_stop(ecc, EDMA_CTLR_CHAN(ecc->id, channel));
860 edma_write_slot(ecc, channel, &dummy_paramset);
861
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +0300862 edma_setup_interrupt(ecc, EDMA_CTLR_CHAN(ecc->id, channel), true);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300863
864 edma_map_dmach_to_queue(ecc, channel, eventq_no);
865
866 return EDMA_CTLR_CHAN(ecc->id, channel);
867}
868
869/**
870 * edma_free_channel - deallocate DMA channel
871 * @ecc: pointer to edma_cc struct
872 * @channel: dma channel returned from edma_alloc_channel()
873 *
874 * This deallocates the DMA channel and associated parameter RAM slot
875 * allocated by edma_alloc_channel().
876 *
877 * Callers are responsible for ensuring the channel is inactive, and
878 * will not be reactivated by linking, chaining, or software calls to
879 * edma_start().
880 */
881static void edma_free_channel(struct edma_cc *ecc, unsigned channel)
882{
883 if (ecc->id != EDMA_CTLR(channel)) {
884 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
885 ecc->id, EDMA_CTLR(channel));
886 return;
887 }
888 channel = EDMA_CHAN_SLOT(channel);
889
890 if (channel >= ecc->num_channels)
891 return;
892
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +0300893 edma_setup_interrupt(ecc, channel, false);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300894 /* REVISIT should probably take out of shadow region 0 */
895
Peter Ujfalusi96f5ff02015-10-14 14:42:58 +0300896 edma_write_slot(ecc, channel, &dummy_paramset);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300897 clear_bit(channel, ecc->edma_inuse);
898}
899
Peter Ujfalusi11c15732015-10-14 14:43:00 +0300900/* Move channel to a specific event queue */
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300901static void edma_assign_channel_eventq(struct edma_cc *ecc, unsigned channel,
902 enum dma_event_q eventq_no)
903{
904 if (ecc->id != EDMA_CTLR(channel)) {
905 dev_err(ecc->dev, "%s: ID mismatch for eDMA%d: %d\n", __func__,
906 ecc->id, EDMA_CTLR(channel));
907 return;
908 }
909 channel = EDMA_CHAN_SLOT(channel);
910
911 if (channel >= ecc->num_channels)
912 return;
913
914 /* default to low priority queue */
915 if (eventq_no == EVENTQ_DEFAULT)
916 eventq_no = ecc->default_queue;
917 if (eventq_no >= ecc->num_tc)
918 return;
919
920 edma_map_dmach_to_queue(ecc, channel, eventq_no);
921}
922
Matt Porterc2dde5f2012-08-22 21:09:34 -0400923static inline struct edma_cc *to_edma_cc(struct dma_device *d)
924{
925 return container_of(d, struct edma_cc, dma_slave);
926}
927
928static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
929{
930 return container_of(c, struct edma_chan, vchan.chan);
931}
932
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300933static inline struct edma_desc *to_edma_desc(struct dma_async_tx_descriptor *tx)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400934{
935 return container_of(tx, struct edma_desc, vdesc.tx);
936}
937
938static void edma_desc_free(struct virt_dma_desc *vdesc)
939{
940 kfree(container_of(vdesc, struct edma_desc, vdesc));
941}
942
943/* Dispatch a queued descriptor to the controller (caller holds lock) */
944static void edma_execute(struct edma_chan *echan)
945{
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300946 struct edma_cc *ecc = echan->ecc;
Joel Fernandes53407062013-09-03 10:02:46 -0500947 struct virt_dma_desc *vdesc;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400948 struct edma_desc *edesc;
Joel Fernandes53407062013-09-03 10:02:46 -0500949 struct device *dev = echan->vchan.chan.device->dev;
950 int i, j, left, nslots;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400951
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300952 if (!echan->edesc) {
953 /* Setup is needed for the first transfer */
Joel Fernandes53407062013-09-03 10:02:46 -0500954 vdesc = vchan_next_desc(&echan->vchan);
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300955 if (!vdesc)
Joel Fernandes53407062013-09-03 10:02:46 -0500956 return;
Joel Fernandes53407062013-09-03 10:02:46 -0500957 list_del(&vdesc->node);
958 echan->edesc = to_edma_desc(&vdesc->tx);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400959 }
960
Joel Fernandes53407062013-09-03 10:02:46 -0500961 edesc = echan->edesc;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400962
Joel Fernandes53407062013-09-03 10:02:46 -0500963 /* Find out how many left */
964 left = edesc->pset_nr - edesc->processed;
965 nslots = min(MAX_NR_SG, left);
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500966 edesc->sg_len = 0;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400967
968 /* Write descriptor PaRAM set(s) */
Joel Fernandes53407062013-09-03 10:02:46 -0500969 for (i = 0; i < nslots; i++) {
970 j = i + edesc->processed;
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300971 edma_write_slot(ecc, echan->slot[i], &edesc->pset[j].param);
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500972 edesc->sg_len += edesc->pset[j].len;
Peter Ujfalusi907f74a2015-10-14 14:42:56 +0300973 dev_vdbg(dev,
974 "\n pset[%d]:\n"
975 " chnum\t%d\n"
976 " slot\t%d\n"
977 " opt\t%08x\n"
978 " src\t%08x\n"
979 " dst\t%08x\n"
980 " abcnt\t%08x\n"
981 " ccnt\t%08x\n"
982 " bidx\t%08x\n"
983 " cidx\t%08x\n"
984 " lkrld\t%08x\n",
985 j, echan->ch_num, echan->slot[i],
986 edesc->pset[j].param.opt,
987 edesc->pset[j].param.src,
988 edesc->pset[j].param.dst,
989 edesc->pset[j].param.a_b_cnt,
990 edesc->pset[j].param.ccnt,
991 edesc->pset[j].param.src_dst_bidx,
992 edesc->pset[j].param.src_dst_cidx,
993 edesc->pset[j].param.link_bcntrld);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400994 /* Link to the previous slot if not the last set */
Joel Fernandes53407062013-09-03 10:02:46 -0500995 if (i != (nslots - 1))
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +0300996 edma_link(ecc, echan->slot[i], echan->slot[i + 1]);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400997 }
998
Joel Fernandes53407062013-09-03 10:02:46 -0500999 edesc->processed += nslots;
1000
Joel Fernandesb267b3b2013-08-29 18:05:44 -05001001 /*
1002 * If this is either the last set in a set of SG-list transactions
1003 * then setup a link to the dummy slot, this results in all future
1004 * events being absorbed and that's OK because we're done
1005 */
Joel Fernandes50a9c702013-10-31 16:31:23 -05001006 if (edesc->processed == edesc->pset_nr) {
1007 if (edesc->cyclic)
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001008 edma_link(ecc, echan->slot[nslots - 1], echan->slot[1]);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001009 else
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001010 edma_link(ecc, echan->slot[nslots - 1],
Joel Fernandes50a9c702013-10-31 16:31:23 -05001011 echan->ecc->dummy_slot);
1012 }
Joel Fernandesb267b3b2013-08-29 18:05:44 -05001013
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +03001014 if (echan->missed) {
1015 /*
1016 * This happens due to setup times between intermediate
1017 * transfers in long SG lists which have to be broken up into
1018 * transfers of MAX_NR_SG
1019 */
1020 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001021 edma_clean_channel(ecc, echan->ch_num);
1022 edma_stop(ecc, echan->ch_num);
1023 edma_start(ecc, echan->ch_num);
1024 edma_trigger_channel(ecc, echan->ch_num);
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +03001025 echan->missed = 0;
1026 } else if (edesc->processed <= MAX_NR_SG) {
Peter Ujfalusi9aac9092014-04-24 10:29:50 +03001027 dev_dbg(dev, "first transfer starting on channel %d\n",
1028 echan->ch_num);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001029 edma_start(ecc, echan->ch_num);
Sekhar Nori5fc68a62014-03-19 11:25:50 +05301030 } else {
1031 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
1032 echan->ch_num, edesc->processed);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001033 edma_resume(ecc, echan->ch_num);
Joel Fernandes53407062013-09-03 10:02:46 -05001034 }
Matt Porterc2dde5f2012-08-22 21:09:34 -04001035}
1036
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001037static int edma_terminate_all(struct dma_chan *chan)
Matt Porterc2dde5f2012-08-22 21:09:34 -04001038{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001039 struct edma_chan *echan = to_edma_chan(chan);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001040 unsigned long flags;
1041 LIST_HEAD(head);
1042
1043 spin_lock_irqsave(&echan->vchan.lock, flags);
1044
1045 /*
1046 * Stop DMA activity: we assume the callback will not be called
1047 * after edma_dma() returns (even if it does, it will see
1048 * echan->edesc is NULL and exit.)
1049 */
1050 if (echan->edesc) {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001051 edma_stop(echan->ecc, echan->ch_num);
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +03001052 /* Move the cyclic channel back to default queue */
1053 if (echan->edesc->cyclic)
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001054 edma_assign_channel_eventq(echan->ecc, echan->ch_num,
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +03001055 EVENTQ_DEFAULT);
Petr Kulhavy5ca9e7c2015-03-27 13:35:51 +02001056 /*
1057 * free the running request descriptor
1058 * since it is not in any of the vdesc lists
1059 */
1060 edma_desc_free(&echan->edesc->vdesc);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001061 echan->edesc = NULL;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001062 }
1063
1064 vchan_get_all_descriptors(&echan->vchan, &head);
1065 spin_unlock_irqrestore(&echan->vchan.lock, flags);
1066 vchan_dma_desc_free_list(&echan->vchan, &head);
1067
1068 return 0;
1069}
1070
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001071static int edma_slave_config(struct dma_chan *chan,
Matt Porter661f7cb2013-01-10 13:41:04 -05001072 struct dma_slave_config *cfg)
Matt Porterc2dde5f2012-08-22 21:09:34 -04001073{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001074 struct edma_chan *echan = to_edma_chan(chan);
1075
Matt Porter661f7cb2013-01-10 13:41:04 -05001076 if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1077 cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
Matt Porterc2dde5f2012-08-22 21:09:34 -04001078 return -EINVAL;
1079
Matt Porter661f7cb2013-01-10 13:41:04 -05001080 memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
Matt Porterc2dde5f2012-08-22 21:09:34 -04001081
1082 return 0;
1083}
1084
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001085static int edma_dma_pause(struct dma_chan *chan)
Peter Ujfalusi72c7b672014-04-14 14:41:59 +03001086{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001087 struct edma_chan *echan = to_edma_chan(chan);
1088
John Ogness02ec6042015-04-27 13:52:25 +02001089 if (!echan->edesc)
Peter Ujfalusi72c7b672014-04-14 14:41:59 +03001090 return -EINVAL;
1091
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001092 edma_pause(echan->ecc, echan->ch_num);
Peter Ujfalusi72c7b672014-04-14 14:41:59 +03001093 return 0;
1094}
1095
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001096static int edma_dma_resume(struct dma_chan *chan)
Peter Ujfalusi72c7b672014-04-14 14:41:59 +03001097{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001098 struct edma_chan *echan = to_edma_chan(chan);
1099
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001100 edma_resume(echan->ecc, echan->ch_num);
Peter Ujfalusi72c7b672014-04-14 14:41:59 +03001101 return 0;
1102}
1103
Joel Fernandesfd009032013-09-23 18:05:13 -05001104/*
1105 * A PaRAM set configuration abstraction used by other modes
1106 * @chan: Channel who's PaRAM set we're configuring
1107 * @pset: PaRAM set to initialize and setup.
1108 * @src_addr: Source address of the DMA
1109 * @dst_addr: Destination address of the DMA
1110 * @burst: In units of dev_width, how much to send
1111 * @dev_width: How much is the dev_width
1112 * @dma_length: Total length of the DMA transfer
1113 * @direction: Direction of the transfer
1114 */
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001115static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001116 dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
1117 enum dma_slave_buswidth dev_width,
1118 unsigned int dma_length,
1119 enum dma_transfer_direction direction)
Joel Fernandesfd009032013-09-23 18:05:13 -05001120{
1121 struct edma_chan *echan = to_edma_chan(chan);
1122 struct device *dev = chan->device->dev;
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001123 struct edmacc_param *param = &epset->param;
Joel Fernandesfd009032013-09-23 18:05:13 -05001124 int acnt, bcnt, ccnt, cidx;
1125 int src_bidx, dst_bidx, src_cidx, dst_cidx;
1126 int absync;
1127
1128 acnt = dev_width;
Peter Ujfalusib2b617d2014-04-14 14:41:58 +03001129
1130 /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
1131 if (!burst)
1132 burst = 1;
Joel Fernandesfd009032013-09-23 18:05:13 -05001133 /*
1134 * If the maxburst is equal to the fifo width, use
1135 * A-synced transfers. This allows for large contiguous
1136 * buffer transfers using only one PaRAM set.
1137 */
1138 if (burst == 1) {
1139 /*
1140 * For the A-sync case, bcnt and ccnt are the remainder
1141 * and quotient respectively of the division of:
1142 * (dma_length / acnt) by (SZ_64K -1). This is so
1143 * that in case bcnt over flows, we have ccnt to use.
1144 * Note: In A-sync tranfer only, bcntrld is used, but it
1145 * only applies for sg_dma_len(sg) >= SZ_64K.
1146 * In this case, the best way adopted is- bccnt for the
1147 * first frame will be the remainder below. Then for
1148 * every successive frame, bcnt will be SZ_64K-1. This
1149 * is assured as bcntrld = 0xffff in end of function.
1150 */
1151 absync = false;
1152 ccnt = dma_length / acnt / (SZ_64K - 1);
1153 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
1154 /*
1155 * If bcnt is non-zero, we have a remainder and hence an
1156 * extra frame to transfer, so increment ccnt.
1157 */
1158 if (bcnt)
1159 ccnt++;
1160 else
1161 bcnt = SZ_64K - 1;
1162 cidx = acnt;
1163 } else {
1164 /*
1165 * If maxburst is greater than the fifo address_width,
1166 * use AB-synced transfers where A count is the fifo
1167 * address_width and B count is the maxburst. In this
1168 * case, we are limited to transfers of C count frames
1169 * of (address_width * maxburst) where C count is limited
1170 * to SZ_64K-1. This places an upper bound on the length
1171 * of an SG segment that can be handled.
1172 */
1173 absync = true;
1174 bcnt = burst;
1175 ccnt = dma_length / (acnt * bcnt);
1176 if (ccnt > (SZ_64K - 1)) {
1177 dev_err(dev, "Exceeded max SG segment size\n");
1178 return -EINVAL;
1179 }
1180 cidx = acnt * bcnt;
1181 }
1182
Thomas Gleixnerc2da2342014-04-28 14:29:57 -05001183 epset->len = dma_length;
1184
Joel Fernandesfd009032013-09-23 18:05:13 -05001185 if (direction == DMA_MEM_TO_DEV) {
1186 src_bidx = acnt;
1187 src_cidx = cidx;
1188 dst_bidx = 0;
1189 dst_cidx = 0;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -05001190 epset->addr = src_addr;
Joel Fernandesfd009032013-09-23 18:05:13 -05001191 } else if (direction == DMA_DEV_TO_MEM) {
1192 src_bidx = 0;
1193 src_cidx = 0;
1194 dst_bidx = acnt;
1195 dst_cidx = cidx;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -05001196 epset->addr = dst_addr;
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001197 } else if (direction == DMA_MEM_TO_MEM) {
1198 src_bidx = acnt;
1199 src_cidx = cidx;
1200 dst_bidx = acnt;
1201 dst_cidx = cidx;
Joel Fernandesfd009032013-09-23 18:05:13 -05001202 } else {
1203 dev_err(dev, "%s: direction not implemented yet\n", __func__);
1204 return -EINVAL;
1205 }
1206
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001207 param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
Joel Fernandesfd009032013-09-23 18:05:13 -05001208 /* Configure A or AB synchronized transfers */
1209 if (absync)
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001210 param->opt |= SYNCDIM;
Joel Fernandesfd009032013-09-23 18:05:13 -05001211
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001212 param->src = src_addr;
1213 param->dst = dst_addr;
Joel Fernandesfd009032013-09-23 18:05:13 -05001214
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001215 param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
1216 param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
Joel Fernandesfd009032013-09-23 18:05:13 -05001217
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001218 param->a_b_cnt = bcnt << 16 | acnt;
1219 param->ccnt = ccnt;
Joel Fernandesfd009032013-09-23 18:05:13 -05001220 /*
1221 * Only time when (bcntrld) auto reload is required is for
1222 * A-sync case, and in this case, a requirement of reload value
1223 * of SZ_64K-1 only is assured. 'link' is initially set to NULL
1224 * and then later will be populated by edma_execute.
1225 */
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001226 param->link_bcntrld = 0xffffffff;
Joel Fernandesfd009032013-09-23 18:05:13 -05001227 return absync;
1228}
1229
Matt Porterc2dde5f2012-08-22 21:09:34 -04001230static struct dma_async_tx_descriptor *edma_prep_slave_sg(
1231 struct dma_chan *chan, struct scatterlist *sgl,
1232 unsigned int sg_len, enum dma_transfer_direction direction,
1233 unsigned long tx_flags, void *context)
1234{
1235 struct edma_chan *echan = to_edma_chan(chan);
1236 struct device *dev = chan->device->dev;
1237 struct edma_desc *edesc;
Joel Fernandesfd009032013-09-23 18:05:13 -05001238 dma_addr_t src_addr = 0, dst_addr = 0;
Matt Porter661f7cb2013-01-10 13:41:04 -05001239 enum dma_slave_buswidth dev_width;
1240 u32 burst;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001241 struct scatterlist *sg;
Joel Fernandesfd009032013-09-23 18:05:13 -05001242 int i, nslots, ret;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001243
1244 if (unlikely(!echan || !sgl || !sg_len))
1245 return NULL;
1246
Matt Porter661f7cb2013-01-10 13:41:04 -05001247 if (direction == DMA_DEV_TO_MEM) {
Joel Fernandesfd009032013-09-23 18:05:13 -05001248 src_addr = echan->cfg.src_addr;
Matt Porter661f7cb2013-01-10 13:41:04 -05001249 dev_width = echan->cfg.src_addr_width;
1250 burst = echan->cfg.src_maxburst;
1251 } else if (direction == DMA_MEM_TO_DEV) {
Joel Fernandesfd009032013-09-23 18:05:13 -05001252 dst_addr = echan->cfg.dst_addr;
Matt Porter661f7cb2013-01-10 13:41:04 -05001253 dev_width = echan->cfg.dst_addr_width;
1254 burst = echan->cfg.dst_maxburst;
1255 } else {
Peter Ujfalusie6fad592014-04-14 14:42:05 +03001256 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
Matt Porter661f7cb2013-01-10 13:41:04 -05001257 return NULL;
1258 }
1259
1260 if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
Peter Ujfalusic594c892014-04-14 14:42:03 +03001261 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001262 return NULL;
1263 }
1264
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001265 edesc = kzalloc(sizeof(*edesc) + sg_len * sizeof(edesc->pset[0]),
1266 GFP_ATOMIC);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001267 if (!edesc) {
Peter Ujfalusic594c892014-04-14 14:42:03 +03001268 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001269 return NULL;
1270 }
1271
1272 edesc->pset_nr = sg_len;
Thomas Gleixnerb6205c32014-04-28 14:18:45 -05001273 edesc->residue = 0;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -05001274 edesc->direction = direction;
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001275 edesc->echan = echan;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001276
Joel Fernandes6fbe24d2013-08-29 18:05:40 -05001277 /* Allocate a PaRAM slot, if needed */
1278 nslots = min_t(unsigned, MAX_NR_SG, sg_len);
1279
1280 for (i = 0; i < nslots; i++) {
Matt Porterc2dde5f2012-08-22 21:09:34 -04001281 if (echan->slot[i] < 0) {
1282 echan->slot[i] =
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001283 edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001284 if (echan->slot[i] < 0) {
Valentin Ilie4b6271a2013-10-24 16:14:22 +03001285 kfree(edesc);
Peter Ujfalusic594c892014-04-14 14:42:03 +03001286 dev_err(dev, "%s: Failed to allocate slot\n",
1287 __func__);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001288 return NULL;
1289 }
1290 }
Joel Fernandes6fbe24d2013-08-29 18:05:40 -05001291 }
1292
1293 /* Configure PaRAM sets for each SG */
1294 for_each_sg(sgl, sg, sg_len, i) {
Joel Fernandesfd009032013-09-23 18:05:13 -05001295 /* Get address for each SG */
1296 if (direction == DMA_DEV_TO_MEM)
1297 dst_addr = sg_dma_address(sg);
1298 else
1299 src_addr = sg_dma_address(sg);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001300
Joel Fernandesfd009032013-09-23 18:05:13 -05001301 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
1302 dst_addr, burst, dev_width,
1303 sg_dma_len(sg), direction);
Vinod Koulb967aec2013-10-30 13:07:18 +05301304 if (ret < 0) {
1305 kfree(edesc);
Joel Fernandesfd009032013-09-23 18:05:13 -05001306 return NULL;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001307 }
1308
Joel Fernandesfd009032013-09-23 18:05:13 -05001309 edesc->absync = ret;
Thomas Gleixnerb6205c32014-04-28 14:18:45 -05001310 edesc->residue += sg_dma_len(sg);
Joel Fernandes6fbe24d2013-08-29 18:05:40 -05001311
1312 /* If this is the last in a current SG set of transactions,
1313 enable interrupts so that next set is processed */
1314 if (!((i+1) % MAX_NR_SG))
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001315 edesc->pset[i].param.opt |= TCINTEN;
Joel Fernandes6fbe24d2013-08-29 18:05:40 -05001316
Matt Porterc2dde5f2012-08-22 21:09:34 -04001317 /* If this is the last set, enable completion interrupt flag */
1318 if (i == sg_len - 1)
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001319 edesc->pset[i].param.opt |= TCINTEN;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001320 }
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001321 edesc->residue_stat = edesc->residue;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001322
Matt Porterc2dde5f2012-08-22 21:09:34 -04001323 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1324}
Matt Porterc2dde5f2012-08-22 21:09:34 -04001325
Lad, Prabhakarb7a4fd52015-02-04 13:03:27 +00001326static struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001327 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1328 size_t len, unsigned long tx_flags)
1329{
1330 int ret;
1331 struct edma_desc *edesc;
1332 struct device *dev = chan->device->dev;
1333 struct edma_chan *echan = to_edma_chan(chan);
1334
1335 if (unlikely(!echan || !len))
1336 return NULL;
1337
1338 edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC);
1339 if (!edesc) {
1340 dev_dbg(dev, "Failed to allocate a descriptor\n");
1341 return NULL;
1342 }
1343
1344 edesc->pset_nr = 1;
1345
1346 ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
1347 DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM);
1348 if (ret < 0)
1349 return NULL;
1350
1351 edesc->absync = ret;
1352
1353 /*
1354 * Enable intermediate transfer chaining to re-trigger channel
1355 * on completion of every TR, and enable transfer-completion
1356 * interrupt on completion of the whole transfer.
1357 */
Joel Fernandesb0cce4c2014-04-28 15:30:32 -05001358 edesc->pset[0].param.opt |= ITCCHEN;
1359 edesc->pset[0].param.opt |= TCINTEN;
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001360
1361 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1362}
1363
Joel Fernandes50a9c702013-10-31 16:31:23 -05001364static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
1365 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1366 size_t period_len, enum dma_transfer_direction direction,
Laurent Pinchart31c1e5a2014-08-01 12:20:10 +02001367 unsigned long tx_flags)
Joel Fernandes50a9c702013-10-31 16:31:23 -05001368{
1369 struct edma_chan *echan = to_edma_chan(chan);
1370 struct device *dev = chan->device->dev;
1371 struct edma_desc *edesc;
1372 dma_addr_t src_addr, dst_addr;
1373 enum dma_slave_buswidth dev_width;
1374 u32 burst;
1375 int i, ret, nslots;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001376
Joel Fernandes50a9c702013-10-31 16:31:23 -05001377 if (unlikely(!echan || !buf_len || !period_len))
1378 return NULL;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001379
Joel Fernandes50a9c702013-10-31 16:31:23 -05001380 if (direction == DMA_DEV_TO_MEM) {
1381 src_addr = echan->cfg.src_addr;
1382 dst_addr = buf_addr;
1383 dev_width = echan->cfg.src_addr_width;
1384 burst = echan->cfg.src_maxburst;
1385 } else if (direction == DMA_MEM_TO_DEV) {
1386 src_addr = buf_addr;
1387 dst_addr = echan->cfg.dst_addr;
1388 dev_width = echan->cfg.dst_addr_width;
1389 burst = echan->cfg.dst_maxburst;
1390 } else {
Peter Ujfalusie6fad592014-04-14 14:42:05 +03001391 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001392 return NULL;
1393 }
1394
1395 if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
Peter Ujfalusic594c892014-04-14 14:42:03 +03001396 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001397 return NULL;
1398 }
1399
1400 if (unlikely(buf_len % period_len)) {
1401 dev_err(dev, "Period should be multiple of Buffer length\n");
1402 return NULL;
1403 }
1404
1405 nslots = (buf_len / period_len) + 1;
1406
1407 /*
1408 * Cyclic DMA users such as audio cannot tolerate delays introduced
1409 * by cases where the number of periods is more than the maximum
1410 * number of SGs the EDMA driver can handle at a time. For DMA types
1411 * such as Slave SGs, such delays are tolerable and synchronized,
1412 * but the synchronization is difficult to achieve with Cyclic and
1413 * cannot be guaranteed, so we error out early.
1414 */
1415 if (nslots > MAX_NR_SG)
1416 return NULL;
1417
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001418 edesc = kzalloc(sizeof(*edesc) + nslots * sizeof(edesc->pset[0]),
1419 GFP_ATOMIC);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001420 if (!edesc) {
Peter Ujfalusic594c892014-04-14 14:42:03 +03001421 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001422 return NULL;
1423 }
1424
1425 edesc->cyclic = 1;
1426 edesc->pset_nr = nslots;
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001427 edesc->residue = edesc->residue_stat = buf_len;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -05001428 edesc->direction = direction;
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001429 edesc->echan = echan;
Joel Fernandes50a9c702013-10-31 16:31:23 -05001430
Peter Ujfalusi83bb3122014-04-14 14:42:02 +03001431 dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
1432 __func__, echan->ch_num, nslots, period_len, buf_len);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001433
1434 for (i = 0; i < nslots; i++) {
1435 /* Allocate a PaRAM slot, if needed */
1436 if (echan->slot[i] < 0) {
1437 echan->slot[i] =
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001438 edma_alloc_slot(echan->ecc, EDMA_SLOT_ANY);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001439 if (echan->slot[i] < 0) {
Christian Engelmayere3ddc972013-12-30 20:48:39 +01001440 kfree(edesc);
Peter Ujfalusic594c892014-04-14 14:42:03 +03001441 dev_err(dev, "%s: Failed to allocate slot\n",
1442 __func__);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001443 return NULL;
1444 }
1445 }
1446
1447 if (i == nslots - 1) {
1448 memcpy(&edesc->pset[i], &edesc->pset[0],
1449 sizeof(edesc->pset[0]));
1450 break;
1451 }
1452
1453 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
1454 dst_addr, burst, dev_width, period_len,
1455 direction);
Christian Engelmayere3ddc972013-12-30 20:48:39 +01001456 if (ret < 0) {
1457 kfree(edesc);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001458 return NULL;
Christian Engelmayere3ddc972013-12-30 20:48:39 +01001459 }
Joel Fernandes50a9c702013-10-31 16:31:23 -05001460
1461 if (direction == DMA_DEV_TO_MEM)
1462 dst_addr += period_len;
1463 else
1464 src_addr += period_len;
1465
Peter Ujfalusi83bb3122014-04-14 14:42:02 +03001466 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
1467 dev_vdbg(dev,
Joel Fernandes50a9c702013-10-31 16:31:23 -05001468 "\n pset[%d]:\n"
1469 " chnum\t%d\n"
1470 " slot\t%d\n"
1471 " opt\t%08x\n"
1472 " src\t%08x\n"
1473 " dst\t%08x\n"
1474 " abcnt\t%08x\n"
1475 " ccnt\t%08x\n"
1476 " bidx\t%08x\n"
1477 " cidx\t%08x\n"
1478 " lkrld\t%08x\n",
1479 i, echan->ch_num, echan->slot[i],
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -05001480 edesc->pset[i].param.opt,
1481 edesc->pset[i].param.src,
1482 edesc->pset[i].param.dst,
1483 edesc->pset[i].param.a_b_cnt,
1484 edesc->pset[i].param.ccnt,
1485 edesc->pset[i].param.src_dst_bidx,
1486 edesc->pset[i].param.src_dst_cidx,
1487 edesc->pset[i].param.link_bcntrld);
Joel Fernandes50a9c702013-10-31 16:31:23 -05001488
1489 edesc->absync = ret;
1490
1491 /*
Peter Ujfalusia1f146f2014-07-16 15:29:21 +03001492 * Enable period interrupt only if it is requested
Joel Fernandes50a9c702013-10-31 16:31:23 -05001493 */
Peter Ujfalusia1f146f2014-07-16 15:29:21 +03001494 if (tx_flags & DMA_PREP_INTERRUPT)
1495 edesc->pset[i].param.opt |= TCINTEN;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001496 }
1497
Peter Ujfalusi8e8805d2014-07-08 13:46:38 +03001498 /* Place the cyclic channel to highest priority queue */
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001499 edma_assign_channel_eventq(echan->ecc, echan->ch_num, EVENTQ_0);
Peter Ujfalusi8e8805d2014-07-08 13:46:38 +03001500
Matt Porterc2dde5f2012-08-22 21:09:34 -04001501 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
1502}
1503
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001504static void edma_completion_handler(struct edma_chan *echan)
Matt Porterc2dde5f2012-08-22 21:09:34 -04001505{
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001506 struct edma_cc *ecc = echan->ecc;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001507 struct device *dev = echan->vchan.chan.device->dev;
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001508 struct edma_desc *edesc = echan->edesc;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001509
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001510 if (!edesc)
1511 return;
Joel Fernandes50a9c702013-10-31 16:31:23 -05001512
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +03001513 spin_lock(&echan->vchan.lock);
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001514 if (edesc->cyclic) {
1515 vchan_cyclic_callback(&edesc->vdesc);
1516 spin_unlock(&echan->vchan.lock);
1517 return;
1518 } else if (edesc->processed == edesc->pset_nr) {
1519 edesc->residue = 0;
1520 edma_stop(ecc, echan->ch_num);
1521 vchan_cookie_complete(&edesc->vdesc);
1522 echan->edesc = NULL;
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001523
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001524 dev_dbg(dev, "Transfer completed on channel %d\n",
1525 echan->ch_num);
1526 } else {
1527 dev_dbg(dev, "Sub transfer completed on channel %d\n",
1528 echan->ch_num);
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +03001529
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001530 edma_pause(ecc, echan->ch_num);
Joel Fernandesc5f47992013-08-29 18:05:43 -05001531
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001532 /* Update statistics for tx_status */
1533 edesc->residue -= edesc->sg_len;
1534 edesc->residue_stat = edesc->residue;
1535 edesc->processed_stat = edesc->processed;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001536 }
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001537 edma_execute(echan);
1538
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +03001539 spin_unlock(&echan->vchan.lock);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001540}
1541
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001542/* eDMA interrupt handler */
1543static irqreturn_t dma_irq_handler(int irq, void *data)
1544{
1545 struct edma_cc *ecc = data;
1546 int ctlr;
1547 u32 sh_ier;
1548 u32 sh_ipr;
1549 u32 bank;
1550
1551 ctlr = ecc->id;
1552 if (ctlr < 0)
1553 return IRQ_NONE;
1554
1555 dev_vdbg(ecc->dev, "dma_irq_handler\n");
1556
1557 sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 0);
1558 if (!sh_ipr) {
1559 sh_ipr = edma_shadow0_read_array(ecc, SH_IPR, 1);
1560 if (!sh_ipr)
1561 return IRQ_NONE;
1562 sh_ier = edma_shadow0_read_array(ecc, SH_IER, 1);
1563 bank = 1;
1564 } else {
1565 sh_ier = edma_shadow0_read_array(ecc, SH_IER, 0);
1566 bank = 0;
1567 }
1568
1569 do {
1570 u32 slot;
1571 u32 channel;
1572
1573 slot = __ffs(sh_ipr);
1574 sh_ipr &= ~(BIT(slot));
1575
1576 if (sh_ier & BIT(slot)) {
1577 channel = (bank << 5) | slot;
1578 /* Clear the corresponding IPR bits */
1579 edma_shadow0_write_array(ecc, SH_ICR, bank, BIT(slot));
1580 edma_completion_handler(&ecc->slave_chans[channel]);
1581 }
1582 } while (sh_ipr);
1583
1584 edma_shadow0_write(ecc, SH_IEVAL, 1);
1585 return IRQ_HANDLED;
1586}
1587
1588static void edma_error_handler(struct edma_chan *echan)
1589{
1590 struct edma_cc *ecc = echan->ecc;
1591 struct device *dev = echan->vchan.chan.device->dev;
1592 struct edmacc_param p;
1593
1594 if (!echan->edesc)
1595 return;
1596
1597 spin_lock(&echan->vchan.lock);
1598
1599 edma_read_slot(ecc, echan->slot[0], &p);
1600 /*
1601 * Issue later based on missed flag which will be sure
1602 * to happen as:
1603 * (1) we finished transmitting an intermediate slot and
1604 * edma_execute is coming up.
1605 * (2) or we finished current transfer and issue will
1606 * call edma_execute.
1607 *
1608 * Important note: issuing can be dangerous here and
1609 * lead to some nasty recursion when we are in a NULL
1610 * slot. So we avoid doing so and set the missed flag.
1611 */
1612 if (p.a_b_cnt == 0 && p.ccnt == 0) {
1613 dev_dbg(dev, "Error on null slot, setting miss\n");
1614 echan->missed = 1;
1615 } else {
1616 /*
1617 * The slot is already programmed but the event got
1618 * missed, so its safe to issue it here.
1619 */
1620 dev_dbg(dev, "Missed event, TRIGGERING\n");
1621 edma_clean_channel(ecc, echan->ch_num);
1622 edma_stop(ecc, echan->ch_num);
1623 edma_start(ecc, echan->ch_num);
1624 edma_trigger_channel(ecc, echan->ch_num);
1625 }
1626 spin_unlock(&echan->vchan.lock);
1627}
1628
Peter Ujfalusi7c3b8b32015-10-14 14:43:02 +03001629static inline bool edma_error_pending(struct edma_cc *ecc)
1630{
1631 if (edma_read_array(ecc, EDMA_EMR, 0) ||
1632 edma_read_array(ecc, EDMA_EMR, 1) ||
1633 edma_read(ecc, EDMA_QEMR) || edma_read(ecc, EDMA_CCERR))
1634 return true;
1635
1636 return false;
1637}
1638
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001639/* eDMA error interrupt handler */
1640static irqreturn_t dma_ccerr_handler(int irq, void *data)
1641{
1642 struct edma_cc *ecc = data;
1643 int i;
1644 int ctlr;
1645 unsigned int cnt = 0;
1646
1647 ctlr = ecc->id;
1648 if (ctlr < 0)
1649 return IRQ_NONE;
1650
1651 dev_vdbg(ecc->dev, "dma_ccerr_handler\n");
1652
Peter Ujfalusi7c3b8b32015-10-14 14:43:02 +03001653 if (!edma_error_pending(ecc))
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001654 return IRQ_NONE;
1655
1656 while (1) {
1657 int j = -1;
1658
1659 if (edma_read_array(ecc, EDMA_EMR, 0))
1660 j = 0;
1661 else if (edma_read_array(ecc, EDMA_EMR, 1))
1662 j = 1;
1663 if (j >= 0) {
1664 dev_dbg(ecc->dev, "EMR%d %08x\n", j,
1665 edma_read_array(ecc, EDMA_EMR, j));
1666 for (i = 0; i < 32; i++) {
1667 int k = (j << 5) + i;
1668
1669 if (edma_read_array(ecc, EDMA_EMR, j) &
1670 BIT(i)) {
1671 /* Clear the corresponding EMR bits */
1672 edma_write_array(ecc, EDMA_EMCR, j,
1673 BIT(i));
1674 /* Clear any SER */
1675 edma_shadow0_write_array(ecc, SH_SECR,
1676 j, BIT(i));
1677 edma_error_handler(&ecc->slave_chans[k]);
1678 }
1679 }
1680 } else if (edma_read(ecc, EDMA_QEMR)) {
1681 dev_dbg(ecc->dev, "QEMR %02x\n",
1682 edma_read(ecc, EDMA_QEMR));
1683 for (i = 0; i < 8; i++) {
1684 if (edma_read(ecc, EDMA_QEMR) & BIT(i)) {
1685 /* Clear the corresponding IPR bits */
1686 edma_write(ecc, EDMA_QEMCR, BIT(i));
1687 edma_shadow0_write(ecc, SH_QSECR,
1688 BIT(i));
1689
1690 /* NOTE: not reported!! */
1691 }
1692 }
1693 } else if (edma_read(ecc, EDMA_CCERR)) {
1694 dev_dbg(ecc->dev, "CCERR %08x\n",
1695 edma_read(ecc, EDMA_CCERR));
1696 /* FIXME: CCERR.BIT(16) ignored! much better
1697 * to just write CCERRCLR with CCERR value...
1698 */
1699 for (i = 0; i < 8; i++) {
1700 if (edma_read(ecc, EDMA_CCERR) & BIT(i)) {
1701 /* Clear the corresponding IPR bits */
1702 edma_write(ecc, EDMA_CCERRCLR, BIT(i));
1703
1704 /* NOTE: not reported!! */
1705 }
1706 }
1707 }
Peter Ujfalusi7c3b8b32015-10-14 14:43:02 +03001708 if (!edma_error_pending(ecc))
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001709 break;
1710 cnt++;
1711 if (cnt > 10)
1712 break;
1713 }
1714 edma_write(ecc, EDMA_EEVAL, 1);
1715 return IRQ_HANDLED;
1716}
1717
Matt Porterc2dde5f2012-08-22 21:09:34 -04001718/* Alloc channel resources */
1719static int edma_alloc_chan_resources(struct dma_chan *chan)
1720{
1721 struct edma_chan *echan = to_edma_chan(chan);
1722 struct device *dev = chan->device->dev;
1723 int ret;
1724 int a_ch_num;
1725 LIST_HEAD(descs);
1726
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03001727 a_ch_num = edma_alloc_channel(echan->ecc, echan->ch_num, EVENTQ_DEFAULT);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001728
1729 if (a_ch_num < 0) {
1730 ret = -ENODEV;
1731 goto err_no_chan;
1732 }
1733
1734 if (a_ch_num != echan->ch_num) {
1735 dev_err(dev, "failed to allocate requested channel %u:%u\n",
1736 EDMA_CTLR(echan->ch_num),
1737 EDMA_CHAN_SLOT(echan->ch_num));
1738 ret = -ENODEV;
1739 goto err_wrong_chan;
1740 }
1741
1742 echan->alloced = true;
1743 echan->slot[0] = echan->ch_num;
1744
Peter Ujfalusi9aac9092014-04-24 10:29:50 +03001745 dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num,
Ezequiel Garcia0e772c62013-12-13 11:06:18 -03001746 EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num));
Matt Porterc2dde5f2012-08-22 21:09:34 -04001747
1748 return 0;
1749
1750err_wrong_chan:
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001751 edma_free_channel(echan->ecc, a_ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001752err_no_chan:
1753 return ret;
1754}
1755
1756/* Free channel resources */
1757static void edma_free_chan_resources(struct dma_chan *chan)
1758{
1759 struct edma_chan *echan = to_edma_chan(chan);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001760 int i;
1761
1762 /* Terminate transfers */
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001763 edma_stop(echan->ecc, echan->ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001764
1765 vchan_free_chan_resources(&echan->vchan);
1766
1767 /* Free EDMA PaRAM slots */
1768 for (i = 1; i < EDMA_MAX_SLOTS; i++) {
1769 if (echan->slot[i] >= 0) {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001770 edma_free_slot(echan->ecc, echan->slot[i]);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001771 echan->slot[i] = -1;
1772 }
1773 }
1774
1775 /* Free EDMA channel */
1776 if (echan->alloced) {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001777 edma_free_channel(echan->ecc, echan->ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001778 echan->alloced = false;
1779 }
1780
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03001781 dev_dbg(chan->device->dev, "freeing channel for %u\n", echan->ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001782}
1783
1784/* Send pending descriptor to hardware */
1785static void edma_issue_pending(struct dma_chan *chan)
1786{
1787 struct edma_chan *echan = to_edma_chan(chan);
1788 unsigned long flags;
1789
1790 spin_lock_irqsave(&echan->vchan.lock, flags);
1791 if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
1792 edma_execute(echan);
1793 spin_unlock_irqrestore(&echan->vchan.lock, flags);
1794}
1795
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001796static u32 edma_residue(struct edma_desc *edesc)
1797{
1798 bool dst = edesc->direction == DMA_DEV_TO_MEM;
1799 struct edma_pset *pset = edesc->pset;
1800 dma_addr_t done, pos;
1801 int i;
1802
1803 /*
1804 * We always read the dst/src position from the first RamPar
1805 * pset. That's the one which is active now.
1806 */
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001807 pos = edma_get_position(edesc->echan->ecc, edesc->echan->slot[0], dst);
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001808
1809 /*
1810 * Cyclic is simple. Just subtract pset[0].addr from pos.
1811 *
1812 * We never update edesc->residue in the cyclic case, so we
1813 * can tell the remaining room to the end of the circular
1814 * buffer.
1815 */
1816 if (edesc->cyclic) {
1817 done = pos - pset->addr;
1818 edesc->residue_stat = edesc->residue - done;
1819 return edesc->residue_stat;
1820 }
1821
1822 /*
1823 * For SG operation we catch up with the last processed
1824 * status.
1825 */
1826 pset += edesc->processed_stat;
1827
1828 for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
1829 /*
1830 * If we are inside this pset address range, we know
1831 * this is the active one. Get the current delta and
1832 * stop walking the psets.
1833 */
1834 if (pos >= pset->addr && pos < pset->addr + pset->len)
1835 return edesc->residue_stat - (pos - pset->addr);
1836
1837 /* Otherwise mark it done and update residue_stat. */
1838 edesc->processed_stat++;
1839 edesc->residue_stat -= pset->len;
1840 }
1841 return edesc->residue_stat;
1842}
1843
Matt Porterc2dde5f2012-08-22 21:09:34 -04001844/* Check request completion status */
1845static enum dma_status edma_tx_status(struct dma_chan *chan,
1846 dma_cookie_t cookie,
1847 struct dma_tx_state *txstate)
1848{
1849 struct edma_chan *echan = to_edma_chan(chan);
1850 struct virt_dma_desc *vdesc;
1851 enum dma_status ret;
1852 unsigned long flags;
1853
1854 ret = dma_cookie_status(chan, cookie, txstate);
Vinod Koul9d386ec2013-10-16 13:42:15 +05301855 if (ret == DMA_COMPLETE || !txstate)
Matt Porterc2dde5f2012-08-22 21:09:34 -04001856 return ret;
1857
1858 spin_lock_irqsave(&echan->vchan.lock, flags);
Thomas Gleixnerde135932014-04-28 14:19:51 -05001859 if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
Thomas Gleixner740b41f2014-04-28 14:34:11 -05001860 txstate->residue = edma_residue(echan->edesc);
Thomas Gleixnerde135932014-04-28 14:19:51 -05001861 else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
1862 txstate->residue = to_edma_desc(&vdesc->tx)->residue;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001863 spin_unlock_irqrestore(&echan->vchan.lock, flags);
1864
1865 return ret;
1866}
1867
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001868static void __init edma_chan_init(struct edma_cc *ecc, struct dma_device *dma,
Matt Porterc2dde5f2012-08-22 21:09:34 -04001869 struct edma_chan *echans)
1870{
1871 int i, j;
1872
Peter Ujfalusicb782052015-10-14 14:42:54 +03001873 for (i = 0; i < ecc->num_channels; i++) {
Matt Porterc2dde5f2012-08-22 21:09:34 -04001874 struct edma_chan *echan = &echans[i];
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001875 echan->ch_num = EDMA_CTLR_CHAN(ecc->id, i);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001876 echan->ecc = ecc;
1877 echan->vchan.desc_free = edma_desc_free;
1878
1879 vchan_init(&echan->vchan, dma);
1880
1881 INIT_LIST_HEAD(&echan->node);
1882 for (j = 0; j < EDMA_MAX_SLOTS; j++)
1883 echan->slot[j] = -1;
1884 }
1885}
1886
Peter Ujfalusi2c88ee62014-04-14 14:42:01 +03001887#define EDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
1888 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
Peter Ujfalusie4a899d2014-07-03 07:51:56 +03001889 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
Peter Ujfalusi2c88ee62014-04-14 14:42:01 +03001890 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
1891
Matt Porterc2dde5f2012-08-22 21:09:34 -04001892static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
1893 struct device *dev)
1894{
1895 dma->device_prep_slave_sg = edma_prep_slave_sg;
Joel Fernandes50a9c702013-10-31 16:31:23 -05001896 dma->device_prep_dma_cyclic = edma_prep_dma_cyclic;
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001897 dma->device_prep_dma_memcpy = edma_prep_dma_memcpy;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001898 dma->device_alloc_chan_resources = edma_alloc_chan_resources;
1899 dma->device_free_chan_resources = edma_free_chan_resources;
1900 dma->device_issue_pending = edma_issue_pending;
1901 dma->device_tx_status = edma_tx_status;
Maxime Ripardaa7c09b2014-11-17 14:42:13 +01001902 dma->device_config = edma_slave_config;
1903 dma->device_pause = edma_dma_pause;
1904 dma->device_resume = edma_dma_resume;
1905 dma->device_terminate_all = edma_terminate_all;
Maxime Ripard9f59cd02014-11-17 14:42:47 +01001906
1907 dma->src_addr_widths = EDMA_DMA_BUSWIDTHS;
1908 dma->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
1909 dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1910 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1911
Matt Porterc2dde5f2012-08-22 21:09:34 -04001912 dma->dev = dev;
1913
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001914 /*
1915 * code using dma memcpy must make sure alignment of
1916 * length is at dma->copy_align boundary.
1917 */
Maxime Ripard77a68e52015-07-20 10:41:32 +02001918 dma->copy_align = DMAENGINE_ALIGN_4_BYTES;
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001919
Matt Porterc2dde5f2012-08-22 21:09:34 -04001920 INIT_LIST_HEAD(&dma->channels);
1921}
1922
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001923static int edma_setup_from_hw(struct device *dev, struct edma_soc_info *pdata,
1924 struct edma_cc *ecc)
1925{
1926 int i;
1927 u32 value, cccfg;
1928 s8 (*queue_priority_map)[2];
1929
1930 /* Decode the eDMA3 configuration from CCCFG register */
1931 cccfg = edma_read(ecc, EDMA_CCCFG);
1932
1933 value = GET_NUM_REGN(cccfg);
1934 ecc->num_region = BIT(value);
1935
1936 value = GET_NUM_DMACH(cccfg);
1937 ecc->num_channels = BIT(value + 1);
1938
1939 value = GET_NUM_PAENTRY(cccfg);
1940 ecc->num_slots = BIT(value + 4);
1941
1942 value = GET_NUM_EVQUE(cccfg);
1943 ecc->num_tc = value + 1;
1944
1945 dev_dbg(dev, "eDMA3 CC HW configuration (cccfg: 0x%08x):\n", cccfg);
1946 dev_dbg(dev, "num_region: %u\n", ecc->num_region);
1947 dev_dbg(dev, "num_channels: %u\n", ecc->num_channels);
1948 dev_dbg(dev, "num_slots: %u\n", ecc->num_slots);
1949 dev_dbg(dev, "num_tc: %u\n", ecc->num_tc);
1950
1951 /* Nothing need to be done if queue priority is provided */
1952 if (pdata->queue_priority_mapping)
1953 return 0;
1954
1955 /*
1956 * Configure TC/queue priority as follows:
1957 * Q0 - priority 0
1958 * Q1 - priority 1
1959 * Q2 - priority 2
1960 * ...
1961 * The meaning of priority numbers: 0 highest priority, 7 lowest
1962 * priority. So Q0 is the highest priority queue and the last queue has
1963 * the lowest priority.
1964 */
Peter Ujfalusi547c6e22015-10-14 14:42:55 +03001965 queue_priority_map = devm_kcalloc(dev, ecc->num_tc + 1, sizeof(s8),
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001966 GFP_KERNEL);
1967 if (!queue_priority_map)
1968 return -ENOMEM;
1969
1970 for (i = 0; i < ecc->num_tc; i++) {
1971 queue_priority_map[i][0] = i;
1972 queue_priority_map[i][1] = i;
1973 }
1974 queue_priority_map[i][0] = -1;
1975 queue_priority_map[i][1] = -1;
1976
1977 pdata->queue_priority_mapping = queue_priority_map;
1978 /* Default queue has the lowest priority */
1979 pdata->default_queue = i - 1;
1980
1981 return 0;
1982}
1983
1984#if IS_ENABLED(CONFIG_OF)
1985static int edma_xbar_event_map(struct device *dev, struct edma_soc_info *pdata,
1986 size_t sz)
1987{
1988 const char pname[] = "ti,edma-xbar-event-map";
1989 struct resource res;
1990 void __iomem *xbar;
1991 s16 (*xbar_chans)[2];
1992 size_t nelm = sz / sizeof(s16);
1993 u32 shift, offset, mux;
1994 int ret, i;
1995
Peter Ujfalusi547c6e22015-10-14 14:42:55 +03001996 xbar_chans = devm_kcalloc(dev, nelm + 2, sizeof(s16), GFP_KERNEL);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03001997 if (!xbar_chans)
1998 return -ENOMEM;
1999
2000 ret = of_address_to_resource(dev->of_node, 1, &res);
2001 if (ret)
2002 return -ENOMEM;
2003
2004 xbar = devm_ioremap(dev, res.start, resource_size(&res));
2005 if (!xbar)
2006 return -ENOMEM;
2007
2008 ret = of_property_read_u16_array(dev->of_node, pname, (u16 *)xbar_chans,
2009 nelm);
2010 if (ret)
2011 return -EIO;
2012
2013 /* Invalidate last entry for the other user of this mess */
2014 nelm >>= 1;
2015 xbar_chans[nelm][0] = -1;
2016 xbar_chans[nelm][1] = -1;
2017
2018 for (i = 0; i < nelm; i++) {
2019 shift = (xbar_chans[i][1] & 0x03) << 3;
2020 offset = xbar_chans[i][1] & 0xfffffffc;
2021 mux = readl(xbar + offset);
2022 mux &= ~(0xff << shift);
2023 mux |= xbar_chans[i][0] << shift;
2024 writel(mux, (xbar + offset));
2025 }
2026
2027 pdata->xbar_chans = (const s16 (*)[2]) xbar_chans;
2028 return 0;
2029}
2030
2031static int edma_of_parse_dt(struct device *dev, struct edma_soc_info *pdata)
2032{
2033 int ret = 0;
2034 struct property *prop;
2035 size_t sz;
2036 struct edma_rsv_info *rsv_info;
2037
2038 rsv_info = devm_kzalloc(dev, sizeof(struct edma_rsv_info), GFP_KERNEL);
2039 if (!rsv_info)
2040 return -ENOMEM;
2041 pdata->rsv = rsv_info;
2042
2043 prop = of_find_property(dev->of_node, "ti,edma-xbar-event-map", &sz);
2044 if (prop)
2045 ret = edma_xbar_event_map(dev, pdata, sz);
2046
2047 return ret;
2048}
2049
2050static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev)
2051{
2052 struct edma_soc_info *info;
2053 int ret;
2054
2055 info = devm_kzalloc(dev, sizeof(struct edma_soc_info), GFP_KERNEL);
2056 if (!info)
2057 return ERR_PTR(-ENOMEM);
2058
2059 ret = edma_of_parse_dt(dev, info);
2060 if (ret)
2061 return ERR_PTR(ret);
2062
2063 return info;
2064}
2065#else
2066static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev)
2067{
2068 return ERR_PTR(-EINVAL);
2069}
2070#endif
2071
Bill Pemberton463a1f82012-11-19 13:22:55 -05002072static int edma_probe(struct platform_device *pdev)
Matt Porterc2dde5f2012-08-22 21:09:34 -04002073{
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002074 struct edma_soc_info *info = pdev->dev.platform_data;
2075 s8 (*queue_priority_mapping)[2];
2076 int i, off, ln;
2077 const s16 (*rsv_chans)[2];
2078 const s16 (*rsv_slots)[2];
2079 const s16 (*xbar_chans)[2];
2080 int irq;
2081 char *irq_name;
2082 struct resource *mem;
2083 struct device_node *node = pdev->dev.of_node;
2084 struct device *dev = &pdev->dev;
2085 struct edma_cc *ecc;
Matt Porterc2dde5f2012-08-22 21:09:34 -04002086 int ret;
2087
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002088 if (node) {
2089 info = edma_setup_info_from_dt(dev);
2090 if (IS_ERR(info)) {
2091 dev_err(dev, "failed to get DT data\n");
2092 return PTR_ERR(info);
2093 }
2094 }
2095
2096 if (!info)
2097 return -ENODEV;
2098
2099 pm_runtime_enable(dev);
2100 ret = pm_runtime_get_sync(dev);
2101 if (ret < 0) {
2102 dev_err(dev, "pm_runtime_get_sync() failed\n");
2103 return ret;
2104 }
2105
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03002106 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
Russell King94cb0e72013-06-27 13:45:16 +01002107 if (ret)
2108 return ret;
2109
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03002110 ecc = devm_kzalloc(dev, sizeof(*ecc), GFP_KERNEL);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002111 if (!ecc) {
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03002112 dev_err(dev, "Can't allocate controller\n");
Matt Porterc2dde5f2012-08-22 21:09:34 -04002113 return -ENOMEM;
2114 }
2115
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002116 ecc->dev = dev;
2117 ecc->id = pdev->id;
2118 /* When booting with DT the pdev->id is -1 */
2119 if (ecc->id < 0)
2120 ecc->id = 0;
Peter Ujfalusica304fa2015-10-14 14:42:49 +03002121
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002122 mem = platform_get_resource_byname(pdev, IORESOURCE_MEM, "edma3_cc");
2123 if (!mem) {
2124 dev_dbg(dev, "mem resource not found, using index 0\n");
2125 mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2126 if (!mem) {
2127 dev_err(dev, "no mem resource?\n");
2128 return -ENODEV;
2129 }
2130 }
2131 ecc->base = devm_ioremap_resource(dev, mem);
2132 if (IS_ERR(ecc->base))
2133 return PTR_ERR(ecc->base);
Peter Ujfalusib2c843a2015-10-14 14:42:50 +03002134
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002135 platform_set_drvdata(pdev, ecc);
2136
2137 /* Get eDMA3 configuration from IP */
2138 ret = edma_setup_from_hw(dev, info, ecc);
2139 if (ret)
2140 return ret;
2141
Peter Ujfalusicb782052015-10-14 14:42:54 +03002142 /* Allocate memory based on the information we got from the IP */
2143 ecc->slave_chans = devm_kcalloc(dev, ecc->num_channels,
2144 sizeof(*ecc->slave_chans), GFP_KERNEL);
2145 if (!ecc->slave_chans)
2146 return -ENOMEM;
2147
Peter Ujfalusicb782052015-10-14 14:42:54 +03002148 ecc->edma_unused = devm_kcalloc(dev, BITS_TO_LONGS(ecc->num_channels),
2149 sizeof(unsigned long), GFP_KERNEL);
2150 if (!ecc->edma_unused)
2151 return -ENOMEM;
2152
2153 ecc->edma_inuse = devm_kcalloc(dev, BITS_TO_LONGS(ecc->num_slots),
2154 sizeof(unsigned long), GFP_KERNEL);
2155 if (!ecc->edma_inuse)
2156 return -ENOMEM;
2157
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002158 ecc->default_queue = info->default_queue;
2159
2160 for (i = 0; i < ecc->num_slots; i++)
2161 edma_write_slot(ecc, i, &dummy_paramset);
2162
2163 /* Mark all channels as unused */
2164 memset(ecc->edma_unused, 0xff, sizeof(ecc->edma_unused));
2165
2166 if (info->rsv) {
2167 /* Clear the reserved channels in unused list */
2168 rsv_chans = info->rsv->rsv_chans;
2169 if (rsv_chans) {
2170 for (i = 0; rsv_chans[i][0] != -1; i++) {
2171 off = rsv_chans[i][0];
2172 ln = rsv_chans[i][1];
2173 clear_bits(off, ln, ecc->edma_unused);
2174 }
2175 }
2176
2177 /* Set the reserved slots in inuse list */
2178 rsv_slots = info->rsv->rsv_slots;
2179 if (rsv_slots) {
2180 for (i = 0; rsv_slots[i][0] != -1; i++) {
2181 off = rsv_slots[i][0];
2182 ln = rsv_slots[i][1];
2183 set_bits(off, ln, ecc->edma_inuse);
2184 }
2185 }
2186 }
2187
2188 /* Clear the xbar mapped channels in unused list */
2189 xbar_chans = info->xbar_chans;
2190 if (xbar_chans) {
2191 for (i = 0; xbar_chans[i][1] != -1; i++) {
2192 off = xbar_chans[i][1];
2193 clear_bits(off, 1, ecc->edma_unused);
2194 }
2195 }
2196
2197 irq = platform_get_irq_byname(pdev, "edma3_ccint");
2198 if (irq < 0 && node)
2199 irq = irq_of_parse_and_map(node, 0);
2200
2201 if (irq >= 0) {
2202 irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccint",
2203 dev_name(dev));
2204 ret = devm_request_irq(dev, irq, dma_irq_handler, 0, irq_name,
2205 ecc);
2206 if (ret) {
2207 dev_err(dev, "CCINT (%d) failed --> %d\n", irq, ret);
2208 return ret;
2209 }
2210 }
2211
2212 irq = platform_get_irq_byname(pdev, "edma3_ccerrint");
2213 if (irq < 0 && node)
2214 irq = irq_of_parse_and_map(node, 2);
2215
2216 if (irq >= 0) {
2217 irq_name = devm_kasprintf(dev, GFP_KERNEL, "%s_ccerrint",
2218 dev_name(dev));
2219 ret = devm_request_irq(dev, irq, dma_ccerr_handler, 0, irq_name,
2220 ecc);
2221 if (ret) {
2222 dev_err(dev, "CCERRINT (%d) failed --> %d\n", irq, ret);
2223 return ret;
2224 }
2225 }
2226
2227 for (i = 0; i < ecc->num_channels; i++)
2228 edma_map_dmach_to_queue(ecc, i, info->default_queue);
2229
2230 queue_priority_mapping = info->queue_priority_mapping;
2231
2232 /* Event queue priority mapping */
2233 for (i = 0; queue_priority_mapping[i][0] != -1; i++)
2234 edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
2235 queue_priority_mapping[i][1]);
2236
2237 /* Map the channel to param entry if channel mapping logic exist */
2238 if (edma_read(ecc, EDMA_CCCFG) & CHMAP_EXIST)
2239 edma_direct_dmach_to_param_mapping(ecc);
2240
2241 for (i = 0; i < ecc->num_region; i++) {
2242 edma_write_array2(ecc, EDMA_DRAE, i, 0, 0x0);
2243 edma_write_array2(ecc, EDMA_DRAE, i, 1, 0x0);
2244 edma_write_array(ecc, EDMA_QRAE, i, 0x0);
2245 }
2246 ecc->info = info;
2247
2248 ecc->dummy_slot = edma_alloc_slot(ecc, EDMA_SLOT_ANY);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002249 if (ecc->dummy_slot < 0) {
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03002250 dev_err(dev, "Can't allocate PaRAM dummy slot\n");
Peter Ujfalusi04d537d2014-07-31 13:12:37 +03002251 return ecc->dummy_slot;
Matt Porterc2dde5f2012-08-22 21:09:34 -04002252 }
2253
2254 dma_cap_zero(ecc->dma_slave.cap_mask);
2255 dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask);
Peter Ujfalusi232b223d2014-04-14 14:42:00 +03002256 dma_cap_set(DMA_CYCLIC, ecc->dma_slave.cap_mask);
Joel Fernandes8cc3e302014-04-18 21:50:33 -05002257 dma_cap_set(DMA_MEMCPY, ecc->dma_slave.cap_mask);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002258
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03002259 edma_dma_init(ecc, &ecc->dma_slave, dev);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002260
2261 edma_chan_init(ecc, &ecc->dma_slave, ecc->slave_chans);
2262
2263 ret = dma_async_device_register(&ecc->dma_slave);
2264 if (ret)
2265 goto err_reg1;
2266
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002267 if (node)
2268 of_dma_controller_register(node, of_dma_xlate_by_chan_id,
Peter Ujfalusib2c843a2015-10-14 14:42:50 +03002269 &ecc->dma_slave);
Peter Ujfalusidc9b60552015-10-14 14:42:47 +03002270
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03002271 dev_info(dev, "TI EDMA DMA engine driver\n");
Matt Porterc2dde5f2012-08-22 21:09:34 -04002272
2273 return 0;
2274
2275err_reg1:
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002276 edma_free_slot(ecc, ecc->dummy_slot);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002277 return ret;
2278}
2279
Greg Kroah-Hartman4bf27b82012-12-21 15:09:59 -08002280static int edma_remove(struct platform_device *pdev)
Matt Porterc2dde5f2012-08-22 21:09:34 -04002281{
2282 struct device *dev = &pdev->dev;
2283 struct edma_cc *ecc = dev_get_drvdata(dev);
2284
Peter Ujfalusi907f74a2015-10-14 14:42:56 +03002285 if (dev->of_node)
2286 of_dma_controller_free(dev->of_node);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002287 dma_async_device_unregister(&ecc->dma_slave);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002288 edma_free_slot(ecc, ecc->dummy_slot);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002289
2290 return 0;
2291}
2292
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002293#ifdef CONFIG_PM_SLEEP
2294static int edma_pm_resume(struct device *dev)
2295{
2296 struct edma_cc *ecc = dev_get_drvdata(dev);
2297 int i;
2298 s8 (*queue_priority_mapping)[2];
2299
2300 queue_priority_mapping = ecc->info->queue_priority_mapping;
2301
2302 /* Event queue priority mapping */
2303 for (i = 0; queue_priority_mapping[i][0] != -1; i++)
2304 edma_assign_priority_to_queue(ecc, queue_priority_mapping[i][0],
2305 queue_priority_mapping[i][1]);
2306
2307 /* Map the channel to param entry if channel mapping logic */
2308 if (edma_read(ecc, EDMA_CCCFG) & CHMAP_EXIST)
2309 edma_direct_dmach_to_param_mapping(ecc);
2310
2311 for (i = 0; i < ecc->num_channels; i++) {
2312 if (test_bit(i, ecc->edma_inuse)) {
2313 /* ensure access through shadow region 0 */
2314 edma_or_array2(ecc, EDMA_DRAE, 0, i >> 5,
2315 BIT(i & 0x1f));
2316
2317 edma_setup_interrupt(ecc, EDMA_CTLR_CHAN(ecc->id, i),
Peter Ujfalusi79ad2e32015-10-14 14:43:01 +03002318 true);
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002319 }
2320 }
2321
2322 return 0;
2323}
2324#endif
2325
2326static const struct dev_pm_ops edma_pm_ops = {
2327 SET_LATE_SYSTEM_SLEEP_PM_OPS(NULL, edma_pm_resume)
2328};
2329
Matt Porterc2dde5f2012-08-22 21:09:34 -04002330static struct platform_driver edma_driver = {
2331 .probe = edma_probe,
Bill Pembertona7d6e3e2012-11-19 13:20:04 -05002332 .remove = edma_remove,
Matt Porterc2dde5f2012-08-22 21:09:34 -04002333 .driver = {
Peter Ujfalusi2b6b3b72015-10-14 14:42:53 +03002334 .name = "edma",
2335 .pm = &edma_pm_ops,
2336 .of_match_table = edma_of_ids,
Matt Porterc2dde5f2012-08-22 21:09:34 -04002337 },
2338};
2339
2340bool edma_filter_fn(struct dma_chan *chan, void *param)
2341{
2342 if (chan->device->dev->driver == &edma_driver.driver) {
2343 struct edma_chan *echan = to_edma_chan(chan);
2344 unsigned ch_req = *(unsigned *)param;
2345 return ch_req == echan->ch_num;
2346 }
2347 return false;
2348}
2349EXPORT_SYMBOL(edma_filter_fn);
2350
Matt Porterc2dde5f2012-08-22 21:09:34 -04002351static int edma_init(void)
2352{
Arnd Bergmann5305e4d2014-10-24 18:14:01 +02002353 return platform_driver_register(&edma_driver);
Matt Porterc2dde5f2012-08-22 21:09:34 -04002354}
2355subsys_initcall(edma_init);
2356
2357static void __exit edma_exit(void)
2358{
Matt Porterc2dde5f2012-08-22 21:09:34 -04002359 platform_driver_unregister(&edma_driver);
2360}
2361module_exit(edma_exit);
2362
Josh Boyerd71505b2013-09-04 10:32:50 -04002363MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
Matt Porterc2dde5f2012-08-22 21:09:34 -04002364MODULE_DESCRIPTION("TI EDMA DMA engine driver");
2365MODULE_LICENSE("GPL v2");