blob: cd04eb7b182e338994f03f8f1d1f84c28d49fc40 [file] [log] [blame]
Matt Porterc2dde5f2012-08-22 21:09:34 -04001/*
2 * TI EDMA DMA engine driver
3 *
4 * Copyright 2012 Texas Instruments
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation version 2.
9 *
10 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
11 * kind, whether express or implied; without even the implied warranty
12 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 */
15
16#include <linux/dmaengine.h>
17#include <linux/dma-mapping.h>
18#include <linux/err.h>
19#include <linux/init.h>
20#include <linux/interrupt.h>
21#include <linux/list.h>
22#include <linux/module.h>
23#include <linux/platform_device.h>
24#include <linux/slab.h>
25#include <linux/spinlock.h>
26
Matt Porter3ad7a422013-03-06 11:15:31 -050027#include <linux/platform_data/edma.h>
Matt Porterc2dde5f2012-08-22 21:09:34 -040028
29#include "dmaengine.h"
30#include "virt-dma.h"
31
32/*
33 * This will go away when the private EDMA API is folded
34 * into this driver and the platform device(s) are
35 * instantiated in the arch code. We can only get away
36 * with this simplification because DA8XX may not be built
37 * in the same kernel image with other DaVinci parts. This
38 * avoids having to sprinkle dmaengine driver platform devices
39 * and data throughout all the existing board files.
40 */
41#ifdef CONFIG_ARCH_DAVINCI_DA8XX
42#define EDMA_CTLRS 2
43#define EDMA_CHANS 32
44#else
45#define EDMA_CTLRS 1
46#define EDMA_CHANS 64
47#endif /* CONFIG_ARCH_DAVINCI_DA8XX */
48
Joel Fernandes2abd5f12013-09-23 18:05:15 -050049/*
50 * Max of 20 segments per channel to conserve PaRAM slots
51 * Also note that MAX_NR_SG should be atleast the no.of periods
52 * that are required for ASoC, otherwise DMA prep calls will
53 * fail. Today davinci-pcm is the only user of this driver and
54 * requires atleast 17 slots, so we setup the default to 20.
55 */
56#define MAX_NR_SG 20
Matt Porterc2dde5f2012-08-22 21:09:34 -040057#define EDMA_MAX_SLOTS MAX_NR_SG
58#define EDMA_DESCRIPTORS 16
59
60struct edma_desc {
61 struct virt_dma_desc vdesc;
62 struct list_head node;
Joel Fernandes50a9c702013-10-31 16:31:23 -050063 int cyclic;
Matt Porterc2dde5f2012-08-22 21:09:34 -040064 int absync;
65 int pset_nr;
Joel Fernandes53407062013-09-03 10:02:46 -050066 int processed;
Matt Porterc2dde5f2012-08-22 21:09:34 -040067 struct edmacc_param pset[0];
68};
69
70struct edma_cc;
71
72struct edma_chan {
73 struct virt_dma_chan vchan;
74 struct list_head node;
75 struct edma_desc *edesc;
76 struct edma_cc *ecc;
77 int ch_num;
78 bool alloced;
79 int slot[EDMA_MAX_SLOTS];
Joel Fernandesc5f47992013-08-29 18:05:43 -050080 int missed;
Matt Porter661f7cb2013-01-10 13:41:04 -050081 struct dma_slave_config cfg;
Matt Porterc2dde5f2012-08-22 21:09:34 -040082};
83
84struct edma_cc {
85 int ctlr;
86 struct dma_device dma_slave;
87 struct edma_chan slave_chans[EDMA_CHANS];
88 int num_slave_chans;
89 int dummy_slot;
90};
91
92static inline struct edma_cc *to_edma_cc(struct dma_device *d)
93{
94 return container_of(d, struct edma_cc, dma_slave);
95}
96
97static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
98{
99 return container_of(c, struct edma_chan, vchan.chan);
100}
101
102static inline struct edma_desc
103*to_edma_desc(struct dma_async_tx_descriptor *tx)
104{
105 return container_of(tx, struct edma_desc, vdesc.tx);
106}
107
108static void edma_desc_free(struct virt_dma_desc *vdesc)
109{
110 kfree(container_of(vdesc, struct edma_desc, vdesc));
111}
112
113/* Dispatch a queued descriptor to the controller (caller holds lock) */
114static void edma_execute(struct edma_chan *echan)
115{
Joel Fernandes53407062013-09-03 10:02:46 -0500116 struct virt_dma_desc *vdesc;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400117 struct edma_desc *edesc;
Joel Fernandes53407062013-09-03 10:02:46 -0500118 struct device *dev = echan->vchan.chan.device->dev;
119 int i, j, left, nslots;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400120
Joel Fernandes53407062013-09-03 10:02:46 -0500121 /* If either we processed all psets or we're still not started */
122 if (!echan->edesc ||
123 echan->edesc->pset_nr == echan->edesc->processed) {
124 /* Get next vdesc */
125 vdesc = vchan_next_desc(&echan->vchan);
126 if (!vdesc) {
127 echan->edesc = NULL;
128 return;
129 }
130 list_del(&vdesc->node);
131 echan->edesc = to_edma_desc(&vdesc->tx);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400132 }
133
Joel Fernandes53407062013-09-03 10:02:46 -0500134 edesc = echan->edesc;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400135
Joel Fernandes53407062013-09-03 10:02:46 -0500136 /* Find out how many left */
137 left = edesc->pset_nr - edesc->processed;
138 nslots = min(MAX_NR_SG, left);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400139
140 /* Write descriptor PaRAM set(s) */
Joel Fernandes53407062013-09-03 10:02:46 -0500141 for (i = 0; i < nslots; i++) {
142 j = i + edesc->processed;
143 edma_write_slot(echan->slot[i], &edesc->pset[j]);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400144 dev_dbg(echan->vchan.chan.device->dev,
145 "\n pset[%d]:\n"
146 " chnum\t%d\n"
147 " slot\t%d\n"
148 " opt\t%08x\n"
149 " src\t%08x\n"
150 " dst\t%08x\n"
151 " abcnt\t%08x\n"
152 " ccnt\t%08x\n"
153 " bidx\t%08x\n"
154 " cidx\t%08x\n"
155 " lkrld\t%08x\n",
Joel Fernandes53407062013-09-03 10:02:46 -0500156 j, echan->ch_num, echan->slot[i],
157 edesc->pset[j].opt,
158 edesc->pset[j].src,
159 edesc->pset[j].dst,
160 edesc->pset[j].a_b_cnt,
161 edesc->pset[j].ccnt,
162 edesc->pset[j].src_dst_bidx,
163 edesc->pset[j].src_dst_cidx,
164 edesc->pset[j].link_bcntrld);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400165 /* Link to the previous slot if not the last set */
Joel Fernandes53407062013-09-03 10:02:46 -0500166 if (i != (nslots - 1))
Matt Porterc2dde5f2012-08-22 21:09:34 -0400167 edma_link(echan->slot[i], echan->slot[i+1]);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400168 }
169
Joel Fernandes53407062013-09-03 10:02:46 -0500170 edesc->processed += nslots;
171
Joel Fernandesb267b3b2013-08-29 18:05:44 -0500172 /*
173 * If this is either the last set in a set of SG-list transactions
174 * then setup a link to the dummy slot, this results in all future
175 * events being absorbed and that's OK because we're done
176 */
Joel Fernandes50a9c702013-10-31 16:31:23 -0500177 if (edesc->processed == edesc->pset_nr) {
178 if (edesc->cyclic)
179 edma_link(echan->slot[nslots-1], echan->slot[1]);
180 else
181 edma_link(echan->slot[nslots-1],
182 echan->ecc->dummy_slot);
183 }
Joel Fernandesb267b3b2013-08-29 18:05:44 -0500184
Joel Fernandes53407062013-09-03 10:02:46 -0500185 edma_resume(echan->ch_num);
186
187 if (edesc->processed <= MAX_NR_SG) {
188 dev_dbg(dev, "first transfer starting %d\n", echan->ch_num);
189 edma_start(echan->ch_num);
190 }
Joel Fernandesc5f47992013-08-29 18:05:43 -0500191
192 /*
193 * This happens due to setup times between intermediate transfers
194 * in long SG lists which have to be broken up into transfers of
195 * MAX_NR_SG
196 */
197 if (echan->missed) {
198 dev_dbg(dev, "missed event in execute detected\n");
199 edma_clean_channel(echan->ch_num);
200 edma_stop(echan->ch_num);
201 edma_start(echan->ch_num);
202 edma_trigger_channel(echan->ch_num);
203 echan->missed = 0;
204 }
Matt Porterc2dde5f2012-08-22 21:09:34 -0400205}
206
207static int edma_terminate_all(struct edma_chan *echan)
208{
209 unsigned long flags;
210 LIST_HEAD(head);
211
212 spin_lock_irqsave(&echan->vchan.lock, flags);
213
214 /*
215 * Stop DMA activity: we assume the callback will not be called
216 * after edma_dma() returns (even if it does, it will see
217 * echan->edesc is NULL and exit.)
218 */
219 if (echan->edesc) {
220 echan->edesc = NULL;
221 edma_stop(echan->ch_num);
222 }
223
224 vchan_get_all_descriptors(&echan->vchan, &head);
225 spin_unlock_irqrestore(&echan->vchan.lock, flags);
226 vchan_dma_desc_free_list(&echan->vchan, &head);
227
228 return 0;
229}
230
Matt Porterc2dde5f2012-08-22 21:09:34 -0400231static int edma_slave_config(struct edma_chan *echan,
Matt Porter661f7cb2013-01-10 13:41:04 -0500232 struct dma_slave_config *cfg)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400233{
Matt Porter661f7cb2013-01-10 13:41:04 -0500234 if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
235 cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400236 return -EINVAL;
237
Matt Porter661f7cb2013-01-10 13:41:04 -0500238 memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
Matt Porterc2dde5f2012-08-22 21:09:34 -0400239
240 return 0;
241}
242
243static int edma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
244 unsigned long arg)
245{
246 int ret = 0;
247 struct dma_slave_config *config;
248 struct edma_chan *echan = to_edma_chan(chan);
249
250 switch (cmd) {
251 case DMA_TERMINATE_ALL:
252 edma_terminate_all(echan);
253 break;
254 case DMA_SLAVE_CONFIG:
255 config = (struct dma_slave_config *)arg;
256 ret = edma_slave_config(echan, config);
257 break;
258 default:
259 ret = -ENOSYS;
260 }
261
262 return ret;
263}
264
Joel Fernandesfd009032013-09-23 18:05:13 -0500265/*
266 * A PaRAM set configuration abstraction used by other modes
267 * @chan: Channel who's PaRAM set we're configuring
268 * @pset: PaRAM set to initialize and setup.
269 * @src_addr: Source address of the DMA
270 * @dst_addr: Destination address of the DMA
271 * @burst: In units of dev_width, how much to send
272 * @dev_width: How much is the dev_width
273 * @dma_length: Total length of the DMA transfer
274 * @direction: Direction of the transfer
275 */
276static int edma_config_pset(struct dma_chan *chan, struct edmacc_param *pset,
277 dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
278 enum dma_slave_buswidth dev_width, unsigned int dma_length,
279 enum dma_transfer_direction direction)
280{
281 struct edma_chan *echan = to_edma_chan(chan);
282 struct device *dev = chan->device->dev;
283 int acnt, bcnt, ccnt, cidx;
284 int src_bidx, dst_bidx, src_cidx, dst_cidx;
285 int absync;
286
287 acnt = dev_width;
288 /*
289 * If the maxburst is equal to the fifo width, use
290 * A-synced transfers. This allows for large contiguous
291 * buffer transfers using only one PaRAM set.
292 */
293 if (burst == 1) {
294 /*
295 * For the A-sync case, bcnt and ccnt are the remainder
296 * and quotient respectively of the division of:
297 * (dma_length / acnt) by (SZ_64K -1). This is so
298 * that in case bcnt over flows, we have ccnt to use.
299 * Note: In A-sync tranfer only, bcntrld is used, but it
300 * only applies for sg_dma_len(sg) >= SZ_64K.
301 * In this case, the best way adopted is- bccnt for the
302 * first frame will be the remainder below. Then for
303 * every successive frame, bcnt will be SZ_64K-1. This
304 * is assured as bcntrld = 0xffff in end of function.
305 */
306 absync = false;
307 ccnt = dma_length / acnt / (SZ_64K - 1);
308 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
309 /*
310 * If bcnt is non-zero, we have a remainder and hence an
311 * extra frame to transfer, so increment ccnt.
312 */
313 if (bcnt)
314 ccnt++;
315 else
316 bcnt = SZ_64K - 1;
317 cidx = acnt;
318 } else {
319 /*
320 * If maxburst is greater than the fifo address_width,
321 * use AB-synced transfers where A count is the fifo
322 * address_width and B count is the maxburst. In this
323 * case, we are limited to transfers of C count frames
324 * of (address_width * maxburst) where C count is limited
325 * to SZ_64K-1. This places an upper bound on the length
326 * of an SG segment that can be handled.
327 */
328 absync = true;
329 bcnt = burst;
330 ccnt = dma_length / (acnt * bcnt);
331 if (ccnt > (SZ_64K - 1)) {
332 dev_err(dev, "Exceeded max SG segment size\n");
333 return -EINVAL;
334 }
335 cidx = acnt * bcnt;
336 }
337
338 if (direction == DMA_MEM_TO_DEV) {
339 src_bidx = acnt;
340 src_cidx = cidx;
341 dst_bidx = 0;
342 dst_cidx = 0;
343 } else if (direction == DMA_DEV_TO_MEM) {
344 src_bidx = 0;
345 src_cidx = 0;
346 dst_bidx = acnt;
347 dst_cidx = cidx;
348 } else {
349 dev_err(dev, "%s: direction not implemented yet\n", __func__);
350 return -EINVAL;
351 }
352
353 pset->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
354 /* Configure A or AB synchronized transfers */
355 if (absync)
356 pset->opt |= SYNCDIM;
357
358 pset->src = src_addr;
359 pset->dst = dst_addr;
360
361 pset->src_dst_bidx = (dst_bidx << 16) | src_bidx;
362 pset->src_dst_cidx = (dst_cidx << 16) | src_cidx;
363
364 pset->a_b_cnt = bcnt << 16 | acnt;
365 pset->ccnt = ccnt;
366 /*
367 * Only time when (bcntrld) auto reload is required is for
368 * A-sync case, and in this case, a requirement of reload value
369 * of SZ_64K-1 only is assured. 'link' is initially set to NULL
370 * and then later will be populated by edma_execute.
371 */
372 pset->link_bcntrld = 0xffffffff;
373 return absync;
374}
375
Matt Porterc2dde5f2012-08-22 21:09:34 -0400376static struct dma_async_tx_descriptor *edma_prep_slave_sg(
377 struct dma_chan *chan, struct scatterlist *sgl,
378 unsigned int sg_len, enum dma_transfer_direction direction,
379 unsigned long tx_flags, void *context)
380{
381 struct edma_chan *echan = to_edma_chan(chan);
382 struct device *dev = chan->device->dev;
383 struct edma_desc *edesc;
Joel Fernandesfd009032013-09-23 18:05:13 -0500384 dma_addr_t src_addr = 0, dst_addr = 0;
Matt Porter661f7cb2013-01-10 13:41:04 -0500385 enum dma_slave_buswidth dev_width;
386 u32 burst;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400387 struct scatterlist *sg;
Joel Fernandesfd009032013-09-23 18:05:13 -0500388 int i, nslots, ret;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400389
390 if (unlikely(!echan || !sgl || !sg_len))
391 return NULL;
392
Matt Porter661f7cb2013-01-10 13:41:04 -0500393 if (direction == DMA_DEV_TO_MEM) {
Joel Fernandesfd009032013-09-23 18:05:13 -0500394 src_addr = echan->cfg.src_addr;
Matt Porter661f7cb2013-01-10 13:41:04 -0500395 dev_width = echan->cfg.src_addr_width;
396 burst = echan->cfg.src_maxburst;
397 } else if (direction == DMA_MEM_TO_DEV) {
Joel Fernandesfd009032013-09-23 18:05:13 -0500398 dst_addr = echan->cfg.dst_addr;
Matt Porter661f7cb2013-01-10 13:41:04 -0500399 dev_width = echan->cfg.dst_addr_width;
400 burst = echan->cfg.dst_maxburst;
401 } else {
402 dev_err(dev, "%s: bad direction?\n", __func__);
403 return NULL;
404 }
405
406 if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
Matt Porterc2dde5f2012-08-22 21:09:34 -0400407 dev_err(dev, "Undefined slave buswidth\n");
408 return NULL;
409 }
410
Matt Porterc2dde5f2012-08-22 21:09:34 -0400411 edesc = kzalloc(sizeof(*edesc) + sg_len *
412 sizeof(edesc->pset[0]), GFP_ATOMIC);
413 if (!edesc) {
414 dev_dbg(dev, "Failed to allocate a descriptor\n");
415 return NULL;
416 }
417
418 edesc->pset_nr = sg_len;
419
Joel Fernandes6fbe24d2013-08-29 18:05:40 -0500420 /* Allocate a PaRAM slot, if needed */
421 nslots = min_t(unsigned, MAX_NR_SG, sg_len);
422
423 for (i = 0; i < nslots; i++) {
Matt Porterc2dde5f2012-08-22 21:09:34 -0400424 if (echan->slot[i] < 0) {
425 echan->slot[i] =
426 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
427 EDMA_SLOT_ANY);
428 if (echan->slot[i] < 0) {
Valentin Ilie4b6271a2013-10-24 16:14:22 +0300429 kfree(edesc);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400430 dev_err(dev, "Failed to allocate slot\n");
431 return NULL;
432 }
433 }
Joel Fernandes6fbe24d2013-08-29 18:05:40 -0500434 }
435
436 /* Configure PaRAM sets for each SG */
437 for_each_sg(sgl, sg, sg_len, i) {
Joel Fernandesfd009032013-09-23 18:05:13 -0500438 /* Get address for each SG */
439 if (direction == DMA_DEV_TO_MEM)
440 dst_addr = sg_dma_address(sg);
441 else
442 src_addr = sg_dma_address(sg);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400443
Joel Fernandesfd009032013-09-23 18:05:13 -0500444 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
445 dst_addr, burst, dev_width,
446 sg_dma_len(sg), direction);
Vinod Koulb967aec2013-10-30 13:07:18 +0530447 if (ret < 0) {
448 kfree(edesc);
Joel Fernandesfd009032013-09-23 18:05:13 -0500449 return NULL;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400450 }
451
Joel Fernandesfd009032013-09-23 18:05:13 -0500452 edesc->absync = ret;
Joel Fernandes6fbe24d2013-08-29 18:05:40 -0500453
454 /* If this is the last in a current SG set of transactions,
455 enable interrupts so that next set is processed */
456 if (!((i+1) % MAX_NR_SG))
457 edesc->pset[i].opt |= TCINTEN;
458
Matt Porterc2dde5f2012-08-22 21:09:34 -0400459 /* If this is the last set, enable completion interrupt flag */
460 if (i == sg_len - 1)
461 edesc->pset[i].opt |= TCINTEN;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400462 }
Matt Porterc2dde5f2012-08-22 21:09:34 -0400463
Matt Porterc2dde5f2012-08-22 21:09:34 -0400464 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
465}
Matt Porterc2dde5f2012-08-22 21:09:34 -0400466
Joel Fernandes50a9c702013-10-31 16:31:23 -0500467static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
468 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
469 size_t period_len, enum dma_transfer_direction direction,
470 unsigned long tx_flags, void *context)
471{
472 struct edma_chan *echan = to_edma_chan(chan);
473 struct device *dev = chan->device->dev;
474 struct edma_desc *edesc;
475 dma_addr_t src_addr, dst_addr;
476 enum dma_slave_buswidth dev_width;
477 u32 burst;
478 int i, ret, nslots;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400479
Joel Fernandes50a9c702013-10-31 16:31:23 -0500480 if (unlikely(!echan || !buf_len || !period_len))
481 return NULL;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400482
Joel Fernandes50a9c702013-10-31 16:31:23 -0500483 if (direction == DMA_DEV_TO_MEM) {
484 src_addr = echan->cfg.src_addr;
485 dst_addr = buf_addr;
486 dev_width = echan->cfg.src_addr_width;
487 burst = echan->cfg.src_maxburst;
488 } else if (direction == DMA_MEM_TO_DEV) {
489 src_addr = buf_addr;
490 dst_addr = echan->cfg.dst_addr;
491 dev_width = echan->cfg.dst_addr_width;
492 burst = echan->cfg.dst_maxburst;
493 } else {
494 dev_err(dev, "%s: bad direction?\n", __func__);
495 return NULL;
496 }
497
498 if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
499 dev_err(dev, "Undefined slave buswidth\n");
500 return NULL;
501 }
502
503 if (unlikely(buf_len % period_len)) {
504 dev_err(dev, "Period should be multiple of Buffer length\n");
505 return NULL;
506 }
507
508 nslots = (buf_len / period_len) + 1;
509
510 /*
511 * Cyclic DMA users such as audio cannot tolerate delays introduced
512 * by cases where the number of periods is more than the maximum
513 * number of SGs the EDMA driver can handle at a time. For DMA types
514 * such as Slave SGs, such delays are tolerable and synchronized,
515 * but the synchronization is difficult to achieve with Cyclic and
516 * cannot be guaranteed, so we error out early.
517 */
518 if (nslots > MAX_NR_SG)
519 return NULL;
520
521 edesc = kzalloc(sizeof(*edesc) + nslots *
522 sizeof(edesc->pset[0]), GFP_ATOMIC);
523 if (!edesc) {
524 dev_dbg(dev, "Failed to allocate a descriptor\n");
525 return NULL;
526 }
527
528 edesc->cyclic = 1;
529 edesc->pset_nr = nslots;
530
531 dev_dbg(dev, "%s: nslots=%d\n", __func__, nslots);
532 dev_dbg(dev, "%s: period_len=%d\n", __func__, period_len);
533 dev_dbg(dev, "%s: buf_len=%d\n", __func__, buf_len);
534
535 for (i = 0; i < nslots; i++) {
536 /* Allocate a PaRAM slot, if needed */
537 if (echan->slot[i] < 0) {
538 echan->slot[i] =
539 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
540 EDMA_SLOT_ANY);
541 if (echan->slot[i] < 0) {
Christian Engelmayere3ddc972013-12-30 20:48:39 +0100542 kfree(edesc);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500543 dev_err(dev, "Failed to allocate slot\n");
544 return NULL;
545 }
546 }
547
548 if (i == nslots - 1) {
549 memcpy(&edesc->pset[i], &edesc->pset[0],
550 sizeof(edesc->pset[0]));
551 break;
552 }
553
554 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
555 dst_addr, burst, dev_width, period_len,
556 direction);
Christian Engelmayere3ddc972013-12-30 20:48:39 +0100557 if (ret < 0) {
558 kfree(edesc);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500559 return NULL;
Christian Engelmayere3ddc972013-12-30 20:48:39 +0100560 }
Joel Fernandes50a9c702013-10-31 16:31:23 -0500561
562 if (direction == DMA_DEV_TO_MEM)
563 dst_addr += period_len;
564 else
565 src_addr += period_len;
566
567 dev_dbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
568 dev_dbg(dev,
569 "\n pset[%d]:\n"
570 " chnum\t%d\n"
571 " slot\t%d\n"
572 " opt\t%08x\n"
573 " src\t%08x\n"
574 " dst\t%08x\n"
575 " abcnt\t%08x\n"
576 " ccnt\t%08x\n"
577 " bidx\t%08x\n"
578 " cidx\t%08x\n"
579 " lkrld\t%08x\n",
580 i, echan->ch_num, echan->slot[i],
581 edesc->pset[i].opt,
582 edesc->pset[i].src,
583 edesc->pset[i].dst,
584 edesc->pset[i].a_b_cnt,
585 edesc->pset[i].ccnt,
586 edesc->pset[i].src_dst_bidx,
587 edesc->pset[i].src_dst_cidx,
588 edesc->pset[i].link_bcntrld);
589
590 edesc->absync = ret;
591
592 /*
593 * Enable interrupts for every period because callback
594 * has to be called for every period.
595 */
596 edesc->pset[i].opt |= TCINTEN;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400597 }
598
599 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
600}
601
602static void edma_callback(unsigned ch_num, u16 ch_status, void *data)
603{
604 struct edma_chan *echan = data;
605 struct device *dev = echan->vchan.chan.device->dev;
606 struct edma_desc *edesc;
607 unsigned long flags;
Joel Fernandesc5f47992013-08-29 18:05:43 -0500608 struct edmacc_param p;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400609
Joel Fernandes50a9c702013-10-31 16:31:23 -0500610 edesc = echan->edesc;
611
612 /* Pause the channel for non-cyclic */
613 if (!edesc || (edesc && !edesc->cyclic))
614 edma_pause(echan->ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400615
616 switch (ch_status) {
Vinod Kouldb60d8d2013-10-30 18:22:30 +0530617 case EDMA_DMA_COMPLETE:
Matt Porterc2dde5f2012-08-22 21:09:34 -0400618 spin_lock_irqsave(&echan->vchan.lock, flags);
619
Matt Porterc2dde5f2012-08-22 21:09:34 -0400620 if (edesc) {
Joel Fernandes50a9c702013-10-31 16:31:23 -0500621 if (edesc->cyclic) {
622 vchan_cyclic_callback(&edesc->vdesc);
623 } else if (edesc->processed == edesc->pset_nr) {
Joel Fernandes53407062013-09-03 10:02:46 -0500624 dev_dbg(dev, "Transfer complete, stopping channel %d\n", ch_num);
625 edma_stop(echan->ch_num);
626 vchan_cookie_complete(&edesc->vdesc);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500627 edma_execute(echan);
Joel Fernandes53407062013-09-03 10:02:46 -0500628 } else {
629 dev_dbg(dev, "Intermediate transfer complete on channel %d\n", ch_num);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500630 edma_execute(echan);
Joel Fernandes53407062013-09-03 10:02:46 -0500631 }
Matt Porterc2dde5f2012-08-22 21:09:34 -0400632 }
633
634 spin_unlock_irqrestore(&echan->vchan.lock, flags);
635
636 break;
Vinod Kouldb60d8d2013-10-30 18:22:30 +0530637 case EDMA_DMA_CC_ERROR:
Joel Fernandesc5f47992013-08-29 18:05:43 -0500638 spin_lock_irqsave(&echan->vchan.lock, flags);
639
640 edma_read_slot(EDMA_CHAN_SLOT(echan->slot[0]), &p);
641
642 /*
643 * Issue later based on missed flag which will be sure
644 * to happen as:
645 * (1) we finished transmitting an intermediate slot and
646 * edma_execute is coming up.
647 * (2) or we finished current transfer and issue will
648 * call edma_execute.
649 *
650 * Important note: issuing can be dangerous here and
651 * lead to some nasty recursion when we are in a NULL
652 * slot. So we avoid doing so and set the missed flag.
653 */
654 if (p.a_b_cnt == 0 && p.ccnt == 0) {
655 dev_dbg(dev, "Error occurred, looks like slot is null, just setting miss\n");
656 echan->missed = 1;
657 } else {
658 /*
659 * The slot is already programmed but the event got
660 * missed, so its safe to issue it here.
661 */
662 dev_dbg(dev, "Error occurred but slot is non-null, TRIGGERING\n");
663 edma_clean_channel(echan->ch_num);
664 edma_stop(echan->ch_num);
665 edma_start(echan->ch_num);
666 edma_trigger_channel(echan->ch_num);
667 }
668
669 spin_unlock_irqrestore(&echan->vchan.lock, flags);
670
Matt Porterc2dde5f2012-08-22 21:09:34 -0400671 break;
672 default:
673 break;
674 }
675}
676
677/* Alloc channel resources */
678static int edma_alloc_chan_resources(struct dma_chan *chan)
679{
680 struct edma_chan *echan = to_edma_chan(chan);
681 struct device *dev = chan->device->dev;
682 int ret;
683 int a_ch_num;
684 LIST_HEAD(descs);
685
686 a_ch_num = edma_alloc_channel(echan->ch_num, edma_callback,
687 chan, EVENTQ_DEFAULT);
688
689 if (a_ch_num < 0) {
690 ret = -ENODEV;
691 goto err_no_chan;
692 }
693
694 if (a_ch_num != echan->ch_num) {
695 dev_err(dev, "failed to allocate requested channel %u:%u\n",
696 EDMA_CTLR(echan->ch_num),
697 EDMA_CHAN_SLOT(echan->ch_num));
698 ret = -ENODEV;
699 goto err_wrong_chan;
700 }
701
702 echan->alloced = true;
703 echan->slot[0] = echan->ch_num;
704
Ezequiel Garcia0e772c62013-12-13 11:06:18 -0300705 dev_dbg(dev, "allocated channel for %u:%u\n",
706 EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num));
Matt Porterc2dde5f2012-08-22 21:09:34 -0400707
708 return 0;
709
710err_wrong_chan:
711 edma_free_channel(a_ch_num);
712err_no_chan:
713 return ret;
714}
715
716/* Free channel resources */
717static void edma_free_chan_resources(struct dma_chan *chan)
718{
719 struct edma_chan *echan = to_edma_chan(chan);
720 struct device *dev = chan->device->dev;
721 int i;
722
723 /* Terminate transfers */
724 edma_stop(echan->ch_num);
725
726 vchan_free_chan_resources(&echan->vchan);
727
728 /* Free EDMA PaRAM slots */
729 for (i = 1; i < EDMA_MAX_SLOTS; i++) {
730 if (echan->slot[i] >= 0) {
731 edma_free_slot(echan->slot[i]);
732 echan->slot[i] = -1;
733 }
734 }
735
736 /* Free EDMA channel */
737 if (echan->alloced) {
738 edma_free_channel(echan->ch_num);
739 echan->alloced = false;
740 }
741
Ezequiel Garcia0e772c62013-12-13 11:06:18 -0300742 dev_dbg(dev, "freeing channel for %u\n", echan->ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400743}
744
745/* Send pending descriptor to hardware */
746static void edma_issue_pending(struct dma_chan *chan)
747{
748 struct edma_chan *echan = to_edma_chan(chan);
749 unsigned long flags;
750
751 spin_lock_irqsave(&echan->vchan.lock, flags);
752 if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
753 edma_execute(echan);
754 spin_unlock_irqrestore(&echan->vchan.lock, flags);
755}
756
757static size_t edma_desc_size(struct edma_desc *edesc)
758{
759 int i;
760 size_t size;
761
762 if (edesc->absync)
763 for (size = i = 0; i < edesc->pset_nr; i++)
764 size += (edesc->pset[i].a_b_cnt & 0xffff) *
765 (edesc->pset[i].a_b_cnt >> 16) *
766 edesc->pset[i].ccnt;
767 else
768 size = (edesc->pset[0].a_b_cnt & 0xffff) *
769 (edesc->pset[0].a_b_cnt >> 16) +
770 (edesc->pset[0].a_b_cnt & 0xffff) *
771 (SZ_64K - 1) * edesc->pset[0].ccnt;
772
773 return size;
774}
775
776/* Check request completion status */
777static enum dma_status edma_tx_status(struct dma_chan *chan,
778 dma_cookie_t cookie,
779 struct dma_tx_state *txstate)
780{
781 struct edma_chan *echan = to_edma_chan(chan);
782 struct virt_dma_desc *vdesc;
783 enum dma_status ret;
784 unsigned long flags;
785
786 ret = dma_cookie_status(chan, cookie, txstate);
Vinod Koul9d386ec2013-10-16 13:42:15 +0530787 if (ret == DMA_COMPLETE || !txstate)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400788 return ret;
789
790 spin_lock_irqsave(&echan->vchan.lock, flags);
791 vdesc = vchan_find_desc(&echan->vchan, cookie);
792 if (vdesc) {
793 txstate->residue = edma_desc_size(to_edma_desc(&vdesc->tx));
794 } else if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie) {
795 struct edma_desc *edesc = echan->edesc;
796 txstate->residue = edma_desc_size(edesc);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400797 }
798 spin_unlock_irqrestore(&echan->vchan.lock, flags);
799
800 return ret;
801}
802
803static void __init edma_chan_init(struct edma_cc *ecc,
804 struct dma_device *dma,
805 struct edma_chan *echans)
806{
807 int i, j;
808
809 for (i = 0; i < EDMA_CHANS; i++) {
810 struct edma_chan *echan = &echans[i];
811 echan->ch_num = EDMA_CTLR_CHAN(ecc->ctlr, i);
812 echan->ecc = ecc;
813 echan->vchan.desc_free = edma_desc_free;
814
815 vchan_init(&echan->vchan, dma);
816
817 INIT_LIST_HEAD(&echan->node);
818 for (j = 0; j < EDMA_MAX_SLOTS; j++)
819 echan->slot[j] = -1;
820 }
821}
822
823static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
824 struct device *dev)
825{
826 dma->device_prep_slave_sg = edma_prep_slave_sg;
Joel Fernandes50a9c702013-10-31 16:31:23 -0500827 dma->device_prep_dma_cyclic = edma_prep_dma_cyclic;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400828 dma->device_alloc_chan_resources = edma_alloc_chan_resources;
829 dma->device_free_chan_resources = edma_free_chan_resources;
830 dma->device_issue_pending = edma_issue_pending;
831 dma->device_tx_status = edma_tx_status;
832 dma->device_control = edma_control;
833 dma->dev = dev;
834
835 INIT_LIST_HEAD(&dma->channels);
836}
837
Bill Pemberton463a1f82012-11-19 13:22:55 -0500838static int edma_probe(struct platform_device *pdev)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400839{
840 struct edma_cc *ecc;
841 int ret;
842
Russell King94cb0e72013-06-27 13:45:16 +0100843 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
844 if (ret)
845 return ret;
846
Matt Porterc2dde5f2012-08-22 21:09:34 -0400847 ecc = devm_kzalloc(&pdev->dev, sizeof(*ecc), GFP_KERNEL);
848 if (!ecc) {
849 dev_err(&pdev->dev, "Can't allocate controller\n");
850 return -ENOMEM;
851 }
852
853 ecc->ctlr = pdev->id;
854 ecc->dummy_slot = edma_alloc_slot(ecc->ctlr, EDMA_SLOT_ANY);
855 if (ecc->dummy_slot < 0) {
856 dev_err(&pdev->dev, "Can't allocate PaRAM dummy slot\n");
857 return -EIO;
858 }
859
860 dma_cap_zero(ecc->dma_slave.cap_mask);
861 dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask);
862
863 edma_dma_init(ecc, &ecc->dma_slave, &pdev->dev);
864
865 edma_chan_init(ecc, &ecc->dma_slave, ecc->slave_chans);
866
867 ret = dma_async_device_register(&ecc->dma_slave);
868 if (ret)
869 goto err_reg1;
870
871 platform_set_drvdata(pdev, ecc);
872
873 dev_info(&pdev->dev, "TI EDMA DMA engine driver\n");
874
875 return 0;
876
877err_reg1:
878 edma_free_slot(ecc->dummy_slot);
879 return ret;
880}
881
Greg Kroah-Hartman4bf27b82012-12-21 15:09:59 -0800882static int edma_remove(struct platform_device *pdev)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400883{
884 struct device *dev = &pdev->dev;
885 struct edma_cc *ecc = dev_get_drvdata(dev);
886
887 dma_async_device_unregister(&ecc->dma_slave);
888 edma_free_slot(ecc->dummy_slot);
889
890 return 0;
891}
892
893static struct platform_driver edma_driver = {
894 .probe = edma_probe,
Bill Pembertona7d6e3e2012-11-19 13:20:04 -0500895 .remove = edma_remove,
Matt Porterc2dde5f2012-08-22 21:09:34 -0400896 .driver = {
897 .name = "edma-dma-engine",
898 .owner = THIS_MODULE,
899 },
900};
901
902bool edma_filter_fn(struct dma_chan *chan, void *param)
903{
904 if (chan->device->dev->driver == &edma_driver.driver) {
905 struct edma_chan *echan = to_edma_chan(chan);
906 unsigned ch_req = *(unsigned *)param;
907 return ch_req == echan->ch_num;
908 }
909 return false;
910}
911EXPORT_SYMBOL(edma_filter_fn);
912
913static struct platform_device *pdev0, *pdev1;
914
915static const struct platform_device_info edma_dev_info0 = {
916 .name = "edma-dma-engine",
917 .id = 0,
Russell King94cb0e72013-06-27 13:45:16 +0100918 .dma_mask = DMA_BIT_MASK(32),
Matt Porterc2dde5f2012-08-22 21:09:34 -0400919};
920
921static const struct platform_device_info edma_dev_info1 = {
922 .name = "edma-dma-engine",
923 .id = 1,
Russell King94cb0e72013-06-27 13:45:16 +0100924 .dma_mask = DMA_BIT_MASK(32),
Matt Porterc2dde5f2012-08-22 21:09:34 -0400925};
926
927static int edma_init(void)
928{
929 int ret = platform_driver_register(&edma_driver);
930
931 if (ret == 0) {
932 pdev0 = platform_device_register_full(&edma_dev_info0);
933 if (IS_ERR(pdev0)) {
934 platform_driver_unregister(&edma_driver);
935 ret = PTR_ERR(pdev0);
936 goto out;
937 }
938 }
939
940 if (EDMA_CTLRS == 2) {
941 pdev1 = platform_device_register_full(&edma_dev_info1);
942 if (IS_ERR(pdev1)) {
943 platform_driver_unregister(&edma_driver);
944 platform_device_unregister(pdev0);
945 ret = PTR_ERR(pdev1);
946 }
Matt Porterc2dde5f2012-08-22 21:09:34 -0400947 }
948
949out:
950 return ret;
951}
952subsys_initcall(edma_init);
953
954static void __exit edma_exit(void)
955{
956 platform_device_unregister(pdev0);
957 if (pdev1)
958 platform_device_unregister(pdev1);
959 platform_driver_unregister(&edma_driver);
960}
961module_exit(edma_exit);
962
Josh Boyerd71505b2013-09-04 10:32:50 -0400963MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
Matt Porterc2dde5f2012-08-22 21:09:34 -0400964MODULE_DESCRIPTION("TI EDMA DMA engine driver");
965MODULE_LICENSE("GPL v2");