blob: bf09db7ca9ee9c691bcb59888575d2679b4ddd3e [file] [log] [blame]
Matt Porterc2dde5f2012-08-22 21:09:34 -04001/*
2 * TI EDMA DMA engine driver
3 *
4 * Copyright 2012 Texas Instruments
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation version 2.
9 *
10 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
11 * kind, whether express or implied; without even the implied warranty
12 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 */
15
16#include <linux/dmaengine.h>
17#include <linux/dma-mapping.h>
Lad, Prabhakarb7a4fd52015-02-04 13:03:27 +000018#include <linux/edma.h>
Matt Porterc2dde5f2012-08-22 21:09:34 -040019#include <linux/err.h>
20#include <linux/init.h>
21#include <linux/interrupt.h>
22#include <linux/list.h>
23#include <linux/module.h>
24#include <linux/platform_device.h>
25#include <linux/slab.h>
26#include <linux/spinlock.h>
Peter Ujfalusied646102014-07-31 13:12:38 +030027#include <linux/of.h>
Matt Porterc2dde5f2012-08-22 21:09:34 -040028
Matt Porter3ad7a422013-03-06 11:15:31 -050029#include <linux/platform_data/edma.h>
Matt Porterc2dde5f2012-08-22 21:09:34 -040030
31#include "dmaengine.h"
32#include "virt-dma.h"
33
34/*
35 * This will go away when the private EDMA API is folded
36 * into this driver and the platform device(s) are
37 * instantiated in the arch code. We can only get away
38 * with this simplification because DA8XX may not be built
39 * in the same kernel image with other DaVinci parts. This
40 * avoids having to sprinkle dmaengine driver platform devices
41 * and data throughout all the existing board files.
42 */
43#ifdef CONFIG_ARCH_DAVINCI_DA8XX
44#define EDMA_CTLRS 2
45#define EDMA_CHANS 32
46#else
47#define EDMA_CTLRS 1
48#define EDMA_CHANS 64
49#endif /* CONFIG_ARCH_DAVINCI_DA8XX */
50
Joel Fernandes2abd5f12013-09-23 18:05:15 -050051/*
52 * Max of 20 segments per channel to conserve PaRAM slots
53 * Also note that MAX_NR_SG should be atleast the no.of periods
54 * that are required for ASoC, otherwise DMA prep calls will
55 * fail. Today davinci-pcm is the only user of this driver and
56 * requires atleast 17 slots, so we setup the default to 20.
57 */
58#define MAX_NR_SG 20
Matt Porterc2dde5f2012-08-22 21:09:34 -040059#define EDMA_MAX_SLOTS MAX_NR_SG
60#define EDMA_DESCRIPTORS 16
61
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -050062struct edma_pset {
Thomas Gleixnerc2da2342014-04-28 14:29:57 -050063 u32 len;
64 dma_addr_t addr;
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -050065 struct edmacc_param param;
66};
67
Matt Porterc2dde5f2012-08-22 21:09:34 -040068struct edma_desc {
69 struct virt_dma_desc vdesc;
70 struct list_head node;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -050071 enum dma_transfer_direction direction;
Joel Fernandes50a9c702013-10-31 16:31:23 -050072 int cyclic;
Matt Porterc2dde5f2012-08-22 21:09:34 -040073 int absync;
74 int pset_nr;
Thomas Gleixner740b41f2014-04-28 14:34:11 -050075 struct edma_chan *echan;
Joel Fernandes04361d82014-04-28 15:19:31 -050076 int processed;
77
78 /*
79 * The following 4 elements are used for residue accounting.
80 *
81 * - processed_stat: the number of SG elements we have traversed
82 * so far to cover accounting. This is updated directly to processed
83 * during edma_callback and is always <= processed, because processed
84 * refers to the number of pending transfer (programmed to EDMA
85 * controller), where as processed_stat tracks number of transfers
86 * accounted for so far.
87 *
88 * - residue: The amount of bytes we have left to transfer for this desc
89 *
90 * - residue_stat: The residue in bytes of data we have covered
91 * so far for accounting. This is updated directly to residue
92 * during callbacks to keep it current.
93 *
94 * - sg_len: Tracks the length of the current intermediate transfer,
95 * this is required to update the residue during intermediate transfer
96 * completion callback.
97 */
98 int processed_stat;
99 u32 sg_len;
100 u32 residue;
101 u32 residue_stat;
102
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500103 struct edma_pset pset[0];
Matt Porterc2dde5f2012-08-22 21:09:34 -0400104};
105
106struct edma_cc;
107
108struct edma_chan {
109 struct virt_dma_chan vchan;
110 struct list_head node;
111 struct edma_desc *edesc;
112 struct edma_cc *ecc;
113 int ch_num;
114 bool alloced;
115 int slot[EDMA_MAX_SLOTS];
Joel Fernandesc5f47992013-08-29 18:05:43 -0500116 int missed;
Matt Porter661f7cb2013-01-10 13:41:04 -0500117 struct dma_slave_config cfg;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400118};
119
120struct edma_cc {
121 int ctlr;
122 struct dma_device dma_slave;
123 struct edma_chan slave_chans[EDMA_CHANS];
124 int num_slave_chans;
125 int dummy_slot;
126};
127
128static inline struct edma_cc *to_edma_cc(struct dma_device *d)
129{
130 return container_of(d, struct edma_cc, dma_slave);
131}
132
133static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
134{
135 return container_of(c, struct edma_chan, vchan.chan);
136}
137
138static inline struct edma_desc
139*to_edma_desc(struct dma_async_tx_descriptor *tx)
140{
141 return container_of(tx, struct edma_desc, vdesc.tx);
142}
143
144static void edma_desc_free(struct virt_dma_desc *vdesc)
145{
146 kfree(container_of(vdesc, struct edma_desc, vdesc));
147}
148
149/* Dispatch a queued descriptor to the controller (caller holds lock) */
150static void edma_execute(struct edma_chan *echan)
151{
Joel Fernandes53407062013-09-03 10:02:46 -0500152 struct virt_dma_desc *vdesc;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400153 struct edma_desc *edesc;
Joel Fernandes53407062013-09-03 10:02:46 -0500154 struct device *dev = echan->vchan.chan.device->dev;
155 int i, j, left, nslots;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400156
Joel Fernandes53407062013-09-03 10:02:46 -0500157 /* If either we processed all psets or we're still not started */
158 if (!echan->edesc ||
159 echan->edesc->pset_nr == echan->edesc->processed) {
160 /* Get next vdesc */
161 vdesc = vchan_next_desc(&echan->vchan);
162 if (!vdesc) {
163 echan->edesc = NULL;
164 return;
165 }
166 list_del(&vdesc->node);
167 echan->edesc = to_edma_desc(&vdesc->tx);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400168 }
169
Joel Fernandes53407062013-09-03 10:02:46 -0500170 edesc = echan->edesc;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400171
Joel Fernandes53407062013-09-03 10:02:46 -0500172 /* Find out how many left */
173 left = edesc->pset_nr - edesc->processed;
174 nslots = min(MAX_NR_SG, left);
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500175 edesc->sg_len = 0;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400176
177 /* Write descriptor PaRAM set(s) */
Joel Fernandes53407062013-09-03 10:02:46 -0500178 for (i = 0; i < nslots; i++) {
179 j = i + edesc->processed;
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500180 edma_write_slot(echan->slot[i], &edesc->pset[j].param);
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500181 edesc->sg_len += edesc->pset[j].len;
Peter Ujfalusi83bb3122014-04-14 14:42:02 +0300182 dev_vdbg(echan->vchan.chan.device->dev,
Matt Porterc2dde5f2012-08-22 21:09:34 -0400183 "\n pset[%d]:\n"
184 " chnum\t%d\n"
185 " slot\t%d\n"
186 " opt\t%08x\n"
187 " src\t%08x\n"
188 " dst\t%08x\n"
189 " abcnt\t%08x\n"
190 " ccnt\t%08x\n"
191 " bidx\t%08x\n"
192 " cidx\t%08x\n"
193 " lkrld\t%08x\n",
Joel Fernandes53407062013-09-03 10:02:46 -0500194 j, echan->ch_num, echan->slot[i],
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500195 edesc->pset[j].param.opt,
196 edesc->pset[j].param.src,
197 edesc->pset[j].param.dst,
198 edesc->pset[j].param.a_b_cnt,
199 edesc->pset[j].param.ccnt,
200 edesc->pset[j].param.src_dst_bidx,
201 edesc->pset[j].param.src_dst_cidx,
202 edesc->pset[j].param.link_bcntrld);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400203 /* Link to the previous slot if not the last set */
Joel Fernandes53407062013-09-03 10:02:46 -0500204 if (i != (nslots - 1))
Matt Porterc2dde5f2012-08-22 21:09:34 -0400205 edma_link(echan->slot[i], echan->slot[i+1]);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400206 }
207
Joel Fernandes53407062013-09-03 10:02:46 -0500208 edesc->processed += nslots;
209
Joel Fernandesb267b3b2013-08-29 18:05:44 -0500210 /*
211 * If this is either the last set in a set of SG-list transactions
212 * then setup a link to the dummy slot, this results in all future
213 * events being absorbed and that's OK because we're done
214 */
Joel Fernandes50a9c702013-10-31 16:31:23 -0500215 if (edesc->processed == edesc->pset_nr) {
216 if (edesc->cyclic)
217 edma_link(echan->slot[nslots-1], echan->slot[1]);
218 else
219 edma_link(echan->slot[nslots-1],
220 echan->ecc->dummy_slot);
221 }
Joel Fernandesb267b3b2013-08-29 18:05:44 -0500222
Joel Fernandes53407062013-09-03 10:02:46 -0500223 if (edesc->processed <= MAX_NR_SG) {
Peter Ujfalusi9aac9092014-04-24 10:29:50 +0300224 dev_dbg(dev, "first transfer starting on channel %d\n",
225 echan->ch_num);
Joel Fernandes53407062013-09-03 10:02:46 -0500226 edma_start(echan->ch_num);
Sekhar Nori5fc68a62014-03-19 11:25:50 +0530227 } else {
228 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
229 echan->ch_num, edesc->processed);
230 edma_resume(echan->ch_num);
Joel Fernandes53407062013-09-03 10:02:46 -0500231 }
Joel Fernandesc5f47992013-08-29 18:05:43 -0500232
233 /*
234 * This happens due to setup times between intermediate transfers
235 * in long SG lists which have to be broken up into transfers of
236 * MAX_NR_SG
237 */
238 if (echan->missed) {
Peter Ujfalusi9aac9092014-04-24 10:29:50 +0300239 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
Joel Fernandesc5f47992013-08-29 18:05:43 -0500240 edma_clean_channel(echan->ch_num);
241 edma_stop(echan->ch_num);
242 edma_start(echan->ch_num);
243 edma_trigger_channel(echan->ch_num);
244 echan->missed = 0;
245 }
Matt Porterc2dde5f2012-08-22 21:09:34 -0400246}
247
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100248static int edma_terminate_all(struct dma_chan *chan)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400249{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100250 struct edma_chan *echan = to_edma_chan(chan);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400251 unsigned long flags;
252 LIST_HEAD(head);
253
254 spin_lock_irqsave(&echan->vchan.lock, flags);
255
256 /*
257 * Stop DMA activity: we assume the callback will not be called
258 * after edma_dma() returns (even if it does, it will see
259 * echan->edesc is NULL and exit.)
260 */
261 if (echan->edesc) {
Peter Ujfalusi8e8805d2014-07-08 13:46:38 +0300262 int cyclic = echan->edesc->cyclic;
Petr Kulhavy5ca9e7c2015-03-27 13:35:51 +0200263
264 /*
265 * free the running request descriptor
266 * since it is not in any of the vdesc lists
267 */
268 edma_desc_free(&echan->edesc->vdesc);
269
Matt Porterc2dde5f2012-08-22 21:09:34 -0400270 echan->edesc = NULL;
271 edma_stop(echan->ch_num);
Peter Ujfalusi8e8805d2014-07-08 13:46:38 +0300272 /* Move the cyclic channel back to default queue */
273 if (cyclic)
274 edma_assign_channel_eventq(echan->ch_num,
275 EVENTQ_DEFAULT);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400276 }
277
278 vchan_get_all_descriptors(&echan->vchan, &head);
279 spin_unlock_irqrestore(&echan->vchan.lock, flags);
280 vchan_dma_desc_free_list(&echan->vchan, &head);
281
282 return 0;
283}
284
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100285static int edma_slave_config(struct dma_chan *chan,
Matt Porter661f7cb2013-01-10 13:41:04 -0500286 struct dma_slave_config *cfg)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400287{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100288 struct edma_chan *echan = to_edma_chan(chan);
289
Matt Porter661f7cb2013-01-10 13:41:04 -0500290 if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
291 cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400292 return -EINVAL;
293
Matt Porter661f7cb2013-01-10 13:41:04 -0500294 memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
Matt Porterc2dde5f2012-08-22 21:09:34 -0400295
296 return 0;
297}
298
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100299static int edma_dma_pause(struct dma_chan *chan)
Peter Ujfalusi72c7b672014-04-14 14:41:59 +0300300{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100301 struct edma_chan *echan = to_edma_chan(chan);
302
Peter Ujfalusi72c7b672014-04-14 14:41:59 +0300303 /* Pause/Resume only allowed with cyclic mode */
Sebastian Andrzej Siewior639559a2014-09-29 20:06:45 +0200304 if (!echan->edesc || !echan->edesc->cyclic)
Peter Ujfalusi72c7b672014-04-14 14:41:59 +0300305 return -EINVAL;
306
307 edma_pause(echan->ch_num);
308 return 0;
309}
310
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100311static int edma_dma_resume(struct dma_chan *chan)
Peter Ujfalusi72c7b672014-04-14 14:41:59 +0300312{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100313 struct edma_chan *echan = to_edma_chan(chan);
314
Peter Ujfalusi72c7b672014-04-14 14:41:59 +0300315 /* Pause/Resume only allowed with cyclic mode */
316 if (!echan->edesc->cyclic)
317 return -EINVAL;
318
319 edma_resume(echan->ch_num);
320 return 0;
321}
322
Joel Fernandesfd009032013-09-23 18:05:13 -0500323/*
324 * A PaRAM set configuration abstraction used by other modes
325 * @chan: Channel who's PaRAM set we're configuring
326 * @pset: PaRAM set to initialize and setup.
327 * @src_addr: Source address of the DMA
328 * @dst_addr: Destination address of the DMA
329 * @burst: In units of dev_width, how much to send
330 * @dev_width: How much is the dev_width
331 * @dma_length: Total length of the DMA transfer
332 * @direction: Direction of the transfer
333 */
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500334static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
Joel Fernandesfd009032013-09-23 18:05:13 -0500335 dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
336 enum dma_slave_buswidth dev_width, unsigned int dma_length,
337 enum dma_transfer_direction direction)
338{
339 struct edma_chan *echan = to_edma_chan(chan);
340 struct device *dev = chan->device->dev;
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500341 struct edmacc_param *param = &epset->param;
Joel Fernandesfd009032013-09-23 18:05:13 -0500342 int acnt, bcnt, ccnt, cidx;
343 int src_bidx, dst_bidx, src_cidx, dst_cidx;
344 int absync;
345
346 acnt = dev_width;
Peter Ujfalusib2b617d2014-04-14 14:41:58 +0300347
348 /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
349 if (!burst)
350 burst = 1;
Joel Fernandesfd009032013-09-23 18:05:13 -0500351 /*
352 * If the maxburst is equal to the fifo width, use
353 * A-synced transfers. This allows for large contiguous
354 * buffer transfers using only one PaRAM set.
355 */
356 if (burst == 1) {
357 /*
358 * For the A-sync case, bcnt and ccnt are the remainder
359 * and quotient respectively of the division of:
360 * (dma_length / acnt) by (SZ_64K -1). This is so
361 * that in case bcnt over flows, we have ccnt to use.
362 * Note: In A-sync tranfer only, bcntrld is used, but it
363 * only applies for sg_dma_len(sg) >= SZ_64K.
364 * In this case, the best way adopted is- bccnt for the
365 * first frame will be the remainder below. Then for
366 * every successive frame, bcnt will be SZ_64K-1. This
367 * is assured as bcntrld = 0xffff in end of function.
368 */
369 absync = false;
370 ccnt = dma_length / acnt / (SZ_64K - 1);
371 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
372 /*
373 * If bcnt is non-zero, we have a remainder and hence an
374 * extra frame to transfer, so increment ccnt.
375 */
376 if (bcnt)
377 ccnt++;
378 else
379 bcnt = SZ_64K - 1;
380 cidx = acnt;
381 } else {
382 /*
383 * If maxburst is greater than the fifo address_width,
384 * use AB-synced transfers where A count is the fifo
385 * address_width and B count is the maxburst. In this
386 * case, we are limited to transfers of C count frames
387 * of (address_width * maxburst) where C count is limited
388 * to SZ_64K-1. This places an upper bound on the length
389 * of an SG segment that can be handled.
390 */
391 absync = true;
392 bcnt = burst;
393 ccnt = dma_length / (acnt * bcnt);
394 if (ccnt > (SZ_64K - 1)) {
395 dev_err(dev, "Exceeded max SG segment size\n");
396 return -EINVAL;
397 }
398 cidx = acnt * bcnt;
399 }
400
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500401 epset->len = dma_length;
402
Joel Fernandesfd009032013-09-23 18:05:13 -0500403 if (direction == DMA_MEM_TO_DEV) {
404 src_bidx = acnt;
405 src_cidx = cidx;
406 dst_bidx = 0;
407 dst_cidx = 0;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500408 epset->addr = src_addr;
Joel Fernandesfd009032013-09-23 18:05:13 -0500409 } else if (direction == DMA_DEV_TO_MEM) {
410 src_bidx = 0;
411 src_cidx = 0;
412 dst_bidx = acnt;
413 dst_cidx = cidx;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500414 epset->addr = dst_addr;
Joel Fernandes8cc3e302014-04-18 21:50:33 -0500415 } else if (direction == DMA_MEM_TO_MEM) {
416 src_bidx = acnt;
417 src_cidx = cidx;
418 dst_bidx = acnt;
419 dst_cidx = cidx;
Joel Fernandesfd009032013-09-23 18:05:13 -0500420 } else {
421 dev_err(dev, "%s: direction not implemented yet\n", __func__);
422 return -EINVAL;
423 }
424
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500425 param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
Joel Fernandesfd009032013-09-23 18:05:13 -0500426 /* Configure A or AB synchronized transfers */
427 if (absync)
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500428 param->opt |= SYNCDIM;
Joel Fernandesfd009032013-09-23 18:05:13 -0500429
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500430 param->src = src_addr;
431 param->dst = dst_addr;
Joel Fernandesfd009032013-09-23 18:05:13 -0500432
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500433 param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
434 param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
Joel Fernandesfd009032013-09-23 18:05:13 -0500435
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500436 param->a_b_cnt = bcnt << 16 | acnt;
437 param->ccnt = ccnt;
Joel Fernandesfd009032013-09-23 18:05:13 -0500438 /*
439 * Only time when (bcntrld) auto reload is required is for
440 * A-sync case, and in this case, a requirement of reload value
441 * of SZ_64K-1 only is assured. 'link' is initially set to NULL
442 * and then later will be populated by edma_execute.
443 */
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500444 param->link_bcntrld = 0xffffffff;
Joel Fernandesfd009032013-09-23 18:05:13 -0500445 return absync;
446}
447
Matt Porterc2dde5f2012-08-22 21:09:34 -0400448static struct dma_async_tx_descriptor *edma_prep_slave_sg(
449 struct dma_chan *chan, struct scatterlist *sgl,
450 unsigned int sg_len, enum dma_transfer_direction direction,
451 unsigned long tx_flags, void *context)
452{
453 struct edma_chan *echan = to_edma_chan(chan);
454 struct device *dev = chan->device->dev;
455 struct edma_desc *edesc;
Joel Fernandesfd009032013-09-23 18:05:13 -0500456 dma_addr_t src_addr = 0, dst_addr = 0;
Matt Porter661f7cb2013-01-10 13:41:04 -0500457 enum dma_slave_buswidth dev_width;
458 u32 burst;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400459 struct scatterlist *sg;
Joel Fernandesfd009032013-09-23 18:05:13 -0500460 int i, nslots, ret;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400461
462 if (unlikely(!echan || !sgl || !sg_len))
463 return NULL;
464
Matt Porter661f7cb2013-01-10 13:41:04 -0500465 if (direction == DMA_DEV_TO_MEM) {
Joel Fernandesfd009032013-09-23 18:05:13 -0500466 src_addr = echan->cfg.src_addr;
Matt Porter661f7cb2013-01-10 13:41:04 -0500467 dev_width = echan->cfg.src_addr_width;
468 burst = echan->cfg.src_maxburst;
469 } else if (direction == DMA_MEM_TO_DEV) {
Joel Fernandesfd009032013-09-23 18:05:13 -0500470 dst_addr = echan->cfg.dst_addr;
Matt Porter661f7cb2013-01-10 13:41:04 -0500471 dev_width = echan->cfg.dst_addr_width;
472 burst = echan->cfg.dst_maxburst;
473 } else {
Peter Ujfalusie6fad592014-04-14 14:42:05 +0300474 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
Matt Porter661f7cb2013-01-10 13:41:04 -0500475 return NULL;
476 }
477
478 if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
Peter Ujfalusic594c892014-04-14 14:42:03 +0300479 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400480 return NULL;
481 }
482
Matt Porterc2dde5f2012-08-22 21:09:34 -0400483 edesc = kzalloc(sizeof(*edesc) + sg_len *
484 sizeof(edesc->pset[0]), GFP_ATOMIC);
485 if (!edesc) {
Peter Ujfalusic594c892014-04-14 14:42:03 +0300486 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400487 return NULL;
488 }
489
490 edesc->pset_nr = sg_len;
Thomas Gleixnerb6205c32014-04-28 14:18:45 -0500491 edesc->residue = 0;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500492 edesc->direction = direction;
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500493 edesc->echan = echan;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400494
Joel Fernandes6fbe24d2013-08-29 18:05:40 -0500495 /* Allocate a PaRAM slot, if needed */
496 nslots = min_t(unsigned, MAX_NR_SG, sg_len);
497
498 for (i = 0; i < nslots; i++) {
Matt Porterc2dde5f2012-08-22 21:09:34 -0400499 if (echan->slot[i] < 0) {
500 echan->slot[i] =
501 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
502 EDMA_SLOT_ANY);
503 if (echan->slot[i] < 0) {
Valentin Ilie4b6271a2013-10-24 16:14:22 +0300504 kfree(edesc);
Peter Ujfalusic594c892014-04-14 14:42:03 +0300505 dev_err(dev, "%s: Failed to allocate slot\n",
506 __func__);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400507 return NULL;
508 }
509 }
Joel Fernandes6fbe24d2013-08-29 18:05:40 -0500510 }
511
512 /* Configure PaRAM sets for each SG */
513 for_each_sg(sgl, sg, sg_len, i) {
Joel Fernandesfd009032013-09-23 18:05:13 -0500514 /* Get address for each SG */
515 if (direction == DMA_DEV_TO_MEM)
516 dst_addr = sg_dma_address(sg);
517 else
518 src_addr = sg_dma_address(sg);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400519
Joel Fernandesfd009032013-09-23 18:05:13 -0500520 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
521 dst_addr, burst, dev_width,
522 sg_dma_len(sg), direction);
Vinod Koulb967aec2013-10-30 13:07:18 +0530523 if (ret < 0) {
524 kfree(edesc);
Joel Fernandesfd009032013-09-23 18:05:13 -0500525 return NULL;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400526 }
527
Joel Fernandesfd009032013-09-23 18:05:13 -0500528 edesc->absync = ret;
Thomas Gleixnerb6205c32014-04-28 14:18:45 -0500529 edesc->residue += sg_dma_len(sg);
Joel Fernandes6fbe24d2013-08-29 18:05:40 -0500530
531 /* If this is the last in a current SG set of transactions,
532 enable interrupts so that next set is processed */
533 if (!((i+1) % MAX_NR_SG))
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500534 edesc->pset[i].param.opt |= TCINTEN;
Joel Fernandes6fbe24d2013-08-29 18:05:40 -0500535
Matt Porterc2dde5f2012-08-22 21:09:34 -0400536 /* If this is the last set, enable completion interrupt flag */
537 if (i == sg_len - 1)
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500538 edesc->pset[i].param.opt |= TCINTEN;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400539 }
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500540 edesc->residue_stat = edesc->residue;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400541
Matt Porterc2dde5f2012-08-22 21:09:34 -0400542 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
543}
Matt Porterc2dde5f2012-08-22 21:09:34 -0400544
Lad, Prabhakarb7a4fd52015-02-04 13:03:27 +0000545static struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
Joel Fernandes8cc3e302014-04-18 21:50:33 -0500546 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
547 size_t len, unsigned long tx_flags)
548{
549 int ret;
550 struct edma_desc *edesc;
551 struct device *dev = chan->device->dev;
552 struct edma_chan *echan = to_edma_chan(chan);
553
554 if (unlikely(!echan || !len))
555 return NULL;
556
557 edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC);
558 if (!edesc) {
559 dev_dbg(dev, "Failed to allocate a descriptor\n");
560 return NULL;
561 }
562
563 edesc->pset_nr = 1;
564
565 ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
566 DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM);
567 if (ret < 0)
568 return NULL;
569
570 edesc->absync = ret;
571
572 /*
573 * Enable intermediate transfer chaining to re-trigger channel
574 * on completion of every TR, and enable transfer-completion
575 * interrupt on completion of the whole transfer.
576 */
Joel Fernandesb0cce4c2014-04-28 15:30:32 -0500577 edesc->pset[0].param.opt |= ITCCHEN;
578 edesc->pset[0].param.opt |= TCINTEN;
Joel Fernandes8cc3e302014-04-18 21:50:33 -0500579
580 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
581}
582
Joel Fernandes50a9c702013-10-31 16:31:23 -0500583static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
584 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
585 size_t period_len, enum dma_transfer_direction direction,
Laurent Pinchart31c1e5a2014-08-01 12:20:10 +0200586 unsigned long tx_flags)
Joel Fernandes50a9c702013-10-31 16:31:23 -0500587{
588 struct edma_chan *echan = to_edma_chan(chan);
589 struct device *dev = chan->device->dev;
590 struct edma_desc *edesc;
591 dma_addr_t src_addr, dst_addr;
592 enum dma_slave_buswidth dev_width;
593 u32 burst;
594 int i, ret, nslots;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400595
Joel Fernandes50a9c702013-10-31 16:31:23 -0500596 if (unlikely(!echan || !buf_len || !period_len))
597 return NULL;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400598
Joel Fernandes50a9c702013-10-31 16:31:23 -0500599 if (direction == DMA_DEV_TO_MEM) {
600 src_addr = echan->cfg.src_addr;
601 dst_addr = buf_addr;
602 dev_width = echan->cfg.src_addr_width;
603 burst = echan->cfg.src_maxburst;
604 } else if (direction == DMA_MEM_TO_DEV) {
605 src_addr = buf_addr;
606 dst_addr = echan->cfg.dst_addr;
607 dev_width = echan->cfg.dst_addr_width;
608 burst = echan->cfg.dst_maxburst;
609 } else {
Peter Ujfalusie6fad592014-04-14 14:42:05 +0300610 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500611 return NULL;
612 }
613
614 if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
Peter Ujfalusic594c892014-04-14 14:42:03 +0300615 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500616 return NULL;
617 }
618
619 if (unlikely(buf_len % period_len)) {
620 dev_err(dev, "Period should be multiple of Buffer length\n");
621 return NULL;
622 }
623
624 nslots = (buf_len / period_len) + 1;
625
626 /*
627 * Cyclic DMA users such as audio cannot tolerate delays introduced
628 * by cases where the number of periods is more than the maximum
629 * number of SGs the EDMA driver can handle at a time. For DMA types
630 * such as Slave SGs, such delays are tolerable and synchronized,
631 * but the synchronization is difficult to achieve with Cyclic and
632 * cannot be guaranteed, so we error out early.
633 */
634 if (nslots > MAX_NR_SG)
635 return NULL;
636
637 edesc = kzalloc(sizeof(*edesc) + nslots *
638 sizeof(edesc->pset[0]), GFP_ATOMIC);
639 if (!edesc) {
Peter Ujfalusic594c892014-04-14 14:42:03 +0300640 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500641 return NULL;
642 }
643
644 edesc->cyclic = 1;
645 edesc->pset_nr = nslots;
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500646 edesc->residue = edesc->residue_stat = buf_len;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500647 edesc->direction = direction;
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500648 edesc->echan = echan;
Joel Fernandes50a9c702013-10-31 16:31:23 -0500649
Peter Ujfalusi83bb3122014-04-14 14:42:02 +0300650 dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
651 __func__, echan->ch_num, nslots, period_len, buf_len);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500652
653 for (i = 0; i < nslots; i++) {
654 /* Allocate a PaRAM slot, if needed */
655 if (echan->slot[i] < 0) {
656 echan->slot[i] =
657 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
658 EDMA_SLOT_ANY);
659 if (echan->slot[i] < 0) {
Christian Engelmayere3ddc972013-12-30 20:48:39 +0100660 kfree(edesc);
Peter Ujfalusic594c892014-04-14 14:42:03 +0300661 dev_err(dev, "%s: Failed to allocate slot\n",
662 __func__);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500663 return NULL;
664 }
665 }
666
667 if (i == nslots - 1) {
668 memcpy(&edesc->pset[i], &edesc->pset[0],
669 sizeof(edesc->pset[0]));
670 break;
671 }
672
673 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
674 dst_addr, burst, dev_width, period_len,
675 direction);
Christian Engelmayere3ddc972013-12-30 20:48:39 +0100676 if (ret < 0) {
677 kfree(edesc);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500678 return NULL;
Christian Engelmayere3ddc972013-12-30 20:48:39 +0100679 }
Joel Fernandes50a9c702013-10-31 16:31:23 -0500680
681 if (direction == DMA_DEV_TO_MEM)
682 dst_addr += period_len;
683 else
684 src_addr += period_len;
685
Peter Ujfalusi83bb3122014-04-14 14:42:02 +0300686 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
687 dev_vdbg(dev,
Joel Fernandes50a9c702013-10-31 16:31:23 -0500688 "\n pset[%d]:\n"
689 " chnum\t%d\n"
690 " slot\t%d\n"
691 " opt\t%08x\n"
692 " src\t%08x\n"
693 " dst\t%08x\n"
694 " abcnt\t%08x\n"
695 " ccnt\t%08x\n"
696 " bidx\t%08x\n"
697 " cidx\t%08x\n"
698 " lkrld\t%08x\n",
699 i, echan->ch_num, echan->slot[i],
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500700 edesc->pset[i].param.opt,
701 edesc->pset[i].param.src,
702 edesc->pset[i].param.dst,
703 edesc->pset[i].param.a_b_cnt,
704 edesc->pset[i].param.ccnt,
705 edesc->pset[i].param.src_dst_bidx,
706 edesc->pset[i].param.src_dst_cidx,
707 edesc->pset[i].param.link_bcntrld);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500708
709 edesc->absync = ret;
710
711 /*
Peter Ujfalusia1f146f2014-07-16 15:29:21 +0300712 * Enable period interrupt only if it is requested
Joel Fernandes50a9c702013-10-31 16:31:23 -0500713 */
Peter Ujfalusia1f146f2014-07-16 15:29:21 +0300714 if (tx_flags & DMA_PREP_INTERRUPT)
715 edesc->pset[i].param.opt |= TCINTEN;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400716 }
717
Peter Ujfalusi8e8805d2014-07-08 13:46:38 +0300718 /* Place the cyclic channel to highest priority queue */
719 edma_assign_channel_eventq(echan->ch_num, EVENTQ_0);
720
Matt Porterc2dde5f2012-08-22 21:09:34 -0400721 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
722}
723
724static void edma_callback(unsigned ch_num, u16 ch_status, void *data)
725{
726 struct edma_chan *echan = data;
727 struct device *dev = echan->vchan.chan.device->dev;
728 struct edma_desc *edesc;
Joel Fernandesc5f47992013-08-29 18:05:43 -0500729 struct edmacc_param p;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400730
Joel Fernandes50a9c702013-10-31 16:31:23 -0500731 edesc = echan->edesc;
732
733 /* Pause the channel for non-cyclic */
734 if (!edesc || (edesc && !edesc->cyclic))
735 edma_pause(echan->ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400736
737 switch (ch_status) {
Vinod Kouldb60d8d2013-10-30 18:22:30 +0530738 case EDMA_DMA_COMPLETE:
Joel Fernandes406efb12014-04-17 00:58:33 -0500739 spin_lock(&echan->vchan.lock);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400740
Matt Porterc2dde5f2012-08-22 21:09:34 -0400741 if (edesc) {
Joel Fernandes50a9c702013-10-31 16:31:23 -0500742 if (edesc->cyclic) {
743 vchan_cyclic_callback(&edesc->vdesc);
744 } else if (edesc->processed == edesc->pset_nr) {
Joel Fernandes53407062013-09-03 10:02:46 -0500745 dev_dbg(dev, "Transfer complete, stopping channel %d\n", ch_num);
Thomas Gleixnerb6205c32014-04-28 14:18:45 -0500746 edesc->residue = 0;
Joel Fernandes53407062013-09-03 10:02:46 -0500747 edma_stop(echan->ch_num);
748 vchan_cookie_complete(&edesc->vdesc);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500749 edma_execute(echan);
Joel Fernandes53407062013-09-03 10:02:46 -0500750 } else {
751 dev_dbg(dev, "Intermediate transfer complete on channel %d\n", ch_num);
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500752
753 /* Update statistics for tx_status */
754 edesc->residue -= edesc->sg_len;
755 edesc->residue_stat = edesc->residue;
756 edesc->processed_stat = edesc->processed;
757
Joel Fernandes50a9c702013-10-31 16:31:23 -0500758 edma_execute(echan);
Joel Fernandes53407062013-09-03 10:02:46 -0500759 }
Matt Porterc2dde5f2012-08-22 21:09:34 -0400760 }
761
Joel Fernandes406efb12014-04-17 00:58:33 -0500762 spin_unlock(&echan->vchan.lock);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400763
764 break;
Vinod Kouldb60d8d2013-10-30 18:22:30 +0530765 case EDMA_DMA_CC_ERROR:
Joel Fernandes406efb12014-04-17 00:58:33 -0500766 spin_lock(&echan->vchan.lock);
Joel Fernandesc5f47992013-08-29 18:05:43 -0500767
768 edma_read_slot(EDMA_CHAN_SLOT(echan->slot[0]), &p);
769
770 /*
771 * Issue later based on missed flag which will be sure
772 * to happen as:
773 * (1) we finished transmitting an intermediate slot and
774 * edma_execute is coming up.
775 * (2) or we finished current transfer and issue will
776 * call edma_execute.
777 *
778 * Important note: issuing can be dangerous here and
779 * lead to some nasty recursion when we are in a NULL
780 * slot. So we avoid doing so and set the missed flag.
781 */
782 if (p.a_b_cnt == 0 && p.ccnt == 0) {
783 dev_dbg(dev, "Error occurred, looks like slot is null, just setting miss\n");
784 echan->missed = 1;
785 } else {
786 /*
787 * The slot is already programmed but the event got
788 * missed, so its safe to issue it here.
789 */
790 dev_dbg(dev, "Error occurred but slot is non-null, TRIGGERING\n");
791 edma_clean_channel(echan->ch_num);
792 edma_stop(echan->ch_num);
793 edma_start(echan->ch_num);
794 edma_trigger_channel(echan->ch_num);
795 }
796
Joel Fernandes406efb12014-04-17 00:58:33 -0500797 spin_unlock(&echan->vchan.lock);
Joel Fernandesc5f47992013-08-29 18:05:43 -0500798
Matt Porterc2dde5f2012-08-22 21:09:34 -0400799 break;
800 default:
801 break;
802 }
803}
804
805/* Alloc channel resources */
806static int edma_alloc_chan_resources(struct dma_chan *chan)
807{
808 struct edma_chan *echan = to_edma_chan(chan);
809 struct device *dev = chan->device->dev;
810 int ret;
811 int a_ch_num;
812 LIST_HEAD(descs);
813
814 a_ch_num = edma_alloc_channel(echan->ch_num, edma_callback,
Petr Kulhavyab7add32015-03-23 21:35:01 +0100815 echan, EVENTQ_DEFAULT);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400816
817 if (a_ch_num < 0) {
818 ret = -ENODEV;
819 goto err_no_chan;
820 }
821
822 if (a_ch_num != echan->ch_num) {
823 dev_err(dev, "failed to allocate requested channel %u:%u\n",
824 EDMA_CTLR(echan->ch_num),
825 EDMA_CHAN_SLOT(echan->ch_num));
826 ret = -ENODEV;
827 goto err_wrong_chan;
828 }
829
830 echan->alloced = true;
831 echan->slot[0] = echan->ch_num;
832
Peter Ujfalusi9aac9092014-04-24 10:29:50 +0300833 dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num,
Ezequiel Garcia0e772c62013-12-13 11:06:18 -0300834 EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num));
Matt Porterc2dde5f2012-08-22 21:09:34 -0400835
836 return 0;
837
838err_wrong_chan:
839 edma_free_channel(a_ch_num);
840err_no_chan:
841 return ret;
842}
843
844/* Free channel resources */
845static void edma_free_chan_resources(struct dma_chan *chan)
846{
847 struct edma_chan *echan = to_edma_chan(chan);
848 struct device *dev = chan->device->dev;
849 int i;
850
851 /* Terminate transfers */
852 edma_stop(echan->ch_num);
853
854 vchan_free_chan_resources(&echan->vchan);
855
856 /* Free EDMA PaRAM slots */
857 for (i = 1; i < EDMA_MAX_SLOTS; i++) {
858 if (echan->slot[i] >= 0) {
859 edma_free_slot(echan->slot[i]);
860 echan->slot[i] = -1;
861 }
862 }
863
864 /* Free EDMA channel */
865 if (echan->alloced) {
866 edma_free_channel(echan->ch_num);
867 echan->alloced = false;
868 }
869
Ezequiel Garcia0e772c62013-12-13 11:06:18 -0300870 dev_dbg(dev, "freeing channel for %u\n", echan->ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400871}
872
873/* Send pending descriptor to hardware */
874static void edma_issue_pending(struct dma_chan *chan)
875{
876 struct edma_chan *echan = to_edma_chan(chan);
877 unsigned long flags;
878
879 spin_lock_irqsave(&echan->vchan.lock, flags);
880 if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
881 edma_execute(echan);
882 spin_unlock_irqrestore(&echan->vchan.lock, flags);
883}
884
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500885static u32 edma_residue(struct edma_desc *edesc)
886{
887 bool dst = edesc->direction == DMA_DEV_TO_MEM;
888 struct edma_pset *pset = edesc->pset;
889 dma_addr_t done, pos;
890 int i;
891
892 /*
893 * We always read the dst/src position from the first RamPar
894 * pset. That's the one which is active now.
895 */
896 pos = edma_get_position(edesc->echan->slot[0], dst);
897
898 /*
899 * Cyclic is simple. Just subtract pset[0].addr from pos.
900 *
901 * We never update edesc->residue in the cyclic case, so we
902 * can tell the remaining room to the end of the circular
903 * buffer.
904 */
905 if (edesc->cyclic) {
906 done = pos - pset->addr;
907 edesc->residue_stat = edesc->residue - done;
908 return edesc->residue_stat;
909 }
910
911 /*
912 * For SG operation we catch up with the last processed
913 * status.
914 */
915 pset += edesc->processed_stat;
916
917 for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
918 /*
919 * If we are inside this pset address range, we know
920 * this is the active one. Get the current delta and
921 * stop walking the psets.
922 */
923 if (pos >= pset->addr && pos < pset->addr + pset->len)
924 return edesc->residue_stat - (pos - pset->addr);
925
926 /* Otherwise mark it done and update residue_stat. */
927 edesc->processed_stat++;
928 edesc->residue_stat -= pset->len;
929 }
930 return edesc->residue_stat;
931}
932
Matt Porterc2dde5f2012-08-22 21:09:34 -0400933/* Check request completion status */
934static enum dma_status edma_tx_status(struct dma_chan *chan,
935 dma_cookie_t cookie,
936 struct dma_tx_state *txstate)
937{
938 struct edma_chan *echan = to_edma_chan(chan);
939 struct virt_dma_desc *vdesc;
940 enum dma_status ret;
941 unsigned long flags;
942
943 ret = dma_cookie_status(chan, cookie, txstate);
Vinod Koul9d386ec2013-10-16 13:42:15 +0530944 if (ret == DMA_COMPLETE || !txstate)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400945 return ret;
946
947 spin_lock_irqsave(&echan->vchan.lock, flags);
Thomas Gleixnerde135932014-04-28 14:19:51 -0500948 if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500949 txstate->residue = edma_residue(echan->edesc);
Thomas Gleixnerde135932014-04-28 14:19:51 -0500950 else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
951 txstate->residue = to_edma_desc(&vdesc->tx)->residue;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400952 spin_unlock_irqrestore(&echan->vchan.lock, flags);
953
954 return ret;
955}
956
957static void __init edma_chan_init(struct edma_cc *ecc,
958 struct dma_device *dma,
959 struct edma_chan *echans)
960{
961 int i, j;
962
963 for (i = 0; i < EDMA_CHANS; i++) {
964 struct edma_chan *echan = &echans[i];
965 echan->ch_num = EDMA_CTLR_CHAN(ecc->ctlr, i);
966 echan->ecc = ecc;
967 echan->vchan.desc_free = edma_desc_free;
968
969 vchan_init(&echan->vchan, dma);
970
971 INIT_LIST_HEAD(&echan->node);
972 for (j = 0; j < EDMA_MAX_SLOTS; j++)
973 echan->slot[j] = -1;
974 }
975}
976
Peter Ujfalusi2c88ee62014-04-14 14:42:01 +0300977#define EDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
978 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
Peter Ujfalusie4a899d2014-07-03 07:51:56 +0300979 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
Peter Ujfalusi2c88ee62014-04-14 14:42:01 +0300980 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
981
Matt Porterc2dde5f2012-08-22 21:09:34 -0400982static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
983 struct device *dev)
984{
985 dma->device_prep_slave_sg = edma_prep_slave_sg;
Joel Fernandes50a9c702013-10-31 16:31:23 -0500986 dma->device_prep_dma_cyclic = edma_prep_dma_cyclic;
Joel Fernandes8cc3e302014-04-18 21:50:33 -0500987 dma->device_prep_dma_memcpy = edma_prep_dma_memcpy;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400988 dma->device_alloc_chan_resources = edma_alloc_chan_resources;
989 dma->device_free_chan_resources = edma_free_chan_resources;
990 dma->device_issue_pending = edma_issue_pending;
991 dma->device_tx_status = edma_tx_status;
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100992 dma->device_config = edma_slave_config;
993 dma->device_pause = edma_dma_pause;
994 dma->device_resume = edma_dma_resume;
995 dma->device_terminate_all = edma_terminate_all;
Maxime Ripard9f59cd02014-11-17 14:42:47 +0100996
997 dma->src_addr_widths = EDMA_DMA_BUSWIDTHS;
998 dma->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
999 dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1000 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1001
Matt Porterc2dde5f2012-08-22 21:09:34 -04001002 dma->dev = dev;
1003
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001004 /*
1005 * code using dma memcpy must make sure alignment of
1006 * length is at dma->copy_align boundary.
1007 */
1008 dma->copy_align = DMA_SLAVE_BUSWIDTH_4_BYTES;
1009
Matt Porterc2dde5f2012-08-22 21:09:34 -04001010 INIT_LIST_HEAD(&dma->channels);
1011}
1012
Bill Pemberton463a1f82012-11-19 13:22:55 -05001013static int edma_probe(struct platform_device *pdev)
Matt Porterc2dde5f2012-08-22 21:09:34 -04001014{
1015 struct edma_cc *ecc;
1016 int ret;
1017
Russell King94cb0e72013-06-27 13:45:16 +01001018 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1019 if (ret)
1020 return ret;
1021
Matt Porterc2dde5f2012-08-22 21:09:34 -04001022 ecc = devm_kzalloc(&pdev->dev, sizeof(*ecc), GFP_KERNEL);
1023 if (!ecc) {
1024 dev_err(&pdev->dev, "Can't allocate controller\n");
1025 return -ENOMEM;
1026 }
1027
1028 ecc->ctlr = pdev->id;
1029 ecc->dummy_slot = edma_alloc_slot(ecc->ctlr, EDMA_SLOT_ANY);
1030 if (ecc->dummy_slot < 0) {
1031 dev_err(&pdev->dev, "Can't allocate PaRAM dummy slot\n");
Peter Ujfalusi04d537d2014-07-31 13:12:37 +03001032 return ecc->dummy_slot;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001033 }
1034
1035 dma_cap_zero(ecc->dma_slave.cap_mask);
1036 dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask);
Peter Ujfalusi232b223d2014-04-14 14:42:00 +03001037 dma_cap_set(DMA_CYCLIC, ecc->dma_slave.cap_mask);
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001038 dma_cap_set(DMA_MEMCPY, ecc->dma_slave.cap_mask);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001039
1040 edma_dma_init(ecc, &ecc->dma_slave, &pdev->dev);
1041
1042 edma_chan_init(ecc, &ecc->dma_slave, ecc->slave_chans);
1043
1044 ret = dma_async_device_register(&ecc->dma_slave);
1045 if (ret)
1046 goto err_reg1;
1047
1048 platform_set_drvdata(pdev, ecc);
1049
1050 dev_info(&pdev->dev, "TI EDMA DMA engine driver\n");
1051
1052 return 0;
1053
1054err_reg1:
1055 edma_free_slot(ecc->dummy_slot);
1056 return ret;
1057}
1058
Greg Kroah-Hartman4bf27b82012-12-21 15:09:59 -08001059static int edma_remove(struct platform_device *pdev)
Matt Porterc2dde5f2012-08-22 21:09:34 -04001060{
1061 struct device *dev = &pdev->dev;
1062 struct edma_cc *ecc = dev_get_drvdata(dev);
1063
1064 dma_async_device_unregister(&ecc->dma_slave);
1065 edma_free_slot(ecc->dummy_slot);
1066
1067 return 0;
1068}
1069
1070static struct platform_driver edma_driver = {
1071 .probe = edma_probe,
Bill Pembertona7d6e3e2012-11-19 13:20:04 -05001072 .remove = edma_remove,
Matt Porterc2dde5f2012-08-22 21:09:34 -04001073 .driver = {
1074 .name = "edma-dma-engine",
Matt Porterc2dde5f2012-08-22 21:09:34 -04001075 },
1076};
1077
1078bool edma_filter_fn(struct dma_chan *chan, void *param)
1079{
1080 if (chan->device->dev->driver == &edma_driver.driver) {
1081 struct edma_chan *echan = to_edma_chan(chan);
1082 unsigned ch_req = *(unsigned *)param;
1083 return ch_req == echan->ch_num;
1084 }
1085 return false;
1086}
1087EXPORT_SYMBOL(edma_filter_fn);
1088
Matt Porterc2dde5f2012-08-22 21:09:34 -04001089static int edma_init(void)
1090{
Arnd Bergmann5305e4d2014-10-24 18:14:01 +02001091 return platform_driver_register(&edma_driver);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001092}
1093subsys_initcall(edma_init);
1094
1095static void __exit edma_exit(void)
1096{
Matt Porterc2dde5f2012-08-22 21:09:34 -04001097 platform_driver_unregister(&edma_driver);
1098}
1099module_exit(edma_exit);
1100
Josh Boyerd71505b2013-09-04 10:32:50 -04001101MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
Matt Porterc2dde5f2012-08-22 21:09:34 -04001102MODULE_DESCRIPTION("TI EDMA DMA engine driver");
1103MODULE_LICENSE("GPL v2");