blob: fc91ab9dd1bb7a773d1d8c676a26110f6a6ea98f [file] [log] [blame]
Matt Porterc2dde5f2012-08-22 21:09:34 -04001/*
2 * TI EDMA DMA engine driver
3 *
4 * Copyright 2012 Texas Instruments
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation version 2.
9 *
10 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
11 * kind, whether express or implied; without even the implied warranty
12 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 */
15
16#include <linux/dmaengine.h>
17#include <linux/dma-mapping.h>
Lad, Prabhakarb7a4fd52015-02-04 13:03:27 +000018#include <linux/edma.h>
Matt Porterc2dde5f2012-08-22 21:09:34 -040019#include <linux/err.h>
20#include <linux/init.h>
21#include <linux/interrupt.h>
22#include <linux/list.h>
23#include <linux/module.h>
24#include <linux/platform_device.h>
25#include <linux/slab.h>
26#include <linux/spinlock.h>
Peter Ujfalusied646102014-07-31 13:12:38 +030027#include <linux/of.h>
Peter Ujfalusidc9b60552015-10-14 14:42:47 +030028#include <linux/of_dma.h>
Matt Porterc2dde5f2012-08-22 21:09:34 -040029
Matt Porter3ad7a422013-03-06 11:15:31 -050030#include <linux/platform_data/edma.h>
Matt Porterc2dde5f2012-08-22 21:09:34 -040031
32#include "dmaengine.h"
33#include "virt-dma.h"
34
35/*
36 * This will go away when the private EDMA API is folded
37 * into this driver and the platform device(s) are
38 * instantiated in the arch code. We can only get away
39 * with this simplification because DA8XX may not be built
40 * in the same kernel image with other DaVinci parts. This
41 * avoids having to sprinkle dmaengine driver platform devices
42 * and data throughout all the existing board files.
43 */
44#ifdef CONFIG_ARCH_DAVINCI_DA8XX
45#define EDMA_CTLRS 2
46#define EDMA_CHANS 32
47#else
48#define EDMA_CTLRS 1
49#define EDMA_CHANS 64
50#endif /* CONFIG_ARCH_DAVINCI_DA8XX */
51
Joel Fernandes2abd5f12013-09-23 18:05:15 -050052/*
53 * Max of 20 segments per channel to conserve PaRAM slots
54 * Also note that MAX_NR_SG should be atleast the no.of periods
55 * that are required for ASoC, otherwise DMA prep calls will
56 * fail. Today davinci-pcm is the only user of this driver and
57 * requires atleast 17 slots, so we setup the default to 20.
58 */
59#define MAX_NR_SG 20
Matt Porterc2dde5f2012-08-22 21:09:34 -040060#define EDMA_MAX_SLOTS MAX_NR_SG
61#define EDMA_DESCRIPTORS 16
62
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -050063struct edma_pset {
Thomas Gleixnerc2da2342014-04-28 14:29:57 -050064 u32 len;
65 dma_addr_t addr;
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -050066 struct edmacc_param param;
67};
68
Matt Porterc2dde5f2012-08-22 21:09:34 -040069struct edma_desc {
70 struct virt_dma_desc vdesc;
71 struct list_head node;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -050072 enum dma_transfer_direction direction;
Joel Fernandes50a9c702013-10-31 16:31:23 -050073 int cyclic;
Matt Porterc2dde5f2012-08-22 21:09:34 -040074 int absync;
75 int pset_nr;
Thomas Gleixner740b41f2014-04-28 14:34:11 -050076 struct edma_chan *echan;
Joel Fernandes04361d82014-04-28 15:19:31 -050077 int processed;
78
79 /*
80 * The following 4 elements are used for residue accounting.
81 *
82 * - processed_stat: the number of SG elements we have traversed
83 * so far to cover accounting. This is updated directly to processed
84 * during edma_callback and is always <= processed, because processed
85 * refers to the number of pending transfer (programmed to EDMA
86 * controller), where as processed_stat tracks number of transfers
87 * accounted for so far.
88 *
89 * - residue: The amount of bytes we have left to transfer for this desc
90 *
91 * - residue_stat: The residue in bytes of data we have covered
92 * so far for accounting. This is updated directly to residue
93 * during callbacks to keep it current.
94 *
95 * - sg_len: Tracks the length of the current intermediate transfer,
96 * this is required to update the residue during intermediate transfer
97 * completion callback.
98 */
99 int processed_stat;
100 u32 sg_len;
101 u32 residue;
102 u32 residue_stat;
103
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500104 struct edma_pset pset[0];
Matt Porterc2dde5f2012-08-22 21:09:34 -0400105};
106
107struct edma_cc;
108
109struct edma_chan {
110 struct virt_dma_chan vchan;
111 struct list_head node;
112 struct edma_desc *edesc;
113 struct edma_cc *ecc;
114 int ch_num;
115 bool alloced;
116 int slot[EDMA_MAX_SLOTS];
Joel Fernandesc5f47992013-08-29 18:05:43 -0500117 int missed;
Matt Porter661f7cb2013-01-10 13:41:04 -0500118 struct dma_slave_config cfg;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400119};
120
121struct edma_cc {
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300122 struct edma *cc;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400123 int ctlr;
124 struct dma_device dma_slave;
125 struct edma_chan slave_chans[EDMA_CHANS];
126 int num_slave_chans;
127 int dummy_slot;
128};
129
130static inline struct edma_cc *to_edma_cc(struct dma_device *d)
131{
132 return container_of(d, struct edma_cc, dma_slave);
133}
134
135static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
136{
137 return container_of(c, struct edma_chan, vchan.chan);
138}
139
140static inline struct edma_desc
141*to_edma_desc(struct dma_async_tx_descriptor *tx)
142{
143 return container_of(tx, struct edma_desc, vdesc.tx);
144}
145
146static void edma_desc_free(struct virt_dma_desc *vdesc)
147{
148 kfree(container_of(vdesc, struct edma_desc, vdesc));
149}
150
151/* Dispatch a queued descriptor to the controller (caller holds lock) */
152static void edma_execute(struct edma_chan *echan)
153{
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300154 struct edma *cc = echan->ecc->cc;
Joel Fernandes53407062013-09-03 10:02:46 -0500155 struct virt_dma_desc *vdesc;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400156 struct edma_desc *edesc;
Joel Fernandes53407062013-09-03 10:02:46 -0500157 struct device *dev = echan->vchan.chan.device->dev;
158 int i, j, left, nslots;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400159
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300160 if (!echan->edesc) {
161 /* Setup is needed for the first transfer */
Joel Fernandes53407062013-09-03 10:02:46 -0500162 vdesc = vchan_next_desc(&echan->vchan);
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300163 if (!vdesc)
Joel Fernandes53407062013-09-03 10:02:46 -0500164 return;
Joel Fernandes53407062013-09-03 10:02:46 -0500165 list_del(&vdesc->node);
166 echan->edesc = to_edma_desc(&vdesc->tx);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400167 }
168
Joel Fernandes53407062013-09-03 10:02:46 -0500169 edesc = echan->edesc;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400170
Joel Fernandes53407062013-09-03 10:02:46 -0500171 /* Find out how many left */
172 left = edesc->pset_nr - edesc->processed;
173 nslots = min(MAX_NR_SG, left);
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500174 edesc->sg_len = 0;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400175
176 /* Write descriptor PaRAM set(s) */
Joel Fernandes53407062013-09-03 10:02:46 -0500177 for (i = 0; i < nslots; i++) {
178 j = i + edesc->processed;
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300179 edma_write_slot(cc, echan->slot[i], &edesc->pset[j].param);
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500180 edesc->sg_len += edesc->pset[j].len;
Peter Ujfalusi83bb3122014-04-14 14:42:02 +0300181 dev_vdbg(echan->vchan.chan.device->dev,
Matt Porterc2dde5f2012-08-22 21:09:34 -0400182 "\n pset[%d]:\n"
183 " chnum\t%d\n"
184 " slot\t%d\n"
185 " opt\t%08x\n"
186 " src\t%08x\n"
187 " dst\t%08x\n"
188 " abcnt\t%08x\n"
189 " ccnt\t%08x\n"
190 " bidx\t%08x\n"
191 " cidx\t%08x\n"
192 " lkrld\t%08x\n",
Joel Fernandes53407062013-09-03 10:02:46 -0500193 j, echan->ch_num, echan->slot[i],
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500194 edesc->pset[j].param.opt,
195 edesc->pset[j].param.src,
196 edesc->pset[j].param.dst,
197 edesc->pset[j].param.a_b_cnt,
198 edesc->pset[j].param.ccnt,
199 edesc->pset[j].param.src_dst_bidx,
200 edesc->pset[j].param.src_dst_cidx,
201 edesc->pset[j].param.link_bcntrld);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400202 /* Link to the previous slot if not the last set */
Joel Fernandes53407062013-09-03 10:02:46 -0500203 if (i != (nslots - 1))
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300204 edma_link(cc, echan->slot[i], echan->slot[i+1]);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400205 }
206
Joel Fernandes53407062013-09-03 10:02:46 -0500207 edesc->processed += nslots;
208
Joel Fernandesb267b3b2013-08-29 18:05:44 -0500209 /*
210 * If this is either the last set in a set of SG-list transactions
211 * then setup a link to the dummy slot, this results in all future
212 * events being absorbed and that's OK because we're done
213 */
Joel Fernandes50a9c702013-10-31 16:31:23 -0500214 if (edesc->processed == edesc->pset_nr) {
215 if (edesc->cyclic)
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300216 edma_link(cc, echan->slot[nslots-1], echan->slot[1]);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500217 else
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300218 edma_link(cc, echan->slot[nslots-1],
Joel Fernandes50a9c702013-10-31 16:31:23 -0500219 echan->ecc->dummy_slot);
220 }
Joel Fernandesb267b3b2013-08-29 18:05:44 -0500221
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300222 if (echan->missed) {
223 /*
224 * This happens due to setup times between intermediate
225 * transfers in long SG lists which have to be broken up into
226 * transfers of MAX_NR_SG
227 */
228 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300229 edma_clean_channel(cc, echan->ch_num);
230 edma_stop(cc, echan->ch_num);
231 edma_start(cc, echan->ch_num);
232 edma_trigger_channel(cc, echan->ch_num);
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300233 echan->missed = 0;
234 } else if (edesc->processed <= MAX_NR_SG) {
Peter Ujfalusi9aac9092014-04-24 10:29:50 +0300235 dev_dbg(dev, "first transfer starting on channel %d\n",
236 echan->ch_num);
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300237 edma_start(cc, echan->ch_num);
Sekhar Nori5fc68a62014-03-19 11:25:50 +0530238 } else {
239 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
240 echan->ch_num, edesc->processed);
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300241 edma_resume(cc, echan->ch_num);
Joel Fernandes53407062013-09-03 10:02:46 -0500242 }
Matt Porterc2dde5f2012-08-22 21:09:34 -0400243}
244
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100245static int edma_terminate_all(struct dma_chan *chan)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400246{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100247 struct edma_chan *echan = to_edma_chan(chan);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400248 unsigned long flags;
249 LIST_HEAD(head);
250
251 spin_lock_irqsave(&echan->vchan.lock, flags);
252
253 /*
254 * Stop DMA activity: we assume the callback will not be called
255 * after edma_dma() returns (even if it does, it will see
256 * echan->edesc is NULL and exit.)
257 */
258 if (echan->edesc) {
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300259 edma_stop(echan->ecc->cc, echan->ch_num);
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300260 /* Move the cyclic channel back to default queue */
261 if (echan->edesc->cyclic)
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300262 edma_assign_channel_eventq(echan->ecc->cc,
263 echan->ch_num,
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300264 EVENTQ_DEFAULT);
Petr Kulhavy5ca9e7c2015-03-27 13:35:51 +0200265 /*
266 * free the running request descriptor
267 * since it is not in any of the vdesc lists
268 */
269 edma_desc_free(&echan->edesc->vdesc);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400270 echan->edesc = NULL;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400271 }
272
273 vchan_get_all_descriptors(&echan->vchan, &head);
274 spin_unlock_irqrestore(&echan->vchan.lock, flags);
275 vchan_dma_desc_free_list(&echan->vchan, &head);
276
277 return 0;
278}
279
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100280static int edma_slave_config(struct dma_chan *chan,
Matt Porter661f7cb2013-01-10 13:41:04 -0500281 struct dma_slave_config *cfg)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400282{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100283 struct edma_chan *echan = to_edma_chan(chan);
284
Matt Porter661f7cb2013-01-10 13:41:04 -0500285 if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
286 cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400287 return -EINVAL;
288
Matt Porter661f7cb2013-01-10 13:41:04 -0500289 memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
Matt Porterc2dde5f2012-08-22 21:09:34 -0400290
291 return 0;
292}
293
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100294static int edma_dma_pause(struct dma_chan *chan)
Peter Ujfalusi72c7b672014-04-14 14:41:59 +0300295{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100296 struct edma_chan *echan = to_edma_chan(chan);
297
John Ogness02ec6042015-04-27 13:52:25 +0200298 if (!echan->edesc)
Peter Ujfalusi72c7b672014-04-14 14:41:59 +0300299 return -EINVAL;
300
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300301 edma_pause(echan->ecc->cc, echan->ch_num);
Peter Ujfalusi72c7b672014-04-14 14:41:59 +0300302 return 0;
303}
304
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100305static int edma_dma_resume(struct dma_chan *chan)
Peter Ujfalusi72c7b672014-04-14 14:41:59 +0300306{
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100307 struct edma_chan *echan = to_edma_chan(chan);
308
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300309 edma_resume(echan->ecc->cc, echan->ch_num);
Peter Ujfalusi72c7b672014-04-14 14:41:59 +0300310 return 0;
311}
312
Joel Fernandesfd009032013-09-23 18:05:13 -0500313/*
314 * A PaRAM set configuration abstraction used by other modes
315 * @chan: Channel who's PaRAM set we're configuring
316 * @pset: PaRAM set to initialize and setup.
317 * @src_addr: Source address of the DMA
318 * @dst_addr: Destination address of the DMA
319 * @burst: In units of dev_width, how much to send
320 * @dev_width: How much is the dev_width
321 * @dma_length: Total length of the DMA transfer
322 * @direction: Direction of the transfer
323 */
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500324static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
Joel Fernandesfd009032013-09-23 18:05:13 -0500325 dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
326 enum dma_slave_buswidth dev_width, unsigned int dma_length,
327 enum dma_transfer_direction direction)
328{
329 struct edma_chan *echan = to_edma_chan(chan);
330 struct device *dev = chan->device->dev;
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500331 struct edmacc_param *param = &epset->param;
Joel Fernandesfd009032013-09-23 18:05:13 -0500332 int acnt, bcnt, ccnt, cidx;
333 int src_bidx, dst_bidx, src_cidx, dst_cidx;
334 int absync;
335
336 acnt = dev_width;
Peter Ujfalusib2b617d2014-04-14 14:41:58 +0300337
338 /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
339 if (!burst)
340 burst = 1;
Joel Fernandesfd009032013-09-23 18:05:13 -0500341 /*
342 * If the maxburst is equal to the fifo width, use
343 * A-synced transfers. This allows for large contiguous
344 * buffer transfers using only one PaRAM set.
345 */
346 if (burst == 1) {
347 /*
348 * For the A-sync case, bcnt and ccnt are the remainder
349 * and quotient respectively of the division of:
350 * (dma_length / acnt) by (SZ_64K -1). This is so
351 * that in case bcnt over flows, we have ccnt to use.
352 * Note: In A-sync tranfer only, bcntrld is used, but it
353 * only applies for sg_dma_len(sg) >= SZ_64K.
354 * In this case, the best way adopted is- bccnt for the
355 * first frame will be the remainder below. Then for
356 * every successive frame, bcnt will be SZ_64K-1. This
357 * is assured as bcntrld = 0xffff in end of function.
358 */
359 absync = false;
360 ccnt = dma_length / acnt / (SZ_64K - 1);
361 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
362 /*
363 * If bcnt is non-zero, we have a remainder and hence an
364 * extra frame to transfer, so increment ccnt.
365 */
366 if (bcnt)
367 ccnt++;
368 else
369 bcnt = SZ_64K - 1;
370 cidx = acnt;
371 } else {
372 /*
373 * If maxburst is greater than the fifo address_width,
374 * use AB-synced transfers where A count is the fifo
375 * address_width and B count is the maxburst. In this
376 * case, we are limited to transfers of C count frames
377 * of (address_width * maxburst) where C count is limited
378 * to SZ_64K-1. This places an upper bound on the length
379 * of an SG segment that can be handled.
380 */
381 absync = true;
382 bcnt = burst;
383 ccnt = dma_length / (acnt * bcnt);
384 if (ccnt > (SZ_64K - 1)) {
385 dev_err(dev, "Exceeded max SG segment size\n");
386 return -EINVAL;
387 }
388 cidx = acnt * bcnt;
389 }
390
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500391 epset->len = dma_length;
392
Joel Fernandesfd009032013-09-23 18:05:13 -0500393 if (direction == DMA_MEM_TO_DEV) {
394 src_bidx = acnt;
395 src_cidx = cidx;
396 dst_bidx = 0;
397 dst_cidx = 0;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500398 epset->addr = src_addr;
Joel Fernandesfd009032013-09-23 18:05:13 -0500399 } else if (direction == DMA_DEV_TO_MEM) {
400 src_bidx = 0;
401 src_cidx = 0;
402 dst_bidx = acnt;
403 dst_cidx = cidx;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500404 epset->addr = dst_addr;
Joel Fernandes8cc3e302014-04-18 21:50:33 -0500405 } else if (direction == DMA_MEM_TO_MEM) {
406 src_bidx = acnt;
407 src_cidx = cidx;
408 dst_bidx = acnt;
409 dst_cidx = cidx;
Joel Fernandesfd009032013-09-23 18:05:13 -0500410 } else {
411 dev_err(dev, "%s: direction not implemented yet\n", __func__);
412 return -EINVAL;
413 }
414
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500415 param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
Joel Fernandesfd009032013-09-23 18:05:13 -0500416 /* Configure A or AB synchronized transfers */
417 if (absync)
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500418 param->opt |= SYNCDIM;
Joel Fernandesfd009032013-09-23 18:05:13 -0500419
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500420 param->src = src_addr;
421 param->dst = dst_addr;
Joel Fernandesfd009032013-09-23 18:05:13 -0500422
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500423 param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
424 param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
Joel Fernandesfd009032013-09-23 18:05:13 -0500425
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500426 param->a_b_cnt = bcnt << 16 | acnt;
427 param->ccnt = ccnt;
Joel Fernandesfd009032013-09-23 18:05:13 -0500428 /*
429 * Only time when (bcntrld) auto reload is required is for
430 * A-sync case, and in this case, a requirement of reload value
431 * of SZ_64K-1 only is assured. 'link' is initially set to NULL
432 * and then later will be populated by edma_execute.
433 */
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500434 param->link_bcntrld = 0xffffffff;
Joel Fernandesfd009032013-09-23 18:05:13 -0500435 return absync;
436}
437
Matt Porterc2dde5f2012-08-22 21:09:34 -0400438static struct dma_async_tx_descriptor *edma_prep_slave_sg(
439 struct dma_chan *chan, struct scatterlist *sgl,
440 unsigned int sg_len, enum dma_transfer_direction direction,
441 unsigned long tx_flags, void *context)
442{
443 struct edma_chan *echan = to_edma_chan(chan);
444 struct device *dev = chan->device->dev;
445 struct edma_desc *edesc;
Joel Fernandesfd009032013-09-23 18:05:13 -0500446 dma_addr_t src_addr = 0, dst_addr = 0;
Matt Porter661f7cb2013-01-10 13:41:04 -0500447 enum dma_slave_buswidth dev_width;
448 u32 burst;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400449 struct scatterlist *sg;
Joel Fernandesfd009032013-09-23 18:05:13 -0500450 int i, nslots, ret;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400451
452 if (unlikely(!echan || !sgl || !sg_len))
453 return NULL;
454
Matt Porter661f7cb2013-01-10 13:41:04 -0500455 if (direction == DMA_DEV_TO_MEM) {
Joel Fernandesfd009032013-09-23 18:05:13 -0500456 src_addr = echan->cfg.src_addr;
Matt Porter661f7cb2013-01-10 13:41:04 -0500457 dev_width = echan->cfg.src_addr_width;
458 burst = echan->cfg.src_maxburst;
459 } else if (direction == DMA_MEM_TO_DEV) {
Joel Fernandesfd009032013-09-23 18:05:13 -0500460 dst_addr = echan->cfg.dst_addr;
Matt Porter661f7cb2013-01-10 13:41:04 -0500461 dev_width = echan->cfg.dst_addr_width;
462 burst = echan->cfg.dst_maxburst;
463 } else {
Peter Ujfalusie6fad592014-04-14 14:42:05 +0300464 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
Matt Porter661f7cb2013-01-10 13:41:04 -0500465 return NULL;
466 }
467
468 if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
Peter Ujfalusic594c892014-04-14 14:42:03 +0300469 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400470 return NULL;
471 }
472
Matt Porterc2dde5f2012-08-22 21:09:34 -0400473 edesc = kzalloc(sizeof(*edesc) + sg_len *
474 sizeof(edesc->pset[0]), GFP_ATOMIC);
475 if (!edesc) {
Peter Ujfalusic594c892014-04-14 14:42:03 +0300476 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400477 return NULL;
478 }
479
480 edesc->pset_nr = sg_len;
Thomas Gleixnerb6205c32014-04-28 14:18:45 -0500481 edesc->residue = 0;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500482 edesc->direction = direction;
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500483 edesc->echan = echan;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400484
Joel Fernandes6fbe24d2013-08-29 18:05:40 -0500485 /* Allocate a PaRAM slot, if needed */
486 nslots = min_t(unsigned, MAX_NR_SG, sg_len);
487
488 for (i = 0; i < nslots; i++) {
Matt Porterc2dde5f2012-08-22 21:09:34 -0400489 if (echan->slot[i] < 0) {
490 echan->slot[i] =
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300491 edma_alloc_slot(echan->ecc->cc, EDMA_SLOT_ANY);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400492 if (echan->slot[i] < 0) {
Valentin Ilie4b6271a2013-10-24 16:14:22 +0300493 kfree(edesc);
Peter Ujfalusic594c892014-04-14 14:42:03 +0300494 dev_err(dev, "%s: Failed to allocate slot\n",
495 __func__);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400496 return NULL;
497 }
498 }
Joel Fernandes6fbe24d2013-08-29 18:05:40 -0500499 }
500
501 /* Configure PaRAM sets for each SG */
502 for_each_sg(sgl, sg, sg_len, i) {
Joel Fernandesfd009032013-09-23 18:05:13 -0500503 /* Get address for each SG */
504 if (direction == DMA_DEV_TO_MEM)
505 dst_addr = sg_dma_address(sg);
506 else
507 src_addr = sg_dma_address(sg);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400508
Joel Fernandesfd009032013-09-23 18:05:13 -0500509 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
510 dst_addr, burst, dev_width,
511 sg_dma_len(sg), direction);
Vinod Koulb967aec2013-10-30 13:07:18 +0530512 if (ret < 0) {
513 kfree(edesc);
Joel Fernandesfd009032013-09-23 18:05:13 -0500514 return NULL;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400515 }
516
Joel Fernandesfd009032013-09-23 18:05:13 -0500517 edesc->absync = ret;
Thomas Gleixnerb6205c32014-04-28 14:18:45 -0500518 edesc->residue += sg_dma_len(sg);
Joel Fernandes6fbe24d2013-08-29 18:05:40 -0500519
520 /* If this is the last in a current SG set of transactions,
521 enable interrupts so that next set is processed */
522 if (!((i+1) % MAX_NR_SG))
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500523 edesc->pset[i].param.opt |= TCINTEN;
Joel Fernandes6fbe24d2013-08-29 18:05:40 -0500524
Matt Porterc2dde5f2012-08-22 21:09:34 -0400525 /* If this is the last set, enable completion interrupt flag */
526 if (i == sg_len - 1)
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500527 edesc->pset[i].param.opt |= TCINTEN;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400528 }
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500529 edesc->residue_stat = edesc->residue;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400530
Matt Porterc2dde5f2012-08-22 21:09:34 -0400531 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
532}
Matt Porterc2dde5f2012-08-22 21:09:34 -0400533
Lad, Prabhakarb7a4fd52015-02-04 13:03:27 +0000534static struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
Joel Fernandes8cc3e302014-04-18 21:50:33 -0500535 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
536 size_t len, unsigned long tx_flags)
537{
538 int ret;
539 struct edma_desc *edesc;
540 struct device *dev = chan->device->dev;
541 struct edma_chan *echan = to_edma_chan(chan);
542
543 if (unlikely(!echan || !len))
544 return NULL;
545
546 edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC);
547 if (!edesc) {
548 dev_dbg(dev, "Failed to allocate a descriptor\n");
549 return NULL;
550 }
551
552 edesc->pset_nr = 1;
553
554 ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
555 DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM);
556 if (ret < 0)
557 return NULL;
558
559 edesc->absync = ret;
560
561 /*
562 * Enable intermediate transfer chaining to re-trigger channel
563 * on completion of every TR, and enable transfer-completion
564 * interrupt on completion of the whole transfer.
565 */
Joel Fernandesb0cce4c2014-04-28 15:30:32 -0500566 edesc->pset[0].param.opt |= ITCCHEN;
567 edesc->pset[0].param.opt |= TCINTEN;
Joel Fernandes8cc3e302014-04-18 21:50:33 -0500568
569 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
570}
571
Joel Fernandes50a9c702013-10-31 16:31:23 -0500572static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
573 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
574 size_t period_len, enum dma_transfer_direction direction,
Laurent Pinchart31c1e5a2014-08-01 12:20:10 +0200575 unsigned long tx_flags)
Joel Fernandes50a9c702013-10-31 16:31:23 -0500576{
577 struct edma_chan *echan = to_edma_chan(chan);
578 struct device *dev = chan->device->dev;
579 struct edma_desc *edesc;
580 dma_addr_t src_addr, dst_addr;
581 enum dma_slave_buswidth dev_width;
582 u32 burst;
583 int i, ret, nslots;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400584
Joel Fernandes50a9c702013-10-31 16:31:23 -0500585 if (unlikely(!echan || !buf_len || !period_len))
586 return NULL;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400587
Joel Fernandes50a9c702013-10-31 16:31:23 -0500588 if (direction == DMA_DEV_TO_MEM) {
589 src_addr = echan->cfg.src_addr;
590 dst_addr = buf_addr;
591 dev_width = echan->cfg.src_addr_width;
592 burst = echan->cfg.src_maxburst;
593 } else if (direction == DMA_MEM_TO_DEV) {
594 src_addr = buf_addr;
595 dst_addr = echan->cfg.dst_addr;
596 dev_width = echan->cfg.dst_addr_width;
597 burst = echan->cfg.dst_maxburst;
598 } else {
Peter Ujfalusie6fad592014-04-14 14:42:05 +0300599 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500600 return NULL;
601 }
602
603 if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
Peter Ujfalusic594c892014-04-14 14:42:03 +0300604 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500605 return NULL;
606 }
607
608 if (unlikely(buf_len % period_len)) {
609 dev_err(dev, "Period should be multiple of Buffer length\n");
610 return NULL;
611 }
612
613 nslots = (buf_len / period_len) + 1;
614
615 /*
616 * Cyclic DMA users such as audio cannot tolerate delays introduced
617 * by cases where the number of periods is more than the maximum
618 * number of SGs the EDMA driver can handle at a time. For DMA types
619 * such as Slave SGs, such delays are tolerable and synchronized,
620 * but the synchronization is difficult to achieve with Cyclic and
621 * cannot be guaranteed, so we error out early.
622 */
623 if (nslots > MAX_NR_SG)
624 return NULL;
625
626 edesc = kzalloc(sizeof(*edesc) + nslots *
627 sizeof(edesc->pset[0]), GFP_ATOMIC);
628 if (!edesc) {
Peter Ujfalusic594c892014-04-14 14:42:03 +0300629 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500630 return NULL;
631 }
632
633 edesc->cyclic = 1;
634 edesc->pset_nr = nslots;
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500635 edesc->residue = edesc->residue_stat = buf_len;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500636 edesc->direction = direction;
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500637 edesc->echan = echan;
Joel Fernandes50a9c702013-10-31 16:31:23 -0500638
Peter Ujfalusi83bb3122014-04-14 14:42:02 +0300639 dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
640 __func__, echan->ch_num, nslots, period_len, buf_len);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500641
642 for (i = 0; i < nslots; i++) {
643 /* Allocate a PaRAM slot, if needed */
644 if (echan->slot[i] < 0) {
645 echan->slot[i] =
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300646 edma_alloc_slot(echan->ecc->cc, EDMA_SLOT_ANY);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500647 if (echan->slot[i] < 0) {
Christian Engelmayere3ddc972013-12-30 20:48:39 +0100648 kfree(edesc);
Peter Ujfalusic594c892014-04-14 14:42:03 +0300649 dev_err(dev, "%s: Failed to allocate slot\n",
650 __func__);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500651 return NULL;
652 }
653 }
654
655 if (i == nslots - 1) {
656 memcpy(&edesc->pset[i], &edesc->pset[0],
657 sizeof(edesc->pset[0]));
658 break;
659 }
660
661 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
662 dst_addr, burst, dev_width, period_len,
663 direction);
Christian Engelmayere3ddc972013-12-30 20:48:39 +0100664 if (ret < 0) {
665 kfree(edesc);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500666 return NULL;
Christian Engelmayere3ddc972013-12-30 20:48:39 +0100667 }
Joel Fernandes50a9c702013-10-31 16:31:23 -0500668
669 if (direction == DMA_DEV_TO_MEM)
670 dst_addr += period_len;
671 else
672 src_addr += period_len;
673
Peter Ujfalusi83bb3122014-04-14 14:42:02 +0300674 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
675 dev_vdbg(dev,
Joel Fernandes50a9c702013-10-31 16:31:23 -0500676 "\n pset[%d]:\n"
677 " chnum\t%d\n"
678 " slot\t%d\n"
679 " opt\t%08x\n"
680 " src\t%08x\n"
681 " dst\t%08x\n"
682 " abcnt\t%08x\n"
683 " ccnt\t%08x\n"
684 " bidx\t%08x\n"
685 " cidx\t%08x\n"
686 " lkrld\t%08x\n",
687 i, echan->ch_num, echan->slot[i],
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500688 edesc->pset[i].param.opt,
689 edesc->pset[i].param.src,
690 edesc->pset[i].param.dst,
691 edesc->pset[i].param.a_b_cnt,
692 edesc->pset[i].param.ccnt,
693 edesc->pset[i].param.src_dst_bidx,
694 edesc->pset[i].param.src_dst_cidx,
695 edesc->pset[i].param.link_bcntrld);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500696
697 edesc->absync = ret;
698
699 /*
Peter Ujfalusia1f146f2014-07-16 15:29:21 +0300700 * Enable period interrupt only if it is requested
Joel Fernandes50a9c702013-10-31 16:31:23 -0500701 */
Peter Ujfalusia1f146f2014-07-16 15:29:21 +0300702 if (tx_flags & DMA_PREP_INTERRUPT)
703 edesc->pset[i].param.opt |= TCINTEN;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400704 }
705
Peter Ujfalusi8e8805d2014-07-08 13:46:38 +0300706 /* Place the cyclic channel to highest priority queue */
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300707 edma_assign_channel_eventq(echan->ecc->cc, echan->ch_num, EVENTQ_0);
Peter Ujfalusi8e8805d2014-07-08 13:46:38 +0300708
Matt Porterc2dde5f2012-08-22 21:09:34 -0400709 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
710}
711
712static void edma_callback(unsigned ch_num, u16 ch_status, void *data)
713{
714 struct edma_chan *echan = data;
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300715 struct edma *cc = echan->ecc->cc;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400716 struct device *dev = echan->vchan.chan.device->dev;
717 struct edma_desc *edesc;
Joel Fernandesc5f47992013-08-29 18:05:43 -0500718 struct edmacc_param p;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400719
Joel Fernandes50a9c702013-10-31 16:31:23 -0500720 edesc = echan->edesc;
721
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300722 spin_lock(&echan->vchan.lock);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400723 switch (ch_status) {
Vinod Kouldb60d8d2013-10-30 18:22:30 +0530724 case EDMA_DMA_COMPLETE:
Matt Porterc2dde5f2012-08-22 21:09:34 -0400725 if (edesc) {
Joel Fernandes50a9c702013-10-31 16:31:23 -0500726 if (edesc->cyclic) {
727 vchan_cyclic_callback(&edesc->vdesc);
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300728 goto out;
Joel Fernandes50a9c702013-10-31 16:31:23 -0500729 } else if (edesc->processed == edesc->pset_nr) {
Joel Fernandes53407062013-09-03 10:02:46 -0500730 dev_dbg(dev, "Transfer complete, stopping channel %d\n", ch_num);
Thomas Gleixnerb6205c32014-04-28 14:18:45 -0500731 edesc->residue = 0;
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300732 edma_stop(cc, echan->ch_num);
Joel Fernandes53407062013-09-03 10:02:46 -0500733 vchan_cookie_complete(&edesc->vdesc);
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300734 echan->edesc = NULL;
Joel Fernandes53407062013-09-03 10:02:46 -0500735 } else {
736 dev_dbg(dev, "Intermediate transfer complete on channel %d\n", ch_num);
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500737
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300738 edma_pause(cc, echan->ch_num);
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300739
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500740 /* Update statistics for tx_status */
741 edesc->residue -= edesc->sg_len;
742 edesc->residue_stat = edesc->residue;
743 edesc->processed_stat = edesc->processed;
Joel Fernandes53407062013-09-03 10:02:46 -0500744 }
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300745 edma_execute(echan);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400746 }
Matt Porterc2dde5f2012-08-22 21:09:34 -0400747 break;
Vinod Kouldb60d8d2013-10-30 18:22:30 +0530748 case EDMA_DMA_CC_ERROR:
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300749 edma_read_slot(cc, echan->slot[0], &p);
Joel Fernandesc5f47992013-08-29 18:05:43 -0500750
751 /*
752 * Issue later based on missed flag which will be sure
753 * to happen as:
754 * (1) we finished transmitting an intermediate slot and
755 * edma_execute is coming up.
756 * (2) or we finished current transfer and issue will
757 * call edma_execute.
758 *
759 * Important note: issuing can be dangerous here and
760 * lead to some nasty recursion when we are in a NULL
761 * slot. So we avoid doing so and set the missed flag.
762 */
763 if (p.a_b_cnt == 0 && p.ccnt == 0) {
764 dev_dbg(dev, "Error occurred, looks like slot is null, just setting miss\n");
765 echan->missed = 1;
766 } else {
767 /*
768 * The slot is already programmed but the event got
769 * missed, so its safe to issue it here.
770 */
771 dev_dbg(dev, "Error occurred but slot is non-null, TRIGGERING\n");
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300772 edma_clean_channel(cc, echan->ch_num);
773 edma_stop(cc, echan->ch_num);
774 edma_start(cc, echan->ch_num);
775 edma_trigger_channel(cc, echan->ch_num);
Joel Fernandesc5f47992013-08-29 18:05:43 -0500776 }
Matt Porterc2dde5f2012-08-22 21:09:34 -0400777 break;
778 default:
779 break;
780 }
Peter Ujfalusi8fa7ff42015-10-14 14:42:45 +0300781out:
782 spin_unlock(&echan->vchan.lock);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400783}
784
785/* Alloc channel resources */
786static int edma_alloc_chan_resources(struct dma_chan *chan)
787{
788 struct edma_chan *echan = to_edma_chan(chan);
789 struct device *dev = chan->device->dev;
790 int ret;
791 int a_ch_num;
792 LIST_HEAD(descs);
793
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300794 a_ch_num = edma_alloc_channel(echan->ecc->cc, echan->ch_num,
795 edma_callback, echan, EVENTQ_DEFAULT);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400796
797 if (a_ch_num < 0) {
798 ret = -ENODEV;
799 goto err_no_chan;
800 }
801
802 if (a_ch_num != echan->ch_num) {
803 dev_err(dev, "failed to allocate requested channel %u:%u\n",
804 EDMA_CTLR(echan->ch_num),
805 EDMA_CHAN_SLOT(echan->ch_num));
806 ret = -ENODEV;
807 goto err_wrong_chan;
808 }
809
810 echan->alloced = true;
811 echan->slot[0] = echan->ch_num;
812
Peter Ujfalusi9aac9092014-04-24 10:29:50 +0300813 dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num,
Ezequiel Garcia0e772c62013-12-13 11:06:18 -0300814 EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num));
Matt Porterc2dde5f2012-08-22 21:09:34 -0400815
816 return 0;
817
818err_wrong_chan:
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300819 edma_free_channel(echan->ecc->cc, a_ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400820err_no_chan:
821 return ret;
822}
823
824/* Free channel resources */
825static void edma_free_chan_resources(struct dma_chan *chan)
826{
827 struct edma_chan *echan = to_edma_chan(chan);
828 struct device *dev = chan->device->dev;
829 int i;
830
831 /* Terminate transfers */
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300832 edma_stop(echan->ecc->cc, echan->ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400833
834 vchan_free_chan_resources(&echan->vchan);
835
836 /* Free EDMA PaRAM slots */
837 for (i = 1; i < EDMA_MAX_SLOTS; i++) {
838 if (echan->slot[i] >= 0) {
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300839 edma_free_slot(echan->ecc->cc, echan->slot[i]);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400840 echan->slot[i] = -1;
841 }
842 }
843
844 /* Free EDMA channel */
845 if (echan->alloced) {
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300846 edma_free_channel(echan->ecc->cc, echan->ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400847 echan->alloced = false;
848 }
849
Ezequiel Garcia0e772c62013-12-13 11:06:18 -0300850 dev_dbg(dev, "freeing channel for %u\n", echan->ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400851}
852
853/* Send pending descriptor to hardware */
854static void edma_issue_pending(struct dma_chan *chan)
855{
856 struct edma_chan *echan = to_edma_chan(chan);
857 unsigned long flags;
858
859 spin_lock_irqsave(&echan->vchan.lock, flags);
860 if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
861 edma_execute(echan);
862 spin_unlock_irqrestore(&echan->vchan.lock, flags);
863}
864
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500865static u32 edma_residue(struct edma_desc *edesc)
866{
867 bool dst = edesc->direction == DMA_DEV_TO_MEM;
868 struct edma_pset *pset = edesc->pset;
869 dma_addr_t done, pos;
870 int i;
871
872 /*
873 * We always read the dst/src position from the first RamPar
874 * pset. That's the one which is active now.
875 */
Peter Ujfalusica304fa2015-10-14 14:42:49 +0300876 pos = edma_get_position(edesc->echan->ecc->cc, edesc->echan->slot[0],
877 dst);
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500878
879 /*
880 * Cyclic is simple. Just subtract pset[0].addr from pos.
881 *
882 * We never update edesc->residue in the cyclic case, so we
883 * can tell the remaining room to the end of the circular
884 * buffer.
885 */
886 if (edesc->cyclic) {
887 done = pos - pset->addr;
888 edesc->residue_stat = edesc->residue - done;
889 return edesc->residue_stat;
890 }
891
892 /*
893 * For SG operation we catch up with the last processed
894 * status.
895 */
896 pset += edesc->processed_stat;
897
898 for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
899 /*
900 * If we are inside this pset address range, we know
901 * this is the active one. Get the current delta and
902 * stop walking the psets.
903 */
904 if (pos >= pset->addr && pos < pset->addr + pset->len)
905 return edesc->residue_stat - (pos - pset->addr);
906
907 /* Otherwise mark it done and update residue_stat. */
908 edesc->processed_stat++;
909 edesc->residue_stat -= pset->len;
910 }
911 return edesc->residue_stat;
912}
913
Matt Porterc2dde5f2012-08-22 21:09:34 -0400914/* Check request completion status */
915static enum dma_status edma_tx_status(struct dma_chan *chan,
916 dma_cookie_t cookie,
917 struct dma_tx_state *txstate)
918{
919 struct edma_chan *echan = to_edma_chan(chan);
920 struct virt_dma_desc *vdesc;
921 enum dma_status ret;
922 unsigned long flags;
923
924 ret = dma_cookie_status(chan, cookie, txstate);
Vinod Koul9d386ec2013-10-16 13:42:15 +0530925 if (ret == DMA_COMPLETE || !txstate)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400926 return ret;
927
928 spin_lock_irqsave(&echan->vchan.lock, flags);
Thomas Gleixnerde135932014-04-28 14:19:51 -0500929 if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500930 txstate->residue = edma_residue(echan->edesc);
Thomas Gleixnerde135932014-04-28 14:19:51 -0500931 else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
932 txstate->residue = to_edma_desc(&vdesc->tx)->residue;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400933 spin_unlock_irqrestore(&echan->vchan.lock, flags);
934
935 return ret;
936}
937
938static void __init edma_chan_init(struct edma_cc *ecc,
939 struct dma_device *dma,
940 struct edma_chan *echans)
941{
942 int i, j;
943
944 for (i = 0; i < EDMA_CHANS; i++) {
945 struct edma_chan *echan = &echans[i];
946 echan->ch_num = EDMA_CTLR_CHAN(ecc->ctlr, i);
947 echan->ecc = ecc;
948 echan->vchan.desc_free = edma_desc_free;
949
950 vchan_init(&echan->vchan, dma);
951
952 INIT_LIST_HEAD(&echan->node);
953 for (j = 0; j < EDMA_MAX_SLOTS; j++)
954 echan->slot[j] = -1;
955 }
956}
957
Peter Ujfalusi2c88ee62014-04-14 14:42:01 +0300958#define EDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
959 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
Peter Ujfalusie4a899d2014-07-03 07:51:56 +0300960 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
Peter Ujfalusi2c88ee62014-04-14 14:42:01 +0300961 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
962
Matt Porterc2dde5f2012-08-22 21:09:34 -0400963static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
964 struct device *dev)
965{
966 dma->device_prep_slave_sg = edma_prep_slave_sg;
Joel Fernandes50a9c702013-10-31 16:31:23 -0500967 dma->device_prep_dma_cyclic = edma_prep_dma_cyclic;
Joel Fernandes8cc3e302014-04-18 21:50:33 -0500968 dma->device_prep_dma_memcpy = edma_prep_dma_memcpy;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400969 dma->device_alloc_chan_resources = edma_alloc_chan_resources;
970 dma->device_free_chan_resources = edma_free_chan_resources;
971 dma->device_issue_pending = edma_issue_pending;
972 dma->device_tx_status = edma_tx_status;
Maxime Ripardaa7c09b2014-11-17 14:42:13 +0100973 dma->device_config = edma_slave_config;
974 dma->device_pause = edma_dma_pause;
975 dma->device_resume = edma_dma_resume;
976 dma->device_terminate_all = edma_terminate_all;
Maxime Ripard9f59cd02014-11-17 14:42:47 +0100977
978 dma->src_addr_widths = EDMA_DMA_BUSWIDTHS;
979 dma->dst_addr_widths = EDMA_DMA_BUSWIDTHS;
980 dma->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
981 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
982
Matt Porterc2dde5f2012-08-22 21:09:34 -0400983 dma->dev = dev;
984
Joel Fernandes8cc3e302014-04-18 21:50:33 -0500985 /*
986 * code using dma memcpy must make sure alignment of
987 * length is at dma->copy_align boundary.
988 */
Maxime Ripard77a68e52015-07-20 10:41:32 +0200989 dma->copy_align = DMAENGINE_ALIGN_4_BYTES;
Joel Fernandes8cc3e302014-04-18 21:50:33 -0500990
Matt Porterc2dde5f2012-08-22 21:09:34 -0400991 INIT_LIST_HEAD(&dma->channels);
992}
993
Bill Pemberton463a1f82012-11-19 13:22:55 -0500994static int edma_probe(struct platform_device *pdev)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400995{
996 struct edma_cc *ecc;
Peter Ujfalusidc9b60552015-10-14 14:42:47 +0300997 struct device_node *parent_node = pdev->dev.parent->of_node;
Peter Ujfalusib2c843a2015-10-14 14:42:50 +0300998 struct platform_device *parent_pdev =
999 to_platform_device(pdev->dev.parent);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001000 int ret;
1001
Russell King94cb0e72013-06-27 13:45:16 +01001002 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1003 if (ret)
1004 return ret;
1005
Matt Porterc2dde5f2012-08-22 21:09:34 -04001006 ecc = devm_kzalloc(&pdev->dev, sizeof(*ecc), GFP_KERNEL);
1007 if (!ecc) {
1008 dev_err(&pdev->dev, "Can't allocate controller\n");
1009 return -ENOMEM;
1010 }
1011
Peter Ujfalusica304fa2015-10-14 14:42:49 +03001012 ecc->cc = edma_get_data(pdev->dev.parent);
1013 if (!ecc->cc)
1014 return -ENODEV;
1015
Peter Ujfalusib2c843a2015-10-14 14:42:50 +03001016 ecc->ctlr = parent_pdev->id;
1017 if (ecc->ctlr < 0)
1018 ecc->ctlr = 0;
1019
Peter Ujfalusica304fa2015-10-14 14:42:49 +03001020 ecc->dummy_slot = edma_alloc_slot(ecc->cc, EDMA_SLOT_ANY);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001021 if (ecc->dummy_slot < 0) {
1022 dev_err(&pdev->dev, "Can't allocate PaRAM dummy slot\n");
Peter Ujfalusi04d537d2014-07-31 13:12:37 +03001023 return ecc->dummy_slot;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001024 }
1025
1026 dma_cap_zero(ecc->dma_slave.cap_mask);
1027 dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask);
Peter Ujfalusi232b223d2014-04-14 14:42:00 +03001028 dma_cap_set(DMA_CYCLIC, ecc->dma_slave.cap_mask);
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001029 dma_cap_set(DMA_MEMCPY, ecc->dma_slave.cap_mask);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001030
1031 edma_dma_init(ecc, &ecc->dma_slave, &pdev->dev);
1032
1033 edma_chan_init(ecc, &ecc->dma_slave, ecc->slave_chans);
1034
1035 ret = dma_async_device_register(&ecc->dma_slave);
1036 if (ret)
1037 goto err_reg1;
1038
1039 platform_set_drvdata(pdev, ecc);
1040
Peter Ujfalusidc9b60552015-10-14 14:42:47 +03001041 if (parent_node) {
Peter Ujfalusib2c843a2015-10-14 14:42:50 +03001042 of_dma_controller_register(parent_node, of_dma_xlate_by_chan_id,
1043 &ecc->dma_slave);
Peter Ujfalusidc9b60552015-10-14 14:42:47 +03001044 }
1045
Matt Porterc2dde5f2012-08-22 21:09:34 -04001046 dev_info(&pdev->dev, "TI EDMA DMA engine driver\n");
1047
1048 return 0;
1049
1050err_reg1:
Peter Ujfalusica304fa2015-10-14 14:42:49 +03001051 edma_free_slot(ecc->cc, ecc->dummy_slot);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001052 return ret;
1053}
1054
Greg Kroah-Hartman4bf27b82012-12-21 15:09:59 -08001055static int edma_remove(struct platform_device *pdev)
Matt Porterc2dde5f2012-08-22 21:09:34 -04001056{
1057 struct device *dev = &pdev->dev;
1058 struct edma_cc *ecc = dev_get_drvdata(dev);
Peter Ujfalusidc9b60552015-10-14 14:42:47 +03001059 struct device_node *parent_node = pdev->dev.parent->of_node;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001060
Peter Ujfalusidc9b60552015-10-14 14:42:47 +03001061 if (parent_node)
1062 of_dma_controller_free(parent_node);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001063 dma_async_device_unregister(&ecc->dma_slave);
Peter Ujfalusica304fa2015-10-14 14:42:49 +03001064 edma_free_slot(ecc->cc, ecc->dummy_slot);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001065
1066 return 0;
1067}
1068
1069static struct platform_driver edma_driver = {
1070 .probe = edma_probe,
Bill Pembertona7d6e3e2012-11-19 13:20:04 -05001071 .remove = edma_remove,
Matt Porterc2dde5f2012-08-22 21:09:34 -04001072 .driver = {
1073 .name = "edma-dma-engine",
Matt Porterc2dde5f2012-08-22 21:09:34 -04001074 },
1075};
1076
1077bool edma_filter_fn(struct dma_chan *chan, void *param)
1078{
1079 if (chan->device->dev->driver == &edma_driver.driver) {
1080 struct edma_chan *echan = to_edma_chan(chan);
1081 unsigned ch_req = *(unsigned *)param;
1082 return ch_req == echan->ch_num;
1083 }
1084 return false;
1085}
1086EXPORT_SYMBOL(edma_filter_fn);
1087
Matt Porterc2dde5f2012-08-22 21:09:34 -04001088static int edma_init(void)
1089{
Arnd Bergmann5305e4d2014-10-24 18:14:01 +02001090 return platform_driver_register(&edma_driver);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001091}
1092subsys_initcall(edma_init);
1093
1094static void __exit edma_exit(void)
1095{
Matt Porterc2dde5f2012-08-22 21:09:34 -04001096 platform_driver_unregister(&edma_driver);
1097}
1098module_exit(edma_exit);
1099
Josh Boyerd71505b2013-09-04 10:32:50 -04001100MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
Matt Porterc2dde5f2012-08-22 21:09:34 -04001101MODULE_DESCRIPTION("TI EDMA DMA engine driver");
1102MODULE_LICENSE("GPL v2");