blob: d08c4dedef3585ba62e6681ec4a778ba65074907 [file] [log] [blame]
Matt Porterc2dde5f2012-08-22 21:09:34 -04001/*
2 * TI EDMA DMA engine driver
3 *
4 * Copyright 2012 Texas Instruments
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation version 2.
9 *
10 * This program is distributed "as is" WITHOUT ANY WARRANTY of any
11 * kind, whether express or implied; without even the implied warranty
12 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 */
15
16#include <linux/dmaengine.h>
17#include <linux/dma-mapping.h>
18#include <linux/err.h>
19#include <linux/init.h>
20#include <linux/interrupt.h>
21#include <linux/list.h>
22#include <linux/module.h>
23#include <linux/platform_device.h>
24#include <linux/slab.h>
25#include <linux/spinlock.h>
26
Matt Porter3ad7a422013-03-06 11:15:31 -050027#include <linux/platform_data/edma.h>
Matt Porterc2dde5f2012-08-22 21:09:34 -040028
29#include "dmaengine.h"
30#include "virt-dma.h"
31
32/*
33 * This will go away when the private EDMA API is folded
34 * into this driver and the platform device(s) are
35 * instantiated in the arch code. We can only get away
36 * with this simplification because DA8XX may not be built
37 * in the same kernel image with other DaVinci parts. This
38 * avoids having to sprinkle dmaengine driver platform devices
39 * and data throughout all the existing board files.
40 */
41#ifdef CONFIG_ARCH_DAVINCI_DA8XX
42#define EDMA_CTLRS 2
43#define EDMA_CHANS 32
44#else
45#define EDMA_CTLRS 1
46#define EDMA_CHANS 64
47#endif /* CONFIG_ARCH_DAVINCI_DA8XX */
48
Joel Fernandes2abd5f12013-09-23 18:05:15 -050049/*
50 * Max of 20 segments per channel to conserve PaRAM slots
51 * Also note that MAX_NR_SG should be atleast the no.of periods
52 * that are required for ASoC, otherwise DMA prep calls will
53 * fail. Today davinci-pcm is the only user of this driver and
54 * requires atleast 17 slots, so we setup the default to 20.
55 */
56#define MAX_NR_SG 20
Matt Porterc2dde5f2012-08-22 21:09:34 -040057#define EDMA_MAX_SLOTS MAX_NR_SG
58#define EDMA_DESCRIPTORS 16
59
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -050060struct edma_pset {
Thomas Gleixnerc2da2342014-04-28 14:29:57 -050061 u32 len;
62 dma_addr_t addr;
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -050063 struct edmacc_param param;
64};
65
Matt Porterc2dde5f2012-08-22 21:09:34 -040066struct edma_desc {
67 struct virt_dma_desc vdesc;
68 struct list_head node;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -050069 enum dma_transfer_direction direction;
Joel Fernandes50a9c702013-10-31 16:31:23 -050070 int cyclic;
Matt Porterc2dde5f2012-08-22 21:09:34 -040071 int absync;
72 int pset_nr;
Thomas Gleixner740b41f2014-04-28 14:34:11 -050073 struct edma_chan *echan;
Joel Fernandes04361d82014-04-28 15:19:31 -050074 int processed;
75
76 /*
77 * The following 4 elements are used for residue accounting.
78 *
79 * - processed_stat: the number of SG elements we have traversed
80 * so far to cover accounting. This is updated directly to processed
81 * during edma_callback and is always <= processed, because processed
82 * refers to the number of pending transfer (programmed to EDMA
83 * controller), where as processed_stat tracks number of transfers
84 * accounted for so far.
85 *
86 * - residue: The amount of bytes we have left to transfer for this desc
87 *
88 * - residue_stat: The residue in bytes of data we have covered
89 * so far for accounting. This is updated directly to residue
90 * during callbacks to keep it current.
91 *
92 * - sg_len: Tracks the length of the current intermediate transfer,
93 * this is required to update the residue during intermediate transfer
94 * completion callback.
95 */
96 int processed_stat;
97 u32 sg_len;
98 u32 residue;
99 u32 residue_stat;
100
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500101 struct edma_pset pset[0];
Matt Porterc2dde5f2012-08-22 21:09:34 -0400102};
103
104struct edma_cc;
105
106struct edma_chan {
107 struct virt_dma_chan vchan;
108 struct list_head node;
109 struct edma_desc *edesc;
110 struct edma_cc *ecc;
111 int ch_num;
112 bool alloced;
113 int slot[EDMA_MAX_SLOTS];
Joel Fernandesc5f47992013-08-29 18:05:43 -0500114 int missed;
Matt Porter661f7cb2013-01-10 13:41:04 -0500115 struct dma_slave_config cfg;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400116};
117
118struct edma_cc {
119 int ctlr;
120 struct dma_device dma_slave;
121 struct edma_chan slave_chans[EDMA_CHANS];
122 int num_slave_chans;
123 int dummy_slot;
124};
125
126static inline struct edma_cc *to_edma_cc(struct dma_device *d)
127{
128 return container_of(d, struct edma_cc, dma_slave);
129}
130
131static inline struct edma_chan *to_edma_chan(struct dma_chan *c)
132{
133 return container_of(c, struct edma_chan, vchan.chan);
134}
135
136static inline struct edma_desc
137*to_edma_desc(struct dma_async_tx_descriptor *tx)
138{
139 return container_of(tx, struct edma_desc, vdesc.tx);
140}
141
142static void edma_desc_free(struct virt_dma_desc *vdesc)
143{
144 kfree(container_of(vdesc, struct edma_desc, vdesc));
145}
146
147/* Dispatch a queued descriptor to the controller (caller holds lock) */
148static void edma_execute(struct edma_chan *echan)
149{
Joel Fernandes53407062013-09-03 10:02:46 -0500150 struct virt_dma_desc *vdesc;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400151 struct edma_desc *edesc;
Joel Fernandes53407062013-09-03 10:02:46 -0500152 struct device *dev = echan->vchan.chan.device->dev;
153 int i, j, left, nslots;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400154
Joel Fernandes53407062013-09-03 10:02:46 -0500155 /* If either we processed all psets or we're still not started */
156 if (!echan->edesc ||
157 echan->edesc->pset_nr == echan->edesc->processed) {
158 /* Get next vdesc */
159 vdesc = vchan_next_desc(&echan->vchan);
160 if (!vdesc) {
161 echan->edesc = NULL;
162 return;
163 }
164 list_del(&vdesc->node);
165 echan->edesc = to_edma_desc(&vdesc->tx);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400166 }
167
Joel Fernandes53407062013-09-03 10:02:46 -0500168 edesc = echan->edesc;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400169
Joel Fernandes53407062013-09-03 10:02:46 -0500170 /* Find out how many left */
171 left = edesc->pset_nr - edesc->processed;
172 nslots = min(MAX_NR_SG, left);
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500173 edesc->sg_len = 0;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400174
175 /* Write descriptor PaRAM set(s) */
Joel Fernandes53407062013-09-03 10:02:46 -0500176 for (i = 0; i < nslots; i++) {
177 j = i + edesc->processed;
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500178 edma_write_slot(echan->slot[i], &edesc->pset[j].param);
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500179 edesc->sg_len += edesc->pset[j].len;
Peter Ujfalusi83bb3122014-04-14 14:42:02 +0300180 dev_vdbg(echan->vchan.chan.device->dev,
Matt Porterc2dde5f2012-08-22 21:09:34 -0400181 "\n pset[%d]:\n"
182 " chnum\t%d\n"
183 " slot\t%d\n"
184 " opt\t%08x\n"
185 " src\t%08x\n"
186 " dst\t%08x\n"
187 " abcnt\t%08x\n"
188 " ccnt\t%08x\n"
189 " bidx\t%08x\n"
190 " cidx\t%08x\n"
191 " lkrld\t%08x\n",
Joel Fernandes53407062013-09-03 10:02:46 -0500192 j, echan->ch_num, echan->slot[i],
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500193 edesc->pset[j].param.opt,
194 edesc->pset[j].param.src,
195 edesc->pset[j].param.dst,
196 edesc->pset[j].param.a_b_cnt,
197 edesc->pset[j].param.ccnt,
198 edesc->pset[j].param.src_dst_bidx,
199 edesc->pset[j].param.src_dst_cidx,
200 edesc->pset[j].param.link_bcntrld);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400201 /* Link to the previous slot if not the last set */
Joel Fernandes53407062013-09-03 10:02:46 -0500202 if (i != (nslots - 1))
Matt Porterc2dde5f2012-08-22 21:09:34 -0400203 edma_link(echan->slot[i], echan->slot[i+1]);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400204 }
205
Joel Fernandes53407062013-09-03 10:02:46 -0500206 edesc->processed += nslots;
207
Joel Fernandesb267b3b2013-08-29 18:05:44 -0500208 /*
209 * If this is either the last set in a set of SG-list transactions
210 * then setup a link to the dummy slot, this results in all future
211 * events being absorbed and that's OK because we're done
212 */
Joel Fernandes50a9c702013-10-31 16:31:23 -0500213 if (edesc->processed == edesc->pset_nr) {
214 if (edesc->cyclic)
215 edma_link(echan->slot[nslots-1], echan->slot[1]);
216 else
217 edma_link(echan->slot[nslots-1],
218 echan->ecc->dummy_slot);
219 }
Joel Fernandesb267b3b2013-08-29 18:05:44 -0500220
Joel Fernandes53407062013-09-03 10:02:46 -0500221 if (edesc->processed <= MAX_NR_SG) {
Peter Ujfalusi9aac9092014-04-24 10:29:50 +0300222 dev_dbg(dev, "first transfer starting on channel %d\n",
223 echan->ch_num);
Joel Fernandes53407062013-09-03 10:02:46 -0500224 edma_start(echan->ch_num);
Sekhar Nori5fc68a62014-03-19 11:25:50 +0530225 } else {
226 dev_dbg(dev, "chan: %d: completed %d elements, resuming\n",
227 echan->ch_num, edesc->processed);
228 edma_resume(echan->ch_num);
Joel Fernandes53407062013-09-03 10:02:46 -0500229 }
Joel Fernandesc5f47992013-08-29 18:05:43 -0500230
231 /*
232 * This happens due to setup times between intermediate transfers
233 * in long SG lists which have to be broken up into transfers of
234 * MAX_NR_SG
235 */
236 if (echan->missed) {
Peter Ujfalusi9aac9092014-04-24 10:29:50 +0300237 dev_dbg(dev, "missed event on channel %d\n", echan->ch_num);
Joel Fernandesc5f47992013-08-29 18:05:43 -0500238 edma_clean_channel(echan->ch_num);
239 edma_stop(echan->ch_num);
240 edma_start(echan->ch_num);
241 edma_trigger_channel(echan->ch_num);
242 echan->missed = 0;
243 }
Matt Porterc2dde5f2012-08-22 21:09:34 -0400244}
245
246static int edma_terminate_all(struct edma_chan *echan)
247{
248 unsigned long flags;
249 LIST_HEAD(head);
250
251 spin_lock_irqsave(&echan->vchan.lock, flags);
252
253 /*
254 * Stop DMA activity: we assume the callback will not be called
255 * after edma_dma() returns (even if it does, it will see
256 * echan->edesc is NULL and exit.)
257 */
258 if (echan->edesc) {
259 echan->edesc = NULL;
260 edma_stop(echan->ch_num);
261 }
262
263 vchan_get_all_descriptors(&echan->vchan, &head);
264 spin_unlock_irqrestore(&echan->vchan.lock, flags);
265 vchan_dma_desc_free_list(&echan->vchan, &head);
266
267 return 0;
268}
269
Matt Porterc2dde5f2012-08-22 21:09:34 -0400270static int edma_slave_config(struct edma_chan *echan,
Matt Porter661f7cb2013-01-10 13:41:04 -0500271 struct dma_slave_config *cfg)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400272{
Matt Porter661f7cb2013-01-10 13:41:04 -0500273 if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
274 cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400275 return -EINVAL;
276
Matt Porter661f7cb2013-01-10 13:41:04 -0500277 memcpy(&echan->cfg, cfg, sizeof(echan->cfg));
Matt Porterc2dde5f2012-08-22 21:09:34 -0400278
279 return 0;
280}
281
Peter Ujfalusi72c7b672014-04-14 14:41:59 +0300282static int edma_dma_pause(struct edma_chan *echan)
283{
284 /* Pause/Resume only allowed with cyclic mode */
285 if (!echan->edesc->cyclic)
286 return -EINVAL;
287
288 edma_pause(echan->ch_num);
289 return 0;
290}
291
292static int edma_dma_resume(struct edma_chan *echan)
293{
294 /* Pause/Resume only allowed with cyclic mode */
295 if (!echan->edesc->cyclic)
296 return -EINVAL;
297
298 edma_resume(echan->ch_num);
299 return 0;
300}
301
Matt Porterc2dde5f2012-08-22 21:09:34 -0400302static int edma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
303 unsigned long arg)
304{
305 int ret = 0;
306 struct dma_slave_config *config;
307 struct edma_chan *echan = to_edma_chan(chan);
308
309 switch (cmd) {
310 case DMA_TERMINATE_ALL:
311 edma_terminate_all(echan);
312 break;
313 case DMA_SLAVE_CONFIG:
314 config = (struct dma_slave_config *)arg;
315 ret = edma_slave_config(echan, config);
316 break;
Peter Ujfalusi72c7b672014-04-14 14:41:59 +0300317 case DMA_PAUSE:
318 ret = edma_dma_pause(echan);
319 break;
320
321 case DMA_RESUME:
322 ret = edma_dma_resume(echan);
323 break;
324
Matt Porterc2dde5f2012-08-22 21:09:34 -0400325 default:
326 ret = -ENOSYS;
327 }
328
329 return ret;
330}
331
Joel Fernandesfd009032013-09-23 18:05:13 -0500332/*
333 * A PaRAM set configuration abstraction used by other modes
334 * @chan: Channel who's PaRAM set we're configuring
335 * @pset: PaRAM set to initialize and setup.
336 * @src_addr: Source address of the DMA
337 * @dst_addr: Destination address of the DMA
338 * @burst: In units of dev_width, how much to send
339 * @dev_width: How much is the dev_width
340 * @dma_length: Total length of the DMA transfer
341 * @direction: Direction of the transfer
342 */
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500343static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset,
Joel Fernandesfd009032013-09-23 18:05:13 -0500344 dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst,
345 enum dma_slave_buswidth dev_width, unsigned int dma_length,
346 enum dma_transfer_direction direction)
347{
348 struct edma_chan *echan = to_edma_chan(chan);
349 struct device *dev = chan->device->dev;
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500350 struct edmacc_param *param = &epset->param;
Joel Fernandesfd009032013-09-23 18:05:13 -0500351 int acnt, bcnt, ccnt, cidx;
352 int src_bidx, dst_bidx, src_cidx, dst_cidx;
353 int absync;
354
355 acnt = dev_width;
Peter Ujfalusib2b617d2014-04-14 14:41:58 +0300356
357 /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */
358 if (!burst)
359 burst = 1;
Joel Fernandesfd009032013-09-23 18:05:13 -0500360 /*
361 * If the maxburst is equal to the fifo width, use
362 * A-synced transfers. This allows for large contiguous
363 * buffer transfers using only one PaRAM set.
364 */
365 if (burst == 1) {
366 /*
367 * For the A-sync case, bcnt and ccnt are the remainder
368 * and quotient respectively of the division of:
369 * (dma_length / acnt) by (SZ_64K -1). This is so
370 * that in case bcnt over flows, we have ccnt to use.
371 * Note: In A-sync tranfer only, bcntrld is used, but it
372 * only applies for sg_dma_len(sg) >= SZ_64K.
373 * In this case, the best way adopted is- bccnt for the
374 * first frame will be the remainder below. Then for
375 * every successive frame, bcnt will be SZ_64K-1. This
376 * is assured as bcntrld = 0xffff in end of function.
377 */
378 absync = false;
379 ccnt = dma_length / acnt / (SZ_64K - 1);
380 bcnt = dma_length / acnt - ccnt * (SZ_64K - 1);
381 /*
382 * If bcnt is non-zero, we have a remainder and hence an
383 * extra frame to transfer, so increment ccnt.
384 */
385 if (bcnt)
386 ccnt++;
387 else
388 bcnt = SZ_64K - 1;
389 cidx = acnt;
390 } else {
391 /*
392 * If maxburst is greater than the fifo address_width,
393 * use AB-synced transfers where A count is the fifo
394 * address_width and B count is the maxburst. In this
395 * case, we are limited to transfers of C count frames
396 * of (address_width * maxburst) where C count is limited
397 * to SZ_64K-1. This places an upper bound on the length
398 * of an SG segment that can be handled.
399 */
400 absync = true;
401 bcnt = burst;
402 ccnt = dma_length / (acnt * bcnt);
403 if (ccnt > (SZ_64K - 1)) {
404 dev_err(dev, "Exceeded max SG segment size\n");
405 return -EINVAL;
406 }
407 cidx = acnt * bcnt;
408 }
409
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500410 epset->len = dma_length;
411
Joel Fernandesfd009032013-09-23 18:05:13 -0500412 if (direction == DMA_MEM_TO_DEV) {
413 src_bidx = acnt;
414 src_cidx = cidx;
415 dst_bidx = 0;
416 dst_cidx = 0;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500417 epset->addr = src_addr;
Joel Fernandesfd009032013-09-23 18:05:13 -0500418 } else if (direction == DMA_DEV_TO_MEM) {
419 src_bidx = 0;
420 src_cidx = 0;
421 dst_bidx = acnt;
422 dst_cidx = cidx;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500423 epset->addr = dst_addr;
Joel Fernandes8cc3e302014-04-18 21:50:33 -0500424 } else if (direction == DMA_MEM_TO_MEM) {
425 src_bidx = acnt;
426 src_cidx = cidx;
427 dst_bidx = acnt;
428 dst_cidx = cidx;
Joel Fernandesfd009032013-09-23 18:05:13 -0500429 } else {
430 dev_err(dev, "%s: direction not implemented yet\n", __func__);
431 return -EINVAL;
432 }
433
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500434 param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num));
Joel Fernandesfd009032013-09-23 18:05:13 -0500435 /* Configure A or AB synchronized transfers */
436 if (absync)
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500437 param->opt |= SYNCDIM;
Joel Fernandesfd009032013-09-23 18:05:13 -0500438
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500439 param->src = src_addr;
440 param->dst = dst_addr;
Joel Fernandesfd009032013-09-23 18:05:13 -0500441
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500442 param->src_dst_bidx = (dst_bidx << 16) | src_bidx;
443 param->src_dst_cidx = (dst_cidx << 16) | src_cidx;
Joel Fernandesfd009032013-09-23 18:05:13 -0500444
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500445 param->a_b_cnt = bcnt << 16 | acnt;
446 param->ccnt = ccnt;
Joel Fernandesfd009032013-09-23 18:05:13 -0500447 /*
448 * Only time when (bcntrld) auto reload is required is for
449 * A-sync case, and in this case, a requirement of reload value
450 * of SZ_64K-1 only is assured. 'link' is initially set to NULL
451 * and then later will be populated by edma_execute.
452 */
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500453 param->link_bcntrld = 0xffffffff;
Joel Fernandesfd009032013-09-23 18:05:13 -0500454 return absync;
455}
456
Matt Porterc2dde5f2012-08-22 21:09:34 -0400457static struct dma_async_tx_descriptor *edma_prep_slave_sg(
458 struct dma_chan *chan, struct scatterlist *sgl,
459 unsigned int sg_len, enum dma_transfer_direction direction,
460 unsigned long tx_flags, void *context)
461{
462 struct edma_chan *echan = to_edma_chan(chan);
463 struct device *dev = chan->device->dev;
464 struct edma_desc *edesc;
Joel Fernandesfd009032013-09-23 18:05:13 -0500465 dma_addr_t src_addr = 0, dst_addr = 0;
Matt Porter661f7cb2013-01-10 13:41:04 -0500466 enum dma_slave_buswidth dev_width;
467 u32 burst;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400468 struct scatterlist *sg;
Joel Fernandesfd009032013-09-23 18:05:13 -0500469 int i, nslots, ret;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400470
471 if (unlikely(!echan || !sgl || !sg_len))
472 return NULL;
473
Matt Porter661f7cb2013-01-10 13:41:04 -0500474 if (direction == DMA_DEV_TO_MEM) {
Joel Fernandesfd009032013-09-23 18:05:13 -0500475 src_addr = echan->cfg.src_addr;
Matt Porter661f7cb2013-01-10 13:41:04 -0500476 dev_width = echan->cfg.src_addr_width;
477 burst = echan->cfg.src_maxburst;
478 } else if (direction == DMA_MEM_TO_DEV) {
Joel Fernandesfd009032013-09-23 18:05:13 -0500479 dst_addr = echan->cfg.dst_addr;
Matt Porter661f7cb2013-01-10 13:41:04 -0500480 dev_width = echan->cfg.dst_addr_width;
481 burst = echan->cfg.dst_maxburst;
482 } else {
Peter Ujfalusie6fad592014-04-14 14:42:05 +0300483 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
Matt Porter661f7cb2013-01-10 13:41:04 -0500484 return NULL;
485 }
486
487 if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
Peter Ujfalusic594c892014-04-14 14:42:03 +0300488 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400489 return NULL;
490 }
491
Matt Porterc2dde5f2012-08-22 21:09:34 -0400492 edesc = kzalloc(sizeof(*edesc) + sg_len *
493 sizeof(edesc->pset[0]), GFP_ATOMIC);
494 if (!edesc) {
Peter Ujfalusic594c892014-04-14 14:42:03 +0300495 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400496 return NULL;
497 }
498
499 edesc->pset_nr = sg_len;
Thomas Gleixnerb6205c32014-04-28 14:18:45 -0500500 edesc->residue = 0;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500501 edesc->direction = direction;
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500502 edesc->echan = echan;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400503
Joel Fernandes6fbe24d2013-08-29 18:05:40 -0500504 /* Allocate a PaRAM slot, if needed */
505 nslots = min_t(unsigned, MAX_NR_SG, sg_len);
506
507 for (i = 0; i < nslots; i++) {
Matt Porterc2dde5f2012-08-22 21:09:34 -0400508 if (echan->slot[i] < 0) {
509 echan->slot[i] =
510 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
511 EDMA_SLOT_ANY);
512 if (echan->slot[i] < 0) {
Valentin Ilie4b6271a2013-10-24 16:14:22 +0300513 kfree(edesc);
Peter Ujfalusic594c892014-04-14 14:42:03 +0300514 dev_err(dev, "%s: Failed to allocate slot\n",
515 __func__);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400516 return NULL;
517 }
518 }
Joel Fernandes6fbe24d2013-08-29 18:05:40 -0500519 }
520
521 /* Configure PaRAM sets for each SG */
522 for_each_sg(sgl, sg, sg_len, i) {
Joel Fernandesfd009032013-09-23 18:05:13 -0500523 /* Get address for each SG */
524 if (direction == DMA_DEV_TO_MEM)
525 dst_addr = sg_dma_address(sg);
526 else
527 src_addr = sg_dma_address(sg);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400528
Joel Fernandesfd009032013-09-23 18:05:13 -0500529 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
530 dst_addr, burst, dev_width,
531 sg_dma_len(sg), direction);
Vinod Koulb967aec2013-10-30 13:07:18 +0530532 if (ret < 0) {
533 kfree(edesc);
Joel Fernandesfd009032013-09-23 18:05:13 -0500534 return NULL;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400535 }
536
Joel Fernandesfd009032013-09-23 18:05:13 -0500537 edesc->absync = ret;
Thomas Gleixnerb6205c32014-04-28 14:18:45 -0500538 edesc->residue += sg_dma_len(sg);
Joel Fernandes6fbe24d2013-08-29 18:05:40 -0500539
540 /* If this is the last in a current SG set of transactions,
541 enable interrupts so that next set is processed */
542 if (!((i+1) % MAX_NR_SG))
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500543 edesc->pset[i].param.opt |= TCINTEN;
Joel Fernandes6fbe24d2013-08-29 18:05:40 -0500544
Matt Porterc2dde5f2012-08-22 21:09:34 -0400545 /* If this is the last set, enable completion interrupt flag */
546 if (i == sg_len - 1)
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500547 edesc->pset[i].param.opt |= TCINTEN;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400548 }
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500549 edesc->residue_stat = edesc->residue;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400550
Matt Porterc2dde5f2012-08-22 21:09:34 -0400551 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
552}
Matt Porterc2dde5f2012-08-22 21:09:34 -0400553
Joel Fernandes8cc3e302014-04-18 21:50:33 -0500554struct dma_async_tx_descriptor *edma_prep_dma_memcpy(
555 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
556 size_t len, unsigned long tx_flags)
557{
558 int ret;
559 struct edma_desc *edesc;
560 struct device *dev = chan->device->dev;
561 struct edma_chan *echan = to_edma_chan(chan);
562
563 if (unlikely(!echan || !len))
564 return NULL;
565
566 edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC);
567 if (!edesc) {
568 dev_dbg(dev, "Failed to allocate a descriptor\n");
569 return NULL;
570 }
571
572 edesc->pset_nr = 1;
573
574 ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,
575 DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM);
576 if (ret < 0)
577 return NULL;
578
579 edesc->absync = ret;
580
581 /*
582 * Enable intermediate transfer chaining to re-trigger channel
583 * on completion of every TR, and enable transfer-completion
584 * interrupt on completion of the whole transfer.
585 */
Joel Fernandesb0cce4c2014-04-28 15:30:32 -0500586 edesc->pset[0].param.opt |= ITCCHEN;
587 edesc->pset[0].param.opt |= TCINTEN;
Joel Fernandes8cc3e302014-04-18 21:50:33 -0500588
589 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
590}
591
Joel Fernandes50a9c702013-10-31 16:31:23 -0500592static struct dma_async_tx_descriptor *edma_prep_dma_cyclic(
593 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
594 size_t period_len, enum dma_transfer_direction direction,
595 unsigned long tx_flags, void *context)
596{
597 struct edma_chan *echan = to_edma_chan(chan);
598 struct device *dev = chan->device->dev;
599 struct edma_desc *edesc;
600 dma_addr_t src_addr, dst_addr;
601 enum dma_slave_buswidth dev_width;
602 u32 burst;
603 int i, ret, nslots;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400604
Joel Fernandes50a9c702013-10-31 16:31:23 -0500605 if (unlikely(!echan || !buf_len || !period_len))
606 return NULL;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400607
Joel Fernandes50a9c702013-10-31 16:31:23 -0500608 if (direction == DMA_DEV_TO_MEM) {
609 src_addr = echan->cfg.src_addr;
610 dst_addr = buf_addr;
611 dev_width = echan->cfg.src_addr_width;
612 burst = echan->cfg.src_maxburst;
613 } else if (direction == DMA_MEM_TO_DEV) {
614 src_addr = buf_addr;
615 dst_addr = echan->cfg.dst_addr;
616 dev_width = echan->cfg.dst_addr_width;
617 burst = echan->cfg.dst_maxburst;
618 } else {
Peter Ujfalusie6fad592014-04-14 14:42:05 +0300619 dev_err(dev, "%s: bad direction: %d\n", __func__, direction);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500620 return NULL;
621 }
622
623 if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) {
Peter Ujfalusic594c892014-04-14 14:42:03 +0300624 dev_err(dev, "%s: Undefined slave buswidth\n", __func__);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500625 return NULL;
626 }
627
628 if (unlikely(buf_len % period_len)) {
629 dev_err(dev, "Period should be multiple of Buffer length\n");
630 return NULL;
631 }
632
633 nslots = (buf_len / period_len) + 1;
634
635 /*
636 * Cyclic DMA users such as audio cannot tolerate delays introduced
637 * by cases where the number of periods is more than the maximum
638 * number of SGs the EDMA driver can handle at a time. For DMA types
639 * such as Slave SGs, such delays are tolerable and synchronized,
640 * but the synchronization is difficult to achieve with Cyclic and
641 * cannot be guaranteed, so we error out early.
642 */
643 if (nslots > MAX_NR_SG)
644 return NULL;
645
646 edesc = kzalloc(sizeof(*edesc) + nslots *
647 sizeof(edesc->pset[0]), GFP_ATOMIC);
648 if (!edesc) {
Peter Ujfalusic594c892014-04-14 14:42:03 +0300649 dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500650 return NULL;
651 }
652
653 edesc->cyclic = 1;
654 edesc->pset_nr = nslots;
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500655 edesc->residue = edesc->residue_stat = buf_len;
Thomas Gleixnerc2da2342014-04-28 14:29:57 -0500656 edesc->direction = direction;
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500657 edesc->echan = echan;
Joel Fernandes50a9c702013-10-31 16:31:23 -0500658
Peter Ujfalusi83bb3122014-04-14 14:42:02 +0300659 dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n",
660 __func__, echan->ch_num, nslots, period_len, buf_len);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500661
662 for (i = 0; i < nslots; i++) {
663 /* Allocate a PaRAM slot, if needed */
664 if (echan->slot[i] < 0) {
665 echan->slot[i] =
666 edma_alloc_slot(EDMA_CTLR(echan->ch_num),
667 EDMA_SLOT_ANY);
668 if (echan->slot[i] < 0) {
Christian Engelmayere3ddc972013-12-30 20:48:39 +0100669 kfree(edesc);
Peter Ujfalusic594c892014-04-14 14:42:03 +0300670 dev_err(dev, "%s: Failed to allocate slot\n",
671 __func__);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500672 return NULL;
673 }
674 }
675
676 if (i == nslots - 1) {
677 memcpy(&edesc->pset[i], &edesc->pset[0],
678 sizeof(edesc->pset[0]));
679 break;
680 }
681
682 ret = edma_config_pset(chan, &edesc->pset[i], src_addr,
683 dst_addr, burst, dev_width, period_len,
684 direction);
Christian Engelmayere3ddc972013-12-30 20:48:39 +0100685 if (ret < 0) {
686 kfree(edesc);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500687 return NULL;
Christian Engelmayere3ddc972013-12-30 20:48:39 +0100688 }
Joel Fernandes50a9c702013-10-31 16:31:23 -0500689
690 if (direction == DMA_DEV_TO_MEM)
691 dst_addr += period_len;
692 else
693 src_addr += period_len;
694
Peter Ujfalusi83bb3122014-04-14 14:42:02 +0300695 dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i);
696 dev_vdbg(dev,
Joel Fernandes50a9c702013-10-31 16:31:23 -0500697 "\n pset[%d]:\n"
698 " chnum\t%d\n"
699 " slot\t%d\n"
700 " opt\t%08x\n"
701 " src\t%08x\n"
702 " dst\t%08x\n"
703 " abcnt\t%08x\n"
704 " ccnt\t%08x\n"
705 " bidx\t%08x\n"
706 " cidx\t%08x\n"
707 " lkrld\t%08x\n",
708 i, echan->ch_num, echan->slot[i],
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500709 edesc->pset[i].param.opt,
710 edesc->pset[i].param.src,
711 edesc->pset[i].param.dst,
712 edesc->pset[i].param.a_b_cnt,
713 edesc->pset[i].param.ccnt,
714 edesc->pset[i].param.src_dst_bidx,
715 edesc->pset[i].param.src_dst_cidx,
716 edesc->pset[i].param.link_bcntrld);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500717
718 edesc->absync = ret;
719
720 /*
721 * Enable interrupts for every period because callback
722 * has to be called for every period.
723 */
Thomas Gleixnerb5088ad2014-04-28 14:23:55 -0500724 edesc->pset[i].param.opt |= TCINTEN;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400725 }
726
727 return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);
728}
729
730static void edma_callback(unsigned ch_num, u16 ch_status, void *data)
731{
732 struct edma_chan *echan = data;
733 struct device *dev = echan->vchan.chan.device->dev;
734 struct edma_desc *edesc;
Joel Fernandesc5f47992013-08-29 18:05:43 -0500735 struct edmacc_param p;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400736
Joel Fernandes50a9c702013-10-31 16:31:23 -0500737 edesc = echan->edesc;
738
739 /* Pause the channel for non-cyclic */
740 if (!edesc || (edesc && !edesc->cyclic))
741 edma_pause(echan->ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400742
743 switch (ch_status) {
Vinod Kouldb60d8d2013-10-30 18:22:30 +0530744 case EDMA_DMA_COMPLETE:
Joel Fernandes406efb12014-04-17 00:58:33 -0500745 spin_lock(&echan->vchan.lock);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400746
Matt Porterc2dde5f2012-08-22 21:09:34 -0400747 if (edesc) {
Joel Fernandes50a9c702013-10-31 16:31:23 -0500748 if (edesc->cyclic) {
749 vchan_cyclic_callback(&edesc->vdesc);
750 } else if (edesc->processed == edesc->pset_nr) {
Joel Fernandes53407062013-09-03 10:02:46 -0500751 dev_dbg(dev, "Transfer complete, stopping channel %d\n", ch_num);
Thomas Gleixnerb6205c32014-04-28 14:18:45 -0500752 edesc->residue = 0;
Joel Fernandes53407062013-09-03 10:02:46 -0500753 edma_stop(echan->ch_num);
754 vchan_cookie_complete(&edesc->vdesc);
Joel Fernandes50a9c702013-10-31 16:31:23 -0500755 edma_execute(echan);
Joel Fernandes53407062013-09-03 10:02:46 -0500756 } else {
757 dev_dbg(dev, "Intermediate transfer complete on channel %d\n", ch_num);
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500758
759 /* Update statistics for tx_status */
760 edesc->residue -= edesc->sg_len;
761 edesc->residue_stat = edesc->residue;
762 edesc->processed_stat = edesc->processed;
763
Joel Fernandes50a9c702013-10-31 16:31:23 -0500764 edma_execute(echan);
Joel Fernandes53407062013-09-03 10:02:46 -0500765 }
Matt Porterc2dde5f2012-08-22 21:09:34 -0400766 }
767
Joel Fernandes406efb12014-04-17 00:58:33 -0500768 spin_unlock(&echan->vchan.lock);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400769
770 break;
Vinod Kouldb60d8d2013-10-30 18:22:30 +0530771 case EDMA_DMA_CC_ERROR:
Joel Fernandes406efb12014-04-17 00:58:33 -0500772 spin_lock(&echan->vchan.lock);
Joel Fernandesc5f47992013-08-29 18:05:43 -0500773
774 edma_read_slot(EDMA_CHAN_SLOT(echan->slot[0]), &p);
775
776 /*
777 * Issue later based on missed flag which will be sure
778 * to happen as:
779 * (1) we finished transmitting an intermediate slot and
780 * edma_execute is coming up.
781 * (2) or we finished current transfer and issue will
782 * call edma_execute.
783 *
784 * Important note: issuing can be dangerous here and
785 * lead to some nasty recursion when we are in a NULL
786 * slot. So we avoid doing so and set the missed flag.
787 */
788 if (p.a_b_cnt == 0 && p.ccnt == 0) {
789 dev_dbg(dev, "Error occurred, looks like slot is null, just setting miss\n");
790 echan->missed = 1;
791 } else {
792 /*
793 * The slot is already programmed but the event got
794 * missed, so its safe to issue it here.
795 */
796 dev_dbg(dev, "Error occurred but slot is non-null, TRIGGERING\n");
797 edma_clean_channel(echan->ch_num);
798 edma_stop(echan->ch_num);
799 edma_start(echan->ch_num);
800 edma_trigger_channel(echan->ch_num);
801 }
802
Joel Fernandes406efb12014-04-17 00:58:33 -0500803 spin_unlock(&echan->vchan.lock);
Joel Fernandesc5f47992013-08-29 18:05:43 -0500804
Matt Porterc2dde5f2012-08-22 21:09:34 -0400805 break;
806 default:
807 break;
808 }
809}
810
811/* Alloc channel resources */
812static int edma_alloc_chan_resources(struct dma_chan *chan)
813{
814 struct edma_chan *echan = to_edma_chan(chan);
815 struct device *dev = chan->device->dev;
816 int ret;
817 int a_ch_num;
818 LIST_HEAD(descs);
819
820 a_ch_num = edma_alloc_channel(echan->ch_num, edma_callback,
821 chan, EVENTQ_DEFAULT);
822
823 if (a_ch_num < 0) {
824 ret = -ENODEV;
825 goto err_no_chan;
826 }
827
828 if (a_ch_num != echan->ch_num) {
829 dev_err(dev, "failed to allocate requested channel %u:%u\n",
830 EDMA_CTLR(echan->ch_num),
831 EDMA_CHAN_SLOT(echan->ch_num));
832 ret = -ENODEV;
833 goto err_wrong_chan;
834 }
835
836 echan->alloced = true;
837 echan->slot[0] = echan->ch_num;
838
Peter Ujfalusi9aac9092014-04-24 10:29:50 +0300839 dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num,
Ezequiel Garcia0e772c62013-12-13 11:06:18 -0300840 EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num));
Matt Porterc2dde5f2012-08-22 21:09:34 -0400841
842 return 0;
843
844err_wrong_chan:
845 edma_free_channel(a_ch_num);
846err_no_chan:
847 return ret;
848}
849
850/* Free channel resources */
851static void edma_free_chan_resources(struct dma_chan *chan)
852{
853 struct edma_chan *echan = to_edma_chan(chan);
854 struct device *dev = chan->device->dev;
855 int i;
856
857 /* Terminate transfers */
858 edma_stop(echan->ch_num);
859
860 vchan_free_chan_resources(&echan->vchan);
861
862 /* Free EDMA PaRAM slots */
863 for (i = 1; i < EDMA_MAX_SLOTS; i++) {
864 if (echan->slot[i] >= 0) {
865 edma_free_slot(echan->slot[i]);
866 echan->slot[i] = -1;
867 }
868 }
869
870 /* Free EDMA channel */
871 if (echan->alloced) {
872 edma_free_channel(echan->ch_num);
873 echan->alloced = false;
874 }
875
Ezequiel Garcia0e772c62013-12-13 11:06:18 -0300876 dev_dbg(dev, "freeing channel for %u\n", echan->ch_num);
Matt Porterc2dde5f2012-08-22 21:09:34 -0400877}
878
879/* Send pending descriptor to hardware */
880static void edma_issue_pending(struct dma_chan *chan)
881{
882 struct edma_chan *echan = to_edma_chan(chan);
883 unsigned long flags;
884
885 spin_lock_irqsave(&echan->vchan.lock, flags);
886 if (vchan_issue_pending(&echan->vchan) && !echan->edesc)
887 edma_execute(echan);
888 spin_unlock_irqrestore(&echan->vchan.lock, flags);
889}
890
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500891static u32 edma_residue(struct edma_desc *edesc)
892{
893 bool dst = edesc->direction == DMA_DEV_TO_MEM;
894 struct edma_pset *pset = edesc->pset;
895 dma_addr_t done, pos;
896 int i;
897
898 /*
899 * We always read the dst/src position from the first RamPar
900 * pset. That's the one which is active now.
901 */
902 pos = edma_get_position(edesc->echan->slot[0], dst);
903
904 /*
905 * Cyclic is simple. Just subtract pset[0].addr from pos.
906 *
907 * We never update edesc->residue in the cyclic case, so we
908 * can tell the remaining room to the end of the circular
909 * buffer.
910 */
911 if (edesc->cyclic) {
912 done = pos - pset->addr;
913 edesc->residue_stat = edesc->residue - done;
914 return edesc->residue_stat;
915 }
916
917 /*
918 * For SG operation we catch up with the last processed
919 * status.
920 */
921 pset += edesc->processed_stat;
922
923 for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {
924 /*
925 * If we are inside this pset address range, we know
926 * this is the active one. Get the current delta and
927 * stop walking the psets.
928 */
929 if (pos >= pset->addr && pos < pset->addr + pset->len)
930 return edesc->residue_stat - (pos - pset->addr);
931
932 /* Otherwise mark it done and update residue_stat. */
933 edesc->processed_stat++;
934 edesc->residue_stat -= pset->len;
935 }
936 return edesc->residue_stat;
937}
938
Matt Porterc2dde5f2012-08-22 21:09:34 -0400939/* Check request completion status */
940static enum dma_status edma_tx_status(struct dma_chan *chan,
941 dma_cookie_t cookie,
942 struct dma_tx_state *txstate)
943{
944 struct edma_chan *echan = to_edma_chan(chan);
945 struct virt_dma_desc *vdesc;
946 enum dma_status ret;
947 unsigned long flags;
948
949 ret = dma_cookie_status(chan, cookie, txstate);
Vinod Koul9d386ec2013-10-16 13:42:15 +0530950 if (ret == DMA_COMPLETE || !txstate)
Matt Porterc2dde5f2012-08-22 21:09:34 -0400951 return ret;
952
953 spin_lock_irqsave(&echan->vchan.lock, flags);
Thomas Gleixnerde135932014-04-28 14:19:51 -0500954 if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie)
Thomas Gleixner740b41f2014-04-28 14:34:11 -0500955 txstate->residue = edma_residue(echan->edesc);
Thomas Gleixnerde135932014-04-28 14:19:51 -0500956 else if ((vdesc = vchan_find_desc(&echan->vchan, cookie)))
957 txstate->residue = to_edma_desc(&vdesc->tx)->residue;
Matt Porterc2dde5f2012-08-22 21:09:34 -0400958 spin_unlock_irqrestore(&echan->vchan.lock, flags);
959
960 return ret;
961}
962
963static void __init edma_chan_init(struct edma_cc *ecc,
964 struct dma_device *dma,
965 struct edma_chan *echans)
966{
967 int i, j;
968
969 for (i = 0; i < EDMA_CHANS; i++) {
970 struct edma_chan *echan = &echans[i];
971 echan->ch_num = EDMA_CTLR_CHAN(ecc->ctlr, i);
972 echan->ecc = ecc;
973 echan->vchan.desc_free = edma_desc_free;
974
975 vchan_init(&echan->vchan, dma);
976
977 INIT_LIST_HEAD(&echan->node);
978 for (j = 0; j < EDMA_MAX_SLOTS; j++)
979 echan->slot[j] = -1;
980 }
981}
982
Peter Ujfalusi2c88ee62014-04-14 14:42:01 +0300983#define EDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
984 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
985 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
986
987static int edma_dma_device_slave_caps(struct dma_chan *dchan,
988 struct dma_slave_caps *caps)
989{
990 caps->src_addr_widths = EDMA_DMA_BUSWIDTHS;
991 caps->dstn_addr_widths = EDMA_DMA_BUSWIDTHS;
992 caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
993 caps->cmd_pause = true;
994 caps->cmd_terminate = true;
995 caps->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
996
997 return 0;
998}
999
Matt Porterc2dde5f2012-08-22 21:09:34 -04001000static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma,
1001 struct device *dev)
1002{
1003 dma->device_prep_slave_sg = edma_prep_slave_sg;
Joel Fernandes50a9c702013-10-31 16:31:23 -05001004 dma->device_prep_dma_cyclic = edma_prep_dma_cyclic;
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001005 dma->device_prep_dma_memcpy = edma_prep_dma_memcpy;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001006 dma->device_alloc_chan_resources = edma_alloc_chan_resources;
1007 dma->device_free_chan_resources = edma_free_chan_resources;
1008 dma->device_issue_pending = edma_issue_pending;
1009 dma->device_tx_status = edma_tx_status;
1010 dma->device_control = edma_control;
Peter Ujfalusi2c88ee62014-04-14 14:42:01 +03001011 dma->device_slave_caps = edma_dma_device_slave_caps;
Matt Porterc2dde5f2012-08-22 21:09:34 -04001012 dma->dev = dev;
1013
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001014 /*
1015 * code using dma memcpy must make sure alignment of
1016 * length is at dma->copy_align boundary.
1017 */
1018 dma->copy_align = DMA_SLAVE_BUSWIDTH_4_BYTES;
1019
Matt Porterc2dde5f2012-08-22 21:09:34 -04001020 INIT_LIST_HEAD(&dma->channels);
1021}
1022
Bill Pemberton463a1f82012-11-19 13:22:55 -05001023static int edma_probe(struct platform_device *pdev)
Matt Porterc2dde5f2012-08-22 21:09:34 -04001024{
1025 struct edma_cc *ecc;
1026 int ret;
1027
Russell King94cb0e72013-06-27 13:45:16 +01001028 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1029 if (ret)
1030 return ret;
1031
Matt Porterc2dde5f2012-08-22 21:09:34 -04001032 ecc = devm_kzalloc(&pdev->dev, sizeof(*ecc), GFP_KERNEL);
1033 if (!ecc) {
1034 dev_err(&pdev->dev, "Can't allocate controller\n");
1035 return -ENOMEM;
1036 }
1037
1038 ecc->ctlr = pdev->id;
1039 ecc->dummy_slot = edma_alloc_slot(ecc->ctlr, EDMA_SLOT_ANY);
1040 if (ecc->dummy_slot < 0) {
1041 dev_err(&pdev->dev, "Can't allocate PaRAM dummy slot\n");
1042 return -EIO;
1043 }
1044
1045 dma_cap_zero(ecc->dma_slave.cap_mask);
1046 dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask);
Peter Ujfalusi232b223d2014-04-14 14:42:00 +03001047 dma_cap_set(DMA_CYCLIC, ecc->dma_slave.cap_mask);
Joel Fernandes8cc3e302014-04-18 21:50:33 -05001048 dma_cap_set(DMA_MEMCPY, ecc->dma_slave.cap_mask);
Matt Porterc2dde5f2012-08-22 21:09:34 -04001049
1050 edma_dma_init(ecc, &ecc->dma_slave, &pdev->dev);
1051
1052 edma_chan_init(ecc, &ecc->dma_slave, ecc->slave_chans);
1053
1054 ret = dma_async_device_register(&ecc->dma_slave);
1055 if (ret)
1056 goto err_reg1;
1057
1058 platform_set_drvdata(pdev, ecc);
1059
1060 dev_info(&pdev->dev, "TI EDMA DMA engine driver\n");
1061
1062 return 0;
1063
1064err_reg1:
1065 edma_free_slot(ecc->dummy_slot);
1066 return ret;
1067}
1068
Greg Kroah-Hartman4bf27b82012-12-21 15:09:59 -08001069static int edma_remove(struct platform_device *pdev)
Matt Porterc2dde5f2012-08-22 21:09:34 -04001070{
1071 struct device *dev = &pdev->dev;
1072 struct edma_cc *ecc = dev_get_drvdata(dev);
1073
1074 dma_async_device_unregister(&ecc->dma_slave);
1075 edma_free_slot(ecc->dummy_slot);
1076
1077 return 0;
1078}
1079
1080static struct platform_driver edma_driver = {
1081 .probe = edma_probe,
Bill Pembertona7d6e3e2012-11-19 13:20:04 -05001082 .remove = edma_remove,
Matt Porterc2dde5f2012-08-22 21:09:34 -04001083 .driver = {
1084 .name = "edma-dma-engine",
1085 .owner = THIS_MODULE,
1086 },
1087};
1088
1089bool edma_filter_fn(struct dma_chan *chan, void *param)
1090{
1091 if (chan->device->dev->driver == &edma_driver.driver) {
1092 struct edma_chan *echan = to_edma_chan(chan);
1093 unsigned ch_req = *(unsigned *)param;
1094 return ch_req == echan->ch_num;
1095 }
1096 return false;
1097}
1098EXPORT_SYMBOL(edma_filter_fn);
1099
1100static struct platform_device *pdev0, *pdev1;
1101
1102static const struct platform_device_info edma_dev_info0 = {
1103 .name = "edma-dma-engine",
1104 .id = 0,
Russell King94cb0e72013-06-27 13:45:16 +01001105 .dma_mask = DMA_BIT_MASK(32),
Matt Porterc2dde5f2012-08-22 21:09:34 -04001106};
1107
1108static const struct platform_device_info edma_dev_info1 = {
1109 .name = "edma-dma-engine",
1110 .id = 1,
Russell King94cb0e72013-06-27 13:45:16 +01001111 .dma_mask = DMA_BIT_MASK(32),
Matt Porterc2dde5f2012-08-22 21:09:34 -04001112};
1113
1114static int edma_init(void)
1115{
1116 int ret = platform_driver_register(&edma_driver);
1117
1118 if (ret == 0) {
1119 pdev0 = platform_device_register_full(&edma_dev_info0);
1120 if (IS_ERR(pdev0)) {
1121 platform_driver_unregister(&edma_driver);
1122 ret = PTR_ERR(pdev0);
1123 goto out;
1124 }
1125 }
1126
1127 if (EDMA_CTLRS == 2) {
1128 pdev1 = platform_device_register_full(&edma_dev_info1);
1129 if (IS_ERR(pdev1)) {
1130 platform_driver_unregister(&edma_driver);
1131 platform_device_unregister(pdev0);
1132 ret = PTR_ERR(pdev1);
1133 }
Matt Porterc2dde5f2012-08-22 21:09:34 -04001134 }
1135
1136out:
1137 return ret;
1138}
1139subsys_initcall(edma_init);
1140
1141static void __exit edma_exit(void)
1142{
1143 platform_device_unregister(pdev0);
1144 if (pdev1)
1145 platform_device_unregister(pdev1);
1146 platform_driver_unregister(&edma_driver);
1147}
1148module_exit(edma_exit);
1149
Josh Boyerd71505b2013-09-04 10:32:50 -04001150MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>");
Matt Porterc2dde5f2012-08-22 21:09:34 -04001151MODULE_DESCRIPTION("TI EDMA DMA engine driver");
1152MODULE_LICENSE("GPL v2");