blob: 4de4f3fd1b4737889072280c0aa82f8597672429 [file] [log] [blame]
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +08001/*
2 * Copyright 2012 Marvell International Ltd.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
Thierry Reding73312052013-01-21 11:09:00 +01008#include <linux/err.h>
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +08009#include <linux/module.h>
10#include <linux/init.h>
11#include <linux/types.h>
12#include <linux/interrupt.h>
13#include <linux/dma-mapping.h>
14#include <linux/slab.h>
15#include <linux/dmaengine.h>
16#include <linux/platform_device.h>
17#include <linux/device.h>
18#include <linux/platform_data/mmp_dma.h>
19#include <linux/dmapool.h>
20#include <linux/of_device.h>
Daniel Macka9a7cf02013-08-10 18:52:19 +020021#include <linux/of_dma.h>
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +080022#include <linux/of.h>
Daniel Mack13b30062013-08-10 18:52:18 +020023#include <linux/dma/mmp-pdma.h>
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +080024
25#include "dmaengine.h"
26
27#define DCSR 0x0000
28#define DALGN 0x00a0
29#define DINT 0x00f0
30#define DDADR 0x0200
31#define DSADR 0x0204
32#define DTADR 0x0208
33#define DCMD 0x020c
34
35#define DCSR_RUN (1 << 31) /* Run Bit (read / write) */
36#define DCSR_NODESC (1 << 30) /* No-Descriptor Fetch (read / write) */
37#define DCSR_STOPIRQEN (1 << 29) /* Stop Interrupt Enable (read / write) */
38#define DCSR_REQPEND (1 << 8) /* Request Pending (read-only) */
39#define DCSR_STOPSTATE (1 << 3) /* Stop State (read-only) */
40#define DCSR_ENDINTR (1 << 2) /* End Interrupt (read / write) */
41#define DCSR_STARTINTR (1 << 1) /* Start Interrupt (read / write) */
42#define DCSR_BUSERR (1 << 0) /* Bus Error Interrupt (read / write) */
43
44#define DCSR_EORIRQEN (1 << 28) /* End of Receive Interrupt Enable (R/W) */
45#define DCSR_EORJMPEN (1 << 27) /* Jump to next descriptor on EOR */
46#define DCSR_EORSTOPEN (1 << 26) /* STOP on an EOR */
47#define DCSR_SETCMPST (1 << 25) /* Set Descriptor Compare Status */
48#define DCSR_CLRCMPST (1 << 24) /* Clear Descriptor Compare Status */
49#define DCSR_CMPST (1 << 10) /* The Descriptor Compare Status */
50#define DCSR_EORINTR (1 << 9) /* The end of Receive */
51
Daniel Mack8b298de2013-08-10 18:52:15 +020052#define DRCMR(n) ((((n) < 64) ? 0x0100 : 0x1100) + \
53 (((n) & 0x3f) << 2))
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +080054#define DRCMR_MAPVLD (1 << 7) /* Map Valid (read / write) */
55#define DRCMR_CHLNUM 0x1f /* mask for Channel Number (read / write) */
56
57#define DDADR_DESCADDR 0xfffffff0 /* Address of next descriptor (mask) */
58#define DDADR_STOP (1 << 0) /* Stop (read / write) */
59
60#define DCMD_INCSRCADDR (1 << 31) /* Source Address Increment Setting. */
61#define DCMD_INCTRGADDR (1 << 30) /* Target Address Increment Setting. */
62#define DCMD_FLOWSRC (1 << 29) /* Flow Control by the source. */
63#define DCMD_FLOWTRG (1 << 28) /* Flow Control by the target. */
64#define DCMD_STARTIRQEN (1 << 22) /* Start Interrupt Enable */
65#define DCMD_ENDIRQEN (1 << 21) /* End Interrupt Enable */
66#define DCMD_ENDIAN (1 << 18) /* Device Endian-ness. */
67#define DCMD_BURST8 (1 << 16) /* 8 byte burst */
68#define DCMD_BURST16 (2 << 16) /* 16 byte burst */
69#define DCMD_BURST32 (3 << 16) /* 32 byte burst */
70#define DCMD_WIDTH1 (1 << 14) /* 1 byte width */
71#define DCMD_WIDTH2 (2 << 14) /* 2 byte width (HalfWord) */
72#define DCMD_WIDTH4 (3 << 14) /* 4 byte width (Word) */
73#define DCMD_LENGTH 0x01fff /* length mask (max = 8K - 1) */
74
75#define PDMA_ALIGNMENT 3
Daniel Mack1ac0e842013-08-10 18:52:17 +020076#define PDMA_MAX_DESC_BYTES DCMD_LENGTH
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +080077
78struct mmp_pdma_desc_hw {
79 u32 ddadr; /* Points to the next descriptor + flags */
80 u32 dsadr; /* DSADR value for the current transfer */
81 u32 dtadr; /* DTADR value for the current transfer */
82 u32 dcmd; /* DCMD value for the current transfer */
83} __aligned(32);
84
85struct mmp_pdma_desc_sw {
86 struct mmp_pdma_desc_hw desc;
87 struct list_head node;
88 struct list_head tx_list;
89 struct dma_async_tx_descriptor async_tx;
90};
91
92struct mmp_pdma_phy;
93
94struct mmp_pdma_chan {
95 struct device *dev;
96 struct dma_chan chan;
97 struct dma_async_tx_descriptor desc;
98 struct mmp_pdma_phy *phy;
99 enum dma_transfer_direction dir;
100
101 /* channel's basic info */
102 struct tasklet_struct tasklet;
103 u32 dcmd;
104 u32 drcmr;
105 u32 dev_addr;
106
107 /* list for desc */
108 spinlock_t desc_lock; /* Descriptor list lock */
109 struct list_head chain_pending; /* Link descriptors queue for pending */
110 struct list_head chain_running; /* Link descriptors queue for running */
111 bool idle; /* channel statue machine */
112
113 struct dma_pool *desc_pool; /* Descriptors pool */
114};
115
116struct mmp_pdma_phy {
117 int idx;
118 void __iomem *base;
119 struct mmp_pdma_chan *vchan;
120};
121
122struct mmp_pdma_device {
123 int dma_channels;
124 void __iomem *base;
125 struct device *dev;
126 struct dma_device device;
127 struct mmp_pdma_phy *phy;
Xiang Wang027f28b2013-06-18 14:55:58 +0800128 spinlock_t phy_lock; /* protect alloc/free phy channels */
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800129};
130
131#define tx_to_mmp_pdma_desc(tx) container_of(tx, struct mmp_pdma_desc_sw, async_tx)
132#define to_mmp_pdma_desc(lh) container_of(lh, struct mmp_pdma_desc_sw, node)
133#define to_mmp_pdma_chan(dchan) container_of(dchan, struct mmp_pdma_chan, chan)
134#define to_mmp_pdma_dev(dmadev) container_of(dmadev, struct mmp_pdma_device, device)
135
136static void set_desc(struct mmp_pdma_phy *phy, dma_addr_t addr)
137{
138 u32 reg = (phy->idx << 4) + DDADR;
139
140 writel(addr, phy->base + reg);
141}
142
143static void enable_chan(struct mmp_pdma_phy *phy)
144{
145 u32 reg;
146
147 if (!phy->vchan)
148 return;
149
Daniel Mack8b298de2013-08-10 18:52:15 +0200150 reg = DRCMR(phy->vchan->drcmr);
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800151 writel(DRCMR_MAPVLD | phy->idx, phy->base + reg);
152
153 reg = (phy->idx << 2) + DCSR;
154 writel(readl(phy->base + reg) | DCSR_RUN,
155 phy->base + reg);
156}
157
158static void disable_chan(struct mmp_pdma_phy *phy)
159{
160 u32 reg;
161
162 if (phy) {
163 reg = (phy->idx << 2) + DCSR;
164 writel(readl(phy->base + reg) & ~DCSR_RUN,
165 phy->base + reg);
166 }
167}
168
169static int clear_chan_irq(struct mmp_pdma_phy *phy)
170{
171 u32 dcsr;
172 u32 dint = readl(phy->base + DINT);
173 u32 reg = (phy->idx << 2) + DCSR;
174
175 if (dint & BIT(phy->idx)) {
176 /* clear irq */
177 dcsr = readl(phy->base + reg);
178 writel(dcsr, phy->base + reg);
179 if ((dcsr & DCSR_BUSERR) && (phy->vchan))
180 dev_warn(phy->vchan->dev, "DCSR_BUSERR\n");
181 return 0;
182 }
183 return -EAGAIN;
184}
185
186static irqreturn_t mmp_pdma_chan_handler(int irq, void *dev_id)
187{
188 struct mmp_pdma_phy *phy = dev_id;
189
190 if (clear_chan_irq(phy) == 0) {
191 tasklet_schedule(&phy->vchan->tasklet);
192 return IRQ_HANDLED;
193 } else
194 return IRQ_NONE;
195}
196
197static irqreturn_t mmp_pdma_int_handler(int irq, void *dev_id)
198{
199 struct mmp_pdma_device *pdev = dev_id;
200 struct mmp_pdma_phy *phy;
201 u32 dint = readl(pdev->base + DINT);
202 int i, ret;
203 int irq_num = 0;
204
205 while (dint) {
206 i = __ffs(dint);
207 dint &= (dint - 1);
208 phy = &pdev->phy[i];
209 ret = mmp_pdma_chan_handler(irq, phy);
210 if (ret == IRQ_HANDLED)
211 irq_num++;
212 }
213
214 if (irq_num)
215 return IRQ_HANDLED;
216 else
217 return IRQ_NONE;
218}
219
220/* lookup free phy channel as descending priority */
221static struct mmp_pdma_phy *lookup_phy(struct mmp_pdma_chan *pchan)
222{
223 int prio, i;
224 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device);
Daniel Mack638a5422013-08-10 18:52:16 +0200225 struct mmp_pdma_phy *phy, *found = NULL;
Xiang Wang027f28b2013-06-18 14:55:58 +0800226 unsigned long flags;
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800227
228 /*
229 * dma channel priorities
230 * ch 0 - 3, 16 - 19 <--> (0)
231 * ch 4 - 7, 20 - 23 <--> (1)
232 * ch 8 - 11, 24 - 27 <--> (2)
233 * ch 12 - 15, 28 - 31 <--> (3)
234 */
Xiang Wang027f28b2013-06-18 14:55:58 +0800235
236 spin_lock_irqsave(&pdev->phy_lock, flags);
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800237 for (prio = 0; prio <= (((pdev->dma_channels - 1) & 0xf) >> 2); prio++) {
238 for (i = 0; i < pdev->dma_channels; i++) {
239 if (prio != ((i & 0xf) >> 2))
240 continue;
241 phy = &pdev->phy[i];
242 if (!phy->vchan) {
243 phy->vchan = pchan;
Daniel Mack638a5422013-08-10 18:52:16 +0200244 found = phy;
245 goto out_unlock;
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800246 }
247 }
248 }
249
Daniel Mack638a5422013-08-10 18:52:16 +0200250out_unlock:
Xiang Wang027f28b2013-06-18 14:55:58 +0800251 spin_unlock_irqrestore(&pdev->phy_lock, flags);
Daniel Mack638a5422013-08-10 18:52:16 +0200252 return found;
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800253}
254
Xiang Wang027f28b2013-06-18 14:55:58 +0800255static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan)
256{
257 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device);
258 unsigned long flags;
Xiang Wang26a2dfd2013-06-18 14:55:59 +0800259 u32 reg;
Xiang Wang027f28b2013-06-18 14:55:58 +0800260
261 if (!pchan->phy)
262 return;
263
Xiang Wang26a2dfd2013-06-18 14:55:59 +0800264 /* clear the channel mapping in DRCMR */
Daniel Mack8b298de2013-08-10 18:52:15 +0200265 reg = DRCMR(pchan->phy->vchan->drcmr);
Xiang Wang26a2dfd2013-06-18 14:55:59 +0800266 writel(0, pchan->phy->base + reg);
267
Xiang Wang027f28b2013-06-18 14:55:58 +0800268 spin_lock_irqsave(&pdev->phy_lock, flags);
269 pchan->phy->vchan = NULL;
270 pchan->phy = NULL;
271 spin_unlock_irqrestore(&pdev->phy_lock, flags);
272}
273
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800274/* desc->tx_list ==> pending list */
275static void append_pending_queue(struct mmp_pdma_chan *chan,
276 struct mmp_pdma_desc_sw *desc)
277{
278 struct mmp_pdma_desc_sw *tail =
279 to_mmp_pdma_desc(chan->chain_pending.prev);
280
281 if (list_empty(&chan->chain_pending))
282 goto out_splice;
283
284 /* one irq per queue, even appended */
285 tail->desc.ddadr = desc->async_tx.phys;
286 tail->desc.dcmd &= ~DCMD_ENDIRQEN;
287
288 /* softly link to pending list */
289out_splice:
290 list_splice_tail_init(&desc->tx_list, &chan->chain_pending);
291}
292
293/**
294 * start_pending_queue - transfer any pending transactions
295 * pending list ==> running list
296 */
297static void start_pending_queue(struct mmp_pdma_chan *chan)
298{
299 struct mmp_pdma_desc_sw *desc;
300
301 /* still in running, irq will start the pending list */
302 if (!chan->idle) {
303 dev_dbg(chan->dev, "DMA controller still busy\n");
304 return;
305 }
306
307 if (list_empty(&chan->chain_pending)) {
308 /* chance to re-fetch phy channel with higher prio */
Xiang Wang027f28b2013-06-18 14:55:58 +0800309 mmp_pdma_free_phy(chan);
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800310 dev_dbg(chan->dev, "no pending list\n");
311 return;
312 }
313
314 if (!chan->phy) {
315 chan->phy = lookup_phy(chan);
316 if (!chan->phy) {
317 dev_dbg(chan->dev, "no free dma channel\n");
318 return;
319 }
320 }
321
322 /*
323 * pending -> running
324 * reintilize pending list
325 */
326 desc = list_first_entry(&chan->chain_pending,
327 struct mmp_pdma_desc_sw, node);
328 list_splice_tail_init(&chan->chain_pending, &chan->chain_running);
329
330 /*
331 * Program the descriptor's address into the DMA controller,
332 * then start the DMA transaction
333 */
334 set_desc(chan->phy, desc->async_tx.phys);
335 enable_chan(chan->phy);
336 chan->idle = false;
337}
338
339
340/* desc->tx_list ==> pending list */
341static dma_cookie_t mmp_pdma_tx_submit(struct dma_async_tx_descriptor *tx)
342{
343 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(tx->chan);
344 struct mmp_pdma_desc_sw *desc = tx_to_mmp_pdma_desc(tx);
345 struct mmp_pdma_desc_sw *child;
346 unsigned long flags;
347 dma_cookie_t cookie = -EBUSY;
348
349 spin_lock_irqsave(&chan->desc_lock, flags);
350
351 list_for_each_entry(child, &desc->tx_list, node) {
352 cookie = dma_cookie_assign(&child->async_tx);
353 }
354
355 append_pending_queue(chan, desc);
356
357 spin_unlock_irqrestore(&chan->desc_lock, flags);
358
359 return cookie;
360}
361
Jingoo Han69c9f0a2013-08-06 19:35:13 +0900362static struct mmp_pdma_desc_sw *
363mmp_pdma_alloc_descriptor(struct mmp_pdma_chan *chan)
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800364{
365 struct mmp_pdma_desc_sw *desc;
366 dma_addr_t pdesc;
367
368 desc = dma_pool_alloc(chan->desc_pool, GFP_ATOMIC, &pdesc);
369 if (!desc) {
370 dev_err(chan->dev, "out of memory for link descriptor\n");
371 return NULL;
372 }
373
374 memset(desc, 0, sizeof(*desc));
375 INIT_LIST_HEAD(&desc->tx_list);
376 dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan);
377 /* each desc has submit */
378 desc->async_tx.tx_submit = mmp_pdma_tx_submit;
379 desc->async_tx.phys = pdesc;
380
381 return desc;
382}
383
384/**
385 * mmp_pdma_alloc_chan_resources - Allocate resources for DMA channel.
386 *
387 * This function will create a dma pool for descriptor allocation.
388 * Request irq only when channel is requested
389 * Return - The number of allocated descriptors.
390 */
391
392static int mmp_pdma_alloc_chan_resources(struct dma_chan *dchan)
393{
394 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
395
396 if (chan->desc_pool)
397 return 1;
398
399 chan->desc_pool =
400 dma_pool_create(dev_name(&dchan->dev->device), chan->dev,
401 sizeof(struct mmp_pdma_desc_sw),
402 __alignof__(struct mmp_pdma_desc_sw), 0);
403 if (!chan->desc_pool) {
404 dev_err(chan->dev, "unable to allocate descriptor pool\n");
405 return -ENOMEM;
406 }
Xiang Wang027f28b2013-06-18 14:55:58 +0800407 mmp_pdma_free_phy(chan);
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800408 chan->idle = true;
409 chan->dev_addr = 0;
410 return 1;
411}
412
413static void mmp_pdma_free_desc_list(struct mmp_pdma_chan *chan,
414 struct list_head *list)
415{
416 struct mmp_pdma_desc_sw *desc, *_desc;
417
418 list_for_each_entry_safe(desc, _desc, list, node) {
419 list_del(&desc->node);
420 dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
421 }
422}
423
424static void mmp_pdma_free_chan_resources(struct dma_chan *dchan)
425{
426 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
427 unsigned long flags;
428
429 spin_lock_irqsave(&chan->desc_lock, flags);
430 mmp_pdma_free_desc_list(chan, &chan->chain_pending);
431 mmp_pdma_free_desc_list(chan, &chan->chain_running);
432 spin_unlock_irqrestore(&chan->desc_lock, flags);
433
434 dma_pool_destroy(chan->desc_pool);
435 chan->desc_pool = NULL;
436 chan->idle = true;
437 chan->dev_addr = 0;
Xiang Wang027f28b2013-06-18 14:55:58 +0800438 mmp_pdma_free_phy(chan);
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800439 return;
440}
441
442static struct dma_async_tx_descriptor *
443mmp_pdma_prep_memcpy(struct dma_chan *dchan,
444 dma_addr_t dma_dst, dma_addr_t dma_src,
445 size_t len, unsigned long flags)
446{
447 struct mmp_pdma_chan *chan;
448 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new;
449 size_t copy = 0;
450
451 if (!dchan)
452 return NULL;
453
454 if (!len)
455 return NULL;
456
457 chan = to_mmp_pdma_chan(dchan);
458
459 if (!chan->dir) {
460 chan->dir = DMA_MEM_TO_MEM;
461 chan->dcmd = DCMD_INCTRGADDR | DCMD_INCSRCADDR;
462 chan->dcmd |= DCMD_BURST32;
463 }
464
465 do {
466 /* Allocate the link descriptor from DMA pool */
467 new = mmp_pdma_alloc_descriptor(chan);
468 if (!new) {
469 dev_err(chan->dev, "no memory for desc\n");
470 goto fail;
471 }
472
473 copy = min_t(size_t, len, PDMA_MAX_DESC_BYTES);
474
475 new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & copy);
476 new->desc.dsadr = dma_src;
477 new->desc.dtadr = dma_dst;
478
479 if (!first)
480 first = new;
481 else
482 prev->desc.ddadr = new->async_tx.phys;
483
484 new->async_tx.cookie = 0;
485 async_tx_ack(&new->async_tx);
486
487 prev = new;
488 len -= copy;
489
490 if (chan->dir == DMA_MEM_TO_DEV) {
491 dma_src += copy;
492 } else if (chan->dir == DMA_DEV_TO_MEM) {
493 dma_dst += copy;
494 } else if (chan->dir == DMA_MEM_TO_MEM) {
495 dma_src += copy;
496 dma_dst += copy;
497 }
498
499 /* Insert the link descriptor to the LD ring */
500 list_add_tail(&new->node, &first->tx_list);
501 } while (len);
502
503 first->async_tx.flags = flags; /* client is in control of this ack */
504 first->async_tx.cookie = -EBUSY;
505
506 /* last desc and fire IRQ */
507 new->desc.ddadr = DDADR_STOP;
508 new->desc.dcmd |= DCMD_ENDIRQEN;
509
510 return &first->async_tx;
511
512fail:
513 if (first)
514 mmp_pdma_free_desc_list(chan, &first->tx_list);
515 return NULL;
516}
517
518static struct dma_async_tx_descriptor *
519mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
520 unsigned int sg_len, enum dma_transfer_direction dir,
521 unsigned long flags, void *context)
522{
523 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
524 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new = NULL;
525 size_t len, avail;
526 struct scatterlist *sg;
527 dma_addr_t addr;
528 int i;
529
530 if ((sgl == NULL) || (sg_len == 0))
531 return NULL;
532
533 for_each_sg(sgl, sg, sg_len, i) {
534 addr = sg_dma_address(sg);
535 avail = sg_dma_len(sgl);
536
537 do {
538 len = min_t(size_t, avail, PDMA_MAX_DESC_BYTES);
539
540 /* allocate and populate the descriptor */
541 new = mmp_pdma_alloc_descriptor(chan);
542 if (!new) {
543 dev_err(chan->dev, "no memory for desc\n");
544 goto fail;
545 }
546
547 new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & len);
548 if (dir == DMA_MEM_TO_DEV) {
549 new->desc.dsadr = addr;
550 new->desc.dtadr = chan->dev_addr;
551 } else {
552 new->desc.dsadr = chan->dev_addr;
553 new->desc.dtadr = addr;
554 }
555
556 if (!first)
557 first = new;
558 else
559 prev->desc.ddadr = new->async_tx.phys;
560
561 new->async_tx.cookie = 0;
562 async_tx_ack(&new->async_tx);
563 prev = new;
564
565 /* Insert the link descriptor to the LD ring */
566 list_add_tail(&new->node, &first->tx_list);
567
568 /* update metadata */
569 addr += len;
570 avail -= len;
571 } while (avail);
572 }
573
574 first->async_tx.cookie = -EBUSY;
575 first->async_tx.flags = flags;
576
577 /* last desc and fire IRQ */
578 new->desc.ddadr = DDADR_STOP;
579 new->desc.dcmd |= DCMD_ENDIRQEN;
580
581 return &first->async_tx;
582
583fail:
584 if (first)
585 mmp_pdma_free_desc_list(chan, &first->tx_list);
586 return NULL;
587}
588
589static int mmp_pdma_control(struct dma_chan *dchan, enum dma_ctrl_cmd cmd,
590 unsigned long arg)
591{
592 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
593 struct dma_slave_config *cfg = (void *)arg;
594 unsigned long flags;
595 int ret = 0;
596 u32 maxburst = 0, addr = 0;
597 enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED;
598
599 if (!dchan)
600 return -EINVAL;
601
602 switch (cmd) {
603 case DMA_TERMINATE_ALL:
604 disable_chan(chan->phy);
Xiang Wang027f28b2013-06-18 14:55:58 +0800605 mmp_pdma_free_phy(chan);
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800606 spin_lock_irqsave(&chan->desc_lock, flags);
607 mmp_pdma_free_desc_list(chan, &chan->chain_pending);
608 mmp_pdma_free_desc_list(chan, &chan->chain_running);
609 spin_unlock_irqrestore(&chan->desc_lock, flags);
610 chan->idle = true;
611 break;
612 case DMA_SLAVE_CONFIG:
613 if (cfg->direction == DMA_DEV_TO_MEM) {
614 chan->dcmd = DCMD_INCTRGADDR | DCMD_FLOWSRC;
615 maxburst = cfg->src_maxburst;
616 width = cfg->src_addr_width;
617 addr = cfg->src_addr;
618 } else if (cfg->direction == DMA_MEM_TO_DEV) {
619 chan->dcmd = DCMD_INCSRCADDR | DCMD_FLOWTRG;
620 maxburst = cfg->dst_maxburst;
621 width = cfg->dst_addr_width;
622 addr = cfg->dst_addr;
623 }
624
625 if (width == DMA_SLAVE_BUSWIDTH_1_BYTE)
626 chan->dcmd |= DCMD_WIDTH1;
627 else if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
628 chan->dcmd |= DCMD_WIDTH2;
629 else if (width == DMA_SLAVE_BUSWIDTH_4_BYTES)
630 chan->dcmd |= DCMD_WIDTH4;
631
632 if (maxburst == 8)
633 chan->dcmd |= DCMD_BURST8;
634 else if (maxburst == 16)
635 chan->dcmd |= DCMD_BURST16;
636 else if (maxburst == 32)
637 chan->dcmd |= DCMD_BURST32;
638
Cong Dinged30933e2013-01-15 01:19:48 +0100639 chan->dir = cfg->direction;
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800640 chan->dev_addr = addr;
Daniel Mack13b30062013-08-10 18:52:18 +0200641 /* FIXME: drivers should be ported over to use the filter
642 * function. Once that's done, the following two lines can
643 * be removed.
644 */
645 if (cfg->slave_id)
646 chan->drcmr = cfg->slave_id;
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800647 break;
648 default:
649 return -ENOSYS;
650 }
651
652 return ret;
653}
654
655static enum dma_status mmp_pdma_tx_status(struct dma_chan *dchan,
656 dma_cookie_t cookie, struct dma_tx_state *txstate)
657{
Andy Shevchenko4aa9fe02013-05-27 15:14:36 +0300658 return dma_cookie_status(dchan, cookie, txstate);
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800659}
660
661/**
662 * mmp_pdma_issue_pending - Issue the DMA start command
663 * pending list ==> running list
664 */
665static void mmp_pdma_issue_pending(struct dma_chan *dchan)
666{
667 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
668 unsigned long flags;
669
670 spin_lock_irqsave(&chan->desc_lock, flags);
671 start_pending_queue(chan);
672 spin_unlock_irqrestore(&chan->desc_lock, flags);
673}
674
675/*
676 * dma_do_tasklet
677 * Do call back
678 * Start pending list
679 */
680static void dma_do_tasklet(unsigned long data)
681{
682 struct mmp_pdma_chan *chan = (struct mmp_pdma_chan *)data;
683 struct mmp_pdma_desc_sw *desc, *_desc;
684 LIST_HEAD(chain_cleanup);
685 unsigned long flags;
686
687 /* submit pending list; callback for each desc; free desc */
688
689 spin_lock_irqsave(&chan->desc_lock, flags);
690
691 /* update the cookie if we have some descriptors to cleanup */
692 if (!list_empty(&chan->chain_running)) {
693 dma_cookie_t cookie;
694
695 desc = to_mmp_pdma_desc(chan->chain_running.prev);
696 cookie = desc->async_tx.cookie;
697 dma_cookie_complete(&desc->async_tx);
698
699 dev_dbg(chan->dev, "completed_cookie=%d\n", cookie);
700 }
701
702 /*
703 * move the descriptors to a temporary list so we can drop the lock
704 * during the entire cleanup operation
705 */
706 list_splice_tail_init(&chan->chain_running, &chain_cleanup);
707
708 /* the hardware is now idle and ready for more */
709 chan->idle = true;
710
711 /* Start any pending transactions automatically */
712 start_pending_queue(chan);
713 spin_unlock_irqrestore(&chan->desc_lock, flags);
714
715 /* Run the callback for each descriptor, in order */
716 list_for_each_entry_safe(desc, _desc, &chain_cleanup, node) {
717 struct dma_async_tx_descriptor *txd = &desc->async_tx;
718
719 /* Remove from the list of transactions */
720 list_del(&desc->node);
721 /* Run the link descriptor callback function */
722 if (txd->callback)
723 txd->callback(txd->callback_param);
724
725 dma_pool_free(chan->desc_pool, desc, txd->phys);
726 }
727}
728
Greg Kroah-Hartman4bf27b82012-12-21 15:09:59 -0800729static int mmp_pdma_remove(struct platform_device *op)
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800730{
731 struct mmp_pdma_device *pdev = platform_get_drvdata(op);
732
733 dma_async_device_unregister(&pdev->device);
734 return 0;
735}
736
Bill Pemberton463a1f82012-11-19 13:22:55 -0500737static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev,
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800738 int idx, int irq)
739{
740 struct mmp_pdma_phy *phy = &pdev->phy[idx];
741 struct mmp_pdma_chan *chan;
742 int ret;
743
744 chan = devm_kzalloc(pdev->dev,
745 sizeof(struct mmp_pdma_chan), GFP_KERNEL);
746 if (chan == NULL)
747 return -ENOMEM;
748
749 phy->idx = idx;
750 phy->base = pdev->base;
751
752 if (irq) {
753 ret = devm_request_irq(pdev->dev, irq,
754 mmp_pdma_chan_handler, IRQF_DISABLED, "pdma", phy);
755 if (ret) {
756 dev_err(pdev->dev, "channel request irq fail!\n");
757 return ret;
758 }
759 }
760
761 spin_lock_init(&chan->desc_lock);
762 chan->dev = pdev->dev;
763 chan->chan.device = &pdev->device;
764 tasklet_init(&chan->tasklet, dma_do_tasklet, (unsigned long)chan);
765 INIT_LIST_HEAD(&chan->chain_pending);
766 INIT_LIST_HEAD(&chan->chain_running);
767
768 /* register virt channel to dma engine */
769 list_add_tail(&chan->chan.device_node,
770 &pdev->device.channels);
771
772 return 0;
773}
774
775static struct of_device_id mmp_pdma_dt_ids[] = {
776 { .compatible = "marvell,pdma-1.0", },
777 {}
778};
779MODULE_DEVICE_TABLE(of, mmp_pdma_dt_ids);
780
Daniel Macka9a7cf02013-08-10 18:52:19 +0200781static struct dma_chan *mmp_pdma_dma_xlate(struct of_phandle_args *dma_spec,
782 struct of_dma *ofdma)
783{
784 struct mmp_pdma_device *d = ofdma->of_dma_data;
785 struct dma_chan *chan, *candidate;
786
787retry:
788 candidate = NULL;
789
790 /* walk the list of channels registered with the current instance and
791 * find one that is currently unused */
792 list_for_each_entry(chan, &d->device.channels, device_node)
793 if (chan->client_count == 0) {
794 candidate = chan;
795 break;
796 }
797
798 if (!candidate)
799 return NULL;
800
801 /* dma_get_slave_channel will return NULL if we lost a race between
802 * the lookup and the reservation */
803 chan = dma_get_slave_channel(candidate);
804
805 if (chan) {
806 struct mmp_pdma_chan *c = to_mmp_pdma_chan(chan);
807 c->drcmr = dma_spec->args[0];
808 return chan;
809 }
810
811 goto retry;
812}
813
Bill Pemberton463a1f82012-11-19 13:22:55 -0500814static int mmp_pdma_probe(struct platform_device *op)
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800815{
816 struct mmp_pdma_device *pdev;
817 const struct of_device_id *of_id;
818 struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev);
819 struct resource *iores;
820 int i, ret, irq = 0;
821 int dma_channels = 0, irq_num = 0;
822
823 pdev = devm_kzalloc(&op->dev, sizeof(*pdev), GFP_KERNEL);
824 if (!pdev)
825 return -ENOMEM;
826 pdev->dev = &op->dev;
827
Xiang Wang027f28b2013-06-18 14:55:58 +0800828 spin_lock_init(&pdev->phy_lock);
829
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800830 iores = platform_get_resource(op, IORESOURCE_MEM, 0);
831 if (!iores)
832 return -EINVAL;
833
Thierry Reding73312052013-01-21 11:09:00 +0100834 pdev->base = devm_ioremap_resource(pdev->dev, iores);
835 if (IS_ERR(pdev->base))
836 return PTR_ERR(pdev->base);
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800837
838 of_id = of_match_device(mmp_pdma_dt_ids, pdev->dev);
839 if (of_id)
840 of_property_read_u32(pdev->dev->of_node,
841 "#dma-channels", &dma_channels);
842 else if (pdata && pdata->dma_channels)
843 dma_channels = pdata->dma_channels;
844 else
845 dma_channels = 32; /* default 32 channel */
846 pdev->dma_channels = dma_channels;
847
848 for (i = 0; i < dma_channels; i++) {
849 if (platform_get_irq(op, i) > 0)
850 irq_num++;
851 }
852
853 pdev->phy = devm_kzalloc(pdev->dev,
854 dma_channels * sizeof(struct mmp_pdma_chan), GFP_KERNEL);
855 if (pdev->phy == NULL)
856 return -ENOMEM;
857
858 INIT_LIST_HEAD(&pdev->device.channels);
859
860 if (irq_num != dma_channels) {
861 /* all chan share one irq, demux inside */
862 irq = platform_get_irq(op, 0);
863 ret = devm_request_irq(pdev->dev, irq,
864 mmp_pdma_int_handler, IRQF_DISABLED, "pdma", pdev);
865 if (ret)
866 return ret;
867 }
868
869 for (i = 0; i < dma_channels; i++) {
870 irq = (irq_num != dma_channels) ? 0 : platform_get_irq(op, i);
871 ret = mmp_pdma_chan_init(pdev, i, irq);
872 if (ret)
873 return ret;
874 }
875
876 dma_cap_set(DMA_SLAVE, pdev->device.cap_mask);
877 dma_cap_set(DMA_MEMCPY, pdev->device.cap_mask);
878 dma_cap_set(DMA_SLAVE, pdev->device.cap_mask);
879 pdev->device.dev = &op->dev;
880 pdev->device.device_alloc_chan_resources = mmp_pdma_alloc_chan_resources;
881 pdev->device.device_free_chan_resources = mmp_pdma_free_chan_resources;
882 pdev->device.device_tx_status = mmp_pdma_tx_status;
883 pdev->device.device_prep_dma_memcpy = mmp_pdma_prep_memcpy;
884 pdev->device.device_prep_slave_sg = mmp_pdma_prep_slave_sg;
885 pdev->device.device_issue_pending = mmp_pdma_issue_pending;
886 pdev->device.device_control = mmp_pdma_control;
887 pdev->device.copy_align = PDMA_ALIGNMENT;
888
889 if (pdev->dev->coherent_dma_mask)
890 dma_set_mask(pdev->dev, pdev->dev->coherent_dma_mask);
891 else
892 dma_set_mask(pdev->dev, DMA_BIT_MASK(64));
893
894 ret = dma_async_device_register(&pdev->device);
895 if (ret) {
896 dev_err(pdev->device.dev, "unable to register\n");
897 return ret;
898 }
899
Daniel Macka9a7cf02013-08-10 18:52:19 +0200900 if (op->dev.of_node) {
901 /* Device-tree DMA controller registration */
902 ret = of_dma_controller_register(op->dev.of_node,
903 mmp_pdma_dma_xlate, pdev);
904 if (ret < 0) {
905 dev_err(&op->dev, "of_dma_controller_register failed\n");
906 return ret;
907 }
908 }
909
Daniel Mack419d1f12013-08-10 18:52:20 +0200910 dev_info(pdev->device.dev, "initialized %d channels\n", dma_channels);
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800911 return 0;
912}
913
914static const struct platform_device_id mmp_pdma_id_table[] = {
915 { "mmp-pdma", },
916 { },
917};
918
919static struct platform_driver mmp_pdma_driver = {
920 .driver = {
921 .name = "mmp-pdma",
922 .owner = THIS_MODULE,
923 .of_match_table = mmp_pdma_dt_ids,
924 },
925 .id_table = mmp_pdma_id_table,
926 .probe = mmp_pdma_probe,
Bill Pembertona7d6e3e2012-11-19 13:20:04 -0500927 .remove = mmp_pdma_remove,
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800928};
929
Daniel Mack13b30062013-08-10 18:52:18 +0200930bool mmp_pdma_filter_fn(struct dma_chan *chan, void *param)
931{
932 struct mmp_pdma_chan *c = to_mmp_pdma_chan(chan);
933
934 if (chan->device->dev->driver != &mmp_pdma_driver.driver)
935 return false;
936
937 c->drcmr = *(unsigned int *) param;
938
939 return true;
940}
941EXPORT_SYMBOL_GPL(mmp_pdma_filter_fn);
942
Zhangfei Gaoc8acd6a2012-09-03 11:03:45 +0800943module_platform_driver(mmp_pdma_driver);
944
945MODULE_DESCRIPTION("MARVELL MMP Periphera DMA Driver");
946MODULE_AUTHOR("Marvell International Ltd.");
947MODULE_LICENSE("GPL v2");