blob: b8ec03ee8e22e495e633ff95bf9faf0d02756dac [file] [log] [blame]
Jassi Brarb3040e42010-05-23 20:28:19 -07001/* linux/drivers/dma/pl330.c
2 *
3 * Copyright (C) 2010 Samsung Electronics Co. Ltd.
4 * Jaswinder Singh <jassi.brar@samsung.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 */
11
12#include <linux/io.h>
13#include <linux/init.h>
14#include <linux/slab.h>
15#include <linux/module.h>
16#include <linux/dmaengine.h>
17#include <linux/interrupt.h>
18#include <linux/amba/bus.h>
19#include <linux/amba/pl330.h>
Boojin Kima2f52032011-09-02 09:44:29 +090020#include <linux/pm_runtime.h>
Boojin Kim1b9bb712011-09-02 09:44:30 +090021#include <linux/scatterlist.h>
Thomas Abraham93ed5542011-10-24 11:43:31 +020022#include <linux/of.h>
Jassi Brarb3040e42010-05-23 20:28:19 -070023
24#define NR_DEFAULT_DESC 16
25
26enum desc_status {
27 /* In the DMAC pool */
28 FREE,
29 /*
30 * Allocted to some channel during prep_xxx
31 * Also may be sitting on the work_list.
32 */
33 PREP,
34 /*
35 * Sitting on the work_list and already submitted
36 * to the PL330 core. Not more than two descriptors
37 * of a channel can be BUSY at any time.
38 */
39 BUSY,
40 /*
41 * Sitting on the channel work_list but xfer done
42 * by PL330 core
43 */
44 DONE,
45};
46
47struct dma_pl330_chan {
48 /* Schedule desc completion */
49 struct tasklet_struct task;
50
51 /* DMA-Engine Channel */
52 struct dma_chan chan;
53
54 /* Last completed cookie */
55 dma_cookie_t completed;
56
57 /* List of to be xfered descriptors */
58 struct list_head work_list;
59
60 /* Pointer to the DMAC that manages this channel,
61 * NULL if the channel is available to be acquired.
62 * As the parent, this DMAC also provides descriptors
63 * to the channel.
64 */
65 struct dma_pl330_dmac *dmac;
66
67 /* To protect channel manipulation */
68 spinlock_t lock;
69
70 /* Token of a hardware channel thread of PL330 DMAC
71 * NULL if the channel is available to be acquired.
72 */
73 void *pl330_chid;
Boojin Kim1b9bb712011-09-02 09:44:30 +090074
75 /* For D-to-M and M-to-D channels */
76 int burst_sz; /* the peripheral fifo width */
Boojin Kim1d0c1d62011-09-02 09:44:31 +090077 int burst_len; /* the number of burst */
Boojin Kim1b9bb712011-09-02 09:44:30 +090078 dma_addr_t fifo_addr;
Boojin Kim42bc9cf2011-09-02 09:44:33 +090079
80 /* for cyclic capability */
81 bool cyclic;
Jassi Brarb3040e42010-05-23 20:28:19 -070082};
83
84struct dma_pl330_dmac {
85 struct pl330_info pif;
86
87 /* DMA-Engine Device */
88 struct dma_device ddma;
89
90 /* Pool of descriptors available for the DMAC's channels */
91 struct list_head desc_pool;
92 /* To protect desc_pool manipulation */
93 spinlock_t pool_lock;
94
95 /* Peripheral channels connected to this DMAC */
Rob Herring4e0e6102011-07-25 16:05:04 -050096 struct dma_pl330_chan *peripherals; /* keep at end */
Boojin Kima2f52032011-09-02 09:44:29 +090097
98 struct clk *clk;
Jassi Brarb3040e42010-05-23 20:28:19 -070099};
100
101struct dma_pl330_desc {
102 /* To attach to a queue as child */
103 struct list_head node;
104
105 /* Descriptor for the DMA Engine API */
106 struct dma_async_tx_descriptor txd;
107
108 /* Xfer for PL330 core */
109 struct pl330_xfer px;
110
111 struct pl330_reqcfg rqcfg;
112 struct pl330_req req;
113
114 enum desc_status status;
115
116 /* The channel which currently holds this desc */
117 struct dma_pl330_chan *pchan;
118};
119
Thomas Abraham3e2ec132011-10-24 11:43:02 +0200120/* forward declaration */
121static struct amba_driver pl330_driver;
122
Jassi Brarb3040e42010-05-23 20:28:19 -0700123static inline struct dma_pl330_chan *
124to_pchan(struct dma_chan *ch)
125{
126 if (!ch)
127 return NULL;
128
129 return container_of(ch, struct dma_pl330_chan, chan);
130}
131
132static inline struct dma_pl330_desc *
133to_desc(struct dma_async_tx_descriptor *tx)
134{
135 return container_of(tx, struct dma_pl330_desc, txd);
136}
137
138static inline void free_desc_list(struct list_head *list)
139{
140 struct dma_pl330_dmac *pdmac;
141 struct dma_pl330_desc *desc;
142 struct dma_pl330_chan *pch;
143 unsigned long flags;
144
145 if (list_empty(list))
146 return;
147
148 /* Finish off the work list */
149 list_for_each_entry(desc, list, node) {
150 dma_async_tx_callback callback;
151 void *param;
152
153 /* All desc in a list belong to same channel */
154 pch = desc->pchan;
155 callback = desc->txd.callback;
156 param = desc->txd.callback_param;
157
158 if (callback)
159 callback(param);
160
161 desc->pchan = NULL;
162 }
163
164 pdmac = pch->dmac;
165
166 spin_lock_irqsave(&pdmac->pool_lock, flags);
167 list_splice_tail_init(list, &pdmac->desc_pool);
168 spin_unlock_irqrestore(&pdmac->pool_lock, flags);
169}
170
Boojin Kim42bc9cf2011-09-02 09:44:33 +0900171static inline void handle_cyclic_desc_list(struct list_head *list)
172{
173 struct dma_pl330_desc *desc;
174 struct dma_pl330_chan *pch;
175 unsigned long flags;
176
177 if (list_empty(list))
178 return;
179
180 list_for_each_entry(desc, list, node) {
181 dma_async_tx_callback callback;
182
183 /* Change status to reload it */
184 desc->status = PREP;
185 pch = desc->pchan;
186 callback = desc->txd.callback;
187 if (callback)
188 callback(desc->txd.callback_param);
189 }
190
191 spin_lock_irqsave(&pch->lock, flags);
192 list_splice_tail_init(list, &pch->work_list);
193 spin_unlock_irqrestore(&pch->lock, flags);
194}
195
Jassi Brarb3040e42010-05-23 20:28:19 -0700196static inline void fill_queue(struct dma_pl330_chan *pch)
197{
198 struct dma_pl330_desc *desc;
199 int ret;
200
201 list_for_each_entry(desc, &pch->work_list, node) {
202
203 /* If already submitted */
204 if (desc->status == BUSY)
205 break;
206
207 ret = pl330_submit_req(pch->pl330_chid,
208 &desc->req);
209 if (!ret) {
210 desc->status = BUSY;
211 break;
212 } else if (ret == -EAGAIN) {
213 /* QFull or DMAC Dying */
214 break;
215 } else {
216 /* Unacceptable request */
217 desc->status = DONE;
218 dev_err(pch->dmac->pif.dev, "%s:%d Bad Desc(%d)\n",
219 __func__, __LINE__, desc->txd.cookie);
220 tasklet_schedule(&pch->task);
221 }
222 }
223}
224
225static void pl330_tasklet(unsigned long data)
226{
227 struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data;
228 struct dma_pl330_desc *desc, *_dt;
229 unsigned long flags;
230 LIST_HEAD(list);
231
232 spin_lock_irqsave(&pch->lock, flags);
233
234 /* Pick up ripe tomatoes */
235 list_for_each_entry_safe(desc, _dt, &pch->work_list, node)
236 if (desc->status == DONE) {
237 pch->completed = desc->txd.cookie;
238 list_move_tail(&desc->node, &list);
239 }
240
241 /* Try to submit a req imm. next to the last completed cookie */
242 fill_queue(pch);
243
244 /* Make sure the PL330 Channel thread is active */
245 pl330_chan_ctrl(pch->pl330_chid, PL330_OP_START);
246
247 spin_unlock_irqrestore(&pch->lock, flags);
248
Boojin Kim42bc9cf2011-09-02 09:44:33 +0900249 if (pch->cyclic)
250 handle_cyclic_desc_list(&list);
251 else
252 free_desc_list(&list);
Jassi Brarb3040e42010-05-23 20:28:19 -0700253}
254
255static void dma_pl330_rqcb(void *token, enum pl330_op_err err)
256{
257 struct dma_pl330_desc *desc = token;
258 struct dma_pl330_chan *pch = desc->pchan;
259 unsigned long flags;
260
261 /* If desc aborted */
262 if (!pch)
263 return;
264
265 spin_lock_irqsave(&pch->lock, flags);
266
267 desc->status = DONE;
268
269 spin_unlock_irqrestore(&pch->lock, flags);
270
271 tasklet_schedule(&pch->task);
272}
273
Thomas Abraham3e2ec132011-10-24 11:43:02 +0200274bool pl330_filter(struct dma_chan *chan, void *param)
275{
Thomas Abrahamcd072512011-10-24 11:43:11 +0200276 u8 *peri_id;
Thomas Abraham3e2ec132011-10-24 11:43:02 +0200277
278 if (chan->device->dev->driver != &pl330_driver.drv)
279 return false;
280
Thomas Abraham93ed5542011-10-24 11:43:31 +0200281#ifdef CONFIG_OF
282 if (chan->device->dev->of_node) {
283 const __be32 *prop_value;
284 phandle phandle;
285 struct device_node *node;
286
287 prop_value = ((struct property *)param)->value;
288 phandle = be32_to_cpup(prop_value++);
289 node = of_find_node_by_phandle(phandle);
290 return ((chan->private == node) &&
291 (chan->chan_id == be32_to_cpup(prop_value)));
292 }
293#endif
294
Thomas Abrahamcd072512011-10-24 11:43:11 +0200295 peri_id = chan->private;
296 return *peri_id == (unsigned)param;
Thomas Abraham3e2ec132011-10-24 11:43:02 +0200297}
298EXPORT_SYMBOL(pl330_filter);
299
Jassi Brarb3040e42010-05-23 20:28:19 -0700300static int pl330_alloc_chan_resources(struct dma_chan *chan)
301{
302 struct dma_pl330_chan *pch = to_pchan(chan);
303 struct dma_pl330_dmac *pdmac = pch->dmac;
304 unsigned long flags;
305
306 spin_lock_irqsave(&pch->lock, flags);
307
308 pch->completed = chan->cookie = 1;
Boojin Kim42bc9cf2011-09-02 09:44:33 +0900309 pch->cyclic = false;
Jassi Brarb3040e42010-05-23 20:28:19 -0700310
311 pch->pl330_chid = pl330_request_channel(&pdmac->pif);
312 if (!pch->pl330_chid) {
313 spin_unlock_irqrestore(&pch->lock, flags);
314 return 0;
315 }
316
317 tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch);
318
319 spin_unlock_irqrestore(&pch->lock, flags);
320
321 return 1;
322}
323
324static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg)
325{
326 struct dma_pl330_chan *pch = to_pchan(chan);
Boojin Kimae43b882011-09-02 09:44:32 +0900327 struct dma_pl330_desc *desc, *_dt;
Jassi Brarb3040e42010-05-23 20:28:19 -0700328 unsigned long flags;
Boojin Kim1d0c1d62011-09-02 09:44:31 +0900329 struct dma_pl330_dmac *pdmac = pch->dmac;
330 struct dma_slave_config *slave_config;
Boojin Kimae43b882011-09-02 09:44:32 +0900331 LIST_HEAD(list);
Jassi Brarb3040e42010-05-23 20:28:19 -0700332
Boojin Kim1d0c1d62011-09-02 09:44:31 +0900333 switch (cmd) {
334 case DMA_TERMINATE_ALL:
335 spin_lock_irqsave(&pch->lock, flags);
336
337 /* FLUSH the PL330 Channel thread */
338 pl330_chan_ctrl(pch->pl330_chid, PL330_OP_FLUSH);
339
340 /* Mark all desc done */
Boojin Kimae43b882011-09-02 09:44:32 +0900341 list_for_each_entry_safe(desc, _dt, &pch->work_list , node) {
Boojin Kim1d0c1d62011-09-02 09:44:31 +0900342 desc->status = DONE;
Boojin Kimae43b882011-09-02 09:44:32 +0900343 pch->completed = desc->txd.cookie;
344 list_move_tail(&desc->node, &list);
345 }
Boojin Kim1d0c1d62011-09-02 09:44:31 +0900346
Boojin Kimae43b882011-09-02 09:44:32 +0900347 list_splice_tail_init(&list, &pdmac->desc_pool);
Boojin Kim1d0c1d62011-09-02 09:44:31 +0900348 spin_unlock_irqrestore(&pch->lock, flags);
Boojin Kim1d0c1d62011-09-02 09:44:31 +0900349 break;
350 case DMA_SLAVE_CONFIG:
351 slave_config = (struct dma_slave_config *)arg;
352
Vinod Kouldb8196d2011-10-13 22:34:23 +0530353 if (slave_config->direction == DMA_MEM_TO_DEV) {
Boojin Kim1d0c1d62011-09-02 09:44:31 +0900354 if (slave_config->dst_addr)
355 pch->fifo_addr = slave_config->dst_addr;
356 if (slave_config->dst_addr_width)
357 pch->burst_sz = __ffs(slave_config->dst_addr_width);
358 if (slave_config->dst_maxburst)
359 pch->burst_len = slave_config->dst_maxburst;
Vinod Kouldb8196d2011-10-13 22:34:23 +0530360 } else if (slave_config->direction == DMA_DEV_TO_MEM) {
Boojin Kim1d0c1d62011-09-02 09:44:31 +0900361 if (slave_config->src_addr)
362 pch->fifo_addr = slave_config->src_addr;
363 if (slave_config->src_addr_width)
364 pch->burst_sz = __ffs(slave_config->src_addr_width);
365 if (slave_config->src_maxburst)
366 pch->burst_len = slave_config->src_maxburst;
367 }
368 break;
369 default:
370 dev_err(pch->dmac->pif.dev, "Not supported command.\n");
Jassi Brarb3040e42010-05-23 20:28:19 -0700371 return -ENXIO;
Boojin Kim1d0c1d62011-09-02 09:44:31 +0900372 }
Jassi Brarb3040e42010-05-23 20:28:19 -0700373
374 return 0;
375}
376
377static void pl330_free_chan_resources(struct dma_chan *chan)
378{
379 struct dma_pl330_chan *pch = to_pchan(chan);
380 unsigned long flags;
381
382 spin_lock_irqsave(&pch->lock, flags);
383
384 tasklet_kill(&pch->task);
385
386 pl330_release_channel(pch->pl330_chid);
387 pch->pl330_chid = NULL;
388
Boojin Kim42bc9cf2011-09-02 09:44:33 +0900389 if (pch->cyclic)
390 list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool);
391
Jassi Brarb3040e42010-05-23 20:28:19 -0700392 spin_unlock_irqrestore(&pch->lock, flags);
393}
394
395static enum dma_status
396pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
397 struct dma_tx_state *txstate)
398{
399 struct dma_pl330_chan *pch = to_pchan(chan);
400 dma_cookie_t last_done, last_used;
401 int ret;
402
403 last_done = pch->completed;
404 last_used = chan->cookie;
405
406 ret = dma_async_is_complete(cookie, last_done, last_used);
407
408 dma_set_tx_state(txstate, last_done, last_used, 0);
409
410 return ret;
411}
412
413static void pl330_issue_pending(struct dma_chan *chan)
414{
415 pl330_tasklet((unsigned long) to_pchan(chan));
416}
417
418/*
419 * We returned the last one of the circular list of descriptor(s)
420 * from prep_xxx, so the argument to submit corresponds to the last
421 * descriptor of the list.
422 */
423static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx)
424{
425 struct dma_pl330_desc *desc, *last = to_desc(tx);
426 struct dma_pl330_chan *pch = to_pchan(tx->chan);
427 dma_cookie_t cookie;
428 unsigned long flags;
429
430 spin_lock_irqsave(&pch->lock, flags);
431
432 /* Assign cookies to all nodes */
433 cookie = tx->chan->cookie;
434
435 while (!list_empty(&last->node)) {
436 desc = list_entry(last->node.next, struct dma_pl330_desc, node);
437
438 if (++cookie < 0)
439 cookie = 1;
440 desc->txd.cookie = cookie;
441
442 list_move_tail(&desc->node, &pch->work_list);
443 }
444
445 if (++cookie < 0)
446 cookie = 1;
447 last->txd.cookie = cookie;
448
449 list_add_tail(&last->node, &pch->work_list);
450
451 tx->chan->cookie = cookie;
452
453 spin_unlock_irqrestore(&pch->lock, flags);
454
455 return cookie;
456}
457
458static inline void _init_desc(struct dma_pl330_desc *desc)
459{
460 desc->pchan = NULL;
461 desc->req.x = &desc->px;
462 desc->req.token = desc;
463 desc->rqcfg.swap = SWAP_NO;
464 desc->rqcfg.privileged = 0;
465 desc->rqcfg.insnaccess = 0;
466 desc->rqcfg.scctl = SCCTRL0;
467 desc->rqcfg.dcctl = DCCTRL0;
468 desc->req.cfg = &desc->rqcfg;
469 desc->req.xfer_cb = dma_pl330_rqcb;
470 desc->txd.tx_submit = pl330_tx_submit;
471
472 INIT_LIST_HEAD(&desc->node);
473}
474
475/* Returns the number of descriptors added to the DMAC pool */
476int add_desc(struct dma_pl330_dmac *pdmac, gfp_t flg, int count)
477{
478 struct dma_pl330_desc *desc;
479 unsigned long flags;
480 int i;
481
482 if (!pdmac)
483 return 0;
484
485 desc = kmalloc(count * sizeof(*desc), flg);
486 if (!desc)
487 return 0;
488
489 spin_lock_irqsave(&pdmac->pool_lock, flags);
490
491 for (i = 0; i < count; i++) {
492 _init_desc(&desc[i]);
493 list_add_tail(&desc[i].node, &pdmac->desc_pool);
494 }
495
496 spin_unlock_irqrestore(&pdmac->pool_lock, flags);
497
498 return count;
499}
500
501static struct dma_pl330_desc *
502pluck_desc(struct dma_pl330_dmac *pdmac)
503{
504 struct dma_pl330_desc *desc = NULL;
505 unsigned long flags;
506
507 if (!pdmac)
508 return NULL;
509
510 spin_lock_irqsave(&pdmac->pool_lock, flags);
511
512 if (!list_empty(&pdmac->desc_pool)) {
513 desc = list_entry(pdmac->desc_pool.next,
514 struct dma_pl330_desc, node);
515
516 list_del_init(&desc->node);
517
518 desc->status = PREP;
519 desc->txd.callback = NULL;
520 }
521
522 spin_unlock_irqrestore(&pdmac->pool_lock, flags);
523
524 return desc;
525}
526
527static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch)
528{
529 struct dma_pl330_dmac *pdmac = pch->dmac;
Thomas Abrahamcd072512011-10-24 11:43:11 +0200530 u8 *peri_id = pch->chan.private;
Jassi Brarb3040e42010-05-23 20:28:19 -0700531 struct dma_pl330_desc *desc;
532
533 /* Pluck one desc from the pool of DMAC */
534 desc = pluck_desc(pdmac);
535
536 /* If the DMAC pool is empty, alloc new */
537 if (!desc) {
538 if (!add_desc(pdmac, GFP_ATOMIC, 1))
539 return NULL;
540
541 /* Try again */
542 desc = pluck_desc(pdmac);
543 if (!desc) {
544 dev_err(pch->dmac->pif.dev,
545 "%s:%d ALERT!\n", __func__, __LINE__);
546 return NULL;
547 }
548 }
549
550 /* Initialize the descriptor */
551 desc->pchan = pch;
552 desc->txd.cookie = 0;
553 async_tx_ack(&desc->txd);
554
Thomas Abrahamcd072512011-10-24 11:43:11 +0200555 desc->req.peri = peri_id ? pch->chan.chan_id : 0;
Jassi Brarb3040e42010-05-23 20:28:19 -0700556
557 dma_async_tx_descriptor_init(&desc->txd, &pch->chan);
558
559 return desc;
560}
561
562static inline void fill_px(struct pl330_xfer *px,
563 dma_addr_t dst, dma_addr_t src, size_t len)
564{
565 px->next = NULL;
566 px->bytes = len;
567 px->dst_addr = dst;
568 px->src_addr = src;
569}
570
571static struct dma_pl330_desc *
572__pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst,
573 dma_addr_t src, size_t len)
574{
575 struct dma_pl330_desc *desc = pl330_get_desc(pch);
576
577 if (!desc) {
578 dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n",
579 __func__, __LINE__);
580 return NULL;
581 }
582
583 /*
584 * Ideally we should lookout for reqs bigger than
585 * those that can be programmed with 256 bytes of
586 * MC buffer, but considering a req size is seldom
587 * going to be word-unaligned and more than 200MB,
588 * we take it easy.
589 * Also, should the limit is reached we'd rather
590 * have the platform increase MC buffer size than
591 * complicating this API driver.
592 */
593 fill_px(&desc->px, dst, src, len);
594
595 return desc;
596}
597
598/* Call after fixing burst size */
599static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len)
600{
601 struct dma_pl330_chan *pch = desc->pchan;
602 struct pl330_info *pi = &pch->dmac->pif;
603 int burst_len;
604
605 burst_len = pi->pcfg.data_bus_width / 8;
606 burst_len *= pi->pcfg.data_buf_dep;
607 burst_len >>= desc->rqcfg.brst_size;
608
609 /* src/dst_burst_len can't be more than 16 */
610 if (burst_len > 16)
611 burst_len = 16;
612
613 while (burst_len > 1) {
614 if (!(len % (burst_len << desc->rqcfg.brst_size)))
615 break;
616 burst_len--;
617 }
618
619 return burst_len;
620}
621
Boojin Kim42bc9cf2011-09-02 09:44:33 +0900622static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
623 struct dma_chan *chan, dma_addr_t dma_addr, size_t len,
Vinod Kouldb8196d2011-10-13 22:34:23 +0530624 size_t period_len, enum dma_transfer_direction direction)
Boojin Kim42bc9cf2011-09-02 09:44:33 +0900625{
626 struct dma_pl330_desc *desc;
627 struct dma_pl330_chan *pch = to_pchan(chan);
628 dma_addr_t dst;
629 dma_addr_t src;
630
631 desc = pl330_get_desc(pch);
632 if (!desc) {
633 dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n",
634 __func__, __LINE__);
635 return NULL;
636 }
637
638 switch (direction) {
Vinod Kouldb8196d2011-10-13 22:34:23 +0530639 case DMA_MEM_TO_DEV:
Boojin Kim42bc9cf2011-09-02 09:44:33 +0900640 desc->rqcfg.src_inc = 1;
641 desc->rqcfg.dst_inc = 0;
Thomas Abrahamcd072512011-10-24 11:43:11 +0200642 desc->req.rqtype = MEMTODEV;
Boojin Kim42bc9cf2011-09-02 09:44:33 +0900643 src = dma_addr;
644 dst = pch->fifo_addr;
645 break;
Vinod Kouldb8196d2011-10-13 22:34:23 +0530646 case DMA_DEV_TO_MEM:
Boojin Kim42bc9cf2011-09-02 09:44:33 +0900647 desc->rqcfg.src_inc = 0;
648 desc->rqcfg.dst_inc = 1;
Thomas Abrahamcd072512011-10-24 11:43:11 +0200649 desc->req.rqtype = DEVTOMEM;
Boojin Kim42bc9cf2011-09-02 09:44:33 +0900650 src = pch->fifo_addr;
651 dst = dma_addr;
652 break;
653 default:
654 dev_err(pch->dmac->pif.dev, "%s:%d Invalid dma direction\n",
655 __func__, __LINE__);
656 return NULL;
657 }
658
659 desc->rqcfg.brst_size = pch->burst_sz;
660 desc->rqcfg.brst_len = 1;
661
662 pch->cyclic = true;
663
664 fill_px(&desc->px, dst, src, period_len);
665
666 return &desc->txd;
667}
668
Jassi Brarb3040e42010-05-23 20:28:19 -0700669static struct dma_async_tx_descriptor *
670pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst,
671 dma_addr_t src, size_t len, unsigned long flags)
672{
673 struct dma_pl330_desc *desc;
674 struct dma_pl330_chan *pch = to_pchan(chan);
Jassi Brarb3040e42010-05-23 20:28:19 -0700675 struct pl330_info *pi;
676 int burst;
677
Rob Herring4e0e6102011-07-25 16:05:04 -0500678 if (unlikely(!pch || !len))
Jassi Brarb3040e42010-05-23 20:28:19 -0700679 return NULL;
680
Jassi Brarb3040e42010-05-23 20:28:19 -0700681 pi = &pch->dmac->pif;
682
683 desc = __pl330_prep_dma_memcpy(pch, dst, src, len);
684 if (!desc)
685 return NULL;
686
687 desc->rqcfg.src_inc = 1;
688 desc->rqcfg.dst_inc = 1;
Thomas Abrahamcd072512011-10-24 11:43:11 +0200689 desc->req.rqtype = MEMTOMEM;
Jassi Brarb3040e42010-05-23 20:28:19 -0700690
691 /* Select max possible burst size */
692 burst = pi->pcfg.data_bus_width / 8;
693
694 while (burst > 1) {
695 if (!(len % burst))
696 break;
697 burst /= 2;
698 }
699
700 desc->rqcfg.brst_size = 0;
701 while (burst != (1 << desc->rqcfg.brst_size))
702 desc->rqcfg.brst_size++;
703
704 desc->rqcfg.brst_len = get_burst_len(desc, len);
705
706 desc->txd.flags = flags;
707
708 return &desc->txd;
709}
710
711static struct dma_async_tx_descriptor *
712pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
Vinod Kouldb8196d2011-10-13 22:34:23 +0530713 unsigned int sg_len, enum dma_transfer_direction direction,
Jassi Brarb3040e42010-05-23 20:28:19 -0700714 unsigned long flg)
715{
716 struct dma_pl330_desc *first, *desc = NULL;
717 struct dma_pl330_chan *pch = to_pchan(chan);
Jassi Brarb3040e42010-05-23 20:28:19 -0700718 struct scatterlist *sg;
719 unsigned long flags;
Boojin Kim1b9bb712011-09-02 09:44:30 +0900720 int i;
Jassi Brarb3040e42010-05-23 20:28:19 -0700721 dma_addr_t addr;
722
Thomas Abrahamcd072512011-10-24 11:43:11 +0200723 if (unlikely(!pch || !sgl || !sg_len))
Jassi Brarb3040e42010-05-23 20:28:19 -0700724 return NULL;
725
Boojin Kim1b9bb712011-09-02 09:44:30 +0900726 addr = pch->fifo_addr;
Jassi Brarb3040e42010-05-23 20:28:19 -0700727
728 first = NULL;
729
730 for_each_sg(sgl, sg, sg_len, i) {
731
732 desc = pl330_get_desc(pch);
733 if (!desc) {
734 struct dma_pl330_dmac *pdmac = pch->dmac;
735
736 dev_err(pch->dmac->pif.dev,
737 "%s:%d Unable to fetch desc\n",
738 __func__, __LINE__);
739 if (!first)
740 return NULL;
741
742 spin_lock_irqsave(&pdmac->pool_lock, flags);
743
744 while (!list_empty(&first->node)) {
745 desc = list_entry(first->node.next,
746 struct dma_pl330_desc, node);
747 list_move_tail(&desc->node, &pdmac->desc_pool);
748 }
749
750 list_move_tail(&first->node, &pdmac->desc_pool);
751
752 spin_unlock_irqrestore(&pdmac->pool_lock, flags);
753
754 return NULL;
755 }
756
757 if (!first)
758 first = desc;
759 else
760 list_add_tail(&desc->node, &first->node);
761
Vinod Kouldb8196d2011-10-13 22:34:23 +0530762 if (direction == DMA_MEM_TO_DEV) {
Jassi Brarb3040e42010-05-23 20:28:19 -0700763 desc->rqcfg.src_inc = 1;
764 desc->rqcfg.dst_inc = 0;
Thomas Abrahamcd072512011-10-24 11:43:11 +0200765 desc->req.rqtype = MEMTODEV;
Jassi Brarb3040e42010-05-23 20:28:19 -0700766 fill_px(&desc->px,
767 addr, sg_dma_address(sg), sg_dma_len(sg));
768 } else {
769 desc->rqcfg.src_inc = 0;
770 desc->rqcfg.dst_inc = 1;
Thomas Abrahamcd072512011-10-24 11:43:11 +0200771 desc->req.rqtype = DEVTOMEM;
Jassi Brarb3040e42010-05-23 20:28:19 -0700772 fill_px(&desc->px,
773 sg_dma_address(sg), addr, sg_dma_len(sg));
774 }
775
Boojin Kim1b9bb712011-09-02 09:44:30 +0900776 desc->rqcfg.brst_size = pch->burst_sz;
Jassi Brarb3040e42010-05-23 20:28:19 -0700777 desc->rqcfg.brst_len = 1;
778 }
779
780 /* Return the last desc in the chain */
781 desc->txd.flags = flg;
782 return &desc->txd;
783}
784
785static irqreturn_t pl330_irq_handler(int irq, void *data)
786{
787 if (pl330_update(data))
788 return IRQ_HANDLED;
789 else
790 return IRQ_NONE;
791}
792
793static int __devinit
Russell Kingaa25afa2011-02-19 15:55:00 +0000794pl330_probe(struct amba_device *adev, const struct amba_id *id)
Jassi Brarb3040e42010-05-23 20:28:19 -0700795{
796 struct dma_pl330_platdata *pdat;
797 struct dma_pl330_dmac *pdmac;
798 struct dma_pl330_chan *pch;
799 struct pl330_info *pi;
800 struct dma_device *pd;
801 struct resource *res;
802 int i, ret, irq;
Rob Herring4e0e6102011-07-25 16:05:04 -0500803 int num_chan;
Jassi Brarb3040e42010-05-23 20:28:19 -0700804
805 pdat = adev->dev.platform_data;
806
Jassi Brarb3040e42010-05-23 20:28:19 -0700807 /* Allocate a new DMAC and its Channels */
Rob Herring4e0e6102011-07-25 16:05:04 -0500808 pdmac = kzalloc(sizeof(*pdmac), GFP_KERNEL);
Jassi Brarb3040e42010-05-23 20:28:19 -0700809 if (!pdmac) {
810 dev_err(&adev->dev, "unable to allocate mem\n");
811 return -ENOMEM;
812 }
813
814 pi = &pdmac->pif;
815 pi->dev = &adev->dev;
816 pi->pl330_data = NULL;
Rob Herring4e0e6102011-07-25 16:05:04 -0500817 pi->mcbufsz = pdat ? pdat->mcbuf_sz : 0;
Jassi Brarb3040e42010-05-23 20:28:19 -0700818
819 res = &adev->res;
820 request_mem_region(res->start, resource_size(res), "dma-pl330");
821
822 pi->base = ioremap(res->start, resource_size(res));
823 if (!pi->base) {
824 ret = -ENXIO;
825 goto probe_err1;
826 }
827
Boojin Kima2f52032011-09-02 09:44:29 +0900828 pdmac->clk = clk_get(&adev->dev, "dma");
829 if (IS_ERR(pdmac->clk)) {
830 dev_err(&adev->dev, "Cannot get operation clock.\n");
831 ret = -EINVAL;
832 goto probe_err1;
833 }
834
835 amba_set_drvdata(adev, pdmac);
836
Tushar Behera3506c0d2011-12-06 16:15:54 +0530837#ifndef CONFIG_PM_RUNTIME
Boojin Kima2f52032011-09-02 09:44:29 +0900838 /* enable dma clk */
839 clk_enable(pdmac->clk);
840#endif
841
Jassi Brarb3040e42010-05-23 20:28:19 -0700842 irq = adev->irq[0];
843 ret = request_irq(irq, pl330_irq_handler, 0,
844 dev_name(&adev->dev), pi);
845 if (ret)
846 goto probe_err2;
847
848 ret = pl330_add(pi);
849 if (ret)
850 goto probe_err3;
851
852 INIT_LIST_HEAD(&pdmac->desc_pool);
853 spin_lock_init(&pdmac->pool_lock);
854
855 /* Create a descriptor pool of default size */
856 if (!add_desc(pdmac, GFP_KERNEL, NR_DEFAULT_DESC))
857 dev_warn(&adev->dev, "unable to allocate desc\n");
858
859 pd = &pdmac->ddma;
860 INIT_LIST_HEAD(&pd->channels);
861
862 /* Initialize channel parameters */
Thomas Abraham93ed5542011-10-24 11:43:31 +0200863 num_chan = max(pdat ? pdat->nr_valid_peri : (u8)pi->pcfg.num_peri,
864 (u8)pi->pcfg.num_chan);
Rob Herring4e0e6102011-07-25 16:05:04 -0500865 pdmac->peripherals = kzalloc(num_chan * sizeof(*pch), GFP_KERNEL);
Jassi Brarb3040e42010-05-23 20:28:19 -0700866
Rob Herring4e0e6102011-07-25 16:05:04 -0500867 for (i = 0; i < num_chan; i++) {
868 pch = &pdmac->peripherals[i];
Thomas Abraham93ed5542011-10-24 11:43:31 +0200869 if (!adev->dev.of_node)
870 pch->chan.private = pdat ? &pdat->peri_id[i] : NULL;
871 else
872 pch->chan.private = adev->dev.of_node;
Jassi Brarb3040e42010-05-23 20:28:19 -0700873
874 INIT_LIST_HEAD(&pch->work_list);
875 spin_lock_init(&pch->lock);
876 pch->pl330_chid = NULL;
Jassi Brarb3040e42010-05-23 20:28:19 -0700877 pch->chan.device = pd;
Jassi Brarb3040e42010-05-23 20:28:19 -0700878 pch->dmac = pdmac;
879
880 /* Add the channel to the DMAC list */
Jassi Brarb3040e42010-05-23 20:28:19 -0700881 list_add_tail(&pch->chan.device_node, &pd->channels);
882 }
883
884 pd->dev = &adev->dev;
Thomas Abraham93ed5542011-10-24 11:43:31 +0200885 if (pdat) {
Thomas Abrahamcd072512011-10-24 11:43:11 +0200886 pd->cap_mask = pdat->cap_mask;
Thomas Abraham93ed5542011-10-24 11:43:31 +0200887 } else {
Thomas Abrahamcd072512011-10-24 11:43:11 +0200888 dma_cap_set(DMA_MEMCPY, pd->cap_mask);
Thomas Abraham93ed5542011-10-24 11:43:31 +0200889 if (pi->pcfg.num_peri) {
890 dma_cap_set(DMA_SLAVE, pd->cap_mask);
891 dma_cap_set(DMA_CYCLIC, pd->cap_mask);
892 }
893 }
Jassi Brarb3040e42010-05-23 20:28:19 -0700894
895 pd->device_alloc_chan_resources = pl330_alloc_chan_resources;
896 pd->device_free_chan_resources = pl330_free_chan_resources;
897 pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy;
Boojin Kim42bc9cf2011-09-02 09:44:33 +0900898 pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic;
Jassi Brarb3040e42010-05-23 20:28:19 -0700899 pd->device_tx_status = pl330_tx_status;
900 pd->device_prep_slave_sg = pl330_prep_slave_sg;
901 pd->device_control = pl330_control;
902 pd->device_issue_pending = pl330_issue_pending;
903
904 ret = dma_async_device_register(pd);
905 if (ret) {
906 dev_err(&adev->dev, "unable to register DMAC\n");
907 goto probe_err4;
908 }
909
Jassi Brarb3040e42010-05-23 20:28:19 -0700910 dev_info(&adev->dev,
911 "Loaded driver for PL330 DMAC-%d\n", adev->periphid);
912 dev_info(&adev->dev,
913 "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n",
914 pi->pcfg.data_buf_dep,
915 pi->pcfg.data_bus_width / 8, pi->pcfg.num_chan,
916 pi->pcfg.num_peri, pi->pcfg.num_events);
917
918 return 0;
919
920probe_err4:
921 pl330_del(pi);
922probe_err3:
923 free_irq(irq, pi);
924probe_err2:
925 iounmap(pi->base);
926probe_err1:
927 release_mem_region(res->start, resource_size(res));
928 kfree(pdmac);
929
930 return ret;
931}
932
933static int __devexit pl330_remove(struct amba_device *adev)
934{
935 struct dma_pl330_dmac *pdmac = amba_get_drvdata(adev);
936 struct dma_pl330_chan *pch, *_p;
937 struct pl330_info *pi;
938 struct resource *res;
939 int irq;
940
941 if (!pdmac)
942 return 0;
943
944 amba_set_drvdata(adev, NULL);
945
946 /* Idle the DMAC */
947 list_for_each_entry_safe(pch, _p, &pdmac->ddma.channels,
948 chan.device_node) {
949
950 /* Remove the channel */
951 list_del(&pch->chan.device_node);
952
953 /* Flush the channel */
954 pl330_control(&pch->chan, DMA_TERMINATE_ALL, 0);
955 pl330_free_chan_resources(&pch->chan);
956 }
957
958 pi = &pdmac->pif;
959
960 pl330_del(pi);
961
962 irq = adev->irq[0];
963 free_irq(irq, pi);
964
965 iounmap(pi->base);
966
967 res = &adev->res;
968 release_mem_region(res->start, resource_size(res));
969
Tushar Behera3506c0d2011-12-06 16:15:54 +0530970#ifndef CONFIG_PM_RUNTIME
Boojin Kima2f52032011-09-02 09:44:29 +0900971 clk_disable(pdmac->clk);
972#endif
973
Jassi Brarb3040e42010-05-23 20:28:19 -0700974 kfree(pdmac);
975
976 return 0;
977}
978
979static struct amba_id pl330_ids[] = {
980 {
981 .id = 0x00041330,
982 .mask = 0x000fffff,
983 },
984 { 0, 0 },
985};
986
Dave Martine8fa5162011-10-05 15:15:20 +0100987MODULE_DEVICE_TABLE(amba, pl330_ids);
988
Boojin Kima2f52032011-09-02 09:44:29 +0900989#ifdef CONFIG_PM_RUNTIME
990static int pl330_runtime_suspend(struct device *dev)
991{
992 struct dma_pl330_dmac *pdmac = dev_get_drvdata(dev);
993
994 if (!pdmac) {
995 dev_err(dev, "failed to get dmac\n");
996 return -ENODEV;
997 }
998
999 clk_disable(pdmac->clk);
1000
1001 return 0;
1002}
1003
1004static int pl330_runtime_resume(struct device *dev)
1005{
1006 struct dma_pl330_dmac *pdmac = dev_get_drvdata(dev);
1007
1008 if (!pdmac) {
1009 dev_err(dev, "failed to get dmac\n");
1010 return -ENODEV;
1011 }
1012
1013 clk_enable(pdmac->clk);
1014
1015 return 0;
1016}
1017#else
1018#define pl330_runtime_suspend NULL
1019#define pl330_runtime_resume NULL
1020#endif /* CONFIG_PM_RUNTIME */
1021
1022static const struct dev_pm_ops pl330_pm_ops = {
1023 .runtime_suspend = pl330_runtime_suspend,
1024 .runtime_resume = pl330_runtime_resume,
1025};
1026
Jassi Brarb3040e42010-05-23 20:28:19 -07001027static struct amba_driver pl330_driver = {
1028 .drv = {
1029 .owner = THIS_MODULE,
1030 .name = "dma-pl330",
Boojin Kima2f52032011-09-02 09:44:29 +09001031 .pm = &pl330_pm_ops,
Jassi Brarb3040e42010-05-23 20:28:19 -07001032 },
1033 .id_table = pl330_ids,
1034 .probe = pl330_probe,
1035 .remove = pl330_remove,
1036};
1037
1038static int __init pl330_init(void)
1039{
1040 return amba_driver_register(&pl330_driver);
1041}
1042module_init(pl330_init);
1043
1044static void __exit pl330_exit(void)
1045{
1046 amba_driver_unregister(&pl330_driver);
1047 return;
1048}
1049module_exit(pl330_exit);
1050
1051MODULE_AUTHOR("Jaswinder Singh <jassi.brar@samsung.com>");
1052MODULE_DESCRIPTION("API Driver for PL330 DMAC");
1053MODULE_LICENSE("GPL");