blob: 4839bfa74a107a1ad4cbdfb09017d86da1fd1a41 [file] [log] [blame]
Boojin Kimb7d861d2011-12-26 18:49:52 +09001/*
2 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
3 * http://www.samsung.com
Jassi Brarb3040e42010-05-23 20:28:19 -07004 *
5 * Copyright (C) 2010 Samsung Electronics Co. Ltd.
6 * Jaswinder Singh <jassi.brar@samsung.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 */
13
Boojin Kimb7d861d2011-12-26 18:49:52 +090014#include <linux/kernel.h>
Jassi Brarb3040e42010-05-23 20:28:19 -070015#include <linux/io.h>
16#include <linux/init.h>
17#include <linux/slab.h>
18#include <linux/module.h>
Boojin Kimb7d861d2011-12-26 18:49:52 +090019#include <linux/string.h>
20#include <linux/delay.h>
21#include <linux/interrupt.h>
22#include <linux/dma-mapping.h>
Jassi Brarb3040e42010-05-23 20:28:19 -070023#include <linux/dmaengine.h>
Jassi Brarb3040e42010-05-23 20:28:19 -070024#include <linux/amba/bus.h>
25#include <linux/amba/pl330.h>
Boojin Kim1b9bb712011-09-02 09:44:30 +090026#include <linux/scatterlist.h>
Thomas Abraham93ed5542011-10-24 11:43:31 +020027#include <linux/of.h>
Padmavathi Vennaa80258f2013-02-14 09:10:06 +053028#include <linux/of_dma.h>
Sachin Kamatbcc7fa92013-03-04 14:36:27 +053029#include <linux/err.h>
Jassi Brarb3040e42010-05-23 20:28:19 -070030
Russell King - ARM Linuxd2ebfb32012-03-06 22:34:26 +000031#include "dmaengine.h"
Boojin Kimb7d861d2011-12-26 18:49:52 +090032#define PL330_MAX_CHAN 8
33#define PL330_MAX_IRQS 32
34#define PL330_MAX_PERI 32
35
Lars-Peter Clausenf0564c72014-07-06 20:32:19 +020036enum pl330_cachectrl {
37 CCTRL0, /* Noncacheable and nonbufferable */
38 CCTRL1, /* Bufferable only */
39 CCTRL2, /* Cacheable, but do not allocate */
40 CCTRL3, /* Cacheable and bufferable, but do not allocate */
41 INVALID1, /* AWCACHE = 0x1000 */
42 INVALID2,
43 CCTRL6, /* Cacheable write-through, allocate on writes only */
44 CCTRL7, /* Cacheable write-back, allocate on writes only */
Boojin Kimb7d861d2011-12-26 18:49:52 +090045};
46
47enum pl330_byteswap {
48 SWAP_NO,
49 SWAP_2,
50 SWAP_4,
51 SWAP_8,
52 SWAP_16,
53};
54
Boojin Kimb7d861d2011-12-26 18:49:52 +090055/* Register and Bit field Definitions */
56#define DS 0x0
57#define DS_ST_STOP 0x0
58#define DS_ST_EXEC 0x1
59#define DS_ST_CMISS 0x2
60#define DS_ST_UPDTPC 0x3
61#define DS_ST_WFE 0x4
62#define DS_ST_ATBRR 0x5
63#define DS_ST_QBUSY 0x6
64#define DS_ST_WFP 0x7
65#define DS_ST_KILL 0x8
66#define DS_ST_CMPLT 0x9
67#define DS_ST_FLTCMP 0xe
68#define DS_ST_FAULT 0xf
69
70#define DPC 0x4
71#define INTEN 0x20
72#define ES 0x24
73#define INTSTATUS 0x28
74#define INTCLR 0x2c
75#define FSM 0x30
76#define FSC 0x34
77#define FTM 0x38
78
79#define _FTC 0x40
80#define FTC(n) (_FTC + (n)*0x4)
81
82#define _CS 0x100
83#define CS(n) (_CS + (n)*0x8)
84#define CS_CNS (1 << 21)
85
86#define _CPC 0x104
87#define CPC(n) (_CPC + (n)*0x8)
88
89#define _SA 0x400
90#define SA(n) (_SA + (n)*0x20)
91
92#define _DA 0x404
93#define DA(n) (_DA + (n)*0x20)
94
95#define _CC 0x408
96#define CC(n) (_CC + (n)*0x20)
97
98#define CC_SRCINC (1 << 0)
99#define CC_DSTINC (1 << 14)
100#define CC_SRCPRI (1 << 8)
101#define CC_DSTPRI (1 << 22)
102#define CC_SRCNS (1 << 9)
103#define CC_DSTNS (1 << 23)
104#define CC_SRCIA (1 << 10)
105#define CC_DSTIA (1 << 24)
106#define CC_SRCBRSTLEN_SHFT 4
107#define CC_DSTBRSTLEN_SHFT 18
108#define CC_SRCBRSTSIZE_SHFT 1
109#define CC_DSTBRSTSIZE_SHFT 15
110#define CC_SRCCCTRL_SHFT 11
111#define CC_SRCCCTRL_MASK 0x7
112#define CC_DSTCCTRL_SHFT 25
113#define CC_DRCCCTRL_MASK 0x7
114#define CC_SWAP_SHFT 28
115
116#define _LC0 0x40c
117#define LC0(n) (_LC0 + (n)*0x20)
118
119#define _LC1 0x410
120#define LC1(n) (_LC1 + (n)*0x20)
121
122#define DBGSTATUS 0xd00
123#define DBG_BUSY (1 << 0)
124
125#define DBGCMD 0xd04
126#define DBGINST0 0xd08
127#define DBGINST1 0xd0c
128
129#define CR0 0xe00
130#define CR1 0xe04
131#define CR2 0xe08
132#define CR3 0xe0c
133#define CR4 0xe10
134#define CRD 0xe14
135
136#define PERIPH_ID 0xfe0
Boojin Kim3ecf51a2011-12-26 18:55:47 +0900137#define PERIPH_REV_SHIFT 20
138#define PERIPH_REV_MASK 0xf
139#define PERIPH_REV_R0P0 0
140#define PERIPH_REV_R1P0 1
141#define PERIPH_REV_R1P1 2
Boojin Kimb7d861d2011-12-26 18:49:52 +0900142
143#define CR0_PERIPH_REQ_SET (1 << 0)
144#define CR0_BOOT_EN_SET (1 << 1)
145#define CR0_BOOT_MAN_NS (1 << 2)
146#define CR0_NUM_CHANS_SHIFT 4
147#define CR0_NUM_CHANS_MASK 0x7
148#define CR0_NUM_PERIPH_SHIFT 12
149#define CR0_NUM_PERIPH_MASK 0x1f
150#define CR0_NUM_EVENTS_SHIFT 17
151#define CR0_NUM_EVENTS_MASK 0x1f
152
153#define CR1_ICACHE_LEN_SHIFT 0
154#define CR1_ICACHE_LEN_MASK 0x7
155#define CR1_NUM_ICACHELINES_SHIFT 4
156#define CR1_NUM_ICACHELINES_MASK 0xf
157
158#define CRD_DATA_WIDTH_SHIFT 0
159#define CRD_DATA_WIDTH_MASK 0x7
160#define CRD_WR_CAP_SHIFT 4
161#define CRD_WR_CAP_MASK 0x7
162#define CRD_WR_Q_DEP_SHIFT 8
163#define CRD_WR_Q_DEP_MASK 0xf
164#define CRD_RD_CAP_SHIFT 12
165#define CRD_RD_CAP_MASK 0x7
166#define CRD_RD_Q_DEP_SHIFT 16
167#define CRD_RD_Q_DEP_MASK 0xf
168#define CRD_DATA_BUFF_SHIFT 20
169#define CRD_DATA_BUFF_MASK 0x3ff
170
171#define PART 0x330
172#define DESIGNER 0x41
173#define REVISION 0x0
174#define INTEG_CFG 0x0
175#define PERIPH_ID_VAL ((PART << 0) | (DESIGNER << 12))
176
Boojin Kimb7d861d2011-12-26 18:49:52 +0900177#define PL330_STATE_STOPPED (1 << 0)
178#define PL330_STATE_EXECUTING (1 << 1)
179#define PL330_STATE_WFE (1 << 2)
180#define PL330_STATE_FAULTING (1 << 3)
181#define PL330_STATE_COMPLETING (1 << 4)
182#define PL330_STATE_WFP (1 << 5)
183#define PL330_STATE_KILLING (1 << 6)
184#define PL330_STATE_FAULT_COMPLETING (1 << 7)
185#define PL330_STATE_CACHEMISS (1 << 8)
186#define PL330_STATE_UPDTPC (1 << 9)
187#define PL330_STATE_ATBARRIER (1 << 10)
188#define PL330_STATE_QUEUEBUSY (1 << 11)
189#define PL330_STATE_INVALID (1 << 15)
190
191#define PL330_STABLE_STATES (PL330_STATE_STOPPED | PL330_STATE_EXECUTING \
192 | PL330_STATE_WFE | PL330_STATE_FAULTING)
193
194#define CMD_DMAADDH 0x54
195#define CMD_DMAEND 0x00
196#define CMD_DMAFLUSHP 0x35
197#define CMD_DMAGO 0xa0
198#define CMD_DMALD 0x04
199#define CMD_DMALDP 0x25
200#define CMD_DMALP 0x20
201#define CMD_DMALPEND 0x28
202#define CMD_DMAKILL 0x01
203#define CMD_DMAMOV 0xbc
204#define CMD_DMANOP 0x18
205#define CMD_DMARMB 0x12
206#define CMD_DMASEV 0x34
207#define CMD_DMAST 0x08
208#define CMD_DMASTP 0x29
209#define CMD_DMASTZ 0x0c
210#define CMD_DMAWFE 0x36
211#define CMD_DMAWFP 0x30
212#define CMD_DMAWMB 0x13
213
214#define SZ_DMAADDH 3
215#define SZ_DMAEND 1
216#define SZ_DMAFLUSHP 2
217#define SZ_DMALD 1
218#define SZ_DMALDP 2
219#define SZ_DMALP 2
220#define SZ_DMALPEND 2
221#define SZ_DMAKILL 1
222#define SZ_DMAMOV 6
223#define SZ_DMANOP 1
224#define SZ_DMARMB 1
225#define SZ_DMASEV 2
226#define SZ_DMAST 1
227#define SZ_DMASTP 2
228#define SZ_DMASTZ 1
229#define SZ_DMAWFE 2
230#define SZ_DMAWFP 2
231#define SZ_DMAWMB 1
232#define SZ_DMAGO 6
233
234#define BRST_LEN(ccr) ((((ccr) >> CC_SRCBRSTLEN_SHFT) & 0xf) + 1)
235#define BRST_SIZE(ccr) (1 << (((ccr) >> CC_SRCBRSTSIZE_SHFT) & 0x7))
236
237#define BYTE_TO_BURST(b, ccr) ((b) / BRST_SIZE(ccr) / BRST_LEN(ccr))
238#define BURST_TO_BYTE(c, ccr) ((c) * BRST_SIZE(ccr) * BRST_LEN(ccr))
239
240/*
241 * With 256 bytes, we can do more than 2.5MB and 5MB xfers per req
242 * at 1byte/burst for P<->M and M<->M respectively.
243 * For typical scenario, at 1word/burst, 10MB and 20MB xfers per req
244 * should be enough for P<->M and M<->M respectively.
245 */
246#define MCODE_BUFF_PER_REQ 256
247
Boojin Kimb7d861d2011-12-26 18:49:52 +0900248/* Use this _only_ to wait on transient states */
249#define UNTIL(t, s) while (!(_state(t) & (s))) cpu_relax();
250
251#ifdef PL330_DEBUG_MCGEN
252static unsigned cmd_line;
253#define PL330_DBGCMD_DUMP(off, x...) do { \
254 printk("%x:", cmd_line); \
255 printk(x); \
256 cmd_line += off; \
257 } while (0)
258#define PL330_DBGMC_START(addr) (cmd_line = addr)
259#else
260#define PL330_DBGCMD_DUMP(off, x...) do {} while (0)
261#define PL330_DBGMC_START(addr) do {} while (0)
262#endif
263
264/* The number of default descriptors */
Russell King - ARM Linuxd2ebfb32012-03-06 22:34:26 +0000265
Jassi Brarb3040e42010-05-23 20:28:19 -0700266#define NR_DEFAULT_DESC 16
267
Boojin Kimb7d861d2011-12-26 18:49:52 +0900268/* Populated by the PL330 core driver for DMA API driver's info */
269struct pl330_config {
270 u32 periph_id;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900271#define DMAC_MODE_NS (1 << 0)
272 unsigned int mode;
273 unsigned int data_bus_width:10; /* In number of bits */
274 unsigned int data_buf_dep:10;
275 unsigned int num_chan:4;
276 unsigned int num_peri:6;
277 u32 peri_ns;
278 unsigned int num_events:6;
279 u32 irq_ns;
280};
281
Boojin Kimb7d861d2011-12-26 18:49:52 +0900282/**
283 * Request Configuration.
284 * The PL330 core does not modify this and uses the last
285 * working configuration if the request doesn't provide any.
286 *
287 * The Client may want to provide this info only for the
288 * first request and a request with new settings.
289 */
290struct pl330_reqcfg {
291 /* Address Incrementing */
292 unsigned dst_inc:1;
293 unsigned src_inc:1;
294
295 /*
296 * For now, the SRC & DST protection levels
297 * and burst size/length are assumed same.
298 */
299 bool nonsecure;
300 bool privileged;
301 bool insnaccess;
302 unsigned brst_len:5;
303 unsigned brst_size:3; /* in power of 2 */
304
Lars-Peter Clausenf0564c72014-07-06 20:32:19 +0200305 enum pl330_cachectrl dcctl;
306 enum pl330_cachectrl scctl;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900307 enum pl330_byteswap swap;
Boojin Kim3ecf51a2011-12-26 18:55:47 +0900308 struct pl330_config *pcfg;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900309};
310
311/*
312 * One cycle of DMAC operation.
313 * There may be more than one xfer in a request.
314 */
315struct pl330_xfer {
316 u32 src_addr;
317 u32 dst_addr;
318 /* Size to xfer */
319 u32 bytes;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900320};
321
322/* The xfer callbacks are made with one of these arguments. */
323enum pl330_op_err {
324 /* The all xfers in the request were success. */
325 PL330_ERR_NONE,
326 /* If req aborted due to global error. */
327 PL330_ERR_ABORT,
328 /* If req failed due to problem with Channel. */
329 PL330_ERR_FAIL,
330};
331
Boojin Kimb7d861d2011-12-26 18:49:52 +0900332enum dmamov_dst {
333 SAR = 0,
334 CCR,
335 DAR,
336};
337
338enum pl330_dst {
339 SRC = 0,
340 DST,
341};
342
343enum pl330_cond {
344 SINGLE,
345 BURST,
346 ALWAYS,
347};
348
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +0200349struct dma_pl330_desc;
350
Boojin Kimb7d861d2011-12-26 18:49:52 +0900351struct _pl330_req {
352 u32 mc_bus;
353 void *mc_cpu;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +0200354 struct dma_pl330_desc *desc;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900355};
356
357/* ToBeDone for tasklet */
358struct _pl330_tbd {
359 bool reset_dmac;
360 bool reset_mngr;
361 u8 reset_chan;
362};
363
364/* A DMAC Thread */
365struct pl330_thread {
366 u8 id;
367 int ev;
368 /* If the channel is not yet acquired by any client */
369 bool free;
370 /* Parent DMAC */
371 struct pl330_dmac *dmac;
372 /* Only two at a time */
373 struct _pl330_req req[2];
374 /* Index of the last enqueued request */
375 unsigned lstenq;
376 /* Index of the last submitted request or -1 if the DMA is stopped */
377 int req_running;
378};
379
380enum pl330_dmac_state {
381 UNINIT,
382 INIT,
383 DYING,
384};
385
Jassi Brarb3040e42010-05-23 20:28:19 -0700386enum desc_status {
387 /* In the DMAC pool */
388 FREE,
389 /*
Masanari Iidad73111c2012-08-04 23:37:53 +0900390 * Allocated to some channel during prep_xxx
Jassi Brarb3040e42010-05-23 20:28:19 -0700391 * Also may be sitting on the work_list.
392 */
393 PREP,
394 /*
395 * Sitting on the work_list and already submitted
396 * to the PL330 core. Not more than two descriptors
397 * of a channel can be BUSY at any time.
398 */
399 BUSY,
400 /*
401 * Sitting on the channel work_list but xfer done
402 * by PL330 core
403 */
404 DONE,
405};
406
407struct dma_pl330_chan {
408 /* Schedule desc completion */
409 struct tasklet_struct task;
410
411 /* DMA-Engine Channel */
412 struct dma_chan chan;
413
Lars-Peter Clausen04abf5d2014-01-11 20:08:38 +0100414 /* List of submitted descriptors */
415 struct list_head submitted_list;
416 /* List of issued descriptors */
Jassi Brarb3040e42010-05-23 20:28:19 -0700417 struct list_head work_list;
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +0200418 /* List of completed descriptors */
419 struct list_head completed_list;
Jassi Brarb3040e42010-05-23 20:28:19 -0700420
421 /* Pointer to the DMAC that manages this channel,
422 * NULL if the channel is available to be acquired.
423 * As the parent, this DMAC also provides descriptors
424 * to the channel.
425 */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200426 struct pl330_dmac *dmac;
Jassi Brarb3040e42010-05-23 20:28:19 -0700427
428 /* To protect channel manipulation */
429 spinlock_t lock;
430
Lars-Peter Clausen65ad6062014-07-06 20:32:26 +0200431 /*
432 * Hardware channel thread of PL330 DMAC. NULL if the channel is
433 * available.
Jassi Brarb3040e42010-05-23 20:28:19 -0700434 */
Lars-Peter Clausen65ad6062014-07-06 20:32:26 +0200435 struct pl330_thread *thread;
Boojin Kim1b9bb712011-09-02 09:44:30 +0900436
437 /* For D-to-M and M-to-D channels */
438 int burst_sz; /* the peripheral fifo width */
Boojin Kim1d0c1d62011-09-02 09:44:31 +0900439 int burst_len; /* the number of burst */
Boojin Kim1b9bb712011-09-02 09:44:30 +0900440 dma_addr_t fifo_addr;
Boojin Kim42bc9cf2011-09-02 09:44:33 +0900441
442 /* for cyclic capability */
443 bool cyclic;
Jassi Brarb3040e42010-05-23 20:28:19 -0700444};
445
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200446struct pl330_dmac {
Jassi Brarb3040e42010-05-23 20:28:19 -0700447 /* DMA-Engine Device */
448 struct dma_device ddma;
449
Lars-Peter Clausenb714b842013-11-25 16:07:46 +0100450 /* Holds info about sg limitations */
451 struct device_dma_parameters dma_parms;
452
Jassi Brarb3040e42010-05-23 20:28:19 -0700453 /* Pool of descriptors available for the DMAC's channels */
454 struct list_head desc_pool;
455 /* To protect desc_pool manipulation */
456 spinlock_t pool_lock;
457
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200458 /* Size of MicroCode buffers for each channel. */
459 unsigned mcbufsz;
460 /* ioremap'ed address of PL330 registers. */
461 void __iomem *base;
462 /* Populated by the PL330 core driver during pl330_add */
463 struct pl330_config pcfg;
464
465 spinlock_t lock;
466 /* Maximum possible events/irqs */
467 int events[32];
468 /* BUS address of MicroCode buffer */
469 dma_addr_t mcode_bus;
470 /* CPU address of MicroCode buffer */
471 void *mcode_cpu;
472 /* List of all Channel threads */
473 struct pl330_thread *channels;
474 /* Pointer to the MANAGER thread */
475 struct pl330_thread *manager;
476 /* To handle bad news in interrupt */
477 struct tasklet_struct tasks;
478 struct _pl330_tbd dmac_tbd;
479 /* State of DMAC operation */
480 enum pl330_dmac_state state;
481 /* Holds list of reqs with due callbacks */
482 struct list_head req_done;
483
Jassi Brarb3040e42010-05-23 20:28:19 -0700484 /* Peripheral channels connected to this DMAC */
Lars-Peter Clausen70cbb162014-01-11 20:08:39 +0100485 unsigned int num_peripherals;
Rob Herring4e0e6102011-07-25 16:05:04 -0500486 struct dma_pl330_chan *peripherals; /* keep at end */
Jassi Brarb3040e42010-05-23 20:28:19 -0700487};
488
489struct dma_pl330_desc {
490 /* To attach to a queue as child */
491 struct list_head node;
492
493 /* Descriptor for the DMA Engine API */
494 struct dma_async_tx_descriptor txd;
495
496 /* Xfer for PL330 core */
497 struct pl330_xfer px;
498
499 struct pl330_reqcfg rqcfg;
Jassi Brarb3040e42010-05-23 20:28:19 -0700500
501 enum desc_status status;
502
503 /* The channel which currently holds this desc */
504 struct dma_pl330_chan *pchan;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +0200505
506 enum dma_transfer_direction rqtype;
507 /* Index of peripheral for the xfer. */
508 unsigned peri:5;
509 /* Hook to attach to DMAC's list of reqs with due callback */
510 struct list_head rqd;
511};
512
513struct _xfer_spec {
514 u32 ccr;
515 struct dma_pl330_desc *desc;
Jassi Brarb3040e42010-05-23 20:28:19 -0700516};
517
Boojin Kimb7d861d2011-12-26 18:49:52 +0900518static inline bool _queue_empty(struct pl330_thread *thrd)
519{
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +0200520 return thrd->req[0].desc == NULL && thrd->req[1].desc == NULL;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900521}
522
523static inline bool _queue_full(struct pl330_thread *thrd)
524{
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +0200525 return thrd->req[0].desc != NULL && thrd->req[1].desc != NULL;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900526}
527
528static inline bool is_manager(struct pl330_thread *thrd)
529{
Lars-Peter Clausenfbbcd9b2014-07-06 20:32:28 +0200530 return thrd->dmac->manager == thrd;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900531}
532
533/* If manager of the thread is in Non-Secure mode */
534static inline bool _manager_ns(struct pl330_thread *thrd)
535{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200536 return (thrd->dmac->pcfg.mode & DMAC_MODE_NS) ? true : false;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900537}
538
Boojin Kim3ecf51a2011-12-26 18:55:47 +0900539static inline u32 get_revision(u32 periph_id)
540{
541 return (periph_id >> PERIPH_REV_SHIFT) & PERIPH_REV_MASK;
542}
543
Boojin Kimb7d861d2011-12-26 18:49:52 +0900544static inline u32 _emit_ADDH(unsigned dry_run, u8 buf[],
545 enum pl330_dst da, u16 val)
546{
547 if (dry_run)
548 return SZ_DMAADDH;
549
550 buf[0] = CMD_DMAADDH;
551 buf[0] |= (da << 1);
552 *((u16 *)&buf[1]) = val;
553
554 PL330_DBGCMD_DUMP(SZ_DMAADDH, "\tDMAADDH %s %u\n",
555 da == 1 ? "DA" : "SA", val);
556
557 return SZ_DMAADDH;
558}
559
560static inline u32 _emit_END(unsigned dry_run, u8 buf[])
561{
562 if (dry_run)
563 return SZ_DMAEND;
564
565 buf[0] = CMD_DMAEND;
566
567 PL330_DBGCMD_DUMP(SZ_DMAEND, "\tDMAEND\n");
568
569 return SZ_DMAEND;
570}
571
572static inline u32 _emit_FLUSHP(unsigned dry_run, u8 buf[], u8 peri)
573{
574 if (dry_run)
575 return SZ_DMAFLUSHP;
576
577 buf[0] = CMD_DMAFLUSHP;
578
579 peri &= 0x1f;
580 peri <<= 3;
581 buf[1] = peri;
582
583 PL330_DBGCMD_DUMP(SZ_DMAFLUSHP, "\tDMAFLUSHP %u\n", peri >> 3);
584
585 return SZ_DMAFLUSHP;
586}
587
588static inline u32 _emit_LD(unsigned dry_run, u8 buf[], enum pl330_cond cond)
589{
590 if (dry_run)
591 return SZ_DMALD;
592
593 buf[0] = CMD_DMALD;
594
595 if (cond == SINGLE)
596 buf[0] |= (0 << 1) | (1 << 0);
597 else if (cond == BURST)
598 buf[0] |= (1 << 1) | (1 << 0);
599
600 PL330_DBGCMD_DUMP(SZ_DMALD, "\tDMALD%c\n",
601 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
602
603 return SZ_DMALD;
604}
605
606static inline u32 _emit_LDP(unsigned dry_run, u8 buf[],
607 enum pl330_cond cond, u8 peri)
608{
609 if (dry_run)
610 return SZ_DMALDP;
611
612 buf[0] = CMD_DMALDP;
613
614 if (cond == BURST)
615 buf[0] |= (1 << 1);
616
617 peri &= 0x1f;
618 peri <<= 3;
619 buf[1] = peri;
620
621 PL330_DBGCMD_DUMP(SZ_DMALDP, "\tDMALDP%c %u\n",
622 cond == SINGLE ? 'S' : 'B', peri >> 3);
623
624 return SZ_DMALDP;
625}
626
627static inline u32 _emit_LP(unsigned dry_run, u8 buf[],
628 unsigned loop, u8 cnt)
629{
630 if (dry_run)
631 return SZ_DMALP;
632
633 buf[0] = CMD_DMALP;
634
635 if (loop)
636 buf[0] |= (1 << 1);
637
638 cnt--; /* DMAC increments by 1 internally */
639 buf[1] = cnt;
640
641 PL330_DBGCMD_DUMP(SZ_DMALP, "\tDMALP_%c %u\n", loop ? '1' : '0', cnt);
642
643 return SZ_DMALP;
644}
645
646struct _arg_LPEND {
647 enum pl330_cond cond;
648 bool forever;
649 unsigned loop;
650 u8 bjump;
651};
652
653static inline u32 _emit_LPEND(unsigned dry_run, u8 buf[],
654 const struct _arg_LPEND *arg)
655{
656 enum pl330_cond cond = arg->cond;
657 bool forever = arg->forever;
658 unsigned loop = arg->loop;
659 u8 bjump = arg->bjump;
660
661 if (dry_run)
662 return SZ_DMALPEND;
663
664 buf[0] = CMD_DMALPEND;
665
666 if (loop)
667 buf[0] |= (1 << 2);
668
669 if (!forever)
670 buf[0] |= (1 << 4);
671
672 if (cond == SINGLE)
673 buf[0] |= (0 << 1) | (1 << 0);
674 else if (cond == BURST)
675 buf[0] |= (1 << 1) | (1 << 0);
676
677 buf[1] = bjump;
678
679 PL330_DBGCMD_DUMP(SZ_DMALPEND, "\tDMALP%s%c_%c bjmpto_%x\n",
680 forever ? "FE" : "END",
681 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'),
682 loop ? '1' : '0',
683 bjump);
684
685 return SZ_DMALPEND;
686}
687
688static inline u32 _emit_KILL(unsigned dry_run, u8 buf[])
689{
690 if (dry_run)
691 return SZ_DMAKILL;
692
693 buf[0] = CMD_DMAKILL;
694
695 return SZ_DMAKILL;
696}
697
698static inline u32 _emit_MOV(unsigned dry_run, u8 buf[],
699 enum dmamov_dst dst, u32 val)
700{
701 if (dry_run)
702 return SZ_DMAMOV;
703
704 buf[0] = CMD_DMAMOV;
705 buf[1] = dst;
706 *((u32 *)&buf[2]) = val;
707
708 PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n",
709 dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val);
710
711 return SZ_DMAMOV;
712}
713
714static inline u32 _emit_NOP(unsigned dry_run, u8 buf[])
715{
716 if (dry_run)
717 return SZ_DMANOP;
718
719 buf[0] = CMD_DMANOP;
720
721 PL330_DBGCMD_DUMP(SZ_DMANOP, "\tDMANOP\n");
722
723 return SZ_DMANOP;
724}
725
726static inline u32 _emit_RMB(unsigned dry_run, u8 buf[])
727{
728 if (dry_run)
729 return SZ_DMARMB;
730
731 buf[0] = CMD_DMARMB;
732
733 PL330_DBGCMD_DUMP(SZ_DMARMB, "\tDMARMB\n");
734
735 return SZ_DMARMB;
736}
737
738static inline u32 _emit_SEV(unsigned dry_run, u8 buf[], u8 ev)
739{
740 if (dry_run)
741 return SZ_DMASEV;
742
743 buf[0] = CMD_DMASEV;
744
745 ev &= 0x1f;
746 ev <<= 3;
747 buf[1] = ev;
748
749 PL330_DBGCMD_DUMP(SZ_DMASEV, "\tDMASEV %u\n", ev >> 3);
750
751 return SZ_DMASEV;
752}
753
754static inline u32 _emit_ST(unsigned dry_run, u8 buf[], enum pl330_cond cond)
755{
756 if (dry_run)
757 return SZ_DMAST;
758
759 buf[0] = CMD_DMAST;
760
761 if (cond == SINGLE)
762 buf[0] |= (0 << 1) | (1 << 0);
763 else if (cond == BURST)
764 buf[0] |= (1 << 1) | (1 << 0);
765
766 PL330_DBGCMD_DUMP(SZ_DMAST, "\tDMAST%c\n",
767 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
768
769 return SZ_DMAST;
770}
771
772static inline u32 _emit_STP(unsigned dry_run, u8 buf[],
773 enum pl330_cond cond, u8 peri)
774{
775 if (dry_run)
776 return SZ_DMASTP;
777
778 buf[0] = CMD_DMASTP;
779
780 if (cond == BURST)
781 buf[0] |= (1 << 1);
782
783 peri &= 0x1f;
784 peri <<= 3;
785 buf[1] = peri;
786
787 PL330_DBGCMD_DUMP(SZ_DMASTP, "\tDMASTP%c %u\n",
788 cond == SINGLE ? 'S' : 'B', peri >> 3);
789
790 return SZ_DMASTP;
791}
792
793static inline u32 _emit_STZ(unsigned dry_run, u8 buf[])
794{
795 if (dry_run)
796 return SZ_DMASTZ;
797
798 buf[0] = CMD_DMASTZ;
799
800 PL330_DBGCMD_DUMP(SZ_DMASTZ, "\tDMASTZ\n");
801
802 return SZ_DMASTZ;
803}
804
805static inline u32 _emit_WFE(unsigned dry_run, u8 buf[], u8 ev,
806 unsigned invalidate)
807{
808 if (dry_run)
809 return SZ_DMAWFE;
810
811 buf[0] = CMD_DMAWFE;
812
813 ev &= 0x1f;
814 ev <<= 3;
815 buf[1] = ev;
816
817 if (invalidate)
818 buf[1] |= (1 << 1);
819
820 PL330_DBGCMD_DUMP(SZ_DMAWFE, "\tDMAWFE %u%s\n",
821 ev >> 3, invalidate ? ", I" : "");
822
823 return SZ_DMAWFE;
824}
825
826static inline u32 _emit_WFP(unsigned dry_run, u8 buf[],
827 enum pl330_cond cond, u8 peri)
828{
829 if (dry_run)
830 return SZ_DMAWFP;
831
832 buf[0] = CMD_DMAWFP;
833
834 if (cond == SINGLE)
835 buf[0] |= (0 << 1) | (0 << 0);
836 else if (cond == BURST)
837 buf[0] |= (1 << 1) | (0 << 0);
838 else
839 buf[0] |= (0 << 1) | (1 << 0);
840
841 peri &= 0x1f;
842 peri <<= 3;
843 buf[1] = peri;
844
845 PL330_DBGCMD_DUMP(SZ_DMAWFP, "\tDMAWFP%c %u\n",
846 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'P'), peri >> 3);
847
848 return SZ_DMAWFP;
849}
850
851static inline u32 _emit_WMB(unsigned dry_run, u8 buf[])
852{
853 if (dry_run)
854 return SZ_DMAWMB;
855
856 buf[0] = CMD_DMAWMB;
857
858 PL330_DBGCMD_DUMP(SZ_DMAWMB, "\tDMAWMB\n");
859
860 return SZ_DMAWMB;
861}
862
863struct _arg_GO {
864 u8 chan;
865 u32 addr;
866 unsigned ns;
867};
868
869static inline u32 _emit_GO(unsigned dry_run, u8 buf[],
870 const struct _arg_GO *arg)
871{
872 u8 chan = arg->chan;
873 u32 addr = arg->addr;
874 unsigned ns = arg->ns;
875
876 if (dry_run)
877 return SZ_DMAGO;
878
879 buf[0] = CMD_DMAGO;
880 buf[0] |= (ns << 1);
881
882 buf[1] = chan & 0x7;
883
884 *((u32 *)&buf[2]) = addr;
885
886 return SZ_DMAGO;
887}
888
889#define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t)
890
891/* Returns Time-Out */
892static bool _until_dmac_idle(struct pl330_thread *thrd)
893{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200894 void __iomem *regs = thrd->dmac->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900895 unsigned long loops = msecs_to_loops(5);
896
897 do {
898 /* Until Manager is Idle */
899 if (!(readl(regs + DBGSTATUS) & DBG_BUSY))
900 break;
901
902 cpu_relax();
903 } while (--loops);
904
905 if (!loops)
906 return true;
907
908 return false;
909}
910
911static inline void _execute_DBGINSN(struct pl330_thread *thrd,
912 u8 insn[], bool as_manager)
913{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200914 void __iomem *regs = thrd->dmac->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900915 u32 val;
916
917 val = (insn[0] << 16) | (insn[1] << 24);
918 if (!as_manager) {
919 val |= (1 << 0);
920 val |= (thrd->id << 8); /* Channel Number */
921 }
922 writel(val, regs + DBGINST0);
923
924 val = *((u32 *)&insn[2]);
925 writel(val, regs + DBGINST1);
926
927 /* If timed out due to halted state-machine */
928 if (_until_dmac_idle(thrd)) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200929 dev_err(thrd->dmac->ddma.dev, "DMAC halted!\n");
Boojin Kimb7d861d2011-12-26 18:49:52 +0900930 return;
931 }
932
933 /* Get going */
934 writel(0, regs + DBGCMD);
935}
936
Boojin Kimb7d861d2011-12-26 18:49:52 +0900937static inline u32 _state(struct pl330_thread *thrd)
938{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200939 void __iomem *regs = thrd->dmac->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900940 u32 val;
941
942 if (is_manager(thrd))
943 val = readl(regs + DS) & 0xf;
944 else
945 val = readl(regs + CS(thrd->id)) & 0xf;
946
947 switch (val) {
948 case DS_ST_STOP:
949 return PL330_STATE_STOPPED;
950 case DS_ST_EXEC:
951 return PL330_STATE_EXECUTING;
952 case DS_ST_CMISS:
953 return PL330_STATE_CACHEMISS;
954 case DS_ST_UPDTPC:
955 return PL330_STATE_UPDTPC;
956 case DS_ST_WFE:
957 return PL330_STATE_WFE;
958 case DS_ST_FAULT:
959 return PL330_STATE_FAULTING;
960 case DS_ST_ATBRR:
961 if (is_manager(thrd))
962 return PL330_STATE_INVALID;
963 else
964 return PL330_STATE_ATBARRIER;
965 case DS_ST_QBUSY:
966 if (is_manager(thrd))
967 return PL330_STATE_INVALID;
968 else
969 return PL330_STATE_QUEUEBUSY;
970 case DS_ST_WFP:
971 if (is_manager(thrd))
972 return PL330_STATE_INVALID;
973 else
974 return PL330_STATE_WFP;
975 case DS_ST_KILL:
976 if (is_manager(thrd))
977 return PL330_STATE_INVALID;
978 else
979 return PL330_STATE_KILLING;
980 case DS_ST_CMPLT:
981 if (is_manager(thrd))
982 return PL330_STATE_INVALID;
983 else
984 return PL330_STATE_COMPLETING;
985 case DS_ST_FLTCMP:
986 if (is_manager(thrd))
987 return PL330_STATE_INVALID;
988 else
989 return PL330_STATE_FAULT_COMPLETING;
990 default:
991 return PL330_STATE_INVALID;
992 }
993}
994
995static void _stop(struct pl330_thread *thrd)
996{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200997 void __iomem *regs = thrd->dmac->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900998 u8 insn[6] = {0, 0, 0, 0, 0, 0};
999
1000 if (_state(thrd) == PL330_STATE_FAULT_COMPLETING)
1001 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
1002
1003 /* Return if nothing needs to be done */
1004 if (_state(thrd) == PL330_STATE_COMPLETING
1005 || _state(thrd) == PL330_STATE_KILLING
1006 || _state(thrd) == PL330_STATE_STOPPED)
1007 return;
1008
1009 _emit_KILL(0, insn);
1010
1011 /* Stop generating interrupts for SEV */
1012 writel(readl(regs + INTEN) & ~(1 << thrd->ev), regs + INTEN);
1013
1014 _execute_DBGINSN(thrd, insn, is_manager(thrd));
1015}
1016
1017/* Start doing req 'idx' of thread 'thrd' */
1018static bool _trigger(struct pl330_thread *thrd)
1019{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001020 void __iomem *regs = thrd->dmac->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001021 struct _pl330_req *req;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001022 struct dma_pl330_desc *desc;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001023 struct _arg_GO go;
1024 unsigned ns;
1025 u8 insn[6] = {0, 0, 0, 0, 0, 0};
1026 int idx;
1027
1028 /* Return if already ACTIVE */
1029 if (_state(thrd) != PL330_STATE_STOPPED)
1030 return true;
1031
1032 idx = 1 - thrd->lstenq;
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001033 if (thrd->req[idx].desc != NULL) {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001034 req = &thrd->req[idx];
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001035 } else {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001036 idx = thrd->lstenq;
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001037 if (thrd->req[idx].desc != NULL)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001038 req = &thrd->req[idx];
1039 else
1040 req = NULL;
1041 }
1042
1043 /* Return if no request */
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001044 if (!req)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001045 return true;
1046
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001047 desc = req->desc;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001048
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001049 ns = desc->rqcfg.nonsecure ? 1 : 0;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001050
1051 /* See 'Abort Sources' point-4 at Page 2-25 */
1052 if (_manager_ns(thrd) && !ns)
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001053 dev_info(thrd->dmac->ddma.dev, "%s:%d Recipe for ABORT!\n",
Boojin Kimb7d861d2011-12-26 18:49:52 +09001054 __func__, __LINE__);
1055
1056 go.chan = thrd->id;
1057 go.addr = req->mc_bus;
1058 go.ns = ns;
1059 _emit_GO(0, insn, &go);
1060
1061 /* Set to generate interrupts for SEV */
1062 writel(readl(regs + INTEN) | (1 << thrd->ev), regs + INTEN);
1063
1064 /* Only manager can execute GO */
1065 _execute_DBGINSN(thrd, insn, true);
1066
1067 thrd->req_running = idx;
1068
1069 return true;
1070}
1071
1072static bool _start(struct pl330_thread *thrd)
1073{
1074 switch (_state(thrd)) {
1075 case PL330_STATE_FAULT_COMPLETING:
1076 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
1077
1078 if (_state(thrd) == PL330_STATE_KILLING)
1079 UNTIL(thrd, PL330_STATE_STOPPED)
1080
1081 case PL330_STATE_FAULTING:
1082 _stop(thrd);
1083
1084 case PL330_STATE_KILLING:
1085 case PL330_STATE_COMPLETING:
1086 UNTIL(thrd, PL330_STATE_STOPPED)
1087
1088 case PL330_STATE_STOPPED:
1089 return _trigger(thrd);
1090
1091 case PL330_STATE_WFP:
1092 case PL330_STATE_QUEUEBUSY:
1093 case PL330_STATE_ATBARRIER:
1094 case PL330_STATE_UPDTPC:
1095 case PL330_STATE_CACHEMISS:
1096 case PL330_STATE_EXECUTING:
1097 return true;
1098
1099 case PL330_STATE_WFE: /* For RESUME, nothing yet */
1100 default:
1101 return false;
1102 }
1103}
1104
1105static inline int _ldst_memtomem(unsigned dry_run, u8 buf[],
1106 const struct _xfer_spec *pxs, int cyc)
1107{
1108 int off = 0;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001109 struct pl330_config *pcfg = pxs->desc->rqcfg.pcfg;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001110
Boojin Kim3ecf51a2011-12-26 18:55:47 +09001111 /* check lock-up free version */
1112 if (get_revision(pcfg->periph_id) >= PERIPH_REV_R1P0) {
1113 while (cyc--) {
1114 off += _emit_LD(dry_run, &buf[off], ALWAYS);
1115 off += _emit_ST(dry_run, &buf[off], ALWAYS);
1116 }
1117 } else {
1118 while (cyc--) {
1119 off += _emit_LD(dry_run, &buf[off], ALWAYS);
1120 off += _emit_RMB(dry_run, &buf[off]);
1121 off += _emit_ST(dry_run, &buf[off], ALWAYS);
1122 off += _emit_WMB(dry_run, &buf[off]);
1123 }
Boojin Kimb7d861d2011-12-26 18:49:52 +09001124 }
1125
1126 return off;
1127}
1128
1129static inline int _ldst_devtomem(unsigned dry_run, u8 buf[],
1130 const struct _xfer_spec *pxs, int cyc)
1131{
1132 int off = 0;
1133
1134 while (cyc--) {
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001135 off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->desc->peri);
1136 off += _emit_LDP(dry_run, &buf[off], SINGLE, pxs->desc->peri);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001137 off += _emit_ST(dry_run, &buf[off], ALWAYS);
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001138 off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001139 }
1140
1141 return off;
1142}
1143
1144static inline int _ldst_memtodev(unsigned dry_run, u8 buf[],
1145 const struct _xfer_spec *pxs, int cyc)
1146{
1147 int off = 0;
1148
1149 while (cyc--) {
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001150 off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->desc->peri);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001151 off += _emit_LD(dry_run, &buf[off], ALWAYS);
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001152 off += _emit_STP(dry_run, &buf[off], SINGLE, pxs->desc->peri);
1153 off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001154 }
1155
1156 return off;
1157}
1158
1159static int _bursts(unsigned dry_run, u8 buf[],
1160 const struct _xfer_spec *pxs, int cyc)
1161{
1162 int off = 0;
1163
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001164 switch (pxs->desc->rqtype) {
Lars-Peter Clausen585a9d02014-07-06 20:32:18 +02001165 case DMA_MEM_TO_DEV:
Boojin Kimb7d861d2011-12-26 18:49:52 +09001166 off += _ldst_memtodev(dry_run, &buf[off], pxs, cyc);
1167 break;
Lars-Peter Clausen585a9d02014-07-06 20:32:18 +02001168 case DMA_DEV_TO_MEM:
Boojin Kimb7d861d2011-12-26 18:49:52 +09001169 off += _ldst_devtomem(dry_run, &buf[off], pxs, cyc);
1170 break;
Lars-Peter Clausen585a9d02014-07-06 20:32:18 +02001171 case DMA_MEM_TO_MEM:
Boojin Kimb7d861d2011-12-26 18:49:52 +09001172 off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc);
1173 break;
1174 default:
1175 off += 0x40000000; /* Scare off the Client */
1176 break;
1177 }
1178
1179 return off;
1180}
1181
1182/* Returns bytes consumed and updates bursts */
1183static inline int _loop(unsigned dry_run, u8 buf[],
1184 unsigned long *bursts, const struct _xfer_spec *pxs)
1185{
1186 int cyc, cycmax, szlp, szlpend, szbrst, off;
1187 unsigned lcnt0, lcnt1, ljmp0, ljmp1;
1188 struct _arg_LPEND lpend;
1189
1190 /* Max iterations possible in DMALP is 256 */
1191 if (*bursts >= 256*256) {
1192 lcnt1 = 256;
1193 lcnt0 = 256;
1194 cyc = *bursts / lcnt1 / lcnt0;
1195 } else if (*bursts > 256) {
1196 lcnt1 = 256;
1197 lcnt0 = *bursts / lcnt1;
1198 cyc = 1;
1199 } else {
1200 lcnt1 = *bursts;
1201 lcnt0 = 0;
1202 cyc = 1;
1203 }
1204
1205 szlp = _emit_LP(1, buf, 0, 0);
1206 szbrst = _bursts(1, buf, pxs, 1);
1207
1208 lpend.cond = ALWAYS;
1209 lpend.forever = false;
1210 lpend.loop = 0;
1211 lpend.bjump = 0;
1212 szlpend = _emit_LPEND(1, buf, &lpend);
1213
1214 if (lcnt0) {
1215 szlp *= 2;
1216 szlpend *= 2;
1217 }
1218
1219 /*
1220 * Max bursts that we can unroll due to limit on the
1221 * size of backward jump that can be encoded in DMALPEND
1222 * which is 8-bits and hence 255
1223 */
1224 cycmax = (255 - (szlp + szlpend)) / szbrst;
1225
1226 cyc = (cycmax < cyc) ? cycmax : cyc;
1227
1228 off = 0;
1229
1230 if (lcnt0) {
1231 off += _emit_LP(dry_run, &buf[off], 0, lcnt0);
1232 ljmp0 = off;
1233 }
1234
1235 off += _emit_LP(dry_run, &buf[off], 1, lcnt1);
1236 ljmp1 = off;
1237
1238 off += _bursts(dry_run, &buf[off], pxs, cyc);
1239
1240 lpend.cond = ALWAYS;
1241 lpend.forever = false;
1242 lpend.loop = 1;
1243 lpend.bjump = off - ljmp1;
1244 off += _emit_LPEND(dry_run, &buf[off], &lpend);
1245
1246 if (lcnt0) {
1247 lpend.cond = ALWAYS;
1248 lpend.forever = false;
1249 lpend.loop = 0;
1250 lpend.bjump = off - ljmp0;
1251 off += _emit_LPEND(dry_run, &buf[off], &lpend);
1252 }
1253
1254 *bursts = lcnt1 * cyc;
1255 if (lcnt0)
1256 *bursts *= lcnt0;
1257
1258 return off;
1259}
1260
1261static inline int _setup_loops(unsigned dry_run, u8 buf[],
1262 const struct _xfer_spec *pxs)
1263{
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001264 struct pl330_xfer *x = &pxs->desc->px;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001265 u32 ccr = pxs->ccr;
1266 unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr);
1267 int off = 0;
1268
1269 while (bursts) {
1270 c = bursts;
1271 off += _loop(dry_run, &buf[off], &c, pxs);
1272 bursts -= c;
1273 }
1274
1275 return off;
1276}
1277
1278static inline int _setup_xfer(unsigned dry_run, u8 buf[],
1279 const struct _xfer_spec *pxs)
1280{
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001281 struct pl330_xfer *x = &pxs->desc->px;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001282 int off = 0;
1283
1284 /* DMAMOV SAR, x->src_addr */
1285 off += _emit_MOV(dry_run, &buf[off], SAR, x->src_addr);
1286 /* DMAMOV DAR, x->dst_addr */
1287 off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr);
1288
1289 /* Setup Loop(s) */
1290 off += _setup_loops(dry_run, &buf[off], pxs);
1291
1292 return off;
1293}
1294
1295/*
1296 * A req is a sequence of one or more xfer units.
1297 * Returns the number of bytes taken to setup the MC for the req.
1298 */
1299static int _setup_req(unsigned dry_run, struct pl330_thread *thrd,
1300 unsigned index, struct _xfer_spec *pxs)
1301{
1302 struct _pl330_req *req = &thrd->req[index];
1303 struct pl330_xfer *x;
1304 u8 *buf = req->mc_cpu;
1305 int off = 0;
1306
1307 PL330_DBGMC_START(req->mc_bus);
1308
1309 /* DMAMOV CCR, ccr */
1310 off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr);
1311
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001312 x = &pxs->desc->px;
Lars-Peter Clausend5cef122014-07-06 20:32:23 +02001313 /* Error if xfer length is not aligned at burst size */
1314 if (x->bytes % (BRST_SIZE(pxs->ccr) * BRST_LEN(pxs->ccr)))
1315 return -EINVAL;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001316
Lars-Peter Clausend5cef122014-07-06 20:32:23 +02001317 off += _setup_xfer(dry_run, &buf[off], pxs);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001318
1319 /* DMASEV peripheral/event */
1320 off += _emit_SEV(dry_run, &buf[off], thrd->ev);
1321 /* DMAEND */
1322 off += _emit_END(dry_run, &buf[off]);
1323
1324 return off;
1325}
1326
1327static inline u32 _prepare_ccr(const struct pl330_reqcfg *rqc)
1328{
1329 u32 ccr = 0;
1330
1331 if (rqc->src_inc)
1332 ccr |= CC_SRCINC;
1333
1334 if (rqc->dst_inc)
1335 ccr |= CC_DSTINC;
1336
1337 /* We set same protection levels for Src and DST for now */
1338 if (rqc->privileged)
1339 ccr |= CC_SRCPRI | CC_DSTPRI;
1340 if (rqc->nonsecure)
1341 ccr |= CC_SRCNS | CC_DSTNS;
1342 if (rqc->insnaccess)
1343 ccr |= CC_SRCIA | CC_DSTIA;
1344
1345 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_SRCBRSTLEN_SHFT);
1346 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_DSTBRSTLEN_SHFT);
1347
1348 ccr |= (rqc->brst_size << CC_SRCBRSTSIZE_SHFT);
1349 ccr |= (rqc->brst_size << CC_DSTBRSTSIZE_SHFT);
1350
1351 ccr |= (rqc->scctl << CC_SRCCCTRL_SHFT);
1352 ccr |= (rqc->dcctl << CC_DSTCCTRL_SHFT);
1353
1354 ccr |= (rqc->swap << CC_SWAP_SHFT);
1355
1356 return ccr;
1357}
1358
Boojin Kimb7d861d2011-12-26 18:49:52 +09001359/*
1360 * Submit a list of xfers after which the client wants notification.
1361 * Client is not notified after each xfer unit, just once after all
1362 * xfer units are done or some error occurs.
1363 */
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001364static int pl330_submit_req(struct pl330_thread *thrd,
1365 struct dma_pl330_desc *desc)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001366{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001367 struct pl330_dmac *pl330 = thrd->dmac;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001368 struct _xfer_spec xs;
1369 unsigned long flags;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001370 unsigned idx;
1371 u32 ccr;
1372 int ret = 0;
1373
Boojin Kimb7d861d2011-12-26 18:49:52 +09001374 if (pl330->state == DYING
1375 || pl330->dmac_tbd.reset_chan & (1 << thrd->id)) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001376 dev_info(thrd->dmac->ddma.dev, "%s:%d\n",
Boojin Kimb7d861d2011-12-26 18:49:52 +09001377 __func__, __LINE__);
1378 return -EAGAIN;
1379 }
1380
1381 /* If request for non-existing peripheral */
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001382 if (desc->rqtype != DMA_MEM_TO_MEM &&
1383 desc->peri >= pl330->pcfg.num_peri) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001384 dev_info(thrd->dmac->ddma.dev,
Boojin Kimb7d861d2011-12-26 18:49:52 +09001385 "%s:%d Invalid peripheral(%u)!\n",
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001386 __func__, __LINE__, desc->peri);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001387 return -EINVAL;
1388 }
1389
1390 spin_lock_irqsave(&pl330->lock, flags);
1391
1392 if (_queue_full(thrd)) {
1393 ret = -EAGAIN;
1394 goto xfer_exit;
1395 }
1396
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001397 /* Prefer Secure Channel */
1398 if (!_manager_ns(thrd))
1399 desc->rqcfg.nonsecure = 0;
1400 else
1401 desc->rqcfg.nonsecure = 1;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001402
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001403 ccr = _prepare_ccr(&desc->rqcfg);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001404
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001405 idx = thrd->req[0].desc == NULL ? 0 : 1;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001406
1407 xs.ccr = ccr;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001408 xs.desc = desc;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001409
1410 /* First dry run to check if req is acceptable */
1411 ret = _setup_req(1, thrd, idx, &xs);
1412 if (ret < 0)
1413 goto xfer_exit;
1414
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001415 if (ret > pl330->mcbufsz / 2) {
1416 dev_info(pl330->ddma.dev, "%s:%d Trying increasing mcbufsz\n",
Boojin Kimb7d861d2011-12-26 18:49:52 +09001417 __func__, __LINE__);
1418 ret = -ENOMEM;
1419 goto xfer_exit;
1420 }
1421
1422 /* Hook the request */
1423 thrd->lstenq = idx;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001424 thrd->req[idx].desc = desc;
Lars-Peter Clausenbe025322014-07-06 20:32:24 +02001425 _setup_req(0, thrd, idx, &xs);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001426
1427 ret = 0;
1428
1429xfer_exit:
1430 spin_unlock_irqrestore(&pl330->lock, flags);
1431
1432 return ret;
1433}
1434
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001435static void dma_pl330_rqcb(struct dma_pl330_desc *desc, enum pl330_op_err err)
Lars-Peter Clausen6079d382014-07-06 20:32:25 +02001436{
Javier Martinez Canillasb1e51d72014-07-19 03:21:47 +02001437 struct dma_pl330_chan *pch;
Lars-Peter Clausen6079d382014-07-06 20:32:25 +02001438 unsigned long flags;
1439
Javier Martinez Canillasb1e51d72014-07-19 03:21:47 +02001440 if (!desc)
1441 return;
1442
1443 pch = desc->pchan;
1444
Lars-Peter Clausen6079d382014-07-06 20:32:25 +02001445 /* If desc aborted */
1446 if (!pch)
1447 return;
1448
1449 spin_lock_irqsave(&pch->lock, flags);
1450
1451 desc->status = DONE;
1452
1453 spin_unlock_irqrestore(&pch->lock, flags);
1454
1455 tasklet_schedule(&pch->task);
1456}
1457
Boojin Kimb7d861d2011-12-26 18:49:52 +09001458static void pl330_dotask(unsigned long data)
1459{
1460 struct pl330_dmac *pl330 = (struct pl330_dmac *) data;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001461 unsigned long flags;
1462 int i;
1463
1464 spin_lock_irqsave(&pl330->lock, flags);
1465
1466 /* The DMAC itself gone nuts */
1467 if (pl330->dmac_tbd.reset_dmac) {
1468 pl330->state = DYING;
1469 /* Reset the manager too */
1470 pl330->dmac_tbd.reset_mngr = true;
1471 /* Clear the reset flag */
1472 pl330->dmac_tbd.reset_dmac = false;
1473 }
1474
1475 if (pl330->dmac_tbd.reset_mngr) {
1476 _stop(pl330->manager);
1477 /* Reset all channels */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001478 pl330->dmac_tbd.reset_chan = (1 << pl330->pcfg.num_chan) - 1;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001479 /* Clear the reset flag */
1480 pl330->dmac_tbd.reset_mngr = false;
1481 }
1482
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001483 for (i = 0; i < pl330->pcfg.num_chan; i++) {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001484
1485 if (pl330->dmac_tbd.reset_chan & (1 << i)) {
1486 struct pl330_thread *thrd = &pl330->channels[i];
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001487 void __iomem *regs = pl330->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001488 enum pl330_op_err err;
1489
1490 _stop(thrd);
1491
1492 if (readl(regs + FSC) & (1 << thrd->id))
1493 err = PL330_ERR_FAIL;
1494 else
1495 err = PL330_ERR_ABORT;
1496
1497 spin_unlock_irqrestore(&pl330->lock, flags);
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001498 dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, err);
1499 dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, err);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001500 spin_lock_irqsave(&pl330->lock, flags);
1501
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001502 thrd->req[0].desc = NULL;
1503 thrd->req[1].desc = NULL;
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001504 thrd->req_running = -1;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001505
1506 /* Clear the reset flag */
1507 pl330->dmac_tbd.reset_chan &= ~(1 << i);
1508 }
1509 }
1510
1511 spin_unlock_irqrestore(&pl330->lock, flags);
1512
1513 return;
1514}
1515
1516/* Returns 1 if state was updated, 0 otherwise */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001517static int pl330_update(struct pl330_dmac *pl330)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001518{
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001519 struct dma_pl330_desc *descdone, *tmp;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001520 unsigned long flags;
1521 void __iomem *regs;
1522 u32 val;
1523 int id, ev, ret = 0;
1524
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001525 regs = pl330->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001526
1527 spin_lock_irqsave(&pl330->lock, flags);
1528
1529 val = readl(regs + FSM) & 0x1;
1530 if (val)
1531 pl330->dmac_tbd.reset_mngr = true;
1532 else
1533 pl330->dmac_tbd.reset_mngr = false;
1534
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001535 val = readl(regs + FSC) & ((1 << pl330->pcfg.num_chan) - 1);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001536 pl330->dmac_tbd.reset_chan |= val;
1537 if (val) {
1538 int i = 0;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001539 while (i < pl330->pcfg.num_chan) {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001540 if (val & (1 << i)) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001541 dev_info(pl330->ddma.dev,
Boojin Kimb7d861d2011-12-26 18:49:52 +09001542 "Reset Channel-%d\t CS-%x FTC-%x\n",
1543 i, readl(regs + CS(i)),
1544 readl(regs + FTC(i)));
1545 _stop(&pl330->channels[i]);
1546 }
1547 i++;
1548 }
1549 }
1550
1551 /* Check which event happened i.e, thread notified */
1552 val = readl(regs + ES);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001553 if (pl330->pcfg.num_events < 32
1554 && val & ~((1 << pl330->pcfg.num_events) - 1)) {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001555 pl330->dmac_tbd.reset_dmac = true;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001556 dev_err(pl330->ddma.dev, "%s:%d Unexpected!\n", __func__,
1557 __LINE__);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001558 ret = 1;
1559 goto updt_exit;
1560 }
1561
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001562 for (ev = 0; ev < pl330->pcfg.num_events; ev++) {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001563 if (val & (1 << ev)) { /* Event occurred */
1564 struct pl330_thread *thrd;
1565 u32 inten = readl(regs + INTEN);
1566 int active;
1567
1568 /* Clear the event */
1569 if (inten & (1 << ev))
1570 writel(1 << ev, regs + INTCLR);
1571
1572 ret = 1;
1573
1574 id = pl330->events[ev];
1575
1576 thrd = &pl330->channels[id];
1577
1578 active = thrd->req_running;
1579 if (active == -1) /* Aborted */
1580 continue;
1581
Javi Merinofdec53d2012-06-13 15:07:00 +01001582 /* Detach the req */
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001583 descdone = thrd->req[active].desc;
1584 thrd->req[active].desc = NULL;
Javi Merinofdec53d2012-06-13 15:07:00 +01001585
Boojin Kimb7d861d2011-12-26 18:49:52 +09001586 /* Get going again ASAP */
1587 _start(thrd);
1588
1589 /* For now, just make a list of callbacks to be done */
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001590 list_add_tail(&descdone->rqd, &pl330->req_done);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001591 }
1592 }
1593
1594 /* Now that we are in no hurry, do the callbacks */
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001595 list_for_each_entry_safe(descdone, tmp, &pl330->req_done, rqd) {
1596 list_del(&descdone->rqd);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001597 spin_unlock_irqrestore(&pl330->lock, flags);
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001598 dma_pl330_rqcb(descdone, PL330_ERR_NONE);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001599 spin_lock_irqsave(&pl330->lock, flags);
1600 }
1601
1602updt_exit:
1603 spin_unlock_irqrestore(&pl330->lock, flags);
1604
1605 if (pl330->dmac_tbd.reset_dmac
1606 || pl330->dmac_tbd.reset_mngr
1607 || pl330->dmac_tbd.reset_chan) {
1608 ret = 1;
1609 tasklet_schedule(&pl330->tasks);
1610 }
1611
1612 return ret;
1613}
1614
Boojin Kimb7d861d2011-12-26 18:49:52 +09001615/* Reserve an event */
1616static inline int _alloc_event(struct pl330_thread *thrd)
1617{
1618 struct pl330_dmac *pl330 = thrd->dmac;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001619 int ev;
1620
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001621 for (ev = 0; ev < pl330->pcfg.num_events; ev++)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001622 if (pl330->events[ev] == -1) {
1623 pl330->events[ev] = thrd->id;
1624 return ev;
1625 }
1626
1627 return -1;
1628}
1629
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001630static bool _chan_ns(const struct pl330_dmac *pl330, int i)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001631{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001632 return pl330->pcfg.irq_ns & (1 << i);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001633}
1634
1635/* Upon success, returns IdentityToken for the
1636 * allocated channel, NULL otherwise.
1637 */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001638static struct pl330_thread *pl330_request_channel(struct pl330_dmac *pl330)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001639{
1640 struct pl330_thread *thrd = NULL;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001641 unsigned long flags;
1642 int chans, i;
1643
Boojin Kimb7d861d2011-12-26 18:49:52 +09001644 if (pl330->state == DYING)
1645 return NULL;
1646
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001647 chans = pl330->pcfg.num_chan;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001648
1649 spin_lock_irqsave(&pl330->lock, flags);
1650
1651 for (i = 0; i < chans; i++) {
1652 thrd = &pl330->channels[i];
1653 if ((thrd->free) && (!_manager_ns(thrd) ||
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001654 _chan_ns(pl330, i))) {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001655 thrd->ev = _alloc_event(thrd);
1656 if (thrd->ev >= 0) {
1657 thrd->free = false;
1658 thrd->lstenq = 1;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001659 thrd->req[0].desc = NULL;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001660 thrd->req[1].desc = NULL;
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001661 thrd->req_running = -1;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001662 break;
1663 }
1664 }
1665 thrd = NULL;
1666 }
1667
1668 spin_unlock_irqrestore(&pl330->lock, flags);
1669
1670 return thrd;
1671}
1672
1673/* Release an event */
1674static inline void _free_event(struct pl330_thread *thrd, int ev)
1675{
1676 struct pl330_dmac *pl330 = thrd->dmac;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001677
1678 /* If the event is valid and was held by the thread */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001679 if (ev >= 0 && ev < pl330->pcfg.num_events
Boojin Kimb7d861d2011-12-26 18:49:52 +09001680 && pl330->events[ev] == thrd->id)
1681 pl330->events[ev] = -1;
1682}
1683
Lars-Peter Clausen65ad6062014-07-06 20:32:26 +02001684static void pl330_release_channel(struct pl330_thread *thrd)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001685{
Boojin Kimb7d861d2011-12-26 18:49:52 +09001686 struct pl330_dmac *pl330;
1687 unsigned long flags;
1688
1689 if (!thrd || thrd->free)
1690 return;
1691
1692 _stop(thrd);
1693
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001694 dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, PL330_ERR_ABORT);
1695 dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, PL330_ERR_ABORT);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001696
1697 pl330 = thrd->dmac;
1698
1699 spin_lock_irqsave(&pl330->lock, flags);
1700 _free_event(thrd, thrd->ev);
1701 thrd->free = true;
1702 spin_unlock_irqrestore(&pl330->lock, flags);
1703}
1704
1705/* Initialize the structure for PL330 configuration, that can be used
1706 * by the client driver the make best use of the DMAC
1707 */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001708static void read_dmac_config(struct pl330_dmac *pl330)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001709{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001710 void __iomem *regs = pl330->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001711 u32 val;
1712
1713 val = readl(regs + CRD) >> CRD_DATA_WIDTH_SHIFT;
1714 val &= CRD_DATA_WIDTH_MASK;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001715 pl330->pcfg.data_bus_width = 8 * (1 << val);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001716
1717 val = readl(regs + CRD) >> CRD_DATA_BUFF_SHIFT;
1718 val &= CRD_DATA_BUFF_MASK;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001719 pl330->pcfg.data_buf_dep = val + 1;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001720
1721 val = readl(regs + CR0) >> CR0_NUM_CHANS_SHIFT;
1722 val &= CR0_NUM_CHANS_MASK;
1723 val += 1;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001724 pl330->pcfg.num_chan = val;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001725
1726 val = readl(regs + CR0);
1727 if (val & CR0_PERIPH_REQ_SET) {
1728 val = (val >> CR0_NUM_PERIPH_SHIFT) & CR0_NUM_PERIPH_MASK;
1729 val += 1;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001730 pl330->pcfg.num_peri = val;
1731 pl330->pcfg.peri_ns = readl(regs + CR4);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001732 } else {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001733 pl330->pcfg.num_peri = 0;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001734 }
1735
1736 val = readl(regs + CR0);
1737 if (val & CR0_BOOT_MAN_NS)
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001738 pl330->pcfg.mode |= DMAC_MODE_NS;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001739 else
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001740 pl330->pcfg.mode &= ~DMAC_MODE_NS;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001741
1742 val = readl(regs + CR0) >> CR0_NUM_EVENTS_SHIFT;
1743 val &= CR0_NUM_EVENTS_MASK;
1744 val += 1;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001745 pl330->pcfg.num_events = val;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001746
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001747 pl330->pcfg.irq_ns = readl(regs + CR3);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001748}
1749
1750static inline void _reset_thread(struct pl330_thread *thrd)
1751{
1752 struct pl330_dmac *pl330 = thrd->dmac;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001753
1754 thrd->req[0].mc_cpu = pl330->mcode_cpu
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001755 + (thrd->id * pl330->mcbufsz);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001756 thrd->req[0].mc_bus = pl330->mcode_bus
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001757 + (thrd->id * pl330->mcbufsz);
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001758 thrd->req[0].desc = NULL;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001759
1760 thrd->req[1].mc_cpu = thrd->req[0].mc_cpu
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001761 + pl330->mcbufsz / 2;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001762 thrd->req[1].mc_bus = thrd->req[0].mc_bus
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001763 + pl330->mcbufsz / 2;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001764 thrd->req[1].desc = NULL;
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001765
1766 thrd->req_running = -1;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001767}
1768
1769static int dmac_alloc_threads(struct pl330_dmac *pl330)
1770{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001771 int chans = pl330->pcfg.num_chan;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001772 struct pl330_thread *thrd;
1773 int i;
1774
1775 /* Allocate 1 Manager and 'chans' Channel threads */
1776 pl330->channels = kzalloc((1 + chans) * sizeof(*thrd),
1777 GFP_KERNEL);
1778 if (!pl330->channels)
1779 return -ENOMEM;
1780
1781 /* Init Channel threads */
1782 for (i = 0; i < chans; i++) {
1783 thrd = &pl330->channels[i];
1784 thrd->id = i;
1785 thrd->dmac = pl330;
1786 _reset_thread(thrd);
1787 thrd->free = true;
1788 }
1789
1790 /* MANAGER is indexed at the end */
1791 thrd = &pl330->channels[chans];
1792 thrd->id = chans;
1793 thrd->dmac = pl330;
1794 thrd->free = false;
1795 pl330->manager = thrd;
1796
1797 return 0;
1798}
1799
1800static int dmac_alloc_resources(struct pl330_dmac *pl330)
1801{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001802 int chans = pl330->pcfg.num_chan;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001803 int ret;
1804
1805 /*
1806 * Alloc MicroCode buffer for 'chans' Channel threads.
1807 * A channel's buffer offset is (Channel_Id * MCODE_BUFF_PERCHAN)
1808 */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001809 pl330->mcode_cpu = dma_alloc_coherent(pl330->ddma.dev,
1810 chans * pl330->mcbufsz,
Boojin Kimb7d861d2011-12-26 18:49:52 +09001811 &pl330->mcode_bus, GFP_KERNEL);
1812 if (!pl330->mcode_cpu) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001813 dev_err(pl330->ddma.dev, "%s:%d Can't allocate memory!\n",
Boojin Kimb7d861d2011-12-26 18:49:52 +09001814 __func__, __LINE__);
1815 return -ENOMEM;
1816 }
1817
1818 ret = dmac_alloc_threads(pl330);
1819 if (ret) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001820 dev_err(pl330->ddma.dev, "%s:%d Can't to create channels for DMAC!\n",
Boojin Kimb7d861d2011-12-26 18:49:52 +09001821 __func__, __LINE__);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001822 dma_free_coherent(pl330->ddma.dev,
1823 chans * pl330->mcbufsz,
Boojin Kimb7d861d2011-12-26 18:49:52 +09001824 pl330->mcode_cpu, pl330->mcode_bus);
1825 return ret;
1826 }
1827
1828 return 0;
1829}
1830
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001831static int pl330_add(struct pl330_dmac *pl330)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001832{
Boojin Kimb7d861d2011-12-26 18:49:52 +09001833 void __iomem *regs;
1834 int i, ret;
1835
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001836 regs = pl330->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001837
1838 /* Check if we can handle this DMAC */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001839 if ((pl330->pcfg.periph_id & 0xfffff) != PERIPH_ID_VAL) {
1840 dev_err(pl330->ddma.dev, "PERIPH_ID 0x%x !\n",
1841 pl330->pcfg.periph_id);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001842 return -EINVAL;
1843 }
1844
1845 /* Read the configuration of the DMAC */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001846 read_dmac_config(pl330);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001847
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001848 if (pl330->pcfg.num_events == 0) {
1849 dev_err(pl330->ddma.dev, "%s:%d Can't work without events!\n",
Boojin Kimb7d861d2011-12-26 18:49:52 +09001850 __func__, __LINE__);
1851 return -EINVAL;
1852 }
1853
Boojin Kimb7d861d2011-12-26 18:49:52 +09001854 spin_lock_init(&pl330->lock);
1855
1856 INIT_LIST_HEAD(&pl330->req_done);
1857
1858 /* Use default MC buffer size if not provided */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001859 if (!pl330->mcbufsz)
1860 pl330->mcbufsz = MCODE_BUFF_PER_REQ * 2;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001861
1862 /* Mark all events as free */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001863 for (i = 0; i < pl330->pcfg.num_events; i++)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001864 pl330->events[i] = -1;
1865
1866 /* Allocate resources needed by the DMAC */
1867 ret = dmac_alloc_resources(pl330);
1868 if (ret) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001869 dev_err(pl330->ddma.dev, "Unable to create channels for DMAC\n");
Boojin Kimb7d861d2011-12-26 18:49:52 +09001870 return ret;
1871 }
1872
1873 tasklet_init(&pl330->tasks, pl330_dotask, (unsigned long) pl330);
1874
1875 pl330->state = INIT;
1876
1877 return 0;
1878}
1879
1880static int dmac_free_threads(struct pl330_dmac *pl330)
1881{
Boojin Kimb7d861d2011-12-26 18:49:52 +09001882 struct pl330_thread *thrd;
1883 int i;
1884
1885 /* Release Channel threads */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001886 for (i = 0; i < pl330->pcfg.num_chan; i++) {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001887 thrd = &pl330->channels[i];
Lars-Peter Clausen65ad6062014-07-06 20:32:26 +02001888 pl330_release_channel(thrd);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001889 }
1890
1891 /* Free memory */
1892 kfree(pl330->channels);
1893
1894 return 0;
1895}
1896
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001897static void pl330_del(struct pl330_dmac *pl330)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001898{
Boojin Kimb7d861d2011-12-26 18:49:52 +09001899 pl330->state = UNINIT;
1900
1901 tasklet_kill(&pl330->tasks);
1902
1903 /* Free DMAC resources */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001904 dmac_free_threads(pl330);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001905
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001906 dma_free_coherent(pl330->ddma.dev,
1907 pl330->pcfg.num_chan * pl330->mcbufsz, pl330->mcode_cpu,
1908 pl330->mcode_bus);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001909}
1910
Thomas Abraham3e2ec132011-10-24 11:43:02 +02001911/* forward declaration */
1912static struct amba_driver pl330_driver;
1913
Jassi Brarb3040e42010-05-23 20:28:19 -07001914static inline struct dma_pl330_chan *
1915to_pchan(struct dma_chan *ch)
1916{
1917 if (!ch)
1918 return NULL;
1919
1920 return container_of(ch, struct dma_pl330_chan, chan);
1921}
1922
1923static inline struct dma_pl330_desc *
1924to_desc(struct dma_async_tx_descriptor *tx)
1925{
1926 return container_of(tx, struct dma_pl330_desc, txd);
1927}
1928
Jassi Brarb3040e42010-05-23 20:28:19 -07001929static inline void fill_queue(struct dma_pl330_chan *pch)
1930{
1931 struct dma_pl330_desc *desc;
1932 int ret;
1933
1934 list_for_each_entry(desc, &pch->work_list, node) {
1935
1936 /* If already submitted */
1937 if (desc->status == BUSY)
Jassi Brar30fb9802013-02-13 16:13:14 +05301938 continue;
Jassi Brarb3040e42010-05-23 20:28:19 -07001939
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001940 ret = pl330_submit_req(pch->thread, desc);
Jassi Brarb3040e42010-05-23 20:28:19 -07001941 if (!ret) {
1942 desc->status = BUSY;
Jassi Brarb3040e42010-05-23 20:28:19 -07001943 } else if (ret == -EAGAIN) {
1944 /* QFull or DMAC Dying */
1945 break;
1946 } else {
1947 /* Unacceptable request */
1948 desc->status = DONE;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001949 dev_err(pch->dmac->ddma.dev, "%s:%d Bad Desc(%d)\n",
Jassi Brarb3040e42010-05-23 20:28:19 -07001950 __func__, __LINE__, desc->txd.cookie);
1951 tasklet_schedule(&pch->task);
1952 }
1953 }
1954}
1955
1956static void pl330_tasklet(unsigned long data)
1957{
1958 struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data;
1959 struct dma_pl330_desc *desc, *_dt;
1960 unsigned long flags;
Jassi Brarb3040e42010-05-23 20:28:19 -07001961
1962 spin_lock_irqsave(&pch->lock, flags);
1963
1964 /* Pick up ripe tomatoes */
1965 list_for_each_entry_safe(desc, _dt, &pch->work_list, node)
1966 if (desc->status == DONE) {
Tushar Behera30c1dc02012-05-23 16:47:31 +05301967 if (!pch->cyclic)
Vinod Kouleab21582012-05-11 11:24:41 +05301968 dma_cookie_complete(&desc->txd);
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02001969 list_move_tail(&desc->node, &pch->completed_list);
Jassi Brarb3040e42010-05-23 20:28:19 -07001970 }
1971
1972 /* Try to submit a req imm. next to the last completed cookie */
1973 fill_queue(pch);
1974
1975 /* Make sure the PL330 Channel thread is active */
Lars-Peter Clausenc26939e2014-07-06 20:32:32 +02001976 spin_lock(&pch->thread->dmac->lock);
1977 _start(pch->thread);
1978 spin_unlock(&pch->thread->dmac->lock);
Jassi Brarb3040e42010-05-23 20:28:19 -07001979
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02001980 while (!list_empty(&pch->completed_list)) {
1981 dma_async_tx_callback callback;
1982 void *callback_param;
Jassi Brarb3040e42010-05-23 20:28:19 -07001983
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02001984 desc = list_first_entry(&pch->completed_list,
1985 struct dma_pl330_desc, node);
1986
1987 callback = desc->txd.callback;
1988 callback_param = desc->txd.callback_param;
1989
1990 if (pch->cyclic) {
1991 desc->status = PREP;
1992 list_move_tail(&desc->node, &pch->work_list);
1993 } else {
1994 desc->status = FREE;
1995 list_move_tail(&desc->node, &pch->dmac->desc_pool);
1996 }
1997
Dan Williamsd38a8c62013-10-18 19:35:23 +02001998 dma_descriptor_unmap(&desc->txd);
1999
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02002000 if (callback) {
2001 spin_unlock_irqrestore(&pch->lock, flags);
2002 callback(callback_param);
2003 spin_lock_irqsave(&pch->lock, flags);
2004 }
2005 }
2006 spin_unlock_irqrestore(&pch->lock, flags);
Jassi Brarb3040e42010-05-23 20:28:19 -07002007}
2008
Thomas Abraham3e2ec132011-10-24 11:43:02 +02002009bool pl330_filter(struct dma_chan *chan, void *param)
2010{
Thomas Abrahamcd072512011-10-24 11:43:11 +02002011 u8 *peri_id;
Thomas Abraham3e2ec132011-10-24 11:43:02 +02002012
2013 if (chan->device->dev->driver != &pl330_driver.drv)
2014 return false;
2015
Thomas Abrahamcd072512011-10-24 11:43:11 +02002016 peri_id = chan->private;
Dan Carpenter2f986ec2013-11-08 12:51:16 +03002017 return *peri_id == (unsigned long)param;
Thomas Abraham3e2ec132011-10-24 11:43:02 +02002018}
2019EXPORT_SYMBOL(pl330_filter);
2020
Padmavathi Vennaa80258f2013-02-14 09:10:06 +05302021static struct dma_chan *of_dma_pl330_xlate(struct of_phandle_args *dma_spec,
2022 struct of_dma *ofdma)
2023{
2024 int count = dma_spec->args_count;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002025 struct pl330_dmac *pl330 = ofdma->of_dma_data;
Lars-Peter Clausen70cbb162014-01-11 20:08:39 +01002026 unsigned int chan_id;
Padmavathi Vennaa80258f2013-02-14 09:10:06 +05302027
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002028 if (!pl330)
2029 return NULL;
2030
Padmavathi Vennaa80258f2013-02-14 09:10:06 +05302031 if (count != 1)
2032 return NULL;
2033
Lars-Peter Clausen70cbb162014-01-11 20:08:39 +01002034 chan_id = dma_spec->args[0];
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002035 if (chan_id >= pl330->num_peripherals)
Lars-Peter Clausen70cbb162014-01-11 20:08:39 +01002036 return NULL;
Padmavathi Vennaa80258f2013-02-14 09:10:06 +05302037
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002038 return dma_get_slave_channel(&pl330->peripherals[chan_id].chan);
Padmavathi Vennaa80258f2013-02-14 09:10:06 +05302039}
2040
Jassi Brarb3040e42010-05-23 20:28:19 -07002041static int pl330_alloc_chan_resources(struct dma_chan *chan)
2042{
2043 struct dma_pl330_chan *pch = to_pchan(chan);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002044 struct pl330_dmac *pl330 = pch->dmac;
Jassi Brarb3040e42010-05-23 20:28:19 -07002045 unsigned long flags;
2046
2047 spin_lock_irqsave(&pch->lock, flags);
2048
Russell King - ARM Linuxd3ee98cdc2012-03-06 22:35:47 +00002049 dma_cookie_init(chan);
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002050 pch->cyclic = false;
Jassi Brarb3040e42010-05-23 20:28:19 -07002051
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002052 pch->thread = pl330_request_channel(pl330);
Lars-Peter Clausen65ad6062014-07-06 20:32:26 +02002053 if (!pch->thread) {
Jassi Brarb3040e42010-05-23 20:28:19 -07002054 spin_unlock_irqrestore(&pch->lock, flags);
Inderpal Singh02747882012-09-17 09:57:45 +05302055 return -ENOMEM;
Jassi Brarb3040e42010-05-23 20:28:19 -07002056 }
2057
2058 tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch);
2059
2060 spin_unlock_irqrestore(&pch->lock, flags);
2061
2062 return 1;
2063}
2064
2065static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg)
2066{
2067 struct dma_pl330_chan *pch = to_pchan(chan);
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02002068 struct dma_pl330_desc *desc;
Jassi Brarb3040e42010-05-23 20:28:19 -07002069 unsigned long flags;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002070 struct pl330_dmac *pl330 = pch->dmac;
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002071 struct dma_slave_config *slave_config;
Boojin Kimae43b882011-09-02 09:44:32 +09002072 LIST_HEAD(list);
Jassi Brarb3040e42010-05-23 20:28:19 -07002073
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002074 switch (cmd) {
2075 case DMA_TERMINATE_ALL:
2076 spin_lock_irqsave(&pch->lock, flags);
2077
Lars-Peter Clausenc26939e2014-07-06 20:32:32 +02002078 spin_lock(&pl330->lock);
2079 _stop(pch->thread);
2080 spin_unlock(&pl330->lock);
2081
2082 pch->thread->req[0].desc = NULL;
2083 pch->thread->req[1].desc = NULL;
2084 pch->thread->req_running = -1;
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002085
2086 /* Mark all desc done */
Lars-Peter Clausen04abf5d2014-01-11 20:08:38 +01002087 list_for_each_entry(desc, &pch->submitted_list, node) {
2088 desc->status = FREE;
2089 dma_cookie_complete(&desc->txd);
2090 }
2091
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02002092 list_for_each_entry(desc, &pch->work_list , node) {
2093 desc->status = FREE;
2094 dma_cookie_complete(&desc->txd);
Boojin Kimae43b882011-09-02 09:44:32 +09002095 }
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002096
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02002097 list_for_each_entry(desc, &pch->completed_list , node) {
2098 desc->status = FREE;
2099 dma_cookie_complete(&desc->txd);
2100 }
2101
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002102 list_splice_tail_init(&pch->submitted_list, &pl330->desc_pool);
2103 list_splice_tail_init(&pch->work_list, &pl330->desc_pool);
2104 list_splice_tail_init(&pch->completed_list, &pl330->desc_pool);
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002105 spin_unlock_irqrestore(&pch->lock, flags);
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002106 break;
2107 case DMA_SLAVE_CONFIG:
2108 slave_config = (struct dma_slave_config *)arg;
2109
Vinod Kouldb8196d2011-10-13 22:34:23 +05302110 if (slave_config->direction == DMA_MEM_TO_DEV) {
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002111 if (slave_config->dst_addr)
2112 pch->fifo_addr = slave_config->dst_addr;
2113 if (slave_config->dst_addr_width)
2114 pch->burst_sz = __ffs(slave_config->dst_addr_width);
2115 if (slave_config->dst_maxburst)
2116 pch->burst_len = slave_config->dst_maxburst;
Vinod Kouldb8196d2011-10-13 22:34:23 +05302117 } else if (slave_config->direction == DMA_DEV_TO_MEM) {
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002118 if (slave_config->src_addr)
2119 pch->fifo_addr = slave_config->src_addr;
2120 if (slave_config->src_addr_width)
2121 pch->burst_sz = __ffs(slave_config->src_addr_width);
2122 if (slave_config->src_maxburst)
2123 pch->burst_len = slave_config->src_maxburst;
2124 }
2125 break;
2126 default:
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002127 dev_err(pch->dmac->ddma.dev, "Not supported command.\n");
Jassi Brarb3040e42010-05-23 20:28:19 -07002128 return -ENXIO;
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002129 }
Jassi Brarb3040e42010-05-23 20:28:19 -07002130
2131 return 0;
2132}
2133
2134static void pl330_free_chan_resources(struct dma_chan *chan)
2135{
2136 struct dma_pl330_chan *pch = to_pchan(chan);
2137 unsigned long flags;
2138
Jassi Brarb3040e42010-05-23 20:28:19 -07002139 tasklet_kill(&pch->task);
2140
Bartlomiej Zolnierkiewiczda331ba2013-07-03 15:00:43 -07002141 spin_lock_irqsave(&pch->lock, flags);
2142
Lars-Peter Clausen65ad6062014-07-06 20:32:26 +02002143 pl330_release_channel(pch->thread);
2144 pch->thread = NULL;
Jassi Brarb3040e42010-05-23 20:28:19 -07002145
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002146 if (pch->cyclic)
2147 list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool);
2148
Jassi Brarb3040e42010-05-23 20:28:19 -07002149 spin_unlock_irqrestore(&pch->lock, flags);
2150}
2151
2152static enum dma_status
2153pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
2154 struct dma_tx_state *txstate)
2155{
Russell King - ARM Linux96a2af42012-03-06 22:35:27 +00002156 return dma_cookie_status(chan, cookie, txstate);
Jassi Brarb3040e42010-05-23 20:28:19 -07002157}
2158
2159static void pl330_issue_pending(struct dma_chan *chan)
2160{
Lars-Peter Clausen04abf5d2014-01-11 20:08:38 +01002161 struct dma_pl330_chan *pch = to_pchan(chan);
2162 unsigned long flags;
2163
2164 spin_lock_irqsave(&pch->lock, flags);
2165 list_splice_tail_init(&pch->submitted_list, &pch->work_list);
2166 spin_unlock_irqrestore(&pch->lock, flags);
2167
2168 pl330_tasklet((unsigned long)pch);
Jassi Brarb3040e42010-05-23 20:28:19 -07002169}
2170
2171/*
2172 * We returned the last one of the circular list of descriptor(s)
2173 * from prep_xxx, so the argument to submit corresponds to the last
2174 * descriptor of the list.
2175 */
2176static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx)
2177{
2178 struct dma_pl330_desc *desc, *last = to_desc(tx);
2179 struct dma_pl330_chan *pch = to_pchan(tx->chan);
2180 dma_cookie_t cookie;
2181 unsigned long flags;
2182
2183 spin_lock_irqsave(&pch->lock, flags);
2184
2185 /* Assign cookies to all nodes */
Jassi Brarb3040e42010-05-23 20:28:19 -07002186 while (!list_empty(&last->node)) {
2187 desc = list_entry(last->node.next, struct dma_pl330_desc, node);
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002188 if (pch->cyclic) {
2189 desc->txd.callback = last->txd.callback;
2190 desc->txd.callback_param = last->txd.callback_param;
2191 }
Jassi Brarb3040e42010-05-23 20:28:19 -07002192
Russell King - ARM Linux884485e2012-03-06 22:34:46 +00002193 dma_cookie_assign(&desc->txd);
Jassi Brarb3040e42010-05-23 20:28:19 -07002194
Lars-Peter Clausen04abf5d2014-01-11 20:08:38 +01002195 list_move_tail(&desc->node, &pch->submitted_list);
Jassi Brarb3040e42010-05-23 20:28:19 -07002196 }
2197
Russell King - ARM Linux884485e2012-03-06 22:34:46 +00002198 cookie = dma_cookie_assign(&last->txd);
Lars-Peter Clausen04abf5d2014-01-11 20:08:38 +01002199 list_add_tail(&last->node, &pch->submitted_list);
Jassi Brarb3040e42010-05-23 20:28:19 -07002200 spin_unlock_irqrestore(&pch->lock, flags);
2201
2202 return cookie;
2203}
2204
2205static inline void _init_desc(struct dma_pl330_desc *desc)
2206{
Jassi Brarb3040e42010-05-23 20:28:19 -07002207 desc->rqcfg.swap = SWAP_NO;
Lars-Peter Clausenf0564c72014-07-06 20:32:19 +02002208 desc->rqcfg.scctl = CCTRL0;
2209 desc->rqcfg.dcctl = CCTRL0;
Jassi Brarb3040e42010-05-23 20:28:19 -07002210 desc->txd.tx_submit = pl330_tx_submit;
2211
2212 INIT_LIST_HEAD(&desc->node);
2213}
2214
2215/* Returns the number of descriptors added to the DMAC pool */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002216static int add_desc(struct pl330_dmac *pl330, gfp_t flg, int count)
Jassi Brarb3040e42010-05-23 20:28:19 -07002217{
2218 struct dma_pl330_desc *desc;
2219 unsigned long flags;
2220 int i;
2221
Will Deacon0baf8f62013-12-02 18:01:30 +00002222 desc = kcalloc(count, sizeof(*desc), flg);
Jassi Brarb3040e42010-05-23 20:28:19 -07002223 if (!desc)
2224 return 0;
2225
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002226 spin_lock_irqsave(&pl330->pool_lock, flags);
Jassi Brarb3040e42010-05-23 20:28:19 -07002227
2228 for (i = 0; i < count; i++) {
2229 _init_desc(&desc[i]);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002230 list_add_tail(&desc[i].node, &pl330->desc_pool);
Jassi Brarb3040e42010-05-23 20:28:19 -07002231 }
2232
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002233 spin_unlock_irqrestore(&pl330->pool_lock, flags);
Jassi Brarb3040e42010-05-23 20:28:19 -07002234
2235 return count;
2236}
2237
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002238static struct dma_pl330_desc *pluck_desc(struct pl330_dmac *pl330)
Jassi Brarb3040e42010-05-23 20:28:19 -07002239{
2240 struct dma_pl330_desc *desc = NULL;
2241 unsigned long flags;
2242
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002243 spin_lock_irqsave(&pl330->pool_lock, flags);
Jassi Brarb3040e42010-05-23 20:28:19 -07002244
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002245 if (!list_empty(&pl330->desc_pool)) {
2246 desc = list_entry(pl330->desc_pool.next,
Jassi Brarb3040e42010-05-23 20:28:19 -07002247 struct dma_pl330_desc, node);
2248
2249 list_del_init(&desc->node);
2250
2251 desc->status = PREP;
2252 desc->txd.callback = NULL;
2253 }
2254
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002255 spin_unlock_irqrestore(&pl330->pool_lock, flags);
Jassi Brarb3040e42010-05-23 20:28:19 -07002256
2257 return desc;
2258}
2259
2260static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch)
2261{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002262 struct pl330_dmac *pl330 = pch->dmac;
Thomas Abrahamcd072512011-10-24 11:43:11 +02002263 u8 *peri_id = pch->chan.private;
Jassi Brarb3040e42010-05-23 20:28:19 -07002264 struct dma_pl330_desc *desc;
2265
2266 /* Pluck one desc from the pool of DMAC */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002267 desc = pluck_desc(pl330);
Jassi Brarb3040e42010-05-23 20:28:19 -07002268
2269 /* If the DMAC pool is empty, alloc new */
2270 if (!desc) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002271 if (!add_desc(pl330, GFP_ATOMIC, 1))
Jassi Brarb3040e42010-05-23 20:28:19 -07002272 return NULL;
2273
2274 /* Try again */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002275 desc = pluck_desc(pl330);
Jassi Brarb3040e42010-05-23 20:28:19 -07002276 if (!desc) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002277 dev_err(pch->dmac->ddma.dev,
Jassi Brarb3040e42010-05-23 20:28:19 -07002278 "%s:%d ALERT!\n", __func__, __LINE__);
2279 return NULL;
2280 }
2281 }
2282
2283 /* Initialize the descriptor */
2284 desc->pchan = pch;
2285 desc->txd.cookie = 0;
2286 async_tx_ack(&desc->txd);
2287
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02002288 desc->peri = peri_id ? pch->chan.chan_id : 0;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002289 desc->rqcfg.pcfg = &pch->dmac->pcfg;
Jassi Brarb3040e42010-05-23 20:28:19 -07002290
2291 dma_async_tx_descriptor_init(&desc->txd, &pch->chan);
2292
2293 return desc;
2294}
2295
2296static inline void fill_px(struct pl330_xfer *px,
2297 dma_addr_t dst, dma_addr_t src, size_t len)
2298{
Jassi Brarb3040e42010-05-23 20:28:19 -07002299 px->bytes = len;
2300 px->dst_addr = dst;
2301 px->src_addr = src;
2302}
2303
2304static struct dma_pl330_desc *
2305__pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst,
2306 dma_addr_t src, size_t len)
2307{
2308 struct dma_pl330_desc *desc = pl330_get_desc(pch);
2309
2310 if (!desc) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002311 dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",
Jassi Brarb3040e42010-05-23 20:28:19 -07002312 __func__, __LINE__);
2313 return NULL;
2314 }
2315
2316 /*
2317 * Ideally we should lookout for reqs bigger than
2318 * those that can be programmed with 256 bytes of
2319 * MC buffer, but considering a req size is seldom
2320 * going to be word-unaligned and more than 200MB,
2321 * we take it easy.
2322 * Also, should the limit is reached we'd rather
2323 * have the platform increase MC buffer size than
2324 * complicating this API driver.
2325 */
2326 fill_px(&desc->px, dst, src, len);
2327
2328 return desc;
2329}
2330
2331/* Call after fixing burst size */
2332static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len)
2333{
2334 struct dma_pl330_chan *pch = desc->pchan;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002335 struct pl330_dmac *pl330 = pch->dmac;
Jassi Brarb3040e42010-05-23 20:28:19 -07002336 int burst_len;
2337
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002338 burst_len = pl330->pcfg.data_bus_width / 8;
2339 burst_len *= pl330->pcfg.data_buf_dep;
Jassi Brarb3040e42010-05-23 20:28:19 -07002340 burst_len >>= desc->rqcfg.brst_size;
2341
2342 /* src/dst_burst_len can't be more than 16 */
2343 if (burst_len > 16)
2344 burst_len = 16;
2345
2346 while (burst_len > 1) {
2347 if (!(len % (burst_len << desc->rqcfg.brst_size)))
2348 break;
2349 burst_len--;
2350 }
2351
2352 return burst_len;
2353}
2354
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002355static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
2356 struct dma_chan *chan, dma_addr_t dma_addr, size_t len,
Alexandre Bounine185ecb52012-03-08 15:35:13 -05002357 size_t period_len, enum dma_transfer_direction direction,
Laurent Pinchart31c1e5a2014-08-01 12:20:10 +02002358 unsigned long flags)
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002359{
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002360 struct dma_pl330_desc *desc = NULL, *first = NULL;
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002361 struct dma_pl330_chan *pch = to_pchan(chan);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002362 struct pl330_dmac *pl330 = pch->dmac;
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002363 unsigned int i;
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002364 dma_addr_t dst;
2365 dma_addr_t src;
2366
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002367 if (len % period_len != 0)
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002368 return NULL;
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002369
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002370 if (!is_slave_direction(direction)) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002371 dev_err(pch->dmac->ddma.dev, "%s:%d Invalid dma direction\n",
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002372 __func__, __LINE__);
2373 return NULL;
2374 }
2375
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002376 for (i = 0; i < len / period_len; i++) {
2377 desc = pl330_get_desc(pch);
2378 if (!desc) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002379 dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002380 __func__, __LINE__);
2381
2382 if (!first)
2383 return NULL;
2384
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002385 spin_lock_irqsave(&pl330->pool_lock, flags);
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002386
2387 while (!list_empty(&first->node)) {
2388 desc = list_entry(first->node.next,
2389 struct dma_pl330_desc, node);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002390 list_move_tail(&desc->node, &pl330->desc_pool);
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002391 }
2392
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002393 list_move_tail(&first->node, &pl330->desc_pool);
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002394
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002395 spin_unlock_irqrestore(&pl330->pool_lock, flags);
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002396
2397 return NULL;
2398 }
2399
2400 switch (direction) {
2401 case DMA_MEM_TO_DEV:
2402 desc->rqcfg.src_inc = 1;
2403 desc->rqcfg.dst_inc = 0;
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002404 src = dma_addr;
2405 dst = pch->fifo_addr;
2406 break;
2407 case DMA_DEV_TO_MEM:
2408 desc->rqcfg.src_inc = 0;
2409 desc->rqcfg.dst_inc = 1;
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002410 src = pch->fifo_addr;
2411 dst = dma_addr;
2412 break;
2413 default:
2414 break;
2415 }
2416
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02002417 desc->rqtype = direction;
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002418 desc->rqcfg.brst_size = pch->burst_sz;
2419 desc->rqcfg.brst_len = 1;
2420 fill_px(&desc->px, dst, src, period_len);
2421
2422 if (!first)
2423 first = desc;
2424 else
2425 list_add_tail(&desc->node, &first->node);
2426
2427 dma_addr += period_len;
2428 }
2429
2430 if (!desc)
2431 return NULL;
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002432
2433 pch->cyclic = true;
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002434 desc->txd.flags = flags;
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002435
2436 return &desc->txd;
2437}
2438
Jassi Brarb3040e42010-05-23 20:28:19 -07002439static struct dma_async_tx_descriptor *
2440pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst,
2441 dma_addr_t src, size_t len, unsigned long flags)
2442{
2443 struct dma_pl330_desc *desc;
2444 struct dma_pl330_chan *pch = to_pchan(chan);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002445 struct pl330_dmac *pl330 = pch->dmac;
Jassi Brarb3040e42010-05-23 20:28:19 -07002446 int burst;
2447
Rob Herring4e0e6102011-07-25 16:05:04 -05002448 if (unlikely(!pch || !len))
Jassi Brarb3040e42010-05-23 20:28:19 -07002449 return NULL;
2450
Jassi Brarb3040e42010-05-23 20:28:19 -07002451 desc = __pl330_prep_dma_memcpy(pch, dst, src, len);
2452 if (!desc)
2453 return NULL;
2454
2455 desc->rqcfg.src_inc = 1;
2456 desc->rqcfg.dst_inc = 1;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02002457 desc->rqtype = DMA_MEM_TO_MEM;
Jassi Brarb3040e42010-05-23 20:28:19 -07002458
2459 /* Select max possible burst size */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002460 burst = pl330->pcfg.data_bus_width / 8;
Jassi Brarb3040e42010-05-23 20:28:19 -07002461
2462 while (burst > 1) {
2463 if (!(len % burst))
2464 break;
2465 burst /= 2;
2466 }
2467
2468 desc->rqcfg.brst_size = 0;
2469 while (burst != (1 << desc->rqcfg.brst_size))
2470 desc->rqcfg.brst_size++;
2471
2472 desc->rqcfg.brst_len = get_burst_len(desc, len);
2473
2474 desc->txd.flags = flags;
2475
2476 return &desc->txd;
2477}
2478
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002479static void __pl330_giveback_desc(struct pl330_dmac *pl330,
Chanho Park52a9d172013-08-09 20:11:33 +09002480 struct dma_pl330_desc *first)
2481{
2482 unsigned long flags;
2483 struct dma_pl330_desc *desc;
2484
2485 if (!first)
2486 return;
2487
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002488 spin_lock_irqsave(&pl330->pool_lock, flags);
Chanho Park52a9d172013-08-09 20:11:33 +09002489
2490 while (!list_empty(&first->node)) {
2491 desc = list_entry(first->node.next,
2492 struct dma_pl330_desc, node);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002493 list_move_tail(&desc->node, &pl330->desc_pool);
Chanho Park52a9d172013-08-09 20:11:33 +09002494 }
2495
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002496 list_move_tail(&first->node, &pl330->desc_pool);
Chanho Park52a9d172013-08-09 20:11:33 +09002497
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002498 spin_unlock_irqrestore(&pl330->pool_lock, flags);
Chanho Park52a9d172013-08-09 20:11:33 +09002499}
2500
Jassi Brarb3040e42010-05-23 20:28:19 -07002501static struct dma_async_tx_descriptor *
2502pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
Vinod Kouldb8196d2011-10-13 22:34:23 +05302503 unsigned int sg_len, enum dma_transfer_direction direction,
Alexandre Bounine185ecb52012-03-08 15:35:13 -05002504 unsigned long flg, void *context)
Jassi Brarb3040e42010-05-23 20:28:19 -07002505{
2506 struct dma_pl330_desc *first, *desc = NULL;
2507 struct dma_pl330_chan *pch = to_pchan(chan);
Jassi Brarb3040e42010-05-23 20:28:19 -07002508 struct scatterlist *sg;
Boojin Kim1b9bb712011-09-02 09:44:30 +09002509 int i;
Jassi Brarb3040e42010-05-23 20:28:19 -07002510 dma_addr_t addr;
2511
Thomas Abrahamcd072512011-10-24 11:43:11 +02002512 if (unlikely(!pch || !sgl || !sg_len))
Jassi Brarb3040e42010-05-23 20:28:19 -07002513 return NULL;
2514
Boojin Kim1b9bb712011-09-02 09:44:30 +09002515 addr = pch->fifo_addr;
Jassi Brarb3040e42010-05-23 20:28:19 -07002516
2517 first = NULL;
2518
2519 for_each_sg(sgl, sg, sg_len, i) {
2520
2521 desc = pl330_get_desc(pch);
2522 if (!desc) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002523 struct pl330_dmac *pl330 = pch->dmac;
Jassi Brarb3040e42010-05-23 20:28:19 -07002524
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002525 dev_err(pch->dmac->ddma.dev,
Jassi Brarb3040e42010-05-23 20:28:19 -07002526 "%s:%d Unable to fetch desc\n",
2527 __func__, __LINE__);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002528 __pl330_giveback_desc(pl330, first);
Jassi Brarb3040e42010-05-23 20:28:19 -07002529
2530 return NULL;
2531 }
2532
2533 if (!first)
2534 first = desc;
2535 else
2536 list_add_tail(&desc->node, &first->node);
2537
Vinod Kouldb8196d2011-10-13 22:34:23 +05302538 if (direction == DMA_MEM_TO_DEV) {
Jassi Brarb3040e42010-05-23 20:28:19 -07002539 desc->rqcfg.src_inc = 1;
2540 desc->rqcfg.dst_inc = 0;
2541 fill_px(&desc->px,
2542 addr, sg_dma_address(sg), sg_dma_len(sg));
2543 } else {
2544 desc->rqcfg.src_inc = 0;
2545 desc->rqcfg.dst_inc = 1;
2546 fill_px(&desc->px,
2547 sg_dma_address(sg), addr, sg_dma_len(sg));
2548 }
2549
Boojin Kim1b9bb712011-09-02 09:44:30 +09002550 desc->rqcfg.brst_size = pch->burst_sz;
Jassi Brarb3040e42010-05-23 20:28:19 -07002551 desc->rqcfg.brst_len = 1;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02002552 desc->rqtype = direction;
Jassi Brarb3040e42010-05-23 20:28:19 -07002553 }
2554
2555 /* Return the last desc in the chain */
2556 desc->txd.flags = flg;
2557 return &desc->txd;
2558}
2559
2560static irqreturn_t pl330_irq_handler(int irq, void *data)
2561{
2562 if (pl330_update(data))
2563 return IRQ_HANDLED;
2564 else
2565 return IRQ_NONE;
2566}
2567
Lars-Peter Clausenca38ff12013-07-15 17:53:08 +02002568#define PL330_DMA_BUSWIDTHS \
2569 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
2570 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
2571 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
2572 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
2573 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)
2574
2575static int pl330_dma_device_slave_caps(struct dma_chan *dchan,
2576 struct dma_slave_caps *caps)
2577{
2578 caps->src_addr_widths = PL330_DMA_BUSWIDTHS;
2579 caps->dstn_addr_widths = PL330_DMA_BUSWIDTHS;
2580 caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
2581 caps->cmd_pause = false;
2582 caps->cmd_terminate = true;
Lars-Peter Clausenbfb9bb42014-01-11 14:02:17 +01002583 caps->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
Lars-Peter Clausenca38ff12013-07-15 17:53:08 +02002584
Lars-Peter Clausenca38ff12013-07-15 17:53:08 +02002585 return 0;
2586}
2587
Bill Pemberton463a1f82012-11-19 13:22:55 -05002588static int
Russell Kingaa25afa2011-02-19 15:55:00 +00002589pl330_probe(struct amba_device *adev, const struct amba_id *id)
Jassi Brarb3040e42010-05-23 20:28:19 -07002590{
2591 struct dma_pl330_platdata *pdat;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002592 struct pl330_config *pcfg;
2593 struct pl330_dmac *pl330;
Padmavathi Venna0b94c572013-03-05 14:55:31 +05302594 struct dma_pl330_chan *pch, *_p;
Jassi Brarb3040e42010-05-23 20:28:19 -07002595 struct dma_device *pd;
2596 struct resource *res;
2597 int i, ret, irq;
Rob Herring4e0e6102011-07-25 16:05:04 -05002598 int num_chan;
Jassi Brarb3040e42010-05-23 20:28:19 -07002599
Jingoo Hand4adcc02013-07-30 17:09:11 +09002600 pdat = dev_get_platdata(&adev->dev);
Jassi Brarb3040e42010-05-23 20:28:19 -07002601
Russell King64113012013-06-27 10:29:32 +01002602 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32));
2603 if (ret)
2604 return ret;
2605
Jassi Brarb3040e42010-05-23 20:28:19 -07002606 /* Allocate a new DMAC and its Channels */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002607 pl330 = devm_kzalloc(&adev->dev, sizeof(*pl330), GFP_KERNEL);
2608 if (!pl330) {
Jassi Brarb3040e42010-05-23 20:28:19 -07002609 dev_err(&adev->dev, "unable to allocate mem\n");
2610 return -ENOMEM;
2611 }
2612
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002613 pl330->mcbufsz = pdat ? pdat->mcbuf_sz : 0;
Jassi Brarb3040e42010-05-23 20:28:19 -07002614
2615 res = &adev->res;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002616 pl330->base = devm_ioremap_resource(&adev->dev, res);
2617 if (IS_ERR(pl330->base))
2618 return PTR_ERR(pl330->base);
Jassi Brarb3040e42010-05-23 20:28:19 -07002619
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002620 amba_set_drvdata(adev, pl330);
Boojin Kima2f52032011-09-02 09:44:29 +09002621
Dan Carpenter02808b42013-11-08 12:50:24 +03002622 for (i = 0; i < AMBA_NR_IRQS; i++) {
Michal Simeke98b3ca2013-09-30 08:50:48 +02002623 irq = adev->irq[i];
2624 if (irq) {
2625 ret = devm_request_irq(&adev->dev, irq,
2626 pl330_irq_handler, 0,
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002627 dev_name(&adev->dev), pl330);
Michal Simeke98b3ca2013-09-30 08:50:48 +02002628 if (ret)
2629 return ret;
2630 } else {
2631 break;
2632 }
2633 }
Jassi Brarb3040e42010-05-23 20:28:19 -07002634
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002635 pcfg = &pl330->pcfg;
2636
2637 pcfg->periph_id = adev->periphid;
2638 ret = pl330_add(pl330);
Jassi Brarb3040e42010-05-23 20:28:19 -07002639 if (ret)
Michal Simek173e8382013-09-04 16:40:17 +02002640 return ret;
Jassi Brarb3040e42010-05-23 20:28:19 -07002641
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002642 INIT_LIST_HEAD(&pl330->desc_pool);
2643 spin_lock_init(&pl330->pool_lock);
Jassi Brarb3040e42010-05-23 20:28:19 -07002644
2645 /* Create a descriptor pool of default size */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002646 if (!add_desc(pl330, GFP_KERNEL, NR_DEFAULT_DESC))
Jassi Brarb3040e42010-05-23 20:28:19 -07002647 dev_warn(&adev->dev, "unable to allocate desc\n");
2648
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002649 pd = &pl330->ddma;
Jassi Brarb3040e42010-05-23 20:28:19 -07002650 INIT_LIST_HEAD(&pd->channels);
2651
2652 /* Initialize channel parameters */
Olof Johanssonc8473822012-04-08 16:26:19 -07002653 if (pdat)
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002654 num_chan = max_t(int, pdat->nr_valid_peri, pcfg->num_chan);
Olof Johanssonc8473822012-04-08 16:26:19 -07002655 else
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002656 num_chan = max_t(int, pcfg->num_peri, pcfg->num_chan);
Olof Johanssonc8473822012-04-08 16:26:19 -07002657
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002658 pl330->num_peripherals = num_chan;
Lars-Peter Clausen70cbb162014-01-11 20:08:39 +01002659
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002660 pl330->peripherals = kzalloc(num_chan * sizeof(*pch), GFP_KERNEL);
2661 if (!pl330->peripherals) {
Sachin Kamat61c6e752012-09-17 15:20:23 +05302662 ret = -ENOMEM;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002663 dev_err(&adev->dev, "unable to allocate pl330->peripherals\n");
Sachin Kamate4d43c12012-11-15 06:27:50 +00002664 goto probe_err2;
Sachin Kamat61c6e752012-09-17 15:20:23 +05302665 }
Jassi Brarb3040e42010-05-23 20:28:19 -07002666
Rob Herring4e0e6102011-07-25 16:05:04 -05002667 for (i = 0; i < num_chan; i++) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002668 pch = &pl330->peripherals[i];
Thomas Abraham93ed5542011-10-24 11:43:31 +02002669 if (!adev->dev.of_node)
2670 pch->chan.private = pdat ? &pdat->peri_id[i] : NULL;
2671 else
2672 pch->chan.private = adev->dev.of_node;
Jassi Brarb3040e42010-05-23 20:28:19 -07002673
Lars-Peter Clausen04abf5d2014-01-11 20:08:38 +01002674 INIT_LIST_HEAD(&pch->submitted_list);
Jassi Brarb3040e42010-05-23 20:28:19 -07002675 INIT_LIST_HEAD(&pch->work_list);
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02002676 INIT_LIST_HEAD(&pch->completed_list);
Jassi Brarb3040e42010-05-23 20:28:19 -07002677 spin_lock_init(&pch->lock);
Lars-Peter Clausen65ad6062014-07-06 20:32:26 +02002678 pch->thread = NULL;
Jassi Brarb3040e42010-05-23 20:28:19 -07002679 pch->chan.device = pd;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002680 pch->dmac = pl330;
Jassi Brarb3040e42010-05-23 20:28:19 -07002681
2682 /* Add the channel to the DMAC list */
Jassi Brarb3040e42010-05-23 20:28:19 -07002683 list_add_tail(&pch->chan.device_node, &pd->channels);
2684 }
2685
2686 pd->dev = &adev->dev;
Thomas Abraham93ed5542011-10-24 11:43:31 +02002687 if (pdat) {
Thomas Abrahamcd072512011-10-24 11:43:11 +02002688 pd->cap_mask = pdat->cap_mask;
Thomas Abraham93ed5542011-10-24 11:43:31 +02002689 } else {
Thomas Abrahamcd072512011-10-24 11:43:11 +02002690 dma_cap_set(DMA_MEMCPY, pd->cap_mask);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002691 if (pcfg->num_peri) {
Thomas Abraham93ed5542011-10-24 11:43:31 +02002692 dma_cap_set(DMA_SLAVE, pd->cap_mask);
2693 dma_cap_set(DMA_CYCLIC, pd->cap_mask);
Tushar Behera5557a412012-08-29 10:16:25 +05302694 dma_cap_set(DMA_PRIVATE, pd->cap_mask);
Thomas Abraham93ed5542011-10-24 11:43:31 +02002695 }
2696 }
Jassi Brarb3040e42010-05-23 20:28:19 -07002697
2698 pd->device_alloc_chan_resources = pl330_alloc_chan_resources;
2699 pd->device_free_chan_resources = pl330_free_chan_resources;
2700 pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy;
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002701 pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic;
Jassi Brarb3040e42010-05-23 20:28:19 -07002702 pd->device_tx_status = pl330_tx_status;
2703 pd->device_prep_slave_sg = pl330_prep_slave_sg;
2704 pd->device_control = pl330_control;
2705 pd->device_issue_pending = pl330_issue_pending;
Lars-Peter Clausenca38ff12013-07-15 17:53:08 +02002706 pd->device_slave_caps = pl330_dma_device_slave_caps;
Jassi Brarb3040e42010-05-23 20:28:19 -07002707
2708 ret = dma_async_device_register(pd);
2709 if (ret) {
2710 dev_err(&adev->dev, "unable to register DMAC\n");
Padmavathi Venna0b94c572013-03-05 14:55:31 +05302711 goto probe_err3;
2712 }
2713
2714 if (adev->dev.of_node) {
2715 ret = of_dma_controller_register(adev->dev.of_node,
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002716 of_dma_pl330_xlate, pl330);
Padmavathi Venna0b94c572013-03-05 14:55:31 +05302717 if (ret) {
2718 dev_err(&adev->dev,
2719 "unable to register DMA to the generic DT DMA helpers\n");
2720 }
Jassi Brarb3040e42010-05-23 20:28:19 -07002721 }
Lars-Peter Clausenb714b842013-11-25 16:07:46 +01002722
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002723 adev->dev.dma_parms = &pl330->dma_parms;
Lars-Peter Clausenb714b842013-11-25 16:07:46 +01002724
Vinod Kouldbaf6d82013-09-02 21:54:48 +05302725 /*
2726 * This is the limit for transfers with a buswidth of 1, larger
2727 * buswidths will have larger limits.
2728 */
2729 ret = dma_set_max_seg_size(&adev->dev, 1900800);
2730 if (ret)
2731 dev_err(&adev->dev, "unable to set the seg size\n");
2732
Jassi Brarb3040e42010-05-23 20:28:19 -07002733
Jassi Brarb3040e42010-05-23 20:28:19 -07002734 dev_info(&adev->dev,
2735 "Loaded driver for PL330 DMAC-%d\n", adev->periphid);
2736 dev_info(&adev->dev,
2737 "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n",
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002738 pcfg->data_buf_dep, pcfg->data_bus_width / 8, pcfg->num_chan,
2739 pcfg->num_peri, pcfg->num_events);
Jassi Brarb3040e42010-05-23 20:28:19 -07002740
2741 return 0;
Padmavathi Venna0b94c572013-03-05 14:55:31 +05302742probe_err3:
Padmavathi Venna0b94c572013-03-05 14:55:31 +05302743 /* Idle the DMAC */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002744 list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,
Padmavathi Venna0b94c572013-03-05 14:55:31 +05302745 chan.device_node) {
2746
2747 /* Remove the channel */
2748 list_del(&pch->chan.device_node);
2749
2750 /* Flush the channel */
Krzysztof Kozlowski0f5ebab2014-09-29 14:42:20 +02002751 if (pch->thread) {
2752 pl330_control(&pch->chan, DMA_TERMINATE_ALL, 0);
2753 pl330_free_chan_resources(&pch->chan);
2754 }
Padmavathi Venna0b94c572013-03-05 14:55:31 +05302755 }
Jassi Brarb3040e42010-05-23 20:28:19 -07002756probe_err2:
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002757 pl330_del(pl330);
Jassi Brarb3040e42010-05-23 20:28:19 -07002758
2759 return ret;
2760}
2761
Greg Kroah-Hartman4bf27b82012-12-21 15:09:59 -08002762static int pl330_remove(struct amba_device *adev)
Jassi Brarb3040e42010-05-23 20:28:19 -07002763{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002764 struct pl330_dmac *pl330 = amba_get_drvdata(adev);
Jassi Brarb3040e42010-05-23 20:28:19 -07002765 struct dma_pl330_chan *pch, *_p;
Jassi Brarb3040e42010-05-23 20:28:19 -07002766
Padmavathi Venna0b94c572013-03-05 14:55:31 +05302767 if (adev->dev.of_node)
2768 of_dma_controller_free(adev->dev.of_node);
Padmavathi Venna421da892013-02-14 09:10:07 +05302769
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002770 dma_async_device_unregister(&pl330->ddma);
Jassi Brarb3040e42010-05-23 20:28:19 -07002771
2772 /* Idle the DMAC */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002773 list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,
Jassi Brarb3040e42010-05-23 20:28:19 -07002774 chan.device_node) {
2775
2776 /* Remove the channel */
2777 list_del(&pch->chan.device_node);
2778
2779 /* Flush the channel */
Krzysztof Kozlowski6e4a2a82014-09-29 14:42:21 +02002780 if (pch->thread) {
2781 pl330_control(&pch->chan, DMA_TERMINATE_ALL, 0);
2782 pl330_free_chan_resources(&pch->chan);
2783 }
Jassi Brarb3040e42010-05-23 20:28:19 -07002784 }
2785
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002786 pl330_del(pl330);
Jassi Brarb3040e42010-05-23 20:28:19 -07002787
Jassi Brarb3040e42010-05-23 20:28:19 -07002788 return 0;
2789}
2790
2791static struct amba_id pl330_ids[] = {
2792 {
2793 .id = 0x00041330,
2794 .mask = 0x000fffff,
2795 },
2796 { 0, 0 },
2797};
2798
Dave Martine8fa5162011-10-05 15:15:20 +01002799MODULE_DEVICE_TABLE(amba, pl330_ids);
2800
Jassi Brarb3040e42010-05-23 20:28:19 -07002801static struct amba_driver pl330_driver = {
2802 .drv = {
2803 .owner = THIS_MODULE,
2804 .name = "dma-pl330",
Jassi Brarb3040e42010-05-23 20:28:19 -07002805 },
2806 .id_table = pl330_ids,
2807 .probe = pl330_probe,
2808 .remove = pl330_remove,
2809};
2810
viresh kumar9e5ed092012-03-15 10:40:38 +01002811module_amba_driver(pl330_driver);
Jassi Brarb3040e42010-05-23 20:28:19 -07002812
2813MODULE_AUTHOR("Jaswinder Singh <jassi.brar@samsung.com>");
2814MODULE_DESCRIPTION("API Driver for PL330 DMAC");
2815MODULE_LICENSE("GPL");