blob: d5149aacd2feefdb9a3d516187e6f5b03f084ada [file] [log] [blame]
Boojin Kimb7d861d2011-12-26 18:49:52 +09001/*
2 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
3 * http://www.samsung.com
Jassi Brarb3040e42010-05-23 20:28:19 -07004 *
5 * Copyright (C) 2010 Samsung Electronics Co. Ltd.
6 * Jaswinder Singh <jassi.brar@samsung.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 */
13
Boojin Kimb7d861d2011-12-26 18:49:52 +090014#include <linux/kernel.h>
Jassi Brarb3040e42010-05-23 20:28:19 -070015#include <linux/io.h>
16#include <linux/init.h>
17#include <linux/slab.h>
18#include <linux/module.h>
Boojin Kimb7d861d2011-12-26 18:49:52 +090019#include <linux/string.h>
20#include <linux/delay.h>
21#include <linux/interrupt.h>
22#include <linux/dma-mapping.h>
Jassi Brarb3040e42010-05-23 20:28:19 -070023#include <linux/dmaengine.h>
Jassi Brarb3040e42010-05-23 20:28:19 -070024#include <linux/amba/bus.h>
25#include <linux/amba/pl330.h>
Boojin Kim1b9bb712011-09-02 09:44:30 +090026#include <linux/scatterlist.h>
Thomas Abraham93ed5542011-10-24 11:43:31 +020027#include <linux/of.h>
Padmavathi Vennaa80258f2013-02-14 09:10:06 +053028#include <linux/of_dma.h>
Sachin Kamatbcc7fa92013-03-04 14:36:27 +053029#include <linux/err.h>
Jassi Brarb3040e42010-05-23 20:28:19 -070030
Russell King - ARM Linuxd2ebfb32012-03-06 22:34:26 +000031#include "dmaengine.h"
Boojin Kimb7d861d2011-12-26 18:49:52 +090032#define PL330_MAX_CHAN 8
33#define PL330_MAX_IRQS 32
34#define PL330_MAX_PERI 32
35
Lars-Peter Clausenf0564c72014-07-06 20:32:19 +020036enum pl330_cachectrl {
37 CCTRL0, /* Noncacheable and nonbufferable */
38 CCTRL1, /* Bufferable only */
39 CCTRL2, /* Cacheable, but do not allocate */
40 CCTRL3, /* Cacheable and bufferable, but do not allocate */
41 INVALID1, /* AWCACHE = 0x1000 */
42 INVALID2,
43 CCTRL6, /* Cacheable write-through, allocate on writes only */
44 CCTRL7, /* Cacheable write-back, allocate on writes only */
Boojin Kimb7d861d2011-12-26 18:49:52 +090045};
46
47enum pl330_byteswap {
48 SWAP_NO,
49 SWAP_2,
50 SWAP_4,
51 SWAP_8,
52 SWAP_16,
53};
54
Boojin Kimb7d861d2011-12-26 18:49:52 +090055/* Register and Bit field Definitions */
56#define DS 0x0
57#define DS_ST_STOP 0x0
58#define DS_ST_EXEC 0x1
59#define DS_ST_CMISS 0x2
60#define DS_ST_UPDTPC 0x3
61#define DS_ST_WFE 0x4
62#define DS_ST_ATBRR 0x5
63#define DS_ST_QBUSY 0x6
64#define DS_ST_WFP 0x7
65#define DS_ST_KILL 0x8
66#define DS_ST_CMPLT 0x9
67#define DS_ST_FLTCMP 0xe
68#define DS_ST_FAULT 0xf
69
70#define DPC 0x4
71#define INTEN 0x20
72#define ES 0x24
73#define INTSTATUS 0x28
74#define INTCLR 0x2c
75#define FSM 0x30
76#define FSC 0x34
77#define FTM 0x38
78
79#define _FTC 0x40
80#define FTC(n) (_FTC + (n)*0x4)
81
82#define _CS 0x100
83#define CS(n) (_CS + (n)*0x8)
84#define CS_CNS (1 << 21)
85
86#define _CPC 0x104
87#define CPC(n) (_CPC + (n)*0x8)
88
89#define _SA 0x400
90#define SA(n) (_SA + (n)*0x20)
91
92#define _DA 0x404
93#define DA(n) (_DA + (n)*0x20)
94
95#define _CC 0x408
96#define CC(n) (_CC + (n)*0x20)
97
98#define CC_SRCINC (1 << 0)
99#define CC_DSTINC (1 << 14)
100#define CC_SRCPRI (1 << 8)
101#define CC_DSTPRI (1 << 22)
102#define CC_SRCNS (1 << 9)
103#define CC_DSTNS (1 << 23)
104#define CC_SRCIA (1 << 10)
105#define CC_DSTIA (1 << 24)
106#define CC_SRCBRSTLEN_SHFT 4
107#define CC_DSTBRSTLEN_SHFT 18
108#define CC_SRCBRSTSIZE_SHFT 1
109#define CC_DSTBRSTSIZE_SHFT 15
110#define CC_SRCCCTRL_SHFT 11
111#define CC_SRCCCTRL_MASK 0x7
112#define CC_DSTCCTRL_SHFT 25
113#define CC_DRCCCTRL_MASK 0x7
114#define CC_SWAP_SHFT 28
115
116#define _LC0 0x40c
117#define LC0(n) (_LC0 + (n)*0x20)
118
119#define _LC1 0x410
120#define LC1(n) (_LC1 + (n)*0x20)
121
122#define DBGSTATUS 0xd00
123#define DBG_BUSY (1 << 0)
124
125#define DBGCMD 0xd04
126#define DBGINST0 0xd08
127#define DBGINST1 0xd0c
128
129#define CR0 0xe00
130#define CR1 0xe04
131#define CR2 0xe08
132#define CR3 0xe0c
133#define CR4 0xe10
134#define CRD 0xe14
135
136#define PERIPH_ID 0xfe0
Boojin Kim3ecf51a2011-12-26 18:55:47 +0900137#define PERIPH_REV_SHIFT 20
138#define PERIPH_REV_MASK 0xf
139#define PERIPH_REV_R0P0 0
140#define PERIPH_REV_R1P0 1
141#define PERIPH_REV_R1P1 2
Boojin Kimb7d861d2011-12-26 18:49:52 +0900142
143#define CR0_PERIPH_REQ_SET (1 << 0)
144#define CR0_BOOT_EN_SET (1 << 1)
145#define CR0_BOOT_MAN_NS (1 << 2)
146#define CR0_NUM_CHANS_SHIFT 4
147#define CR0_NUM_CHANS_MASK 0x7
148#define CR0_NUM_PERIPH_SHIFT 12
149#define CR0_NUM_PERIPH_MASK 0x1f
150#define CR0_NUM_EVENTS_SHIFT 17
151#define CR0_NUM_EVENTS_MASK 0x1f
152
153#define CR1_ICACHE_LEN_SHIFT 0
154#define CR1_ICACHE_LEN_MASK 0x7
155#define CR1_NUM_ICACHELINES_SHIFT 4
156#define CR1_NUM_ICACHELINES_MASK 0xf
157
158#define CRD_DATA_WIDTH_SHIFT 0
159#define CRD_DATA_WIDTH_MASK 0x7
160#define CRD_WR_CAP_SHIFT 4
161#define CRD_WR_CAP_MASK 0x7
162#define CRD_WR_Q_DEP_SHIFT 8
163#define CRD_WR_Q_DEP_MASK 0xf
164#define CRD_RD_CAP_SHIFT 12
165#define CRD_RD_CAP_MASK 0x7
166#define CRD_RD_Q_DEP_SHIFT 16
167#define CRD_RD_Q_DEP_MASK 0xf
168#define CRD_DATA_BUFF_SHIFT 20
169#define CRD_DATA_BUFF_MASK 0x3ff
170
171#define PART 0x330
172#define DESIGNER 0x41
173#define REVISION 0x0
174#define INTEG_CFG 0x0
175#define PERIPH_ID_VAL ((PART << 0) | (DESIGNER << 12))
176
Boojin Kimb7d861d2011-12-26 18:49:52 +0900177#define PL330_STATE_STOPPED (1 << 0)
178#define PL330_STATE_EXECUTING (1 << 1)
179#define PL330_STATE_WFE (1 << 2)
180#define PL330_STATE_FAULTING (1 << 3)
181#define PL330_STATE_COMPLETING (1 << 4)
182#define PL330_STATE_WFP (1 << 5)
183#define PL330_STATE_KILLING (1 << 6)
184#define PL330_STATE_FAULT_COMPLETING (1 << 7)
185#define PL330_STATE_CACHEMISS (1 << 8)
186#define PL330_STATE_UPDTPC (1 << 9)
187#define PL330_STATE_ATBARRIER (1 << 10)
188#define PL330_STATE_QUEUEBUSY (1 << 11)
189#define PL330_STATE_INVALID (1 << 15)
190
191#define PL330_STABLE_STATES (PL330_STATE_STOPPED | PL330_STATE_EXECUTING \
192 | PL330_STATE_WFE | PL330_STATE_FAULTING)
193
194#define CMD_DMAADDH 0x54
195#define CMD_DMAEND 0x00
196#define CMD_DMAFLUSHP 0x35
197#define CMD_DMAGO 0xa0
198#define CMD_DMALD 0x04
199#define CMD_DMALDP 0x25
200#define CMD_DMALP 0x20
201#define CMD_DMALPEND 0x28
202#define CMD_DMAKILL 0x01
203#define CMD_DMAMOV 0xbc
204#define CMD_DMANOP 0x18
205#define CMD_DMARMB 0x12
206#define CMD_DMASEV 0x34
207#define CMD_DMAST 0x08
208#define CMD_DMASTP 0x29
209#define CMD_DMASTZ 0x0c
210#define CMD_DMAWFE 0x36
211#define CMD_DMAWFP 0x30
212#define CMD_DMAWMB 0x13
213
214#define SZ_DMAADDH 3
215#define SZ_DMAEND 1
216#define SZ_DMAFLUSHP 2
217#define SZ_DMALD 1
218#define SZ_DMALDP 2
219#define SZ_DMALP 2
220#define SZ_DMALPEND 2
221#define SZ_DMAKILL 1
222#define SZ_DMAMOV 6
223#define SZ_DMANOP 1
224#define SZ_DMARMB 1
225#define SZ_DMASEV 2
226#define SZ_DMAST 1
227#define SZ_DMASTP 2
228#define SZ_DMASTZ 1
229#define SZ_DMAWFE 2
230#define SZ_DMAWFP 2
231#define SZ_DMAWMB 1
232#define SZ_DMAGO 6
233
234#define BRST_LEN(ccr) ((((ccr) >> CC_SRCBRSTLEN_SHFT) & 0xf) + 1)
235#define BRST_SIZE(ccr) (1 << (((ccr) >> CC_SRCBRSTSIZE_SHFT) & 0x7))
236
237#define BYTE_TO_BURST(b, ccr) ((b) / BRST_SIZE(ccr) / BRST_LEN(ccr))
238#define BURST_TO_BYTE(c, ccr) ((c) * BRST_SIZE(ccr) * BRST_LEN(ccr))
239
240/*
241 * With 256 bytes, we can do more than 2.5MB and 5MB xfers per req
242 * at 1byte/burst for P<->M and M<->M respectively.
243 * For typical scenario, at 1word/burst, 10MB and 20MB xfers per req
244 * should be enough for P<->M and M<->M respectively.
245 */
246#define MCODE_BUFF_PER_REQ 256
247
Boojin Kimb7d861d2011-12-26 18:49:52 +0900248/* Use this _only_ to wait on transient states */
249#define UNTIL(t, s) while (!(_state(t) & (s))) cpu_relax();
250
251#ifdef PL330_DEBUG_MCGEN
252static unsigned cmd_line;
253#define PL330_DBGCMD_DUMP(off, x...) do { \
254 printk("%x:", cmd_line); \
255 printk(x); \
256 cmd_line += off; \
257 } while (0)
258#define PL330_DBGMC_START(addr) (cmd_line = addr)
259#else
260#define PL330_DBGCMD_DUMP(off, x...) do {} while (0)
261#define PL330_DBGMC_START(addr) do {} while (0)
262#endif
263
264/* The number of default descriptors */
Russell King - ARM Linuxd2ebfb32012-03-06 22:34:26 +0000265
Jassi Brarb3040e42010-05-23 20:28:19 -0700266#define NR_DEFAULT_DESC 16
267
Boojin Kimb7d861d2011-12-26 18:49:52 +0900268/* Populated by the PL330 core driver for DMA API driver's info */
269struct pl330_config {
270 u32 periph_id;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900271#define DMAC_MODE_NS (1 << 0)
272 unsigned int mode;
273 unsigned int data_bus_width:10; /* In number of bits */
274 unsigned int data_buf_dep:10;
275 unsigned int num_chan:4;
276 unsigned int num_peri:6;
277 u32 peri_ns;
278 unsigned int num_events:6;
279 u32 irq_ns;
280};
281
Boojin Kimb7d861d2011-12-26 18:49:52 +0900282/**
283 * Request Configuration.
284 * The PL330 core does not modify this and uses the last
285 * working configuration if the request doesn't provide any.
286 *
287 * The Client may want to provide this info only for the
288 * first request and a request with new settings.
289 */
290struct pl330_reqcfg {
291 /* Address Incrementing */
292 unsigned dst_inc:1;
293 unsigned src_inc:1;
294
295 /*
296 * For now, the SRC & DST protection levels
297 * and burst size/length are assumed same.
298 */
299 bool nonsecure;
300 bool privileged;
301 bool insnaccess;
302 unsigned brst_len:5;
303 unsigned brst_size:3; /* in power of 2 */
304
Lars-Peter Clausenf0564c72014-07-06 20:32:19 +0200305 enum pl330_cachectrl dcctl;
306 enum pl330_cachectrl scctl;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900307 enum pl330_byteswap swap;
Boojin Kim3ecf51a2011-12-26 18:55:47 +0900308 struct pl330_config *pcfg;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900309};
310
311/*
312 * One cycle of DMAC operation.
313 * There may be more than one xfer in a request.
314 */
315struct pl330_xfer {
316 u32 src_addr;
317 u32 dst_addr;
318 /* Size to xfer */
319 u32 bytes;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900320};
321
322/* The xfer callbacks are made with one of these arguments. */
323enum pl330_op_err {
324 /* The all xfers in the request were success. */
325 PL330_ERR_NONE,
326 /* If req aborted due to global error. */
327 PL330_ERR_ABORT,
328 /* If req failed due to problem with Channel. */
329 PL330_ERR_FAIL,
330};
331
Boojin Kimb7d861d2011-12-26 18:49:52 +0900332enum dmamov_dst {
333 SAR = 0,
334 CCR,
335 DAR,
336};
337
338enum pl330_dst {
339 SRC = 0,
340 DST,
341};
342
343enum pl330_cond {
344 SINGLE,
345 BURST,
346 ALWAYS,
347};
348
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +0200349struct dma_pl330_desc;
350
Boojin Kimb7d861d2011-12-26 18:49:52 +0900351struct _pl330_req {
352 u32 mc_bus;
353 void *mc_cpu;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +0200354 struct dma_pl330_desc *desc;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900355};
356
357/* ToBeDone for tasklet */
358struct _pl330_tbd {
359 bool reset_dmac;
360 bool reset_mngr;
361 u8 reset_chan;
362};
363
364/* A DMAC Thread */
365struct pl330_thread {
366 u8 id;
367 int ev;
368 /* If the channel is not yet acquired by any client */
369 bool free;
370 /* Parent DMAC */
371 struct pl330_dmac *dmac;
372 /* Only two at a time */
373 struct _pl330_req req[2];
374 /* Index of the last enqueued request */
375 unsigned lstenq;
376 /* Index of the last submitted request or -1 if the DMA is stopped */
377 int req_running;
378};
379
380enum pl330_dmac_state {
381 UNINIT,
382 INIT,
383 DYING,
384};
385
Jassi Brarb3040e42010-05-23 20:28:19 -0700386enum desc_status {
387 /* In the DMAC pool */
388 FREE,
389 /*
Masanari Iidad73111c2012-08-04 23:37:53 +0900390 * Allocated to some channel during prep_xxx
Jassi Brarb3040e42010-05-23 20:28:19 -0700391 * Also may be sitting on the work_list.
392 */
393 PREP,
394 /*
395 * Sitting on the work_list and already submitted
396 * to the PL330 core. Not more than two descriptors
397 * of a channel can be BUSY at any time.
398 */
399 BUSY,
400 /*
401 * Sitting on the channel work_list but xfer done
402 * by PL330 core
403 */
404 DONE,
405};
406
407struct dma_pl330_chan {
408 /* Schedule desc completion */
409 struct tasklet_struct task;
410
411 /* DMA-Engine Channel */
412 struct dma_chan chan;
413
Lars-Peter Clausen04abf5d2014-01-11 20:08:38 +0100414 /* List of submitted descriptors */
415 struct list_head submitted_list;
416 /* List of issued descriptors */
Jassi Brarb3040e42010-05-23 20:28:19 -0700417 struct list_head work_list;
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +0200418 /* List of completed descriptors */
419 struct list_head completed_list;
Jassi Brarb3040e42010-05-23 20:28:19 -0700420
421 /* Pointer to the DMAC that manages this channel,
422 * NULL if the channel is available to be acquired.
423 * As the parent, this DMAC also provides descriptors
424 * to the channel.
425 */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200426 struct pl330_dmac *dmac;
Jassi Brarb3040e42010-05-23 20:28:19 -0700427
428 /* To protect channel manipulation */
429 spinlock_t lock;
430
Lars-Peter Clausen65ad6062014-07-06 20:32:26 +0200431 /*
432 * Hardware channel thread of PL330 DMAC. NULL if the channel is
433 * available.
Jassi Brarb3040e42010-05-23 20:28:19 -0700434 */
Lars-Peter Clausen65ad6062014-07-06 20:32:26 +0200435 struct pl330_thread *thread;
Boojin Kim1b9bb712011-09-02 09:44:30 +0900436
437 /* For D-to-M and M-to-D channels */
438 int burst_sz; /* the peripheral fifo width */
Boojin Kim1d0c1d62011-09-02 09:44:31 +0900439 int burst_len; /* the number of burst */
Boojin Kim1b9bb712011-09-02 09:44:30 +0900440 dma_addr_t fifo_addr;
Boojin Kim42bc9cf2011-09-02 09:44:33 +0900441
442 /* for cyclic capability */
443 bool cyclic;
Jassi Brarb3040e42010-05-23 20:28:19 -0700444};
445
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200446struct pl330_dmac {
Jassi Brarb3040e42010-05-23 20:28:19 -0700447 /* DMA-Engine Device */
448 struct dma_device ddma;
449
Lars-Peter Clausenb714b842013-11-25 16:07:46 +0100450 /* Holds info about sg limitations */
451 struct device_dma_parameters dma_parms;
452
Jassi Brarb3040e42010-05-23 20:28:19 -0700453 /* Pool of descriptors available for the DMAC's channels */
454 struct list_head desc_pool;
455 /* To protect desc_pool manipulation */
456 spinlock_t pool_lock;
457
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200458 /* Size of MicroCode buffers for each channel. */
459 unsigned mcbufsz;
460 /* ioremap'ed address of PL330 registers. */
461 void __iomem *base;
462 /* Populated by the PL330 core driver during pl330_add */
463 struct pl330_config pcfg;
464
465 spinlock_t lock;
466 /* Maximum possible events/irqs */
467 int events[32];
468 /* BUS address of MicroCode buffer */
469 dma_addr_t mcode_bus;
470 /* CPU address of MicroCode buffer */
471 void *mcode_cpu;
472 /* List of all Channel threads */
473 struct pl330_thread *channels;
474 /* Pointer to the MANAGER thread */
475 struct pl330_thread *manager;
476 /* To handle bad news in interrupt */
477 struct tasklet_struct tasks;
478 struct _pl330_tbd dmac_tbd;
479 /* State of DMAC operation */
480 enum pl330_dmac_state state;
481 /* Holds list of reqs with due callbacks */
482 struct list_head req_done;
483
Jassi Brarb3040e42010-05-23 20:28:19 -0700484 /* Peripheral channels connected to this DMAC */
Lars-Peter Clausen70cbb162014-01-11 20:08:39 +0100485 unsigned int num_peripherals;
Rob Herring4e0e6102011-07-25 16:05:04 -0500486 struct dma_pl330_chan *peripherals; /* keep at end */
Jassi Brarb3040e42010-05-23 20:28:19 -0700487};
488
489struct dma_pl330_desc {
490 /* To attach to a queue as child */
491 struct list_head node;
492
493 /* Descriptor for the DMA Engine API */
494 struct dma_async_tx_descriptor txd;
495
496 /* Xfer for PL330 core */
497 struct pl330_xfer px;
498
499 struct pl330_reqcfg rqcfg;
Jassi Brarb3040e42010-05-23 20:28:19 -0700500
501 enum desc_status status;
502
503 /* The channel which currently holds this desc */
504 struct dma_pl330_chan *pchan;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +0200505
506 enum dma_transfer_direction rqtype;
507 /* Index of peripheral for the xfer. */
508 unsigned peri:5;
509 /* Hook to attach to DMAC's list of reqs with due callback */
510 struct list_head rqd;
511};
512
513struct _xfer_spec {
514 u32 ccr;
515 struct dma_pl330_desc *desc;
Jassi Brarb3040e42010-05-23 20:28:19 -0700516};
517
Boojin Kimb7d861d2011-12-26 18:49:52 +0900518static inline bool _queue_empty(struct pl330_thread *thrd)
519{
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +0200520 return thrd->req[0].desc == NULL && thrd->req[1].desc == NULL;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900521}
522
523static inline bool _queue_full(struct pl330_thread *thrd)
524{
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +0200525 return thrd->req[0].desc != NULL && thrd->req[1].desc != NULL;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900526}
527
528static inline bool is_manager(struct pl330_thread *thrd)
529{
Lars-Peter Clausenfbbcd9b2014-07-06 20:32:28 +0200530 return thrd->dmac->manager == thrd;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900531}
532
533/* If manager of the thread is in Non-Secure mode */
534static inline bool _manager_ns(struct pl330_thread *thrd)
535{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200536 return (thrd->dmac->pcfg.mode & DMAC_MODE_NS) ? true : false;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900537}
538
Boojin Kim3ecf51a2011-12-26 18:55:47 +0900539static inline u32 get_revision(u32 periph_id)
540{
541 return (periph_id >> PERIPH_REV_SHIFT) & PERIPH_REV_MASK;
542}
543
Boojin Kimb7d861d2011-12-26 18:49:52 +0900544static inline u32 _emit_ADDH(unsigned dry_run, u8 buf[],
545 enum pl330_dst da, u16 val)
546{
547 if (dry_run)
548 return SZ_DMAADDH;
549
550 buf[0] = CMD_DMAADDH;
551 buf[0] |= (da << 1);
552 *((u16 *)&buf[1]) = val;
553
554 PL330_DBGCMD_DUMP(SZ_DMAADDH, "\tDMAADDH %s %u\n",
555 da == 1 ? "DA" : "SA", val);
556
557 return SZ_DMAADDH;
558}
559
560static inline u32 _emit_END(unsigned dry_run, u8 buf[])
561{
562 if (dry_run)
563 return SZ_DMAEND;
564
565 buf[0] = CMD_DMAEND;
566
567 PL330_DBGCMD_DUMP(SZ_DMAEND, "\tDMAEND\n");
568
569 return SZ_DMAEND;
570}
571
572static inline u32 _emit_FLUSHP(unsigned dry_run, u8 buf[], u8 peri)
573{
574 if (dry_run)
575 return SZ_DMAFLUSHP;
576
577 buf[0] = CMD_DMAFLUSHP;
578
579 peri &= 0x1f;
580 peri <<= 3;
581 buf[1] = peri;
582
583 PL330_DBGCMD_DUMP(SZ_DMAFLUSHP, "\tDMAFLUSHP %u\n", peri >> 3);
584
585 return SZ_DMAFLUSHP;
586}
587
588static inline u32 _emit_LD(unsigned dry_run, u8 buf[], enum pl330_cond cond)
589{
590 if (dry_run)
591 return SZ_DMALD;
592
593 buf[0] = CMD_DMALD;
594
595 if (cond == SINGLE)
596 buf[0] |= (0 << 1) | (1 << 0);
597 else if (cond == BURST)
598 buf[0] |= (1 << 1) | (1 << 0);
599
600 PL330_DBGCMD_DUMP(SZ_DMALD, "\tDMALD%c\n",
601 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
602
603 return SZ_DMALD;
604}
605
606static inline u32 _emit_LDP(unsigned dry_run, u8 buf[],
607 enum pl330_cond cond, u8 peri)
608{
609 if (dry_run)
610 return SZ_DMALDP;
611
612 buf[0] = CMD_DMALDP;
613
614 if (cond == BURST)
615 buf[0] |= (1 << 1);
616
617 peri &= 0x1f;
618 peri <<= 3;
619 buf[1] = peri;
620
621 PL330_DBGCMD_DUMP(SZ_DMALDP, "\tDMALDP%c %u\n",
622 cond == SINGLE ? 'S' : 'B', peri >> 3);
623
624 return SZ_DMALDP;
625}
626
627static inline u32 _emit_LP(unsigned dry_run, u8 buf[],
628 unsigned loop, u8 cnt)
629{
630 if (dry_run)
631 return SZ_DMALP;
632
633 buf[0] = CMD_DMALP;
634
635 if (loop)
636 buf[0] |= (1 << 1);
637
638 cnt--; /* DMAC increments by 1 internally */
639 buf[1] = cnt;
640
641 PL330_DBGCMD_DUMP(SZ_DMALP, "\tDMALP_%c %u\n", loop ? '1' : '0', cnt);
642
643 return SZ_DMALP;
644}
645
646struct _arg_LPEND {
647 enum pl330_cond cond;
648 bool forever;
649 unsigned loop;
650 u8 bjump;
651};
652
653static inline u32 _emit_LPEND(unsigned dry_run, u8 buf[],
654 const struct _arg_LPEND *arg)
655{
656 enum pl330_cond cond = arg->cond;
657 bool forever = arg->forever;
658 unsigned loop = arg->loop;
659 u8 bjump = arg->bjump;
660
661 if (dry_run)
662 return SZ_DMALPEND;
663
664 buf[0] = CMD_DMALPEND;
665
666 if (loop)
667 buf[0] |= (1 << 2);
668
669 if (!forever)
670 buf[0] |= (1 << 4);
671
672 if (cond == SINGLE)
673 buf[0] |= (0 << 1) | (1 << 0);
674 else if (cond == BURST)
675 buf[0] |= (1 << 1) | (1 << 0);
676
677 buf[1] = bjump;
678
679 PL330_DBGCMD_DUMP(SZ_DMALPEND, "\tDMALP%s%c_%c bjmpto_%x\n",
680 forever ? "FE" : "END",
681 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'),
682 loop ? '1' : '0',
683 bjump);
684
685 return SZ_DMALPEND;
686}
687
688static inline u32 _emit_KILL(unsigned dry_run, u8 buf[])
689{
690 if (dry_run)
691 return SZ_DMAKILL;
692
693 buf[0] = CMD_DMAKILL;
694
695 return SZ_DMAKILL;
696}
697
698static inline u32 _emit_MOV(unsigned dry_run, u8 buf[],
699 enum dmamov_dst dst, u32 val)
700{
701 if (dry_run)
702 return SZ_DMAMOV;
703
704 buf[0] = CMD_DMAMOV;
705 buf[1] = dst;
706 *((u32 *)&buf[2]) = val;
707
708 PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n",
709 dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val);
710
711 return SZ_DMAMOV;
712}
713
714static inline u32 _emit_NOP(unsigned dry_run, u8 buf[])
715{
716 if (dry_run)
717 return SZ_DMANOP;
718
719 buf[0] = CMD_DMANOP;
720
721 PL330_DBGCMD_DUMP(SZ_DMANOP, "\tDMANOP\n");
722
723 return SZ_DMANOP;
724}
725
726static inline u32 _emit_RMB(unsigned dry_run, u8 buf[])
727{
728 if (dry_run)
729 return SZ_DMARMB;
730
731 buf[0] = CMD_DMARMB;
732
733 PL330_DBGCMD_DUMP(SZ_DMARMB, "\tDMARMB\n");
734
735 return SZ_DMARMB;
736}
737
738static inline u32 _emit_SEV(unsigned dry_run, u8 buf[], u8 ev)
739{
740 if (dry_run)
741 return SZ_DMASEV;
742
743 buf[0] = CMD_DMASEV;
744
745 ev &= 0x1f;
746 ev <<= 3;
747 buf[1] = ev;
748
749 PL330_DBGCMD_DUMP(SZ_DMASEV, "\tDMASEV %u\n", ev >> 3);
750
751 return SZ_DMASEV;
752}
753
754static inline u32 _emit_ST(unsigned dry_run, u8 buf[], enum pl330_cond cond)
755{
756 if (dry_run)
757 return SZ_DMAST;
758
759 buf[0] = CMD_DMAST;
760
761 if (cond == SINGLE)
762 buf[0] |= (0 << 1) | (1 << 0);
763 else if (cond == BURST)
764 buf[0] |= (1 << 1) | (1 << 0);
765
766 PL330_DBGCMD_DUMP(SZ_DMAST, "\tDMAST%c\n",
767 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
768
769 return SZ_DMAST;
770}
771
772static inline u32 _emit_STP(unsigned dry_run, u8 buf[],
773 enum pl330_cond cond, u8 peri)
774{
775 if (dry_run)
776 return SZ_DMASTP;
777
778 buf[0] = CMD_DMASTP;
779
780 if (cond == BURST)
781 buf[0] |= (1 << 1);
782
783 peri &= 0x1f;
784 peri <<= 3;
785 buf[1] = peri;
786
787 PL330_DBGCMD_DUMP(SZ_DMASTP, "\tDMASTP%c %u\n",
788 cond == SINGLE ? 'S' : 'B', peri >> 3);
789
790 return SZ_DMASTP;
791}
792
793static inline u32 _emit_STZ(unsigned dry_run, u8 buf[])
794{
795 if (dry_run)
796 return SZ_DMASTZ;
797
798 buf[0] = CMD_DMASTZ;
799
800 PL330_DBGCMD_DUMP(SZ_DMASTZ, "\tDMASTZ\n");
801
802 return SZ_DMASTZ;
803}
804
805static inline u32 _emit_WFE(unsigned dry_run, u8 buf[], u8 ev,
806 unsigned invalidate)
807{
808 if (dry_run)
809 return SZ_DMAWFE;
810
811 buf[0] = CMD_DMAWFE;
812
813 ev &= 0x1f;
814 ev <<= 3;
815 buf[1] = ev;
816
817 if (invalidate)
818 buf[1] |= (1 << 1);
819
820 PL330_DBGCMD_DUMP(SZ_DMAWFE, "\tDMAWFE %u%s\n",
821 ev >> 3, invalidate ? ", I" : "");
822
823 return SZ_DMAWFE;
824}
825
826static inline u32 _emit_WFP(unsigned dry_run, u8 buf[],
827 enum pl330_cond cond, u8 peri)
828{
829 if (dry_run)
830 return SZ_DMAWFP;
831
832 buf[0] = CMD_DMAWFP;
833
834 if (cond == SINGLE)
835 buf[0] |= (0 << 1) | (0 << 0);
836 else if (cond == BURST)
837 buf[0] |= (1 << 1) | (0 << 0);
838 else
839 buf[0] |= (0 << 1) | (1 << 0);
840
841 peri &= 0x1f;
842 peri <<= 3;
843 buf[1] = peri;
844
845 PL330_DBGCMD_DUMP(SZ_DMAWFP, "\tDMAWFP%c %u\n",
846 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'P'), peri >> 3);
847
848 return SZ_DMAWFP;
849}
850
851static inline u32 _emit_WMB(unsigned dry_run, u8 buf[])
852{
853 if (dry_run)
854 return SZ_DMAWMB;
855
856 buf[0] = CMD_DMAWMB;
857
858 PL330_DBGCMD_DUMP(SZ_DMAWMB, "\tDMAWMB\n");
859
860 return SZ_DMAWMB;
861}
862
863struct _arg_GO {
864 u8 chan;
865 u32 addr;
866 unsigned ns;
867};
868
869static inline u32 _emit_GO(unsigned dry_run, u8 buf[],
870 const struct _arg_GO *arg)
871{
872 u8 chan = arg->chan;
873 u32 addr = arg->addr;
874 unsigned ns = arg->ns;
875
876 if (dry_run)
877 return SZ_DMAGO;
878
879 buf[0] = CMD_DMAGO;
880 buf[0] |= (ns << 1);
881
882 buf[1] = chan & 0x7;
883
884 *((u32 *)&buf[2]) = addr;
885
886 return SZ_DMAGO;
887}
888
889#define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t)
890
891/* Returns Time-Out */
892static bool _until_dmac_idle(struct pl330_thread *thrd)
893{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200894 void __iomem *regs = thrd->dmac->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900895 unsigned long loops = msecs_to_loops(5);
896
897 do {
898 /* Until Manager is Idle */
899 if (!(readl(regs + DBGSTATUS) & DBG_BUSY))
900 break;
901
902 cpu_relax();
903 } while (--loops);
904
905 if (!loops)
906 return true;
907
908 return false;
909}
910
911static inline void _execute_DBGINSN(struct pl330_thread *thrd,
912 u8 insn[], bool as_manager)
913{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200914 void __iomem *regs = thrd->dmac->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900915 u32 val;
916
917 val = (insn[0] << 16) | (insn[1] << 24);
918 if (!as_manager) {
919 val |= (1 << 0);
920 val |= (thrd->id << 8); /* Channel Number */
921 }
922 writel(val, regs + DBGINST0);
923
924 val = *((u32 *)&insn[2]);
925 writel(val, regs + DBGINST1);
926
927 /* If timed out due to halted state-machine */
928 if (_until_dmac_idle(thrd)) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200929 dev_err(thrd->dmac->ddma.dev, "DMAC halted!\n");
Boojin Kimb7d861d2011-12-26 18:49:52 +0900930 return;
931 }
932
933 /* Get going */
934 writel(0, regs + DBGCMD);
935}
936
Boojin Kimb7d861d2011-12-26 18:49:52 +0900937static inline u32 _state(struct pl330_thread *thrd)
938{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200939 void __iomem *regs = thrd->dmac->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900940 u32 val;
941
942 if (is_manager(thrd))
943 val = readl(regs + DS) & 0xf;
944 else
945 val = readl(regs + CS(thrd->id)) & 0xf;
946
947 switch (val) {
948 case DS_ST_STOP:
949 return PL330_STATE_STOPPED;
950 case DS_ST_EXEC:
951 return PL330_STATE_EXECUTING;
952 case DS_ST_CMISS:
953 return PL330_STATE_CACHEMISS;
954 case DS_ST_UPDTPC:
955 return PL330_STATE_UPDTPC;
956 case DS_ST_WFE:
957 return PL330_STATE_WFE;
958 case DS_ST_FAULT:
959 return PL330_STATE_FAULTING;
960 case DS_ST_ATBRR:
961 if (is_manager(thrd))
962 return PL330_STATE_INVALID;
963 else
964 return PL330_STATE_ATBARRIER;
965 case DS_ST_QBUSY:
966 if (is_manager(thrd))
967 return PL330_STATE_INVALID;
968 else
969 return PL330_STATE_QUEUEBUSY;
970 case DS_ST_WFP:
971 if (is_manager(thrd))
972 return PL330_STATE_INVALID;
973 else
974 return PL330_STATE_WFP;
975 case DS_ST_KILL:
976 if (is_manager(thrd))
977 return PL330_STATE_INVALID;
978 else
979 return PL330_STATE_KILLING;
980 case DS_ST_CMPLT:
981 if (is_manager(thrd))
982 return PL330_STATE_INVALID;
983 else
984 return PL330_STATE_COMPLETING;
985 case DS_ST_FLTCMP:
986 if (is_manager(thrd))
987 return PL330_STATE_INVALID;
988 else
989 return PL330_STATE_FAULT_COMPLETING;
990 default:
991 return PL330_STATE_INVALID;
992 }
993}
994
995static void _stop(struct pl330_thread *thrd)
996{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +0200997 void __iomem *regs = thrd->dmac->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +0900998 u8 insn[6] = {0, 0, 0, 0, 0, 0};
999
1000 if (_state(thrd) == PL330_STATE_FAULT_COMPLETING)
1001 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
1002
1003 /* Return if nothing needs to be done */
1004 if (_state(thrd) == PL330_STATE_COMPLETING
1005 || _state(thrd) == PL330_STATE_KILLING
1006 || _state(thrd) == PL330_STATE_STOPPED)
1007 return;
1008
1009 _emit_KILL(0, insn);
1010
1011 /* Stop generating interrupts for SEV */
1012 writel(readl(regs + INTEN) & ~(1 << thrd->ev), regs + INTEN);
1013
1014 _execute_DBGINSN(thrd, insn, is_manager(thrd));
1015}
1016
1017/* Start doing req 'idx' of thread 'thrd' */
1018static bool _trigger(struct pl330_thread *thrd)
1019{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001020 void __iomem *regs = thrd->dmac->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001021 struct _pl330_req *req;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001022 struct dma_pl330_desc *desc;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001023 struct _arg_GO go;
1024 unsigned ns;
1025 u8 insn[6] = {0, 0, 0, 0, 0, 0};
1026 int idx;
1027
1028 /* Return if already ACTIVE */
1029 if (_state(thrd) != PL330_STATE_STOPPED)
1030 return true;
1031
1032 idx = 1 - thrd->lstenq;
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001033 if (thrd->req[idx].desc != NULL) {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001034 req = &thrd->req[idx];
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001035 } else {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001036 idx = thrd->lstenq;
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001037 if (thrd->req[idx].desc != NULL)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001038 req = &thrd->req[idx];
1039 else
1040 req = NULL;
1041 }
1042
1043 /* Return if no request */
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001044 if (!req)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001045 return true;
1046
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001047 desc = req->desc;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001048
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001049 ns = desc->rqcfg.nonsecure ? 1 : 0;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001050
1051 /* See 'Abort Sources' point-4 at Page 2-25 */
1052 if (_manager_ns(thrd) && !ns)
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001053 dev_info(thrd->dmac->ddma.dev, "%s:%d Recipe for ABORT!\n",
Boojin Kimb7d861d2011-12-26 18:49:52 +09001054 __func__, __LINE__);
1055
1056 go.chan = thrd->id;
1057 go.addr = req->mc_bus;
1058 go.ns = ns;
1059 _emit_GO(0, insn, &go);
1060
1061 /* Set to generate interrupts for SEV */
1062 writel(readl(regs + INTEN) | (1 << thrd->ev), regs + INTEN);
1063
1064 /* Only manager can execute GO */
1065 _execute_DBGINSN(thrd, insn, true);
1066
1067 thrd->req_running = idx;
1068
1069 return true;
1070}
1071
1072static bool _start(struct pl330_thread *thrd)
1073{
1074 switch (_state(thrd)) {
1075 case PL330_STATE_FAULT_COMPLETING:
1076 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
1077
1078 if (_state(thrd) == PL330_STATE_KILLING)
1079 UNTIL(thrd, PL330_STATE_STOPPED)
1080
1081 case PL330_STATE_FAULTING:
1082 _stop(thrd);
1083
1084 case PL330_STATE_KILLING:
1085 case PL330_STATE_COMPLETING:
1086 UNTIL(thrd, PL330_STATE_STOPPED)
1087
1088 case PL330_STATE_STOPPED:
1089 return _trigger(thrd);
1090
1091 case PL330_STATE_WFP:
1092 case PL330_STATE_QUEUEBUSY:
1093 case PL330_STATE_ATBARRIER:
1094 case PL330_STATE_UPDTPC:
1095 case PL330_STATE_CACHEMISS:
1096 case PL330_STATE_EXECUTING:
1097 return true;
1098
1099 case PL330_STATE_WFE: /* For RESUME, nothing yet */
1100 default:
1101 return false;
1102 }
1103}
1104
1105static inline int _ldst_memtomem(unsigned dry_run, u8 buf[],
1106 const struct _xfer_spec *pxs, int cyc)
1107{
1108 int off = 0;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001109 struct pl330_config *pcfg = pxs->desc->rqcfg.pcfg;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001110
Boojin Kim3ecf51a2011-12-26 18:55:47 +09001111 /* check lock-up free version */
1112 if (get_revision(pcfg->periph_id) >= PERIPH_REV_R1P0) {
1113 while (cyc--) {
1114 off += _emit_LD(dry_run, &buf[off], ALWAYS);
1115 off += _emit_ST(dry_run, &buf[off], ALWAYS);
1116 }
1117 } else {
1118 while (cyc--) {
1119 off += _emit_LD(dry_run, &buf[off], ALWAYS);
1120 off += _emit_RMB(dry_run, &buf[off]);
1121 off += _emit_ST(dry_run, &buf[off], ALWAYS);
1122 off += _emit_WMB(dry_run, &buf[off]);
1123 }
Boojin Kimb7d861d2011-12-26 18:49:52 +09001124 }
1125
1126 return off;
1127}
1128
1129static inline int _ldst_devtomem(unsigned dry_run, u8 buf[],
1130 const struct _xfer_spec *pxs, int cyc)
1131{
1132 int off = 0;
1133
1134 while (cyc--) {
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001135 off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->desc->peri);
1136 off += _emit_LDP(dry_run, &buf[off], SINGLE, pxs->desc->peri);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001137 off += _emit_ST(dry_run, &buf[off], ALWAYS);
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001138 off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001139 }
1140
1141 return off;
1142}
1143
1144static inline int _ldst_memtodev(unsigned dry_run, u8 buf[],
1145 const struct _xfer_spec *pxs, int cyc)
1146{
1147 int off = 0;
1148
1149 while (cyc--) {
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001150 off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->desc->peri);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001151 off += _emit_LD(dry_run, &buf[off], ALWAYS);
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001152 off += _emit_STP(dry_run, &buf[off], SINGLE, pxs->desc->peri);
1153 off += _emit_FLUSHP(dry_run, &buf[off], pxs->desc->peri);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001154 }
1155
1156 return off;
1157}
1158
1159static int _bursts(unsigned dry_run, u8 buf[],
1160 const struct _xfer_spec *pxs, int cyc)
1161{
1162 int off = 0;
1163
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001164 switch (pxs->desc->rqtype) {
Lars-Peter Clausen585a9d02014-07-06 20:32:18 +02001165 case DMA_MEM_TO_DEV:
Boojin Kimb7d861d2011-12-26 18:49:52 +09001166 off += _ldst_memtodev(dry_run, &buf[off], pxs, cyc);
1167 break;
Lars-Peter Clausen585a9d02014-07-06 20:32:18 +02001168 case DMA_DEV_TO_MEM:
Boojin Kimb7d861d2011-12-26 18:49:52 +09001169 off += _ldst_devtomem(dry_run, &buf[off], pxs, cyc);
1170 break;
Lars-Peter Clausen585a9d02014-07-06 20:32:18 +02001171 case DMA_MEM_TO_MEM:
Boojin Kimb7d861d2011-12-26 18:49:52 +09001172 off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc);
1173 break;
1174 default:
1175 off += 0x40000000; /* Scare off the Client */
1176 break;
1177 }
1178
1179 return off;
1180}
1181
1182/* Returns bytes consumed and updates bursts */
1183static inline int _loop(unsigned dry_run, u8 buf[],
1184 unsigned long *bursts, const struct _xfer_spec *pxs)
1185{
1186 int cyc, cycmax, szlp, szlpend, szbrst, off;
1187 unsigned lcnt0, lcnt1, ljmp0, ljmp1;
1188 struct _arg_LPEND lpend;
1189
1190 /* Max iterations possible in DMALP is 256 */
1191 if (*bursts >= 256*256) {
1192 lcnt1 = 256;
1193 lcnt0 = 256;
1194 cyc = *bursts / lcnt1 / lcnt0;
1195 } else if (*bursts > 256) {
1196 lcnt1 = 256;
1197 lcnt0 = *bursts / lcnt1;
1198 cyc = 1;
1199 } else {
1200 lcnt1 = *bursts;
1201 lcnt0 = 0;
1202 cyc = 1;
1203 }
1204
1205 szlp = _emit_LP(1, buf, 0, 0);
1206 szbrst = _bursts(1, buf, pxs, 1);
1207
1208 lpend.cond = ALWAYS;
1209 lpend.forever = false;
1210 lpend.loop = 0;
1211 lpend.bjump = 0;
1212 szlpend = _emit_LPEND(1, buf, &lpend);
1213
1214 if (lcnt0) {
1215 szlp *= 2;
1216 szlpend *= 2;
1217 }
1218
1219 /*
1220 * Max bursts that we can unroll due to limit on the
1221 * size of backward jump that can be encoded in DMALPEND
1222 * which is 8-bits and hence 255
1223 */
1224 cycmax = (255 - (szlp + szlpend)) / szbrst;
1225
1226 cyc = (cycmax < cyc) ? cycmax : cyc;
1227
1228 off = 0;
1229
1230 if (lcnt0) {
1231 off += _emit_LP(dry_run, &buf[off], 0, lcnt0);
1232 ljmp0 = off;
1233 }
1234
1235 off += _emit_LP(dry_run, &buf[off], 1, lcnt1);
1236 ljmp1 = off;
1237
1238 off += _bursts(dry_run, &buf[off], pxs, cyc);
1239
1240 lpend.cond = ALWAYS;
1241 lpend.forever = false;
1242 lpend.loop = 1;
1243 lpend.bjump = off - ljmp1;
1244 off += _emit_LPEND(dry_run, &buf[off], &lpend);
1245
1246 if (lcnt0) {
1247 lpend.cond = ALWAYS;
1248 lpend.forever = false;
1249 lpend.loop = 0;
1250 lpend.bjump = off - ljmp0;
1251 off += _emit_LPEND(dry_run, &buf[off], &lpend);
1252 }
1253
1254 *bursts = lcnt1 * cyc;
1255 if (lcnt0)
1256 *bursts *= lcnt0;
1257
1258 return off;
1259}
1260
1261static inline int _setup_loops(unsigned dry_run, u8 buf[],
1262 const struct _xfer_spec *pxs)
1263{
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001264 struct pl330_xfer *x = &pxs->desc->px;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001265 u32 ccr = pxs->ccr;
1266 unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr);
1267 int off = 0;
1268
1269 while (bursts) {
1270 c = bursts;
1271 off += _loop(dry_run, &buf[off], &c, pxs);
1272 bursts -= c;
1273 }
1274
1275 return off;
1276}
1277
1278static inline int _setup_xfer(unsigned dry_run, u8 buf[],
1279 const struct _xfer_spec *pxs)
1280{
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001281 struct pl330_xfer *x = &pxs->desc->px;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001282 int off = 0;
1283
1284 /* DMAMOV SAR, x->src_addr */
1285 off += _emit_MOV(dry_run, &buf[off], SAR, x->src_addr);
1286 /* DMAMOV DAR, x->dst_addr */
1287 off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr);
1288
1289 /* Setup Loop(s) */
1290 off += _setup_loops(dry_run, &buf[off], pxs);
1291
1292 return off;
1293}
1294
1295/*
1296 * A req is a sequence of one or more xfer units.
1297 * Returns the number of bytes taken to setup the MC for the req.
1298 */
1299static int _setup_req(unsigned dry_run, struct pl330_thread *thrd,
1300 unsigned index, struct _xfer_spec *pxs)
1301{
1302 struct _pl330_req *req = &thrd->req[index];
1303 struct pl330_xfer *x;
1304 u8 *buf = req->mc_cpu;
1305 int off = 0;
1306
1307 PL330_DBGMC_START(req->mc_bus);
1308
1309 /* DMAMOV CCR, ccr */
1310 off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr);
1311
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001312 x = &pxs->desc->px;
Lars-Peter Clausend5cef122014-07-06 20:32:23 +02001313 /* Error if xfer length is not aligned at burst size */
1314 if (x->bytes % (BRST_SIZE(pxs->ccr) * BRST_LEN(pxs->ccr)))
1315 return -EINVAL;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001316
Lars-Peter Clausend5cef122014-07-06 20:32:23 +02001317 off += _setup_xfer(dry_run, &buf[off], pxs);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001318
1319 /* DMASEV peripheral/event */
1320 off += _emit_SEV(dry_run, &buf[off], thrd->ev);
1321 /* DMAEND */
1322 off += _emit_END(dry_run, &buf[off]);
1323
1324 return off;
1325}
1326
1327static inline u32 _prepare_ccr(const struct pl330_reqcfg *rqc)
1328{
1329 u32 ccr = 0;
1330
1331 if (rqc->src_inc)
1332 ccr |= CC_SRCINC;
1333
1334 if (rqc->dst_inc)
1335 ccr |= CC_DSTINC;
1336
1337 /* We set same protection levels for Src and DST for now */
1338 if (rqc->privileged)
1339 ccr |= CC_SRCPRI | CC_DSTPRI;
1340 if (rqc->nonsecure)
1341 ccr |= CC_SRCNS | CC_DSTNS;
1342 if (rqc->insnaccess)
1343 ccr |= CC_SRCIA | CC_DSTIA;
1344
1345 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_SRCBRSTLEN_SHFT);
1346 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_DSTBRSTLEN_SHFT);
1347
1348 ccr |= (rqc->brst_size << CC_SRCBRSTSIZE_SHFT);
1349 ccr |= (rqc->brst_size << CC_DSTBRSTSIZE_SHFT);
1350
1351 ccr |= (rqc->scctl << CC_SRCCCTRL_SHFT);
1352 ccr |= (rqc->dcctl << CC_DSTCCTRL_SHFT);
1353
1354 ccr |= (rqc->swap << CC_SWAP_SHFT);
1355
1356 return ccr;
1357}
1358
Boojin Kimb7d861d2011-12-26 18:49:52 +09001359/*
1360 * Submit a list of xfers after which the client wants notification.
1361 * Client is not notified after each xfer unit, just once after all
1362 * xfer units are done or some error occurs.
1363 */
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001364static int pl330_submit_req(struct pl330_thread *thrd,
1365 struct dma_pl330_desc *desc)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001366{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001367 struct pl330_dmac *pl330 = thrd->dmac;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001368 struct _xfer_spec xs;
1369 unsigned long flags;
1370 void __iomem *regs;
1371 unsigned idx;
1372 u32 ccr;
1373 int ret = 0;
1374
1375 /* No Req or Unacquired Channel or DMAC */
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001376 if (!desc || !thrd || thrd->free)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001377 return -EINVAL;
1378
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001379 regs = thrd->dmac->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001380
1381 if (pl330->state == DYING
1382 || pl330->dmac_tbd.reset_chan & (1 << thrd->id)) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001383 dev_info(thrd->dmac->ddma.dev, "%s:%d\n",
Boojin Kimb7d861d2011-12-26 18:49:52 +09001384 __func__, __LINE__);
1385 return -EAGAIN;
1386 }
1387
1388 /* If request for non-existing peripheral */
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001389 if (desc->rqtype != DMA_MEM_TO_MEM &&
1390 desc->peri >= pl330->pcfg.num_peri) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001391 dev_info(thrd->dmac->ddma.dev,
Boojin Kimb7d861d2011-12-26 18:49:52 +09001392 "%s:%d Invalid peripheral(%u)!\n",
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001393 __func__, __LINE__, desc->peri);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001394 return -EINVAL;
1395 }
1396
1397 spin_lock_irqsave(&pl330->lock, flags);
1398
1399 if (_queue_full(thrd)) {
1400 ret = -EAGAIN;
1401 goto xfer_exit;
1402 }
1403
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001404 /* Prefer Secure Channel */
1405 if (!_manager_ns(thrd))
1406 desc->rqcfg.nonsecure = 0;
1407 else
1408 desc->rqcfg.nonsecure = 1;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001409
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001410 ccr = _prepare_ccr(&desc->rqcfg);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001411
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001412 idx = thrd->req[0].desc == NULL ? 0 : 1;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001413
1414 xs.ccr = ccr;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001415 xs.desc = desc;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001416
1417 /* First dry run to check if req is acceptable */
1418 ret = _setup_req(1, thrd, idx, &xs);
1419 if (ret < 0)
1420 goto xfer_exit;
1421
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001422 if (ret > pl330->mcbufsz / 2) {
1423 dev_info(pl330->ddma.dev, "%s:%d Trying increasing mcbufsz\n",
Boojin Kimb7d861d2011-12-26 18:49:52 +09001424 __func__, __LINE__);
1425 ret = -ENOMEM;
1426 goto xfer_exit;
1427 }
1428
1429 /* Hook the request */
1430 thrd->lstenq = idx;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001431 thrd->req[idx].desc = desc;
Lars-Peter Clausenbe025322014-07-06 20:32:24 +02001432 _setup_req(0, thrd, idx, &xs);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001433
1434 ret = 0;
1435
1436xfer_exit:
1437 spin_unlock_irqrestore(&pl330->lock, flags);
1438
1439 return ret;
1440}
1441
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001442static void dma_pl330_rqcb(struct dma_pl330_desc *desc, enum pl330_op_err err)
Lars-Peter Clausen6079d382014-07-06 20:32:25 +02001443{
Javier Martinez Canillasb1e51d72014-07-19 03:21:47 +02001444 struct dma_pl330_chan *pch;
Lars-Peter Clausen6079d382014-07-06 20:32:25 +02001445 unsigned long flags;
1446
Javier Martinez Canillasb1e51d72014-07-19 03:21:47 +02001447 if (!desc)
1448 return;
1449
1450 pch = desc->pchan;
1451
Lars-Peter Clausen6079d382014-07-06 20:32:25 +02001452 /* If desc aborted */
1453 if (!pch)
1454 return;
1455
1456 spin_lock_irqsave(&pch->lock, flags);
1457
1458 desc->status = DONE;
1459
1460 spin_unlock_irqrestore(&pch->lock, flags);
1461
1462 tasklet_schedule(&pch->task);
1463}
1464
Boojin Kimb7d861d2011-12-26 18:49:52 +09001465static void pl330_dotask(unsigned long data)
1466{
1467 struct pl330_dmac *pl330 = (struct pl330_dmac *) data;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001468 unsigned long flags;
1469 int i;
1470
1471 spin_lock_irqsave(&pl330->lock, flags);
1472
1473 /* The DMAC itself gone nuts */
1474 if (pl330->dmac_tbd.reset_dmac) {
1475 pl330->state = DYING;
1476 /* Reset the manager too */
1477 pl330->dmac_tbd.reset_mngr = true;
1478 /* Clear the reset flag */
1479 pl330->dmac_tbd.reset_dmac = false;
1480 }
1481
1482 if (pl330->dmac_tbd.reset_mngr) {
1483 _stop(pl330->manager);
1484 /* Reset all channels */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001485 pl330->dmac_tbd.reset_chan = (1 << pl330->pcfg.num_chan) - 1;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001486 /* Clear the reset flag */
1487 pl330->dmac_tbd.reset_mngr = false;
1488 }
1489
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001490 for (i = 0; i < pl330->pcfg.num_chan; i++) {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001491
1492 if (pl330->dmac_tbd.reset_chan & (1 << i)) {
1493 struct pl330_thread *thrd = &pl330->channels[i];
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001494 void __iomem *regs = pl330->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001495 enum pl330_op_err err;
1496
1497 _stop(thrd);
1498
1499 if (readl(regs + FSC) & (1 << thrd->id))
1500 err = PL330_ERR_FAIL;
1501 else
1502 err = PL330_ERR_ABORT;
1503
1504 spin_unlock_irqrestore(&pl330->lock, flags);
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001505 dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, err);
1506 dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, err);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001507 spin_lock_irqsave(&pl330->lock, flags);
1508
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001509 thrd->req[0].desc = NULL;
1510 thrd->req[1].desc = NULL;
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001511 thrd->req_running = -1;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001512
1513 /* Clear the reset flag */
1514 pl330->dmac_tbd.reset_chan &= ~(1 << i);
1515 }
1516 }
1517
1518 spin_unlock_irqrestore(&pl330->lock, flags);
1519
1520 return;
1521}
1522
1523/* Returns 1 if state was updated, 0 otherwise */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001524static int pl330_update(struct pl330_dmac *pl330)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001525{
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001526 struct dma_pl330_desc *descdone, *tmp;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001527 unsigned long flags;
1528 void __iomem *regs;
1529 u32 val;
1530 int id, ev, ret = 0;
1531
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001532 regs = pl330->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001533
1534 spin_lock_irqsave(&pl330->lock, flags);
1535
1536 val = readl(regs + FSM) & 0x1;
1537 if (val)
1538 pl330->dmac_tbd.reset_mngr = true;
1539 else
1540 pl330->dmac_tbd.reset_mngr = false;
1541
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001542 val = readl(regs + FSC) & ((1 << pl330->pcfg.num_chan) - 1);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001543 pl330->dmac_tbd.reset_chan |= val;
1544 if (val) {
1545 int i = 0;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001546 while (i < pl330->pcfg.num_chan) {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001547 if (val & (1 << i)) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001548 dev_info(pl330->ddma.dev,
Boojin Kimb7d861d2011-12-26 18:49:52 +09001549 "Reset Channel-%d\t CS-%x FTC-%x\n",
1550 i, readl(regs + CS(i)),
1551 readl(regs + FTC(i)));
1552 _stop(&pl330->channels[i]);
1553 }
1554 i++;
1555 }
1556 }
1557
1558 /* Check which event happened i.e, thread notified */
1559 val = readl(regs + ES);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001560 if (pl330->pcfg.num_events < 32
1561 && val & ~((1 << pl330->pcfg.num_events) - 1)) {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001562 pl330->dmac_tbd.reset_dmac = true;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001563 dev_err(pl330->ddma.dev, "%s:%d Unexpected!\n", __func__,
1564 __LINE__);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001565 ret = 1;
1566 goto updt_exit;
1567 }
1568
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001569 for (ev = 0; ev < pl330->pcfg.num_events; ev++) {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001570 if (val & (1 << ev)) { /* Event occurred */
1571 struct pl330_thread *thrd;
1572 u32 inten = readl(regs + INTEN);
1573 int active;
1574
1575 /* Clear the event */
1576 if (inten & (1 << ev))
1577 writel(1 << ev, regs + INTCLR);
1578
1579 ret = 1;
1580
1581 id = pl330->events[ev];
1582
1583 thrd = &pl330->channels[id];
1584
1585 active = thrd->req_running;
1586 if (active == -1) /* Aborted */
1587 continue;
1588
Javi Merinofdec53d2012-06-13 15:07:00 +01001589 /* Detach the req */
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001590 descdone = thrd->req[active].desc;
1591 thrd->req[active].desc = NULL;
Javi Merinofdec53d2012-06-13 15:07:00 +01001592
Boojin Kimb7d861d2011-12-26 18:49:52 +09001593 /* Get going again ASAP */
1594 _start(thrd);
1595
1596 /* For now, just make a list of callbacks to be done */
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001597 list_add_tail(&descdone->rqd, &pl330->req_done);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001598 }
1599 }
1600
1601 /* Now that we are in no hurry, do the callbacks */
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001602 list_for_each_entry_safe(descdone, tmp, &pl330->req_done, rqd) {
1603 list_del(&descdone->rqd);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001604 spin_unlock_irqrestore(&pl330->lock, flags);
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001605 dma_pl330_rqcb(descdone, PL330_ERR_NONE);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001606 spin_lock_irqsave(&pl330->lock, flags);
1607 }
1608
1609updt_exit:
1610 spin_unlock_irqrestore(&pl330->lock, flags);
1611
1612 if (pl330->dmac_tbd.reset_dmac
1613 || pl330->dmac_tbd.reset_mngr
1614 || pl330->dmac_tbd.reset_chan) {
1615 ret = 1;
1616 tasklet_schedule(&pl330->tasks);
1617 }
1618
1619 return ret;
1620}
1621
Boojin Kimb7d861d2011-12-26 18:49:52 +09001622/* Reserve an event */
1623static inline int _alloc_event(struct pl330_thread *thrd)
1624{
1625 struct pl330_dmac *pl330 = thrd->dmac;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001626 int ev;
1627
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001628 for (ev = 0; ev < pl330->pcfg.num_events; ev++)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001629 if (pl330->events[ev] == -1) {
1630 pl330->events[ev] = thrd->id;
1631 return ev;
1632 }
1633
1634 return -1;
1635}
1636
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001637static bool _chan_ns(const struct pl330_dmac *pl330, int i)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001638{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001639 return pl330->pcfg.irq_ns & (1 << i);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001640}
1641
1642/* Upon success, returns IdentityToken for the
1643 * allocated channel, NULL otherwise.
1644 */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001645static struct pl330_thread *pl330_request_channel(struct pl330_dmac *pl330)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001646{
1647 struct pl330_thread *thrd = NULL;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001648 unsigned long flags;
1649 int chans, i;
1650
Boojin Kimb7d861d2011-12-26 18:49:52 +09001651 if (pl330->state == DYING)
1652 return NULL;
1653
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001654 chans = pl330->pcfg.num_chan;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001655
1656 spin_lock_irqsave(&pl330->lock, flags);
1657
1658 for (i = 0; i < chans; i++) {
1659 thrd = &pl330->channels[i];
1660 if ((thrd->free) && (!_manager_ns(thrd) ||
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001661 _chan_ns(pl330, i))) {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001662 thrd->ev = _alloc_event(thrd);
1663 if (thrd->ev >= 0) {
1664 thrd->free = false;
1665 thrd->lstenq = 1;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001666 thrd->req[0].desc = NULL;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001667 thrd->req[1].desc = NULL;
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001668 thrd->req_running = -1;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001669 break;
1670 }
1671 }
1672 thrd = NULL;
1673 }
1674
1675 spin_unlock_irqrestore(&pl330->lock, flags);
1676
1677 return thrd;
1678}
1679
1680/* Release an event */
1681static inline void _free_event(struct pl330_thread *thrd, int ev)
1682{
1683 struct pl330_dmac *pl330 = thrd->dmac;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001684
1685 /* If the event is valid and was held by the thread */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001686 if (ev >= 0 && ev < pl330->pcfg.num_events
Boojin Kimb7d861d2011-12-26 18:49:52 +09001687 && pl330->events[ev] == thrd->id)
1688 pl330->events[ev] = -1;
1689}
1690
Lars-Peter Clausen65ad6062014-07-06 20:32:26 +02001691static void pl330_release_channel(struct pl330_thread *thrd)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001692{
Boojin Kimb7d861d2011-12-26 18:49:52 +09001693 struct pl330_dmac *pl330;
1694 unsigned long flags;
1695
1696 if (!thrd || thrd->free)
1697 return;
1698
1699 _stop(thrd);
1700
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001701 dma_pl330_rqcb(thrd->req[1 - thrd->lstenq].desc, PL330_ERR_ABORT);
1702 dma_pl330_rqcb(thrd->req[thrd->lstenq].desc, PL330_ERR_ABORT);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001703
1704 pl330 = thrd->dmac;
1705
1706 spin_lock_irqsave(&pl330->lock, flags);
1707 _free_event(thrd, thrd->ev);
1708 thrd->free = true;
1709 spin_unlock_irqrestore(&pl330->lock, flags);
1710}
1711
1712/* Initialize the structure for PL330 configuration, that can be used
1713 * by the client driver the make best use of the DMAC
1714 */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001715static void read_dmac_config(struct pl330_dmac *pl330)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001716{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001717 void __iomem *regs = pl330->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001718 u32 val;
1719
1720 val = readl(regs + CRD) >> CRD_DATA_WIDTH_SHIFT;
1721 val &= CRD_DATA_WIDTH_MASK;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001722 pl330->pcfg.data_bus_width = 8 * (1 << val);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001723
1724 val = readl(regs + CRD) >> CRD_DATA_BUFF_SHIFT;
1725 val &= CRD_DATA_BUFF_MASK;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001726 pl330->pcfg.data_buf_dep = val + 1;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001727
1728 val = readl(regs + CR0) >> CR0_NUM_CHANS_SHIFT;
1729 val &= CR0_NUM_CHANS_MASK;
1730 val += 1;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001731 pl330->pcfg.num_chan = val;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001732
1733 val = readl(regs + CR0);
1734 if (val & CR0_PERIPH_REQ_SET) {
1735 val = (val >> CR0_NUM_PERIPH_SHIFT) & CR0_NUM_PERIPH_MASK;
1736 val += 1;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001737 pl330->pcfg.num_peri = val;
1738 pl330->pcfg.peri_ns = readl(regs + CR4);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001739 } else {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001740 pl330->pcfg.num_peri = 0;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001741 }
1742
1743 val = readl(regs + CR0);
1744 if (val & CR0_BOOT_MAN_NS)
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001745 pl330->pcfg.mode |= DMAC_MODE_NS;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001746 else
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001747 pl330->pcfg.mode &= ~DMAC_MODE_NS;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001748
1749 val = readl(regs + CR0) >> CR0_NUM_EVENTS_SHIFT;
1750 val &= CR0_NUM_EVENTS_MASK;
1751 val += 1;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001752 pl330->pcfg.num_events = val;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001753
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001754 pl330->pcfg.irq_ns = readl(regs + CR3);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001755}
1756
1757static inline void _reset_thread(struct pl330_thread *thrd)
1758{
1759 struct pl330_dmac *pl330 = thrd->dmac;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001760
1761 thrd->req[0].mc_cpu = pl330->mcode_cpu
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001762 + (thrd->id * pl330->mcbufsz);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001763 thrd->req[0].mc_bus = pl330->mcode_bus
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001764 + (thrd->id * pl330->mcbufsz);
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001765 thrd->req[0].desc = NULL;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001766
1767 thrd->req[1].mc_cpu = thrd->req[0].mc_cpu
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001768 + pl330->mcbufsz / 2;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001769 thrd->req[1].mc_bus = thrd->req[0].mc_bus
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001770 + pl330->mcbufsz / 2;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001771 thrd->req[1].desc = NULL;
Lars-Peter Clausen8ed30a12014-07-06 20:32:31 +02001772
1773 thrd->req_running = -1;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001774}
1775
1776static int dmac_alloc_threads(struct pl330_dmac *pl330)
1777{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001778 int chans = pl330->pcfg.num_chan;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001779 struct pl330_thread *thrd;
1780 int i;
1781
1782 /* Allocate 1 Manager and 'chans' Channel threads */
1783 pl330->channels = kzalloc((1 + chans) * sizeof(*thrd),
1784 GFP_KERNEL);
1785 if (!pl330->channels)
1786 return -ENOMEM;
1787
1788 /* Init Channel threads */
1789 for (i = 0; i < chans; i++) {
1790 thrd = &pl330->channels[i];
1791 thrd->id = i;
1792 thrd->dmac = pl330;
1793 _reset_thread(thrd);
1794 thrd->free = true;
1795 }
1796
1797 /* MANAGER is indexed at the end */
1798 thrd = &pl330->channels[chans];
1799 thrd->id = chans;
1800 thrd->dmac = pl330;
1801 thrd->free = false;
1802 pl330->manager = thrd;
1803
1804 return 0;
1805}
1806
1807static int dmac_alloc_resources(struct pl330_dmac *pl330)
1808{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001809 int chans = pl330->pcfg.num_chan;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001810 int ret;
1811
1812 /*
1813 * Alloc MicroCode buffer for 'chans' Channel threads.
1814 * A channel's buffer offset is (Channel_Id * MCODE_BUFF_PERCHAN)
1815 */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001816 pl330->mcode_cpu = dma_alloc_coherent(pl330->ddma.dev,
1817 chans * pl330->mcbufsz,
Boojin Kimb7d861d2011-12-26 18:49:52 +09001818 &pl330->mcode_bus, GFP_KERNEL);
1819 if (!pl330->mcode_cpu) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001820 dev_err(pl330->ddma.dev, "%s:%d Can't allocate memory!\n",
Boojin Kimb7d861d2011-12-26 18:49:52 +09001821 __func__, __LINE__);
1822 return -ENOMEM;
1823 }
1824
1825 ret = dmac_alloc_threads(pl330);
1826 if (ret) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001827 dev_err(pl330->ddma.dev, "%s:%d Can't to create channels for DMAC!\n",
Boojin Kimb7d861d2011-12-26 18:49:52 +09001828 __func__, __LINE__);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001829 dma_free_coherent(pl330->ddma.dev,
1830 chans * pl330->mcbufsz,
Boojin Kimb7d861d2011-12-26 18:49:52 +09001831 pl330->mcode_cpu, pl330->mcode_bus);
1832 return ret;
1833 }
1834
1835 return 0;
1836}
1837
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001838static int pl330_add(struct pl330_dmac *pl330)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001839{
Boojin Kimb7d861d2011-12-26 18:49:52 +09001840 void __iomem *regs;
1841 int i, ret;
1842
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001843 regs = pl330->base;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001844
1845 /* Check if we can handle this DMAC */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001846 if ((pl330->pcfg.periph_id & 0xfffff) != PERIPH_ID_VAL) {
1847 dev_err(pl330->ddma.dev, "PERIPH_ID 0x%x !\n",
1848 pl330->pcfg.periph_id);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001849 return -EINVAL;
1850 }
1851
1852 /* Read the configuration of the DMAC */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001853 read_dmac_config(pl330);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001854
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001855 if (pl330->pcfg.num_events == 0) {
1856 dev_err(pl330->ddma.dev, "%s:%d Can't work without events!\n",
Boojin Kimb7d861d2011-12-26 18:49:52 +09001857 __func__, __LINE__);
1858 return -EINVAL;
1859 }
1860
Boojin Kimb7d861d2011-12-26 18:49:52 +09001861 spin_lock_init(&pl330->lock);
1862
1863 INIT_LIST_HEAD(&pl330->req_done);
1864
1865 /* Use default MC buffer size if not provided */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001866 if (!pl330->mcbufsz)
1867 pl330->mcbufsz = MCODE_BUFF_PER_REQ * 2;
Boojin Kimb7d861d2011-12-26 18:49:52 +09001868
1869 /* Mark all events as free */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001870 for (i = 0; i < pl330->pcfg.num_events; i++)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001871 pl330->events[i] = -1;
1872
1873 /* Allocate resources needed by the DMAC */
1874 ret = dmac_alloc_resources(pl330);
1875 if (ret) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001876 dev_err(pl330->ddma.dev, "Unable to create channels for DMAC\n");
Boojin Kimb7d861d2011-12-26 18:49:52 +09001877 return ret;
1878 }
1879
1880 tasklet_init(&pl330->tasks, pl330_dotask, (unsigned long) pl330);
1881
1882 pl330->state = INIT;
1883
1884 return 0;
1885}
1886
1887static int dmac_free_threads(struct pl330_dmac *pl330)
1888{
Boojin Kimb7d861d2011-12-26 18:49:52 +09001889 struct pl330_thread *thrd;
1890 int i;
1891
1892 /* Release Channel threads */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001893 for (i = 0; i < pl330->pcfg.num_chan; i++) {
Boojin Kimb7d861d2011-12-26 18:49:52 +09001894 thrd = &pl330->channels[i];
Lars-Peter Clausen65ad6062014-07-06 20:32:26 +02001895 pl330_release_channel(thrd);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001896 }
1897
1898 /* Free memory */
1899 kfree(pl330->channels);
1900
1901 return 0;
1902}
1903
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001904static void pl330_del(struct pl330_dmac *pl330)
Boojin Kimb7d861d2011-12-26 18:49:52 +09001905{
Boojin Kimb7d861d2011-12-26 18:49:52 +09001906 pl330->state = UNINIT;
1907
1908 tasklet_kill(&pl330->tasks);
1909
1910 /* Free DMAC resources */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001911 dmac_free_threads(pl330);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001912
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001913 dma_free_coherent(pl330->ddma.dev,
1914 pl330->pcfg.num_chan * pl330->mcbufsz, pl330->mcode_cpu,
1915 pl330->mcode_bus);
Boojin Kimb7d861d2011-12-26 18:49:52 +09001916}
1917
Thomas Abraham3e2ec132011-10-24 11:43:02 +02001918/* forward declaration */
1919static struct amba_driver pl330_driver;
1920
Jassi Brarb3040e42010-05-23 20:28:19 -07001921static inline struct dma_pl330_chan *
1922to_pchan(struct dma_chan *ch)
1923{
1924 if (!ch)
1925 return NULL;
1926
1927 return container_of(ch, struct dma_pl330_chan, chan);
1928}
1929
1930static inline struct dma_pl330_desc *
1931to_desc(struct dma_async_tx_descriptor *tx)
1932{
1933 return container_of(tx, struct dma_pl330_desc, txd);
1934}
1935
Jassi Brarb3040e42010-05-23 20:28:19 -07001936static inline void fill_queue(struct dma_pl330_chan *pch)
1937{
1938 struct dma_pl330_desc *desc;
1939 int ret;
1940
1941 list_for_each_entry(desc, &pch->work_list, node) {
1942
1943 /* If already submitted */
1944 if (desc->status == BUSY)
Jassi Brar30fb9802013-02-13 16:13:14 +05301945 continue;
Jassi Brarb3040e42010-05-23 20:28:19 -07001946
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02001947 ret = pl330_submit_req(pch->thread, desc);
Jassi Brarb3040e42010-05-23 20:28:19 -07001948 if (!ret) {
1949 desc->status = BUSY;
Jassi Brarb3040e42010-05-23 20:28:19 -07001950 } else if (ret == -EAGAIN) {
1951 /* QFull or DMAC Dying */
1952 break;
1953 } else {
1954 /* Unacceptable request */
1955 desc->status = DONE;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02001956 dev_err(pch->dmac->ddma.dev, "%s:%d Bad Desc(%d)\n",
Jassi Brarb3040e42010-05-23 20:28:19 -07001957 __func__, __LINE__, desc->txd.cookie);
1958 tasklet_schedule(&pch->task);
1959 }
1960 }
1961}
1962
1963static void pl330_tasklet(unsigned long data)
1964{
1965 struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data;
1966 struct dma_pl330_desc *desc, *_dt;
1967 unsigned long flags;
Jassi Brarb3040e42010-05-23 20:28:19 -07001968
1969 spin_lock_irqsave(&pch->lock, flags);
1970
1971 /* Pick up ripe tomatoes */
1972 list_for_each_entry_safe(desc, _dt, &pch->work_list, node)
1973 if (desc->status == DONE) {
Tushar Behera30c1dc02012-05-23 16:47:31 +05301974 if (!pch->cyclic)
Vinod Kouleab21582012-05-11 11:24:41 +05301975 dma_cookie_complete(&desc->txd);
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02001976 list_move_tail(&desc->node, &pch->completed_list);
Jassi Brarb3040e42010-05-23 20:28:19 -07001977 }
1978
1979 /* Try to submit a req imm. next to the last completed cookie */
1980 fill_queue(pch);
1981
1982 /* Make sure the PL330 Channel thread is active */
Lars-Peter Clausenc26939e2014-07-06 20:32:32 +02001983 spin_lock(&pch->thread->dmac->lock);
1984 _start(pch->thread);
1985 spin_unlock(&pch->thread->dmac->lock);
Jassi Brarb3040e42010-05-23 20:28:19 -07001986
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02001987 while (!list_empty(&pch->completed_list)) {
1988 dma_async_tx_callback callback;
1989 void *callback_param;
Jassi Brarb3040e42010-05-23 20:28:19 -07001990
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02001991 desc = list_first_entry(&pch->completed_list,
1992 struct dma_pl330_desc, node);
1993
1994 callback = desc->txd.callback;
1995 callback_param = desc->txd.callback_param;
1996
1997 if (pch->cyclic) {
1998 desc->status = PREP;
1999 list_move_tail(&desc->node, &pch->work_list);
2000 } else {
2001 desc->status = FREE;
2002 list_move_tail(&desc->node, &pch->dmac->desc_pool);
2003 }
2004
Dan Williamsd38a8c62013-10-18 19:35:23 +02002005 dma_descriptor_unmap(&desc->txd);
2006
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02002007 if (callback) {
2008 spin_unlock_irqrestore(&pch->lock, flags);
2009 callback(callback_param);
2010 spin_lock_irqsave(&pch->lock, flags);
2011 }
2012 }
2013 spin_unlock_irqrestore(&pch->lock, flags);
Jassi Brarb3040e42010-05-23 20:28:19 -07002014}
2015
Thomas Abraham3e2ec132011-10-24 11:43:02 +02002016bool pl330_filter(struct dma_chan *chan, void *param)
2017{
Thomas Abrahamcd072512011-10-24 11:43:11 +02002018 u8 *peri_id;
Thomas Abraham3e2ec132011-10-24 11:43:02 +02002019
2020 if (chan->device->dev->driver != &pl330_driver.drv)
2021 return false;
2022
Thomas Abrahamcd072512011-10-24 11:43:11 +02002023 peri_id = chan->private;
Dan Carpenter2f986ec2013-11-08 12:51:16 +03002024 return *peri_id == (unsigned long)param;
Thomas Abraham3e2ec132011-10-24 11:43:02 +02002025}
2026EXPORT_SYMBOL(pl330_filter);
2027
Padmavathi Vennaa80258f2013-02-14 09:10:06 +05302028static struct dma_chan *of_dma_pl330_xlate(struct of_phandle_args *dma_spec,
2029 struct of_dma *ofdma)
2030{
2031 int count = dma_spec->args_count;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002032 struct pl330_dmac *pl330 = ofdma->of_dma_data;
Lars-Peter Clausen70cbb162014-01-11 20:08:39 +01002033 unsigned int chan_id;
Padmavathi Vennaa80258f2013-02-14 09:10:06 +05302034
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002035 if (!pl330)
2036 return NULL;
2037
Padmavathi Vennaa80258f2013-02-14 09:10:06 +05302038 if (count != 1)
2039 return NULL;
2040
Lars-Peter Clausen70cbb162014-01-11 20:08:39 +01002041 chan_id = dma_spec->args[0];
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002042 if (chan_id >= pl330->num_peripherals)
Lars-Peter Clausen70cbb162014-01-11 20:08:39 +01002043 return NULL;
Padmavathi Vennaa80258f2013-02-14 09:10:06 +05302044
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002045 return dma_get_slave_channel(&pl330->peripherals[chan_id].chan);
Padmavathi Vennaa80258f2013-02-14 09:10:06 +05302046}
2047
Jassi Brarb3040e42010-05-23 20:28:19 -07002048static int pl330_alloc_chan_resources(struct dma_chan *chan)
2049{
2050 struct dma_pl330_chan *pch = to_pchan(chan);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002051 struct pl330_dmac *pl330 = pch->dmac;
Jassi Brarb3040e42010-05-23 20:28:19 -07002052 unsigned long flags;
2053
2054 spin_lock_irqsave(&pch->lock, flags);
2055
Russell King - ARM Linuxd3ee98cdc2012-03-06 22:35:47 +00002056 dma_cookie_init(chan);
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002057 pch->cyclic = false;
Jassi Brarb3040e42010-05-23 20:28:19 -07002058
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002059 pch->thread = pl330_request_channel(pl330);
Lars-Peter Clausen65ad6062014-07-06 20:32:26 +02002060 if (!pch->thread) {
Jassi Brarb3040e42010-05-23 20:28:19 -07002061 spin_unlock_irqrestore(&pch->lock, flags);
Inderpal Singh02747882012-09-17 09:57:45 +05302062 return -ENOMEM;
Jassi Brarb3040e42010-05-23 20:28:19 -07002063 }
2064
2065 tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch);
2066
2067 spin_unlock_irqrestore(&pch->lock, flags);
2068
2069 return 1;
2070}
2071
2072static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg)
2073{
2074 struct dma_pl330_chan *pch = to_pchan(chan);
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02002075 struct dma_pl330_desc *desc;
Jassi Brarb3040e42010-05-23 20:28:19 -07002076 unsigned long flags;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002077 struct pl330_dmac *pl330 = pch->dmac;
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002078 struct dma_slave_config *slave_config;
Boojin Kimae43b882011-09-02 09:44:32 +09002079 LIST_HEAD(list);
Jassi Brarb3040e42010-05-23 20:28:19 -07002080
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002081 switch (cmd) {
2082 case DMA_TERMINATE_ALL:
2083 spin_lock_irqsave(&pch->lock, flags);
2084
Lars-Peter Clausenc26939e2014-07-06 20:32:32 +02002085 spin_lock(&pl330->lock);
2086 _stop(pch->thread);
2087 spin_unlock(&pl330->lock);
2088
2089 pch->thread->req[0].desc = NULL;
2090 pch->thread->req[1].desc = NULL;
2091 pch->thread->req_running = -1;
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002092
2093 /* Mark all desc done */
Lars-Peter Clausen04abf5d2014-01-11 20:08:38 +01002094 list_for_each_entry(desc, &pch->submitted_list, node) {
2095 desc->status = FREE;
2096 dma_cookie_complete(&desc->txd);
2097 }
2098
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02002099 list_for_each_entry(desc, &pch->work_list , node) {
2100 desc->status = FREE;
2101 dma_cookie_complete(&desc->txd);
Boojin Kimae43b882011-09-02 09:44:32 +09002102 }
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002103
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02002104 list_for_each_entry(desc, &pch->completed_list , node) {
2105 desc->status = FREE;
2106 dma_cookie_complete(&desc->txd);
2107 }
2108
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002109 list_splice_tail_init(&pch->submitted_list, &pl330->desc_pool);
2110 list_splice_tail_init(&pch->work_list, &pl330->desc_pool);
2111 list_splice_tail_init(&pch->completed_list, &pl330->desc_pool);
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002112 spin_unlock_irqrestore(&pch->lock, flags);
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002113 break;
2114 case DMA_SLAVE_CONFIG:
2115 slave_config = (struct dma_slave_config *)arg;
2116
Vinod Kouldb8196d2011-10-13 22:34:23 +05302117 if (slave_config->direction == DMA_MEM_TO_DEV) {
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002118 if (slave_config->dst_addr)
2119 pch->fifo_addr = slave_config->dst_addr;
2120 if (slave_config->dst_addr_width)
2121 pch->burst_sz = __ffs(slave_config->dst_addr_width);
2122 if (slave_config->dst_maxburst)
2123 pch->burst_len = slave_config->dst_maxburst;
Vinod Kouldb8196d2011-10-13 22:34:23 +05302124 } else if (slave_config->direction == DMA_DEV_TO_MEM) {
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002125 if (slave_config->src_addr)
2126 pch->fifo_addr = slave_config->src_addr;
2127 if (slave_config->src_addr_width)
2128 pch->burst_sz = __ffs(slave_config->src_addr_width);
2129 if (slave_config->src_maxburst)
2130 pch->burst_len = slave_config->src_maxburst;
2131 }
2132 break;
2133 default:
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002134 dev_err(pch->dmac->ddma.dev, "Not supported command.\n");
Jassi Brarb3040e42010-05-23 20:28:19 -07002135 return -ENXIO;
Boojin Kim1d0c1d62011-09-02 09:44:31 +09002136 }
Jassi Brarb3040e42010-05-23 20:28:19 -07002137
2138 return 0;
2139}
2140
2141static void pl330_free_chan_resources(struct dma_chan *chan)
2142{
2143 struct dma_pl330_chan *pch = to_pchan(chan);
2144 unsigned long flags;
2145
Jassi Brarb3040e42010-05-23 20:28:19 -07002146 tasklet_kill(&pch->task);
2147
Bartlomiej Zolnierkiewiczda331ba2013-07-03 15:00:43 -07002148 spin_lock_irqsave(&pch->lock, flags);
2149
Lars-Peter Clausen65ad6062014-07-06 20:32:26 +02002150 pl330_release_channel(pch->thread);
2151 pch->thread = NULL;
Jassi Brarb3040e42010-05-23 20:28:19 -07002152
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002153 if (pch->cyclic)
2154 list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool);
2155
Jassi Brarb3040e42010-05-23 20:28:19 -07002156 spin_unlock_irqrestore(&pch->lock, flags);
2157}
2158
2159static enum dma_status
2160pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
2161 struct dma_tx_state *txstate)
2162{
Russell King - ARM Linux96a2af42012-03-06 22:35:27 +00002163 return dma_cookie_status(chan, cookie, txstate);
Jassi Brarb3040e42010-05-23 20:28:19 -07002164}
2165
2166static void pl330_issue_pending(struct dma_chan *chan)
2167{
Lars-Peter Clausen04abf5d2014-01-11 20:08:38 +01002168 struct dma_pl330_chan *pch = to_pchan(chan);
2169 unsigned long flags;
2170
2171 spin_lock_irqsave(&pch->lock, flags);
2172 list_splice_tail_init(&pch->submitted_list, &pch->work_list);
2173 spin_unlock_irqrestore(&pch->lock, flags);
2174
2175 pl330_tasklet((unsigned long)pch);
Jassi Brarb3040e42010-05-23 20:28:19 -07002176}
2177
2178/*
2179 * We returned the last one of the circular list of descriptor(s)
2180 * from prep_xxx, so the argument to submit corresponds to the last
2181 * descriptor of the list.
2182 */
2183static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx)
2184{
2185 struct dma_pl330_desc *desc, *last = to_desc(tx);
2186 struct dma_pl330_chan *pch = to_pchan(tx->chan);
2187 dma_cookie_t cookie;
2188 unsigned long flags;
2189
2190 spin_lock_irqsave(&pch->lock, flags);
2191
2192 /* Assign cookies to all nodes */
Jassi Brarb3040e42010-05-23 20:28:19 -07002193 while (!list_empty(&last->node)) {
2194 desc = list_entry(last->node.next, struct dma_pl330_desc, node);
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002195 if (pch->cyclic) {
2196 desc->txd.callback = last->txd.callback;
2197 desc->txd.callback_param = last->txd.callback_param;
2198 }
Jassi Brarb3040e42010-05-23 20:28:19 -07002199
Russell King - ARM Linux884485e2012-03-06 22:34:46 +00002200 dma_cookie_assign(&desc->txd);
Jassi Brarb3040e42010-05-23 20:28:19 -07002201
Lars-Peter Clausen04abf5d2014-01-11 20:08:38 +01002202 list_move_tail(&desc->node, &pch->submitted_list);
Jassi Brarb3040e42010-05-23 20:28:19 -07002203 }
2204
Russell King - ARM Linux884485e2012-03-06 22:34:46 +00002205 cookie = dma_cookie_assign(&last->txd);
Lars-Peter Clausen04abf5d2014-01-11 20:08:38 +01002206 list_add_tail(&last->node, &pch->submitted_list);
Jassi Brarb3040e42010-05-23 20:28:19 -07002207 spin_unlock_irqrestore(&pch->lock, flags);
2208
2209 return cookie;
2210}
2211
2212static inline void _init_desc(struct dma_pl330_desc *desc)
2213{
Jassi Brarb3040e42010-05-23 20:28:19 -07002214 desc->rqcfg.swap = SWAP_NO;
Lars-Peter Clausenf0564c72014-07-06 20:32:19 +02002215 desc->rqcfg.scctl = CCTRL0;
2216 desc->rqcfg.dcctl = CCTRL0;
Jassi Brarb3040e42010-05-23 20:28:19 -07002217 desc->txd.tx_submit = pl330_tx_submit;
2218
2219 INIT_LIST_HEAD(&desc->node);
2220}
2221
2222/* Returns the number of descriptors added to the DMAC pool */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002223static int add_desc(struct pl330_dmac *pl330, gfp_t flg, int count)
Jassi Brarb3040e42010-05-23 20:28:19 -07002224{
2225 struct dma_pl330_desc *desc;
2226 unsigned long flags;
2227 int i;
2228
Will Deacon0baf8f62013-12-02 18:01:30 +00002229 desc = kcalloc(count, sizeof(*desc), flg);
Jassi Brarb3040e42010-05-23 20:28:19 -07002230 if (!desc)
2231 return 0;
2232
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002233 spin_lock_irqsave(&pl330->pool_lock, flags);
Jassi Brarb3040e42010-05-23 20:28:19 -07002234
2235 for (i = 0; i < count; i++) {
2236 _init_desc(&desc[i]);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002237 list_add_tail(&desc[i].node, &pl330->desc_pool);
Jassi Brarb3040e42010-05-23 20:28:19 -07002238 }
2239
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002240 spin_unlock_irqrestore(&pl330->pool_lock, flags);
Jassi Brarb3040e42010-05-23 20:28:19 -07002241
2242 return count;
2243}
2244
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002245static struct dma_pl330_desc *pluck_desc(struct pl330_dmac *pl330)
Jassi Brarb3040e42010-05-23 20:28:19 -07002246{
2247 struct dma_pl330_desc *desc = NULL;
2248 unsigned long flags;
2249
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002250 spin_lock_irqsave(&pl330->pool_lock, flags);
Jassi Brarb3040e42010-05-23 20:28:19 -07002251
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002252 if (!list_empty(&pl330->desc_pool)) {
2253 desc = list_entry(pl330->desc_pool.next,
Jassi Brarb3040e42010-05-23 20:28:19 -07002254 struct dma_pl330_desc, node);
2255
2256 list_del_init(&desc->node);
2257
2258 desc->status = PREP;
2259 desc->txd.callback = NULL;
2260 }
2261
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002262 spin_unlock_irqrestore(&pl330->pool_lock, flags);
Jassi Brarb3040e42010-05-23 20:28:19 -07002263
2264 return desc;
2265}
2266
2267static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch)
2268{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002269 struct pl330_dmac *pl330 = pch->dmac;
Thomas Abrahamcd072512011-10-24 11:43:11 +02002270 u8 *peri_id = pch->chan.private;
Jassi Brarb3040e42010-05-23 20:28:19 -07002271 struct dma_pl330_desc *desc;
2272
2273 /* Pluck one desc from the pool of DMAC */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002274 desc = pluck_desc(pl330);
Jassi Brarb3040e42010-05-23 20:28:19 -07002275
2276 /* If the DMAC pool is empty, alloc new */
2277 if (!desc) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002278 if (!add_desc(pl330, GFP_ATOMIC, 1))
Jassi Brarb3040e42010-05-23 20:28:19 -07002279 return NULL;
2280
2281 /* Try again */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002282 desc = pluck_desc(pl330);
Jassi Brarb3040e42010-05-23 20:28:19 -07002283 if (!desc) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002284 dev_err(pch->dmac->ddma.dev,
Jassi Brarb3040e42010-05-23 20:28:19 -07002285 "%s:%d ALERT!\n", __func__, __LINE__);
2286 return NULL;
2287 }
2288 }
2289
2290 /* Initialize the descriptor */
2291 desc->pchan = pch;
2292 desc->txd.cookie = 0;
2293 async_tx_ack(&desc->txd);
2294
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02002295 desc->peri = peri_id ? pch->chan.chan_id : 0;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002296 desc->rqcfg.pcfg = &pch->dmac->pcfg;
Jassi Brarb3040e42010-05-23 20:28:19 -07002297
2298 dma_async_tx_descriptor_init(&desc->txd, &pch->chan);
2299
2300 return desc;
2301}
2302
2303static inline void fill_px(struct pl330_xfer *px,
2304 dma_addr_t dst, dma_addr_t src, size_t len)
2305{
Jassi Brarb3040e42010-05-23 20:28:19 -07002306 px->bytes = len;
2307 px->dst_addr = dst;
2308 px->src_addr = src;
2309}
2310
2311static struct dma_pl330_desc *
2312__pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst,
2313 dma_addr_t src, size_t len)
2314{
2315 struct dma_pl330_desc *desc = pl330_get_desc(pch);
2316
2317 if (!desc) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002318 dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",
Jassi Brarb3040e42010-05-23 20:28:19 -07002319 __func__, __LINE__);
2320 return NULL;
2321 }
2322
2323 /*
2324 * Ideally we should lookout for reqs bigger than
2325 * those that can be programmed with 256 bytes of
2326 * MC buffer, but considering a req size is seldom
2327 * going to be word-unaligned and more than 200MB,
2328 * we take it easy.
2329 * Also, should the limit is reached we'd rather
2330 * have the platform increase MC buffer size than
2331 * complicating this API driver.
2332 */
2333 fill_px(&desc->px, dst, src, len);
2334
2335 return desc;
2336}
2337
2338/* Call after fixing burst size */
2339static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len)
2340{
2341 struct dma_pl330_chan *pch = desc->pchan;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002342 struct pl330_dmac *pl330 = pch->dmac;
Jassi Brarb3040e42010-05-23 20:28:19 -07002343 int burst_len;
2344
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002345 burst_len = pl330->pcfg.data_bus_width / 8;
2346 burst_len *= pl330->pcfg.data_buf_dep;
Jassi Brarb3040e42010-05-23 20:28:19 -07002347 burst_len >>= desc->rqcfg.brst_size;
2348
2349 /* src/dst_burst_len can't be more than 16 */
2350 if (burst_len > 16)
2351 burst_len = 16;
2352
2353 while (burst_len > 1) {
2354 if (!(len % (burst_len << desc->rqcfg.brst_size)))
2355 break;
2356 burst_len--;
2357 }
2358
2359 return burst_len;
2360}
2361
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002362static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
2363 struct dma_chan *chan, dma_addr_t dma_addr, size_t len,
Alexandre Bounine185ecb52012-03-08 15:35:13 -05002364 size_t period_len, enum dma_transfer_direction direction,
Laurent Pinchart31c1e5a2014-08-01 12:20:10 +02002365 unsigned long flags)
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002366{
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002367 struct dma_pl330_desc *desc = NULL, *first = NULL;
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002368 struct dma_pl330_chan *pch = to_pchan(chan);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002369 struct pl330_dmac *pl330 = pch->dmac;
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002370 unsigned int i;
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002371 dma_addr_t dst;
2372 dma_addr_t src;
2373
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002374 if (len % period_len != 0)
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002375 return NULL;
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002376
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002377 if (!is_slave_direction(direction)) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002378 dev_err(pch->dmac->ddma.dev, "%s:%d Invalid dma direction\n",
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002379 __func__, __LINE__);
2380 return NULL;
2381 }
2382
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002383 for (i = 0; i < len / period_len; i++) {
2384 desc = pl330_get_desc(pch);
2385 if (!desc) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002386 dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002387 __func__, __LINE__);
2388
2389 if (!first)
2390 return NULL;
2391
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002392 spin_lock_irqsave(&pl330->pool_lock, flags);
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002393
2394 while (!list_empty(&first->node)) {
2395 desc = list_entry(first->node.next,
2396 struct dma_pl330_desc, node);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002397 list_move_tail(&desc->node, &pl330->desc_pool);
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002398 }
2399
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002400 list_move_tail(&first->node, &pl330->desc_pool);
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002401
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002402 spin_unlock_irqrestore(&pl330->pool_lock, flags);
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002403
2404 return NULL;
2405 }
2406
2407 switch (direction) {
2408 case DMA_MEM_TO_DEV:
2409 desc->rqcfg.src_inc = 1;
2410 desc->rqcfg.dst_inc = 0;
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002411 src = dma_addr;
2412 dst = pch->fifo_addr;
2413 break;
2414 case DMA_DEV_TO_MEM:
2415 desc->rqcfg.src_inc = 0;
2416 desc->rqcfg.dst_inc = 1;
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002417 src = pch->fifo_addr;
2418 dst = dma_addr;
2419 break;
2420 default:
2421 break;
2422 }
2423
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02002424 desc->rqtype = direction;
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002425 desc->rqcfg.brst_size = pch->burst_sz;
2426 desc->rqcfg.brst_len = 1;
2427 fill_px(&desc->px, dst, src, period_len);
2428
2429 if (!first)
2430 first = desc;
2431 else
2432 list_add_tail(&desc->node, &first->node);
2433
2434 dma_addr += period_len;
2435 }
2436
2437 if (!desc)
2438 return NULL;
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002439
2440 pch->cyclic = true;
Lars-Peter Clausenfc514462013-07-23 10:24:50 +02002441 desc->txd.flags = flags;
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002442
2443 return &desc->txd;
2444}
2445
Jassi Brarb3040e42010-05-23 20:28:19 -07002446static struct dma_async_tx_descriptor *
2447pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst,
2448 dma_addr_t src, size_t len, unsigned long flags)
2449{
2450 struct dma_pl330_desc *desc;
2451 struct dma_pl330_chan *pch = to_pchan(chan);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002452 struct pl330_dmac *pl330 = pch->dmac;
Jassi Brarb3040e42010-05-23 20:28:19 -07002453 int burst;
2454
Rob Herring4e0e6102011-07-25 16:05:04 -05002455 if (unlikely(!pch || !len))
Jassi Brarb3040e42010-05-23 20:28:19 -07002456 return NULL;
2457
Jassi Brarb3040e42010-05-23 20:28:19 -07002458 desc = __pl330_prep_dma_memcpy(pch, dst, src, len);
2459 if (!desc)
2460 return NULL;
2461
2462 desc->rqcfg.src_inc = 1;
2463 desc->rqcfg.dst_inc = 1;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02002464 desc->rqtype = DMA_MEM_TO_MEM;
Jassi Brarb3040e42010-05-23 20:28:19 -07002465
2466 /* Select max possible burst size */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002467 burst = pl330->pcfg.data_bus_width / 8;
Jassi Brarb3040e42010-05-23 20:28:19 -07002468
2469 while (burst > 1) {
2470 if (!(len % burst))
2471 break;
2472 burst /= 2;
2473 }
2474
2475 desc->rqcfg.brst_size = 0;
2476 while (burst != (1 << desc->rqcfg.brst_size))
2477 desc->rqcfg.brst_size++;
2478
2479 desc->rqcfg.brst_len = get_burst_len(desc, len);
2480
2481 desc->txd.flags = flags;
2482
2483 return &desc->txd;
2484}
2485
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002486static void __pl330_giveback_desc(struct pl330_dmac *pl330,
Chanho Park52a9d172013-08-09 20:11:33 +09002487 struct dma_pl330_desc *first)
2488{
2489 unsigned long flags;
2490 struct dma_pl330_desc *desc;
2491
2492 if (!first)
2493 return;
2494
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002495 spin_lock_irqsave(&pl330->pool_lock, flags);
Chanho Park52a9d172013-08-09 20:11:33 +09002496
2497 while (!list_empty(&first->node)) {
2498 desc = list_entry(first->node.next,
2499 struct dma_pl330_desc, node);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002500 list_move_tail(&desc->node, &pl330->desc_pool);
Chanho Park52a9d172013-08-09 20:11:33 +09002501 }
2502
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002503 list_move_tail(&first->node, &pl330->desc_pool);
Chanho Park52a9d172013-08-09 20:11:33 +09002504
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002505 spin_unlock_irqrestore(&pl330->pool_lock, flags);
Chanho Park52a9d172013-08-09 20:11:33 +09002506}
2507
Jassi Brarb3040e42010-05-23 20:28:19 -07002508static struct dma_async_tx_descriptor *
2509pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
Vinod Kouldb8196d2011-10-13 22:34:23 +05302510 unsigned int sg_len, enum dma_transfer_direction direction,
Alexandre Bounine185ecb52012-03-08 15:35:13 -05002511 unsigned long flg, void *context)
Jassi Brarb3040e42010-05-23 20:28:19 -07002512{
2513 struct dma_pl330_desc *first, *desc = NULL;
2514 struct dma_pl330_chan *pch = to_pchan(chan);
Jassi Brarb3040e42010-05-23 20:28:19 -07002515 struct scatterlist *sg;
Boojin Kim1b9bb712011-09-02 09:44:30 +09002516 int i;
Jassi Brarb3040e42010-05-23 20:28:19 -07002517 dma_addr_t addr;
2518
Thomas Abrahamcd072512011-10-24 11:43:11 +02002519 if (unlikely(!pch || !sgl || !sg_len))
Jassi Brarb3040e42010-05-23 20:28:19 -07002520 return NULL;
2521
Boojin Kim1b9bb712011-09-02 09:44:30 +09002522 addr = pch->fifo_addr;
Jassi Brarb3040e42010-05-23 20:28:19 -07002523
2524 first = NULL;
2525
2526 for_each_sg(sgl, sg, sg_len, i) {
2527
2528 desc = pl330_get_desc(pch);
2529 if (!desc) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002530 struct pl330_dmac *pl330 = pch->dmac;
Jassi Brarb3040e42010-05-23 20:28:19 -07002531
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002532 dev_err(pch->dmac->ddma.dev,
Jassi Brarb3040e42010-05-23 20:28:19 -07002533 "%s:%d Unable to fetch desc\n",
2534 __func__, __LINE__);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002535 __pl330_giveback_desc(pl330, first);
Jassi Brarb3040e42010-05-23 20:28:19 -07002536
2537 return NULL;
2538 }
2539
2540 if (!first)
2541 first = desc;
2542 else
2543 list_add_tail(&desc->node, &first->node);
2544
Vinod Kouldb8196d2011-10-13 22:34:23 +05302545 if (direction == DMA_MEM_TO_DEV) {
Jassi Brarb3040e42010-05-23 20:28:19 -07002546 desc->rqcfg.src_inc = 1;
2547 desc->rqcfg.dst_inc = 0;
2548 fill_px(&desc->px,
2549 addr, sg_dma_address(sg), sg_dma_len(sg));
2550 } else {
2551 desc->rqcfg.src_inc = 0;
2552 desc->rqcfg.dst_inc = 1;
2553 fill_px(&desc->px,
2554 sg_dma_address(sg), addr, sg_dma_len(sg));
2555 }
2556
Boojin Kim1b9bb712011-09-02 09:44:30 +09002557 desc->rqcfg.brst_size = pch->burst_sz;
Jassi Brarb3040e42010-05-23 20:28:19 -07002558 desc->rqcfg.brst_len = 1;
Lars-Peter Clausen9dc5a312014-07-06 20:32:30 +02002559 desc->rqtype = direction;
Jassi Brarb3040e42010-05-23 20:28:19 -07002560 }
2561
2562 /* Return the last desc in the chain */
2563 desc->txd.flags = flg;
2564 return &desc->txd;
2565}
2566
2567static irqreturn_t pl330_irq_handler(int irq, void *data)
2568{
2569 if (pl330_update(data))
2570 return IRQ_HANDLED;
2571 else
2572 return IRQ_NONE;
2573}
2574
Lars-Peter Clausenca38ff12013-07-15 17:53:08 +02002575#define PL330_DMA_BUSWIDTHS \
2576 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
2577 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
2578 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
2579 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
2580 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)
2581
2582static int pl330_dma_device_slave_caps(struct dma_chan *dchan,
2583 struct dma_slave_caps *caps)
2584{
2585 caps->src_addr_widths = PL330_DMA_BUSWIDTHS;
2586 caps->dstn_addr_widths = PL330_DMA_BUSWIDTHS;
2587 caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
2588 caps->cmd_pause = false;
2589 caps->cmd_terminate = true;
Lars-Peter Clausenbfb9bb42014-01-11 14:02:17 +01002590 caps->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
Lars-Peter Clausenca38ff12013-07-15 17:53:08 +02002591
Lars-Peter Clausenca38ff12013-07-15 17:53:08 +02002592 return 0;
2593}
2594
Bill Pemberton463a1f82012-11-19 13:22:55 -05002595static int
Russell Kingaa25afa2011-02-19 15:55:00 +00002596pl330_probe(struct amba_device *adev, const struct amba_id *id)
Jassi Brarb3040e42010-05-23 20:28:19 -07002597{
2598 struct dma_pl330_platdata *pdat;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002599 struct pl330_config *pcfg;
2600 struct pl330_dmac *pl330;
Padmavathi Venna0b94c572013-03-05 14:55:31 +05302601 struct dma_pl330_chan *pch, *_p;
Jassi Brarb3040e42010-05-23 20:28:19 -07002602 struct dma_device *pd;
2603 struct resource *res;
2604 int i, ret, irq;
Rob Herring4e0e6102011-07-25 16:05:04 -05002605 int num_chan;
Jassi Brarb3040e42010-05-23 20:28:19 -07002606
Jingoo Hand4adcc02013-07-30 17:09:11 +09002607 pdat = dev_get_platdata(&adev->dev);
Jassi Brarb3040e42010-05-23 20:28:19 -07002608
Russell King64113012013-06-27 10:29:32 +01002609 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32));
2610 if (ret)
2611 return ret;
2612
Jassi Brarb3040e42010-05-23 20:28:19 -07002613 /* Allocate a new DMAC and its Channels */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002614 pl330 = devm_kzalloc(&adev->dev, sizeof(*pl330), GFP_KERNEL);
2615 if (!pl330) {
Jassi Brarb3040e42010-05-23 20:28:19 -07002616 dev_err(&adev->dev, "unable to allocate mem\n");
2617 return -ENOMEM;
2618 }
2619
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002620 pl330->mcbufsz = pdat ? pdat->mcbuf_sz : 0;
Jassi Brarb3040e42010-05-23 20:28:19 -07002621
2622 res = &adev->res;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002623 pl330->base = devm_ioremap_resource(&adev->dev, res);
2624 if (IS_ERR(pl330->base))
2625 return PTR_ERR(pl330->base);
Jassi Brarb3040e42010-05-23 20:28:19 -07002626
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002627 amba_set_drvdata(adev, pl330);
Boojin Kima2f52032011-09-02 09:44:29 +09002628
Dan Carpenter02808b42013-11-08 12:50:24 +03002629 for (i = 0; i < AMBA_NR_IRQS; i++) {
Michal Simeke98b3ca2013-09-30 08:50:48 +02002630 irq = adev->irq[i];
2631 if (irq) {
2632 ret = devm_request_irq(&adev->dev, irq,
2633 pl330_irq_handler, 0,
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002634 dev_name(&adev->dev), pl330);
Michal Simeke98b3ca2013-09-30 08:50:48 +02002635 if (ret)
2636 return ret;
2637 } else {
2638 break;
2639 }
2640 }
Jassi Brarb3040e42010-05-23 20:28:19 -07002641
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002642 pcfg = &pl330->pcfg;
2643
2644 pcfg->periph_id = adev->periphid;
2645 ret = pl330_add(pl330);
Jassi Brarb3040e42010-05-23 20:28:19 -07002646 if (ret)
Michal Simek173e8382013-09-04 16:40:17 +02002647 return ret;
Jassi Brarb3040e42010-05-23 20:28:19 -07002648
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002649 INIT_LIST_HEAD(&pl330->desc_pool);
2650 spin_lock_init(&pl330->pool_lock);
Jassi Brarb3040e42010-05-23 20:28:19 -07002651
2652 /* Create a descriptor pool of default size */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002653 if (!add_desc(pl330, GFP_KERNEL, NR_DEFAULT_DESC))
Jassi Brarb3040e42010-05-23 20:28:19 -07002654 dev_warn(&adev->dev, "unable to allocate desc\n");
2655
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002656 pd = &pl330->ddma;
Jassi Brarb3040e42010-05-23 20:28:19 -07002657 INIT_LIST_HEAD(&pd->channels);
2658
2659 /* Initialize channel parameters */
Olof Johanssonc8473822012-04-08 16:26:19 -07002660 if (pdat)
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002661 num_chan = max_t(int, pdat->nr_valid_peri, pcfg->num_chan);
Olof Johanssonc8473822012-04-08 16:26:19 -07002662 else
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002663 num_chan = max_t(int, pcfg->num_peri, pcfg->num_chan);
Olof Johanssonc8473822012-04-08 16:26:19 -07002664
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002665 pl330->num_peripherals = num_chan;
Lars-Peter Clausen70cbb162014-01-11 20:08:39 +01002666
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002667 pl330->peripherals = kzalloc(num_chan * sizeof(*pch), GFP_KERNEL);
2668 if (!pl330->peripherals) {
Sachin Kamat61c6e752012-09-17 15:20:23 +05302669 ret = -ENOMEM;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002670 dev_err(&adev->dev, "unable to allocate pl330->peripherals\n");
Sachin Kamate4d43c12012-11-15 06:27:50 +00002671 goto probe_err2;
Sachin Kamat61c6e752012-09-17 15:20:23 +05302672 }
Jassi Brarb3040e42010-05-23 20:28:19 -07002673
Rob Herring4e0e6102011-07-25 16:05:04 -05002674 for (i = 0; i < num_chan; i++) {
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002675 pch = &pl330->peripherals[i];
Thomas Abraham93ed5542011-10-24 11:43:31 +02002676 if (!adev->dev.of_node)
2677 pch->chan.private = pdat ? &pdat->peri_id[i] : NULL;
2678 else
2679 pch->chan.private = adev->dev.of_node;
Jassi Brarb3040e42010-05-23 20:28:19 -07002680
Lars-Peter Clausen04abf5d2014-01-11 20:08:38 +01002681 INIT_LIST_HEAD(&pch->submitted_list);
Jassi Brarb3040e42010-05-23 20:28:19 -07002682 INIT_LIST_HEAD(&pch->work_list);
Lars-Peter Clausen39ff8612013-08-27 20:34:05 +02002683 INIT_LIST_HEAD(&pch->completed_list);
Jassi Brarb3040e42010-05-23 20:28:19 -07002684 spin_lock_init(&pch->lock);
Lars-Peter Clausen65ad6062014-07-06 20:32:26 +02002685 pch->thread = NULL;
Jassi Brarb3040e42010-05-23 20:28:19 -07002686 pch->chan.device = pd;
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002687 pch->dmac = pl330;
Jassi Brarb3040e42010-05-23 20:28:19 -07002688
2689 /* Add the channel to the DMAC list */
Jassi Brarb3040e42010-05-23 20:28:19 -07002690 list_add_tail(&pch->chan.device_node, &pd->channels);
2691 }
2692
2693 pd->dev = &adev->dev;
Thomas Abraham93ed5542011-10-24 11:43:31 +02002694 if (pdat) {
Thomas Abrahamcd072512011-10-24 11:43:11 +02002695 pd->cap_mask = pdat->cap_mask;
Thomas Abraham93ed5542011-10-24 11:43:31 +02002696 } else {
Thomas Abrahamcd072512011-10-24 11:43:11 +02002697 dma_cap_set(DMA_MEMCPY, pd->cap_mask);
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002698 if (pcfg->num_peri) {
Thomas Abraham93ed5542011-10-24 11:43:31 +02002699 dma_cap_set(DMA_SLAVE, pd->cap_mask);
2700 dma_cap_set(DMA_CYCLIC, pd->cap_mask);
Tushar Behera5557a412012-08-29 10:16:25 +05302701 dma_cap_set(DMA_PRIVATE, pd->cap_mask);
Thomas Abraham93ed5542011-10-24 11:43:31 +02002702 }
2703 }
Jassi Brarb3040e42010-05-23 20:28:19 -07002704
2705 pd->device_alloc_chan_resources = pl330_alloc_chan_resources;
2706 pd->device_free_chan_resources = pl330_free_chan_resources;
2707 pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy;
Boojin Kim42bc9cf2011-09-02 09:44:33 +09002708 pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic;
Jassi Brarb3040e42010-05-23 20:28:19 -07002709 pd->device_tx_status = pl330_tx_status;
2710 pd->device_prep_slave_sg = pl330_prep_slave_sg;
2711 pd->device_control = pl330_control;
2712 pd->device_issue_pending = pl330_issue_pending;
Lars-Peter Clausenca38ff12013-07-15 17:53:08 +02002713 pd->device_slave_caps = pl330_dma_device_slave_caps;
Jassi Brarb3040e42010-05-23 20:28:19 -07002714
2715 ret = dma_async_device_register(pd);
2716 if (ret) {
2717 dev_err(&adev->dev, "unable to register DMAC\n");
Padmavathi Venna0b94c572013-03-05 14:55:31 +05302718 goto probe_err3;
2719 }
2720
2721 if (adev->dev.of_node) {
2722 ret = of_dma_controller_register(adev->dev.of_node,
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002723 of_dma_pl330_xlate, pl330);
Padmavathi Venna0b94c572013-03-05 14:55:31 +05302724 if (ret) {
2725 dev_err(&adev->dev,
2726 "unable to register DMA to the generic DT DMA helpers\n");
2727 }
Jassi Brarb3040e42010-05-23 20:28:19 -07002728 }
Lars-Peter Clausenb714b842013-11-25 16:07:46 +01002729
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002730 adev->dev.dma_parms = &pl330->dma_parms;
Lars-Peter Clausenb714b842013-11-25 16:07:46 +01002731
Vinod Kouldbaf6d82013-09-02 21:54:48 +05302732 /*
2733 * This is the limit for transfers with a buswidth of 1, larger
2734 * buswidths will have larger limits.
2735 */
2736 ret = dma_set_max_seg_size(&adev->dev, 1900800);
2737 if (ret)
2738 dev_err(&adev->dev, "unable to set the seg size\n");
2739
Jassi Brarb3040e42010-05-23 20:28:19 -07002740
Jassi Brarb3040e42010-05-23 20:28:19 -07002741 dev_info(&adev->dev,
2742 "Loaded driver for PL330 DMAC-%d\n", adev->periphid);
2743 dev_info(&adev->dev,
2744 "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n",
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002745 pcfg->data_buf_dep, pcfg->data_bus_width / 8, pcfg->num_chan,
2746 pcfg->num_peri, pcfg->num_events);
Jassi Brarb3040e42010-05-23 20:28:19 -07002747
2748 return 0;
Padmavathi Venna0b94c572013-03-05 14:55:31 +05302749probe_err3:
Padmavathi Venna0b94c572013-03-05 14:55:31 +05302750 /* Idle the DMAC */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002751 list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,
Padmavathi Venna0b94c572013-03-05 14:55:31 +05302752 chan.device_node) {
2753
2754 /* Remove the channel */
2755 list_del(&pch->chan.device_node);
2756
2757 /* Flush the channel */
2758 pl330_control(&pch->chan, DMA_TERMINATE_ALL, 0);
2759 pl330_free_chan_resources(&pch->chan);
2760 }
Jassi Brarb3040e42010-05-23 20:28:19 -07002761probe_err2:
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002762 pl330_del(pl330);
Jassi Brarb3040e42010-05-23 20:28:19 -07002763
2764 return ret;
2765}
2766
Greg Kroah-Hartman4bf27b82012-12-21 15:09:59 -08002767static int pl330_remove(struct amba_device *adev)
Jassi Brarb3040e42010-05-23 20:28:19 -07002768{
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002769 struct pl330_dmac *pl330 = amba_get_drvdata(adev);
Jassi Brarb3040e42010-05-23 20:28:19 -07002770 struct dma_pl330_chan *pch, *_p;
Jassi Brarb3040e42010-05-23 20:28:19 -07002771
Padmavathi Venna0b94c572013-03-05 14:55:31 +05302772 if (adev->dev.of_node)
2773 of_dma_controller_free(adev->dev.of_node);
Padmavathi Venna421da892013-02-14 09:10:07 +05302774
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002775 dma_async_device_unregister(&pl330->ddma);
Jassi Brarb3040e42010-05-23 20:28:19 -07002776
2777 /* Idle the DMAC */
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002778 list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,
Jassi Brarb3040e42010-05-23 20:28:19 -07002779 chan.device_node) {
2780
2781 /* Remove the channel */
2782 list_del(&pch->chan.device_node);
2783
2784 /* Flush the channel */
2785 pl330_control(&pch->chan, DMA_TERMINATE_ALL, 0);
2786 pl330_free_chan_resources(&pch->chan);
2787 }
2788
Lars-Peter Clausenf6f24212014-07-06 20:32:29 +02002789 pl330_del(pl330);
Jassi Brarb3040e42010-05-23 20:28:19 -07002790
Jassi Brarb3040e42010-05-23 20:28:19 -07002791 return 0;
2792}
2793
2794static struct amba_id pl330_ids[] = {
2795 {
2796 .id = 0x00041330,
2797 .mask = 0x000fffff,
2798 },
2799 { 0, 0 },
2800};
2801
Dave Martine8fa5162011-10-05 15:15:20 +01002802MODULE_DEVICE_TABLE(amba, pl330_ids);
2803
Jassi Brarb3040e42010-05-23 20:28:19 -07002804static struct amba_driver pl330_driver = {
2805 .drv = {
2806 .owner = THIS_MODULE,
2807 .name = "dma-pl330",
Jassi Brarb3040e42010-05-23 20:28:19 -07002808 },
2809 .id_table = pl330_ids,
2810 .probe = pl330_probe,
2811 .remove = pl330_remove,
2812};
2813
viresh kumar9e5ed092012-03-15 10:40:38 +01002814module_amba_driver(pl330_driver);
Jassi Brarb3040e42010-05-23 20:28:19 -07002815
2816MODULE_AUTHOR("Jaswinder Singh <jassi.brar@samsung.com>");
2817MODULE_DESCRIPTION("API Driver for PL330 DMAC");
2818MODULE_LICENSE("GPL");