blob: 8114731a1c62d6450cd5f8bbd490f87c7800b1ea [file] [log] [blame]
Linus Walleije8689e62010-09-28 15:57:37 +02001/*
2 * Copyright (c) 2006 ARM Ltd.
3 * Copyright (c) 2010 ST-Ericsson SA
4 *
5 * Author: Peter Pearse <peter.pearse@arm.com>
6 * Author: Linus Walleij <linus.walleij@stericsson.com>
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along with
19 * this program; if not, write to the Free Software Foundation, Inc., 59
20 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
21 *
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +000022 * The full GNU General Public License is in this distribution in the file
23 * called COPYING.
Linus Walleije8689e62010-09-28 15:57:37 +020024 *
25 * Documentation: ARM DDI 0196G == PL080
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +000026 * Documentation: ARM DDI 0218E == PL081
Tomasz Figada1b6c02013-08-11 19:59:17 +020027 * Documentation: S3C6410 User's Manual == PL080S
Linus Walleije8689e62010-09-28 15:57:37 +020028 *
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +000029 * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any
30 * channel.
Linus Walleije8689e62010-09-28 15:57:37 +020031 *
32 * The PL080 has 8 channels available for simultaneous use, and the PL081
33 * has only two channels. So on these DMA controllers the number of channels
34 * and the number of incoming DMA signals are two totally different things.
35 * It is usually not possible to theoretically handle all physical signals,
36 * so a multiplexing scheme with possible denial of use is necessary.
37 *
38 * The PL080 has a dual bus master, PL081 has a single master.
39 *
Tomasz Figada1b6c02013-08-11 19:59:17 +020040 * PL080S is a version modified by Samsung and used in S3C64xx SoCs.
41 * It differs in following aspects:
42 * - CH_CONFIG register at different offset,
43 * - separate CH_CONTROL2 register for transfer size,
44 * - bigger maximum transfer size,
45 * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word,
46 * - no support for peripheral flow control.
47 *
Linus Walleije8689e62010-09-28 15:57:37 +020048 * Memory to peripheral transfer may be visualized as
49 * Get data from memory to DMAC
50 * Until no data left
51 * On burst request from peripheral
52 * Destination burst from DMAC to peripheral
53 * Clear burst request
54 * Raise terminal count interrupt
55 *
56 * For peripherals with a FIFO:
57 * Source burst size == half the depth of the peripheral FIFO
58 * Destination burst size == the depth of the peripheral FIFO
59 *
60 * (Bursts are irrelevant for mem to mem transfers - there are no burst
61 * signals, the DMA controller will simply facilitate its AHB master.)
62 *
63 * ASSUMES default (little) endianness for DMA transfers
64 *
Russell King - ARM Linux9dc2c202011-01-03 22:33:06 +000065 * The PL08x has two flow control settings:
66 * - DMAC flow control: the transfer size defines the number of transfers
67 * which occur for the current LLI entry, and the DMAC raises TC at the
68 * end of every LLI entry. Observed behaviour shows the DMAC listening
69 * to both the BREQ and SREQ signals (contrary to documented),
70 * transferring data if either is active. The LBREQ and LSREQ signals
71 * are ignored.
72 *
73 * - Peripheral flow control: the transfer size is ignored (and should be
74 * zero). The data is transferred from the current LLI entry, until
75 * after the final transfer signalled by LBREQ or LSREQ. The DMAC
Tomasz Figada1b6c02013-08-11 19:59:17 +020076 * will then move to the next LLI entry. Unsupported by PL080S.
Linus Walleije8689e62010-09-28 15:57:37 +020077 */
Russell King - ARM Linux730404a2011-01-03 22:34:07 +000078#include <linux/amba/bus.h>
Linus Walleije8689e62010-09-28 15:57:37 +020079#include <linux/amba/pl08x.h>
80#include <linux/debugfs.h>
Viresh Kumar0c38d702011-08-05 15:32:28 +053081#include <linux/delay.h>
82#include <linux/device.h>
83#include <linux/dmaengine.h>
84#include <linux/dmapool.h>
Vinod Koul8516f522011-09-02 16:43:44 +053085#include <linux/dma-mapping.h>
Sachin Kamat6d05c9f2014-01-23 16:10:07 +053086#include <linux/export.h>
Viresh Kumar0c38d702011-08-05 15:32:28 +053087#include <linux/init.h>
88#include <linux/interrupt.h>
89#include <linux/module.h>
Viresh Kumarb7b60182011-08-05 15:32:33 +053090#include <linux/pm_runtime.h>
Linus Walleije8689e62010-09-28 15:57:37 +020091#include <linux/seq_file.h>
Viresh Kumar0c38d702011-08-05 15:32:28 +053092#include <linux/slab.h>
Alessandro Rubini3a95b9f2012-11-24 00:22:56 +000093#include <linux/amba/pl080.h>
Linus Walleije8689e62010-09-28 15:57:37 +020094
Russell King - ARM Linuxd2ebfb32012-03-06 22:34:26 +000095#include "dmaengine.h"
Russell King01d8dc62012-05-26 14:04:29 +010096#include "virt-dma.h"
Russell King - ARM Linuxd2ebfb32012-03-06 22:34:26 +000097
Linus Walleije8689e62010-09-28 15:57:37 +020098#define DRIVER_NAME "pl08xdmac"
99
Russell King - ARM Linux7703eac2011-08-31 09:34:35 +0100100static struct amba_driver pl08x_amba_driver;
Russell Kingb23f2042012-05-16 10:48:44 +0100101struct pl08x_driver_data;
Russell King - ARM Linux7703eac2011-08-31 09:34:35 +0100102
Linus Walleije8689e62010-09-28 15:57:37 +0200103/**
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +0000104 * struct vendor_data - vendor-specific config parameters for PL08x derivatives
Linus Walleije8689e62010-09-28 15:57:37 +0200105 * @channels: the number of channels available in this variant
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +0000106 * @dualmaster: whether this version supports dual AHB masters or not.
Linus Walleijaffa1152012-04-12 09:01:49 +0200107 * @nomadik: whether the channels have Nomadik security extension bits
108 * that need to be checked for permission before use and some registers are
109 * missing
Tomasz Figada1b6c02013-08-11 19:59:17 +0200110 * @pl080s: whether this version is a PL080S, which has separate register and
111 * LLI word for transfer size.
Linus Walleije8689e62010-09-28 15:57:37 +0200112 */
113struct vendor_data {
Tomasz Figad86ccea2013-08-11 19:59:14 +0200114 u8 config_offset;
Linus Walleije8689e62010-09-28 15:57:37 +0200115 u8 channels;
116 bool dualmaster;
Linus Walleijaffa1152012-04-12 09:01:49 +0200117 bool nomadik;
Tomasz Figada1b6c02013-08-11 19:59:17 +0200118 bool pl080s;
Tomasz Figa5110e512013-08-11 19:59:18 +0200119 u32 max_transfer_size;
Linus Walleije8689e62010-09-28 15:57:37 +0200120};
121
122/**
Russell Kingb23f2042012-05-16 10:48:44 +0100123 * struct pl08x_bus_data - information of source or destination
124 * busses for a transfer
125 * @addr: current address
126 * @maxwidth: the maximum width of a transfer on this bus
127 * @buswidth: the width of this bus in bytes: 1, 2 or 4
128 */
129struct pl08x_bus_data {
130 dma_addr_t addr;
131 u8 maxwidth;
132 u8 buswidth;
133};
134
Andre Przywara1c38b282013-08-19 12:19:28 +0200135#define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth)
136
Russell Kingb23f2042012-05-16 10:48:44 +0100137/**
138 * struct pl08x_phy_chan - holder for the physical channels
139 * @id: physical index to this channel
140 * @lock: a lock to use when altering an instance of this struct
Russell Kingb23f2042012-05-16 10:48:44 +0100141 * @serving: the virtual channel currently being served by this physical
142 * channel
Russell Kingad0de2a2012-05-25 11:15:15 +0100143 * @locked: channel unavailable for the system, e.g. dedicated to secure
144 * world
Russell Kingb23f2042012-05-16 10:48:44 +0100145 */
146struct pl08x_phy_chan {
147 unsigned int id;
148 void __iomem *base;
Tomasz Figad86ccea2013-08-11 19:59:14 +0200149 void __iomem *reg_config;
Russell Kingb23f2042012-05-16 10:48:44 +0100150 spinlock_t lock;
Russell Kingb23f2042012-05-16 10:48:44 +0100151 struct pl08x_dma_chan *serving;
Russell Kingad0de2a2012-05-25 11:15:15 +0100152 bool locked;
Russell Kingb23f2042012-05-16 10:48:44 +0100153};
154
155/**
156 * struct pl08x_sg - structure containing data per sg
157 * @src_addr: src address of sg
158 * @dst_addr: dst address of sg
159 * @len: transfer len in bytes
160 * @node: node for txd's dsg_list
161 */
162struct pl08x_sg {
163 dma_addr_t src_addr;
164 dma_addr_t dst_addr;
165 size_t len;
166 struct list_head node;
167};
168
169/**
170 * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor
Russell King01d8dc62012-05-26 14:04:29 +0100171 * @vd: virtual DMA descriptor
Russell Kingb23f2042012-05-16 10:48:44 +0100172 * @dsg_list: list of children sg's
Russell Kingb23f2042012-05-16 10:48:44 +0100173 * @llis_bus: DMA memory address (physical) start for the LLIs
174 * @llis_va: virtual memory address start for the LLIs
175 * @cctl: control reg values for current txd
176 * @ccfg: config reg values for current txd
Russell King18536132012-05-26 14:42:23 +0100177 * @done: this marks completed descriptors, which should not have their
178 * mux released.
Alban Bedel3b24c202013-08-11 19:59:20 +0200179 * @cyclic: indicate cyclic transfers
Russell Kingb23f2042012-05-16 10:48:44 +0100180 */
181struct pl08x_txd {
Russell King01d8dc62012-05-26 14:04:29 +0100182 struct virt_dma_desc vd;
Russell Kingb23f2042012-05-16 10:48:44 +0100183 struct list_head dsg_list;
Russell Kingb23f2042012-05-16 10:48:44 +0100184 dma_addr_t llis_bus;
Tomasz Figaba6785f2013-08-11 19:59:15 +0200185 u32 *llis_va;
Russell Kingb23f2042012-05-16 10:48:44 +0100186 /* Default cctl value for LLIs */
187 u32 cctl;
188 /*
189 * Settings to be put into the physical channel when we
190 * trigger this txd. Other registers are in llis_va[0].
191 */
192 u32 ccfg;
Russell King18536132012-05-26 14:42:23 +0100193 bool done;
Alban Bedel3b24c202013-08-11 19:59:20 +0200194 bool cyclic;
Russell Kingb23f2042012-05-16 10:48:44 +0100195};
196
197/**
198 * struct pl08x_dma_chan_state - holds the PL08x specific virtual channel
199 * states
200 * @PL08X_CHAN_IDLE: the channel is idle
201 * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport
202 * channel and is running a transfer on it
203 * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport
204 * channel, but the transfer is currently paused
205 * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport
206 * channel to become available (only pertains to memcpy channels)
207 */
208enum pl08x_dma_chan_state {
209 PL08X_CHAN_IDLE,
210 PL08X_CHAN_RUNNING,
211 PL08X_CHAN_PAUSED,
212 PL08X_CHAN_WAITING,
213};
214
215/**
216 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel
Russell King01d8dc62012-05-26 14:04:29 +0100217 * @vc: wrappped virtual channel
Russell Kingb23f2042012-05-16 10:48:44 +0100218 * @phychan: the physical channel utilized by this channel, if there is one
Russell Kingb23f2042012-05-16 10:48:44 +0100219 * @name: name of channel
220 * @cd: channel platform data
221 * @runtime_addr: address for RX/TX according to the runtime config
Russell Kingb23f2042012-05-16 10:48:44 +0100222 * @at: active transaction on this channel
223 * @lock: a lock for this channel data
224 * @host: a pointer to the host (internal use)
225 * @state: whether the channel is idle, paused, running etc
226 * @slave: whether this channel is a device (slave) or for memcpy
Russell Kingad0de2a2012-05-25 11:15:15 +0100227 * @signal: the physical DMA request signal which this channel is using
Russell King5e2479b2012-05-25 11:32:45 +0100228 * @mux_use: count of descriptors using this DMA request signal setting
Russell Kingb23f2042012-05-16 10:48:44 +0100229 */
230struct pl08x_dma_chan {
Russell King01d8dc62012-05-26 14:04:29 +0100231 struct virt_dma_chan vc;
Russell Kingb23f2042012-05-16 10:48:44 +0100232 struct pl08x_phy_chan *phychan;
Russell King550ec362012-05-28 10:18:55 +0100233 const char *name;
Russell Kingb23f2042012-05-16 10:48:44 +0100234 const struct pl08x_channel_data *cd;
Russell Kinged91c132012-05-16 11:02:40 +0100235 struct dma_slave_config cfg;
Russell Kingb23f2042012-05-16 10:48:44 +0100236 struct pl08x_txd *at;
Russell Kingb23f2042012-05-16 10:48:44 +0100237 struct pl08x_driver_data *host;
238 enum pl08x_dma_chan_state state;
239 bool slave;
Russell Kingad0de2a2012-05-25 11:15:15 +0100240 int signal;
Russell King5e2479b2012-05-25 11:32:45 +0100241 unsigned mux_use;
Russell Kingb23f2042012-05-16 10:48:44 +0100242};
243
244/**
Linus Walleije8689e62010-09-28 15:57:37 +0200245 * struct pl08x_driver_data - the local state holder for the PL08x
246 * @slave: slave engine for this instance
247 * @memcpy: memcpy engine for this instance
248 * @base: virtual memory base (remapped) for the PL08x
249 * @adev: the corresponding AMBA (PrimeCell) bus entry
250 * @vd: vendor data for this PL08x variant
251 * @pd: platform data passed in from the platform/machine
252 * @phy_chans: array of data for the physical channels
253 * @pool: a pool for the LLI descriptors
Viresh Kumar3e27ee82011-08-05 15:32:27 +0530254 * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI
255 * fetches
Russell King - ARM Linux30749cb2011-01-03 22:41:13 +0000256 * @mem_buses: set to indicate memory transfers on AHB2.
Linus Walleije8689e62010-09-28 15:57:37 +0200257 * @lock: a spinlock for this struct
258 */
259struct pl08x_driver_data {
260 struct dma_device slave;
261 struct dma_device memcpy;
262 void __iomem *base;
263 struct amba_device *adev;
Russell King - ARM Linuxf96ca9ec2011-01-03 22:35:08 +0000264 const struct vendor_data *vd;
Linus Walleije8689e62010-09-28 15:57:37 +0200265 struct pl08x_platform_data *pd;
266 struct pl08x_phy_chan *phy_chans;
267 struct dma_pool *pool;
Russell King - ARM Linux30749cb2011-01-03 22:41:13 +0000268 u8 lli_buses;
269 u8 mem_buses;
Tomasz Figaba6785f2013-08-11 19:59:15 +0200270 u8 lli_words;
Linus Walleije8689e62010-09-28 15:57:37 +0200271};
272
273/*
274 * PL08X specific defines
275 */
276
Tomasz Figaba6785f2013-08-11 19:59:15 +0200277/* The order of words in an LLI. */
278#define PL080_LLI_SRC 0
279#define PL080_LLI_DST 1
280#define PL080_LLI_LLI 2
281#define PL080_LLI_CCTL 3
Tomasz Figada1b6c02013-08-11 19:59:17 +0200282#define PL080S_LLI_CCTL2 4
Linus Walleije8689e62010-09-28 15:57:37 +0200283
Tomasz Figaba6785f2013-08-11 19:59:15 +0200284/* Total words in an LLI. */
285#define PL080_LLI_WORDS 4
Tomasz Figada1b6c02013-08-11 19:59:17 +0200286#define PL080S_LLI_WORDS 8
Tomasz Figaba6785f2013-08-11 19:59:15 +0200287
288/*
289 * Number of LLIs in each LLI buffer allocated for one transfer
290 * (maximum times we call dma_pool_alloc on this pool without freeing)
291 */
292#define MAX_NUM_TSFR_LLIS 512
Linus Walleije8689e62010-09-28 15:57:37 +0200293#define PL08X_ALIGN 8
294
295static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)
296{
Russell King01d8dc62012-05-26 14:04:29 +0100297 return container_of(chan, struct pl08x_dma_chan, vc.chan);
Linus Walleije8689e62010-09-28 15:57:37 +0200298}
299
Russell King - ARM Linux501e67e2011-01-03 22:44:57 +0000300static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx)
301{
Russell King01d8dc62012-05-26 14:04:29 +0100302 return container_of(tx, struct pl08x_txd, vd.tx);
Russell King - ARM Linux501e67e2011-01-03 22:44:57 +0000303}
304
Linus Walleije8689e62010-09-28 15:57:37 +0200305/*
Russell King6b16c8b2012-05-25 11:10:58 +0100306 * Mux handling.
307 *
308 * This gives us the DMA request input to the PL08x primecell which the
309 * peripheral described by the channel data will be routed to, possibly
310 * via a board/SoC specific external MUX. One important point to note
311 * here is that this does not depend on the physical channel.
312 */
Russell Kingad0de2a2012-05-25 11:15:15 +0100313static int pl08x_request_mux(struct pl08x_dma_chan *plchan)
Russell King6b16c8b2012-05-25 11:10:58 +0100314{
315 const struct pl08x_platform_data *pd = plchan->host->pd;
316 int ret;
317
Mark Brownd7cabee2013-06-19 20:38:28 +0100318 if (plchan->mux_use++ == 0 && pd->get_xfer_signal) {
319 ret = pd->get_xfer_signal(plchan->cd);
Russell King5e2479b2012-05-25 11:32:45 +0100320 if (ret < 0) {
321 plchan->mux_use = 0;
Russell King6b16c8b2012-05-25 11:10:58 +0100322 return ret;
Russell King5e2479b2012-05-25 11:32:45 +0100323 }
Russell King6b16c8b2012-05-25 11:10:58 +0100324
Russell Kingad0de2a2012-05-25 11:15:15 +0100325 plchan->signal = ret;
Russell King6b16c8b2012-05-25 11:10:58 +0100326 }
327 return 0;
328}
329
330static void pl08x_release_mux(struct pl08x_dma_chan *plchan)
331{
332 const struct pl08x_platform_data *pd = plchan->host->pd;
333
Russell King5e2479b2012-05-25 11:32:45 +0100334 if (plchan->signal >= 0) {
335 WARN_ON(plchan->mux_use == 0);
336
Mark Brownd7cabee2013-06-19 20:38:28 +0100337 if (--plchan->mux_use == 0 && pd->put_xfer_signal) {
338 pd->put_xfer_signal(plchan->cd, plchan->signal);
Russell King5e2479b2012-05-25 11:32:45 +0100339 plchan->signal = -1;
340 }
Russell King6b16c8b2012-05-25 11:10:58 +0100341 }
342}
343
344/*
Linus Walleije8689e62010-09-28 15:57:37 +0200345 * Physical channel handling
346 */
347
348/* Whether a certain channel is busy or not */
349static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch)
350{
351 unsigned int val;
352
Tomasz Figad86ccea2013-08-11 19:59:14 +0200353 val = readl(ch->reg_config);
Linus Walleije8689e62010-09-28 15:57:37 +0200354 return val & PL080_CONFIG_ACTIVE;
355}
356
Tomasz Figaba6785f2013-08-11 19:59:15 +0200357static void pl08x_write_lli(struct pl08x_driver_data *pl08x,
358 struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg)
359{
Tomasz Figada1b6c02013-08-11 19:59:17 +0200360 if (pl08x->vd->pl080s)
361 dev_vdbg(&pl08x->adev->dev,
362 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
363 "clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n",
364 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
365 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL],
366 lli[PL080S_LLI_CCTL2], ccfg);
367 else
368 dev_vdbg(&pl08x->adev->dev,
369 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
370 "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n",
371 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
372 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg);
Tomasz Figaba6785f2013-08-11 19:59:15 +0200373
374 writel_relaxed(lli[PL080_LLI_SRC], phychan->base + PL080_CH_SRC_ADDR);
375 writel_relaxed(lli[PL080_LLI_DST], phychan->base + PL080_CH_DST_ADDR);
376 writel_relaxed(lli[PL080_LLI_LLI], phychan->base + PL080_CH_LLI);
377 writel_relaxed(lli[PL080_LLI_CCTL], phychan->base + PL080_CH_CONTROL);
378
Tomasz Figada1b6c02013-08-11 19:59:17 +0200379 if (pl08x->vd->pl080s)
380 writel_relaxed(lli[PL080S_LLI_CCTL2],
381 phychan->base + PL080S_CH_CONTROL2);
382
Tomasz Figaba6785f2013-08-11 19:59:15 +0200383 writel(ccfg, phychan->reg_config);
384}
385
Linus Walleije8689e62010-09-28 15:57:37 +0200386/*
387 * Set the initial DMA register values i.e. those for the first LLI
Russell King - ARM Linuxe8b5e112011-01-03 22:30:24 +0000388 * The next LLI pointer and the configuration interrupt bit have
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000389 * been set when the LLIs were constructed. Poke them into the hardware
390 * and start the transfer.
Linus Walleije8689e62010-09-28 15:57:37 +0200391 */
Russell Kingeab82532012-05-25 12:32:00 +0100392static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan)
Linus Walleije8689e62010-09-28 15:57:37 +0200393{
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000394 struct pl08x_driver_data *pl08x = plchan->host;
Linus Walleije8689e62010-09-28 15:57:37 +0200395 struct pl08x_phy_chan *phychan = plchan->phychan;
Russell King879f1272012-05-26 14:27:40 +0100396 struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc);
397 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
Russell King - ARM Linux09b3c322011-01-03 22:39:53 +0000398 u32 val;
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000399
Russell King879f1272012-05-26 14:27:40 +0100400 list_del(&txd->vd.node);
Russell Kingeab82532012-05-25 12:32:00 +0100401
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000402 plchan->at = txd;
Linus Walleije8689e62010-09-28 15:57:37 +0200403
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000404 /* Wait for channel inactive */
405 while (pl08x_phy_channel_busy(phychan))
Russell King - ARM Linux19386b322011-01-03 22:36:29 +0000406 cpu_relax();
Linus Walleije8689e62010-09-28 15:57:37 +0200407
Tomasz Figaba6785f2013-08-11 19:59:15 +0200408 pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg);
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000409
410 /* Enable the DMA channel */
411 /* Do not access config register until channel shows as disabled */
412 while (readl(pl08x->base + PL080_EN_CHAN) & (1 << phychan->id))
413 cpu_relax();
414
415 /* Do not access config register until channel shows as inactive */
Tomasz Figad86ccea2013-08-11 19:59:14 +0200416 val = readl(phychan->reg_config);
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000417 while ((val & PL080_CONFIG_ACTIVE) || (val & PL080_CONFIG_ENABLE))
Tomasz Figad86ccea2013-08-11 19:59:14 +0200418 val = readl(phychan->reg_config);
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000419
Tomasz Figad86ccea2013-08-11 19:59:14 +0200420 writel(val | PL080_CONFIG_ENABLE, phychan->reg_config);
Linus Walleije8689e62010-09-28 15:57:37 +0200421}
422
423/*
Russell King - ARM Linux81796612011-01-27 12:37:44 +0000424 * Pause the channel by setting the HALT bit.
Linus Walleije8689e62010-09-28 15:57:37 +0200425 *
Russell King - ARM Linux81796612011-01-27 12:37:44 +0000426 * For M->P transfers, pause the DMAC first and then stop the peripheral -
427 * the FIFO can only drain if the peripheral is still requesting data.
428 * (note: this can still timeout if the DMAC FIFO never drains of data.)
Linus Walleije8689e62010-09-28 15:57:37 +0200429 *
Russell King - ARM Linux81796612011-01-27 12:37:44 +0000430 * For P->M transfers, disable the peripheral first to stop it filling
431 * the DMAC FIFO, and then pause the DMAC.
Linus Walleije8689e62010-09-28 15:57:37 +0200432 */
433static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch)
434{
435 u32 val;
Russell King - ARM Linux81796612011-01-27 12:37:44 +0000436 int timeout;
Linus Walleije8689e62010-09-28 15:57:37 +0200437
438 /* Set the HALT bit and wait for the FIFO to drain */
Tomasz Figad86ccea2013-08-11 19:59:14 +0200439 val = readl(ch->reg_config);
Linus Walleije8689e62010-09-28 15:57:37 +0200440 val |= PL080_CONFIG_HALT;
Tomasz Figad86ccea2013-08-11 19:59:14 +0200441 writel(val, ch->reg_config);
Linus Walleije8689e62010-09-28 15:57:37 +0200442
443 /* Wait for channel inactive */
Russell King - ARM Linux81796612011-01-27 12:37:44 +0000444 for (timeout = 1000; timeout; timeout--) {
445 if (!pl08x_phy_channel_busy(ch))
446 break;
447 udelay(1);
448 }
449 if (pl08x_phy_channel_busy(ch))
450 pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id);
Linus Walleije8689e62010-09-28 15:57:37 +0200451}
452
453static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch)
454{
455 u32 val;
456
457 /* Clear the HALT bit */
Tomasz Figad86ccea2013-08-11 19:59:14 +0200458 val = readl(ch->reg_config);
Linus Walleije8689e62010-09-28 15:57:37 +0200459 val &= ~PL080_CONFIG_HALT;
Tomasz Figad86ccea2013-08-11 19:59:14 +0200460 writel(val, ch->reg_config);
Linus Walleije8689e62010-09-28 15:57:37 +0200461}
462
Russell King - ARM Linuxfb526212011-01-27 12:32:53 +0000463/*
464 * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and
465 * clears any pending interrupt status. This should not be used for
466 * an on-going transfer, but as a method of shutting down a channel
467 * (eg, when it's no longer used) or terminating a transfer.
468 */
469static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x,
470 struct pl08x_phy_chan *ch)
Linus Walleije8689e62010-09-28 15:57:37 +0200471{
Tomasz Figad86ccea2013-08-11 19:59:14 +0200472 u32 val = readl(ch->reg_config);
Linus Walleije8689e62010-09-28 15:57:37 +0200473
Russell King - ARM Linuxfb526212011-01-27 12:32:53 +0000474 val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK |
475 PL080_CONFIG_TC_IRQ_MASK);
Linus Walleije8689e62010-09-28 15:57:37 +0200476
Tomasz Figad86ccea2013-08-11 19:59:14 +0200477 writel(val, ch->reg_config);
Russell King - ARM Linuxfb526212011-01-27 12:32:53 +0000478
479 writel(1 << ch->id, pl08x->base + PL080_ERR_CLEAR);
480 writel(1 << ch->id, pl08x->base + PL080_TC_CLEAR);
Linus Walleije8689e62010-09-28 15:57:37 +0200481}
482
483static inline u32 get_bytes_in_cctl(u32 cctl)
484{
485 /* The source width defines the number of bytes */
486 u32 bytes = cctl & PL080_CONTROL_TRANSFER_SIZE_MASK;
487
Alban Bedelf3287a52013-08-11 19:59:19 +0200488 cctl &= PL080_CONTROL_SWIDTH_MASK;
489
Linus Walleije8689e62010-09-28 15:57:37 +0200490 switch (cctl >> PL080_CONTROL_SWIDTH_SHIFT) {
491 case PL080_WIDTH_8BIT:
492 break;
493 case PL080_WIDTH_16BIT:
494 bytes *= 2;
495 break;
496 case PL080_WIDTH_32BIT:
497 bytes *= 4;
498 break;
499 }
500 return bytes;
501}
502
Tomasz Figada1b6c02013-08-11 19:59:17 +0200503static inline u32 get_bytes_in_cctl_pl080s(u32 cctl, u32 cctl1)
504{
505 /* The source width defines the number of bytes */
506 u32 bytes = cctl1 & PL080S_CONTROL_TRANSFER_SIZE_MASK;
507
Alban Bedelf3287a52013-08-11 19:59:19 +0200508 cctl &= PL080_CONTROL_SWIDTH_MASK;
509
Linus Walleije8689e62010-09-28 15:57:37 +0200510 switch (cctl >> PL080_CONTROL_SWIDTH_SHIFT) {
511 case PL080_WIDTH_8BIT:
512 break;
513 case PL080_WIDTH_16BIT:
514 bytes *= 2;
515 break;
516 case PL080_WIDTH_32BIT:
517 bytes *= 4;
518 break;
519 }
520 return bytes;
521}
522
523/* The channel should be paused when calling this */
524static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan)
525{
Tomasz Figaba6785f2013-08-11 19:59:15 +0200526 struct pl08x_driver_data *pl08x = plchan->host;
527 const u32 *llis_va, *llis_va_limit;
Linus Walleije8689e62010-09-28 15:57:37 +0200528 struct pl08x_phy_chan *ch;
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200529 dma_addr_t llis_bus;
Linus Walleije8689e62010-09-28 15:57:37 +0200530 struct pl08x_txd *txd;
Tomasz Figaba6785f2013-08-11 19:59:15 +0200531 u32 llis_max_words;
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200532 size_t bytes;
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200533 u32 clli;
Linus Walleije8689e62010-09-28 15:57:37 +0200534
Linus Walleije8689e62010-09-28 15:57:37 +0200535 ch = plchan->phychan;
536 txd = plchan->at;
537
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200538 if (!ch || !txd)
539 return 0;
540
Linus Walleije8689e62010-09-28 15:57:37 +0200541 /*
Russell King - ARM Linuxdb9f1362011-01-03 22:38:32 +0000542 * Follow the LLIs to get the number of remaining
543 * bytes in the currently active transaction.
Linus Walleije8689e62010-09-28 15:57:37 +0200544 */
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200545 clli = readl(ch->base + PL080_CH_LLI) & ~PL080_LLI_LM_AHB2;
Linus Walleije8689e62010-09-28 15:57:37 +0200546
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200547 /* First get the remaining bytes in the active transfer */
Tomasz Figada1b6c02013-08-11 19:59:17 +0200548 if (pl08x->vd->pl080s)
549 bytes = get_bytes_in_cctl_pl080s(
550 readl(ch->base + PL080_CH_CONTROL),
551 readl(ch->base + PL080S_CH_CONTROL2));
552 else
Linus Walleije8689e62010-09-28 15:57:37 +0200553 bytes = get_bytes_in_cctl(readl(ch->base + PL080_CH_CONTROL));
554
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200555 if (!clli)
556 return bytes;
Linus Walleije8689e62010-09-28 15:57:37 +0200557
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200558 llis_va = txd->llis_va;
559 llis_bus = txd->llis_bus;
Linus Walleije8689e62010-09-28 15:57:37 +0200560
Tomasz Figaba6785f2013-08-11 19:59:15 +0200561 llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS;
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200562 BUG_ON(clli < llis_bus || clli >= llis_bus +
Tomasz Figaba6785f2013-08-11 19:59:15 +0200563 sizeof(u32) * llis_max_words);
Russell King - ARM Linuxdb9f1362011-01-03 22:38:32 +0000564
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200565 /*
566 * Locate the next LLI - as this is an array,
567 * it's simple maths to find.
568 */
Tomasz Figaba6785f2013-08-11 19:59:15 +0200569 llis_va += (clli - llis_bus) / sizeof(u32);
Russell King - ARM Linuxdb9f1362011-01-03 22:38:32 +0000570
Tomasz Figaba6785f2013-08-11 19:59:15 +0200571 llis_va_limit = llis_va + llis_max_words;
572
573 for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) {
Tomasz Figada1b6c02013-08-11 19:59:17 +0200574 if (pl08x->vd->pl080s)
575 bytes += get_bytes_in_cctl_pl080s(
576 llis_va[PL080_LLI_CCTL],
577 llis_va[PL080S_LLI_CCTL2]);
578 else
579 bytes += get_bytes_in_cctl(llis_va[PL080_LLI_CCTL]);
Linus Walleije8689e62010-09-28 15:57:37 +0200580
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200581 /*
Alban Bedel3b24c202013-08-11 19:59:20 +0200582 * A LLI pointer going backward terminates the LLI list
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200583 */
Alban Bedel3b24c202013-08-11 19:59:20 +0200584 if (llis_va[PL080_LLI_LLI] <= clli)
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200585 break;
Linus Walleije8689e62010-09-28 15:57:37 +0200586 }
587
Linus Walleije8689e62010-09-28 15:57:37 +0200588 return bytes;
589}
590
591/*
592 * Allocate a physical channel for a virtual channel
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +0000593 *
594 * Try to locate a physical channel to be used for this transfer. If all
595 * are taken return NULL and the requester will have to cope by using
596 * some fallback PIO mode or retrying later.
Linus Walleije8689e62010-09-28 15:57:37 +0200597 */
598static struct pl08x_phy_chan *
599pl08x_get_phy_channel(struct pl08x_driver_data *pl08x,
600 struct pl08x_dma_chan *virt_chan)
601{
602 struct pl08x_phy_chan *ch = NULL;
603 unsigned long flags;
604 int i;
605
Linus Walleije8689e62010-09-28 15:57:37 +0200606 for (i = 0; i < pl08x->vd->channels; i++) {
607 ch = &pl08x->phy_chans[i];
608
609 spin_lock_irqsave(&ch->lock, flags);
610
Linus Walleijaffa1152012-04-12 09:01:49 +0200611 if (!ch->locked && !ch->serving) {
Linus Walleije8689e62010-09-28 15:57:37 +0200612 ch->serving = virt_chan;
Linus Walleije8689e62010-09-28 15:57:37 +0200613 spin_unlock_irqrestore(&ch->lock, flags);
614 break;
615 }
616
617 spin_unlock_irqrestore(&ch->lock, flags);
618 }
619
620 if (i == pl08x->vd->channels) {
621 /* No physical channel available, cope with it */
622 return NULL;
623 }
624
625 return ch;
626}
627
Russell Kinga5a488d2012-05-26 13:54:15 +0100628/* Mark the physical channel as free. Note, this write is atomic. */
Linus Walleije8689e62010-09-28 15:57:37 +0200629static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x,
630 struct pl08x_phy_chan *ch)
631{
Linus Walleije8689e62010-09-28 15:57:37 +0200632 ch->serving = NULL;
Russell Kinga5a488d2012-05-26 13:54:15 +0100633}
634
635/*
636 * Try to allocate a physical channel. When successful, assign it to
637 * this virtual channel, and initiate the next descriptor. The
638 * virtual channel lock must be held at this point.
639 */
640static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan)
641{
642 struct pl08x_driver_data *pl08x = plchan->host;
643 struct pl08x_phy_chan *ch;
644
645 ch = pl08x_get_phy_channel(pl08x, plchan);
646 if (!ch) {
647 dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name);
648 plchan->state = PL08X_CHAN_WAITING;
649 return;
650 }
651
652 dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n",
653 ch->id, plchan->name);
654
655 plchan->phychan = ch;
656 plchan->state = PL08X_CHAN_RUNNING;
657 pl08x_start_next_txd(plchan);
658}
659
660static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch,
661 struct pl08x_dma_chan *plchan)
662{
663 struct pl08x_driver_data *pl08x = plchan->host;
664
665 dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n",
666 ch->id, plchan->name);
667
668 /*
669 * We do this without taking the lock; we're really only concerned
670 * about whether this pointer is NULL or not, and we're guaranteed
671 * that this will only be called when it _already_ is non-NULL.
672 */
673 ch->serving = plchan;
674 plchan->phychan = ch;
675 plchan->state = PL08X_CHAN_RUNNING;
676 pl08x_start_next_txd(plchan);
677}
678
679/*
680 * Free a physical DMA channel, potentially reallocating it to another
681 * virtual channel if we have any pending.
682 */
683static void pl08x_phy_free(struct pl08x_dma_chan *plchan)
684{
685 struct pl08x_driver_data *pl08x = plchan->host;
686 struct pl08x_dma_chan *p, *next;
687
688 retry:
689 next = NULL;
690
691 /* Find a waiting virtual channel for the next transfer. */
Russell King01d8dc62012-05-26 14:04:29 +0100692 list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node)
Russell Kinga5a488d2012-05-26 13:54:15 +0100693 if (p->state == PL08X_CHAN_WAITING) {
694 next = p;
695 break;
696 }
697
698 if (!next) {
Russell King01d8dc62012-05-26 14:04:29 +0100699 list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node)
Russell Kinga5a488d2012-05-26 13:54:15 +0100700 if (p->state == PL08X_CHAN_WAITING) {
701 next = p;
702 break;
703 }
704 }
705
706 /* Ensure that the physical channel is stopped */
707 pl08x_terminate_phy_chan(pl08x, plchan->phychan);
708
709 if (next) {
710 bool success;
711
712 /*
713 * Eww. We know this isn't going to deadlock
714 * but lockdep probably doesn't.
715 */
Russell King083be282012-05-26 14:09:53 +0100716 spin_lock(&next->vc.lock);
Russell Kinga5a488d2012-05-26 13:54:15 +0100717 /* Re-check the state now that we have the lock */
718 success = next->state == PL08X_CHAN_WAITING;
719 if (success)
720 pl08x_phy_reassign_start(plchan->phychan, next);
Russell King083be282012-05-26 14:09:53 +0100721 spin_unlock(&next->vc.lock);
Russell Kinga5a488d2012-05-26 13:54:15 +0100722
723 /* If the state changed, try to find another channel */
724 if (!success)
725 goto retry;
726 } else {
727 /* No more jobs, so free up the physical channel */
728 pl08x_put_phy_channel(pl08x, plchan->phychan);
729 }
730
731 plchan->phychan = NULL;
732 plchan->state = PL08X_CHAN_IDLE;
Linus Walleije8689e62010-09-28 15:57:37 +0200733}
734
735/*
736 * LLI handling
737 */
738
739static inline unsigned int pl08x_get_bytes_for_cctl(unsigned int coded)
740{
741 switch (coded) {
742 case PL080_WIDTH_8BIT:
743 return 1;
744 case PL080_WIDTH_16BIT:
745 return 2;
746 case PL080_WIDTH_32BIT:
747 return 4;
748 default:
749 break;
750 }
751 BUG();
752 return 0;
753}
754
755static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth,
Russell King - ARM Linuxcace6582011-01-03 22:37:31 +0000756 size_t tsize)
Linus Walleije8689e62010-09-28 15:57:37 +0200757{
758 u32 retbits = cctl;
759
Russell King - ARM Linuxe8b5e112011-01-03 22:30:24 +0000760 /* Remove all src, dst and transfer size bits */
Linus Walleije8689e62010-09-28 15:57:37 +0200761 retbits &= ~PL080_CONTROL_DWIDTH_MASK;
762 retbits &= ~PL080_CONTROL_SWIDTH_MASK;
763 retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK;
764
765 /* Then set the bits according to the parameters */
766 switch (srcwidth) {
767 case 1:
768 retbits |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT;
769 break;
770 case 2:
771 retbits |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT;
772 break;
773 case 4:
774 retbits |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT;
775 break;
776 default:
777 BUG();
778 break;
779 }
780
781 switch (dstwidth) {
782 case 1:
783 retbits |= PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT;
784 break;
785 case 2:
786 retbits |= PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT;
787 break;
788 case 4:
789 retbits |= PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT;
790 break;
791 default:
792 BUG();
793 break;
794 }
795
Tomasz Figa5110e512013-08-11 19:59:18 +0200796 tsize &= PL080_CONTROL_TRANSFER_SIZE_MASK;
Linus Walleije8689e62010-09-28 15:57:37 +0200797 retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT;
798 return retbits;
799}
800
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000801struct pl08x_lli_build_data {
802 struct pl08x_txd *txd;
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000803 struct pl08x_bus_data srcbus;
804 struct pl08x_bus_data dstbus;
805 size_t remainder;
Russell King - ARM Linux25c94f72011-07-21 17:11:46 +0100806 u32 lli_bus;
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000807};
808
Linus Walleije8689e62010-09-28 15:57:37 +0200809/*
Viresh Kumar0532e6f2011-08-05 15:32:31 +0530810 * Autoselect a master bus to use for the transfer. Slave will be the chosen as
811 * victim in case src & dest are not similarly aligned. i.e. If after aligning
812 * masters address with width requirements of transfer (by sending few byte by
813 * byte data), slave is still not aligned, then its width will be reduced to
814 * BYTE.
815 * - prefers the destination bus if both available
Viresh Kumar036f05f2011-08-05 15:32:41 +0530816 * - prefers bus with fixed address (i.e. peripheral)
Linus Walleije8689e62010-09-28 15:57:37 +0200817 */
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000818static void pl08x_choose_master_bus(struct pl08x_lli_build_data *bd,
819 struct pl08x_bus_data **mbus, struct pl08x_bus_data **sbus, u32 cctl)
Linus Walleije8689e62010-09-28 15:57:37 +0200820{
821 if (!(cctl & PL080_CONTROL_DST_INCR)) {
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000822 *mbus = &bd->dstbus;
823 *sbus = &bd->srcbus;
Viresh Kumar036f05f2011-08-05 15:32:41 +0530824 } else if (!(cctl & PL080_CONTROL_SRC_INCR)) {
825 *mbus = &bd->srcbus;
826 *sbus = &bd->dstbus;
Linus Walleije8689e62010-09-28 15:57:37 +0200827 } else {
Viresh Kumar036f05f2011-08-05 15:32:41 +0530828 if (bd->dstbus.buswidth >= bd->srcbus.buswidth) {
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000829 *mbus = &bd->dstbus;
830 *sbus = &bd->srcbus;
Linus Walleije8689e62010-09-28 15:57:37 +0200831 } else {
Viresh Kumar036f05f2011-08-05 15:32:41 +0530832 *mbus = &bd->srcbus;
833 *sbus = &bd->dstbus;
Linus Walleije8689e62010-09-28 15:57:37 +0200834 }
835 }
836}
837
838/*
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +0000839 * Fills in one LLI for a certain transfer descriptor and advance the counter
Linus Walleije8689e62010-09-28 15:57:37 +0200840 */
Tomasz Figaba6785f2013-08-11 19:59:15 +0200841static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x,
842 struct pl08x_lli_build_data *bd,
Tomasz Figada1b6c02013-08-11 19:59:17 +0200843 int num_llis, int len, u32 cctl, u32 cctl2)
Linus Walleije8689e62010-09-28 15:57:37 +0200844{
Tomasz Figaba6785f2013-08-11 19:59:15 +0200845 u32 offset = num_llis * pl08x->lli_words;
846 u32 *llis_va = bd->txd->llis_va + offset;
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000847 dma_addr_t llis_bus = bd->txd->llis_bus;
Linus Walleije8689e62010-09-28 15:57:37 +0200848
849 BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS);
850
Tomasz Figaba6785f2013-08-11 19:59:15 +0200851 /* Advance the offset to next LLI. */
852 offset += pl08x->lli_words;
853
854 llis_va[PL080_LLI_SRC] = bd->srcbus.addr;
855 llis_va[PL080_LLI_DST] = bd->dstbus.addr;
856 llis_va[PL080_LLI_LLI] = (llis_bus + sizeof(u32) * offset);
857 llis_va[PL080_LLI_LLI] |= bd->lli_bus;
858 llis_va[PL080_LLI_CCTL] = cctl;
Tomasz Figada1b6c02013-08-11 19:59:17 +0200859 if (pl08x->vd->pl080s)
860 llis_va[PL080S_LLI_CCTL2] = cctl2;
Linus Walleije8689e62010-09-28 15:57:37 +0200861
862 if (cctl & PL080_CONTROL_SRC_INCR)
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000863 bd->srcbus.addr += len;
Linus Walleije8689e62010-09-28 15:57:37 +0200864 if (cctl & PL080_CONTROL_DST_INCR)
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000865 bd->dstbus.addr += len;
Linus Walleije8689e62010-09-28 15:57:37 +0200866
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000867 BUG_ON(bd->remainder < len);
Russell King - ARM Linuxcace6582011-01-03 22:37:31 +0000868
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000869 bd->remainder -= len;
Linus Walleije8689e62010-09-28 15:57:37 +0200870}
871
Tomasz Figaba6785f2013-08-11 19:59:15 +0200872static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x,
873 struct pl08x_lli_build_data *bd, u32 *cctl, u32 len,
874 int num_llis, size_t *total_bytes)
Linus Walleije8689e62010-09-28 15:57:37 +0200875{
Viresh Kumar03af5002011-08-05 15:32:39 +0530876 *cctl = pl08x_cctl_bits(*cctl, 1, 1, len);
Tomasz Figada1b6c02013-08-11 19:59:17 +0200877 pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len);
Viresh Kumar03af5002011-08-05 15:32:39 +0530878 (*total_bytes) += len;
Linus Walleije8689e62010-09-28 15:57:37 +0200879}
880
Tomasz Figa48924e42013-08-11 19:59:16 +0200881#ifdef VERBOSE_DEBUG
882static void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
883 const u32 *llis_va, int num_llis)
884{
885 int i;
886
Tomasz Figada1b6c02013-08-11 19:59:17 +0200887 if (pl08x->vd->pl080s) {
Tomasz Figa48924e42013-08-11 19:59:16 +0200888 dev_vdbg(&pl08x->adev->dev,
Tomasz Figada1b6c02013-08-11 19:59:17 +0200889 "%-3s %-9s %-10s %-10s %-10s %-10s %s\n",
890 "lli", "", "csrc", "cdst", "clli", "cctl", "cctl2");
891 for (i = 0; i < num_llis; i++) {
892 dev_vdbg(&pl08x->adev->dev,
893 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
894 i, llis_va, llis_va[PL080_LLI_SRC],
895 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
896 llis_va[PL080_LLI_CCTL],
897 llis_va[PL080S_LLI_CCTL2]);
898 llis_va += pl08x->lli_words;
899 }
900 } else {
901 dev_vdbg(&pl08x->adev->dev,
902 "%-3s %-9s %-10s %-10s %-10s %s\n",
903 "lli", "", "csrc", "cdst", "clli", "cctl");
904 for (i = 0; i < num_llis; i++) {
905 dev_vdbg(&pl08x->adev->dev,
906 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n",
907 i, llis_va, llis_va[PL080_LLI_SRC],
908 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
909 llis_va[PL080_LLI_CCTL]);
910 llis_va += pl08x->lli_words;
911 }
Tomasz Figa48924e42013-08-11 19:59:16 +0200912 }
913}
914#else
915static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
916 const u32 *llis_va, int num_llis) {}
917#endif
918
Linus Walleije8689e62010-09-28 15:57:37 +0200919/*
920 * This fills in the table of LLIs for the transfer descriptor
921 * Note that we assume we never have to change the burst sizes
922 * Return 0 for error
923 */
924static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
925 struct pl08x_txd *txd)
926{
Linus Walleije8689e62010-09-28 15:57:37 +0200927 struct pl08x_bus_data *mbus, *sbus;
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000928 struct pl08x_lli_build_data bd;
Linus Walleije8689e62010-09-28 15:57:37 +0200929 int num_llis = 0;
Viresh Kumar03af5002011-08-05 15:32:39 +0530930 u32 cctl, early_bytes = 0;
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530931 size_t max_bytes_per_lli, total_bytes;
Tomasz Figaba6785f2013-08-11 19:59:15 +0200932 u32 *llis_va, *last_lli;
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530933 struct pl08x_sg *dsg;
Linus Walleije8689e62010-09-28 15:57:37 +0200934
Viresh Kumar3e27ee82011-08-05 15:32:27 +0530935 txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus);
Linus Walleije8689e62010-09-28 15:57:37 +0200936 if (!txd->llis_va) {
937 dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__);
938 return 0;
939 }
940
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000941 bd.txd = txd;
Russell King - ARM Linux25c94f72011-07-21 17:11:46 +0100942 bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0;
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530943 cctl = txd->cctl;
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000944
Linus Walleije8689e62010-09-28 15:57:37 +0200945 /* Find maximum width of the source bus */
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000946 bd.srcbus.maxwidth =
Linus Walleije8689e62010-09-28 15:57:37 +0200947 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_SWIDTH_MASK) >>
948 PL080_CONTROL_SWIDTH_SHIFT);
949
950 /* Find maximum width of the destination bus */
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000951 bd.dstbus.maxwidth =
Linus Walleije8689e62010-09-28 15:57:37 +0200952 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_DWIDTH_MASK) >>
953 PL080_CONTROL_DWIDTH_SHIFT);
954
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530955 list_for_each_entry(dsg, &txd->dsg_list, node) {
956 total_bytes = 0;
957 cctl = txd->cctl;
Linus Walleije8689e62010-09-28 15:57:37 +0200958
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530959 bd.srcbus.addr = dsg->src_addr;
960 bd.dstbus.addr = dsg->dst_addr;
961 bd.remainder = dsg->len;
962 bd.srcbus.buswidth = bd.srcbus.maxwidth;
963 bd.dstbus.buswidth = bd.dstbus.maxwidth;
Linus Walleije8689e62010-09-28 15:57:37 +0200964
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530965 pl08x_choose_master_bus(&bd, &mbus, &sbus, cctl);
Linus Walleije8689e62010-09-28 15:57:37 +0200966
Andre Przywarab90ca062013-08-14 14:52:09 +0200967 dev_vdbg(&pl08x->adev->dev,
968 "src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n",
969 (u64)bd.srcbus.addr,
970 cctl & PL080_CONTROL_SRC_INCR ? "+" : "",
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530971 bd.srcbus.buswidth,
Andre Przywarab90ca062013-08-14 14:52:09 +0200972 (u64)bd.dstbus.addr,
973 cctl & PL080_CONTROL_DST_INCR ? "+" : "",
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530974 bd.dstbus.buswidth,
975 bd.remainder);
976 dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n",
977 mbus == &bd.srcbus ? "src" : "dst",
978 sbus == &bd.srcbus ? "src" : "dst");
Russell King - ARM Linuxfc74eb72011-07-21 17:12:06 +0100979
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530980 /*
981 * Zero length is only allowed if all these requirements are
982 * met:
983 * - flow controller is peripheral.
984 * - src.addr is aligned to src.width
985 * - dst.addr is aligned to dst.width
986 *
987 * sg_len == 1 should be true, as there can be two cases here:
988 *
989 * - Memory addresses are contiguous and are not scattered.
990 * Here, Only one sg will be passed by user driver, with
991 * memory address and zero length. We pass this to controller
992 * and after the transfer it will receive the last burst
993 * request from peripheral and so transfer finishes.
994 *
995 * - Memory addresses are scattered and are not contiguous.
996 * Here, Obviously as DMA controller doesn't know when a lli's
997 * transfer gets over, it can't load next lli. So in this
998 * case, there has to be an assumption that only one lli is
999 * supported. Thus, we can't have scattered addresses.
1000 */
1001 if (!bd.remainder) {
1002 u32 fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >>
1003 PL080_CONFIG_FLOW_CONTROL_SHIFT;
1004 if (!((fc >= PL080_FLOW_SRC2DST_DST) &&
Viresh Kumar0a235652011-08-05 15:32:42 +05301005 (fc <= PL080_FLOW_SRC2DST_SRC))) {
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301006 dev_err(&pl08x->adev->dev, "%s sg len can't be zero",
1007 __func__);
1008 return 0;
1009 }
Linus Walleije8689e62010-09-28 15:57:37 +02001010
Andre Przywara1c38b282013-08-19 12:19:28 +02001011 if (!IS_BUS_ALIGNED(&bd.srcbus) ||
1012 !IS_BUS_ALIGNED(&bd.dstbus)) {
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301013 dev_err(&pl08x->adev->dev,
1014 "%s src & dst address must be aligned to src"
1015 " & dst width if peripheral is flow controller",
1016 __func__);
1017 return 0;
1018 }
Linus Walleije8689e62010-09-28 15:57:37 +02001019
Viresh Kumar16a2e7d2011-08-05 15:32:37 +05301020 cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth,
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301021 bd.dstbus.buswidth, 0);
Tomasz Figaba6785f2013-08-11 19:59:15 +02001022 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,
Tomasz Figada1b6c02013-08-11 19:59:17 +02001023 0, cctl, 0);
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301024 break;
Linus Walleije8689e62010-09-28 15:57:37 +02001025 }
1026
1027 /*
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301028 * Send byte by byte for following cases
1029 * - Less than a bus width available
1030 * - until master bus is aligned
Linus Walleije8689e62010-09-28 15:57:37 +02001031 */
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301032 if (bd.remainder < mbus->buswidth)
1033 early_bytes = bd.remainder;
Andre Przywara1c38b282013-08-19 12:19:28 +02001034 else if (!IS_BUS_ALIGNED(mbus)) {
1035 early_bytes = mbus->buswidth -
1036 (mbus->addr & (mbus->buswidth - 1));
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301037 if ((bd.remainder - early_bytes) < mbus->buswidth)
1038 early_bytes = bd.remainder;
Linus Walleije8689e62010-09-28 15:57:37 +02001039 }
Viresh Kumar16a2e7d2011-08-05 15:32:37 +05301040
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301041 if (early_bytes) {
1042 dev_vdbg(&pl08x->adev->dev,
1043 "%s byte width LLIs (remain 0x%08x)\n",
1044 __func__, bd.remainder);
Tomasz Figaba6785f2013-08-11 19:59:15 +02001045 prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes,
1046 num_llis++, &total_bytes);
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301047 }
Linus Walleije8689e62010-09-28 15:57:37 +02001048
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301049 if (bd.remainder) {
1050 /*
1051 * Master now aligned
1052 * - if slave is not then we must set its width down
1053 */
Andre Przywara1c38b282013-08-19 12:19:28 +02001054 if (!IS_BUS_ALIGNED(sbus)) {
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301055 dev_dbg(&pl08x->adev->dev,
1056 "%s set down bus width to one byte\n",
1057 __func__);
1058
1059 sbus->buswidth = 1;
1060 }
1061
1062 /*
1063 * Bytes transferred = tsize * src width, not
1064 * MIN(buswidths)
1065 */
1066 max_bytes_per_lli = bd.srcbus.buswidth *
Tomasz Figa5110e512013-08-11 19:59:18 +02001067 pl08x->vd->max_transfer_size;
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301068 dev_vdbg(&pl08x->adev->dev,
1069 "%s max bytes per lli = %zu\n",
1070 __func__, max_bytes_per_lli);
1071
1072 /*
1073 * Make largest possible LLIs until less than one bus
1074 * width left
1075 */
1076 while (bd.remainder > (mbus->buswidth - 1)) {
1077 size_t lli_len, tsize, width;
1078
1079 /*
1080 * If enough left try to send max possible,
1081 * otherwise try to send the remainder
1082 */
1083 lli_len = min(bd.remainder, max_bytes_per_lli);
1084
1085 /*
1086 * Check against maximum bus alignment:
1087 * Calculate actual transfer size in relation to
1088 * bus width an get a maximum remainder of the
1089 * highest bus width - 1
1090 */
1091 width = max(mbus->buswidth, sbus->buswidth);
1092 lli_len = (lli_len / width) * width;
1093 tsize = lli_len / bd.srcbus.buswidth;
1094
1095 dev_vdbg(&pl08x->adev->dev,
1096 "%s fill lli with single lli chunk of "
1097 "size 0x%08zx (remainder 0x%08zx)\n",
1098 __func__, lli_len, bd.remainder);
1099
1100 cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth,
1101 bd.dstbus.buswidth, tsize);
Tomasz Figaba6785f2013-08-11 19:59:15 +02001102 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,
Tomasz Figada1b6c02013-08-11 19:59:17 +02001103 lli_len, cctl, tsize);
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301104 total_bytes += lli_len;
1105 }
1106
1107 /*
1108 * Send any odd bytes
1109 */
1110 if (bd.remainder) {
1111 dev_vdbg(&pl08x->adev->dev,
1112 "%s align with boundary, send odd bytes (remain %zu)\n",
1113 __func__, bd.remainder);
Tomasz Figaba6785f2013-08-11 19:59:15 +02001114 prep_byte_width_lli(pl08x, &bd, &cctl,
1115 bd.remainder, num_llis++, &total_bytes);
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301116 }
1117 }
1118
1119 if (total_bytes != dsg->len) {
1120 dev_err(&pl08x->adev->dev,
1121 "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n",
1122 __func__, total_bytes, dsg->len);
1123 return 0;
1124 }
1125
1126 if (num_llis >= MAX_NUM_TSFR_LLIS) {
1127 dev_err(&pl08x->adev->dev,
1128 "%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n",
Tomasz Figaba6785f2013-08-11 19:59:15 +02001129 __func__, MAX_NUM_TSFR_LLIS);
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301130 return 0;
1131 }
Linus Walleije8689e62010-09-28 15:57:37 +02001132 }
Linus Walleije8689e62010-09-28 15:57:37 +02001133
Russell King - ARM Linuxb58b6b52011-01-03 22:34:48 +00001134 llis_va = txd->llis_va;
Tomasz Figaba6785f2013-08-11 19:59:15 +02001135 last_lli = llis_va + (num_llis - 1) * pl08x->lli_words;
Linus Walleije8689e62010-09-28 15:57:37 +02001136
Alban Bedel3b24c202013-08-11 19:59:20 +02001137 if (txd->cyclic) {
1138 /* Link back to the first LLI. */
1139 last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus;
1140 } else {
1141 /* The final LLI terminates the LLI. */
1142 last_lli[PL080_LLI_LLI] = 0;
1143 /* The final LLI element shall also fire an interrupt. */
1144 last_lli[PL080_LLI_CCTL] |= PL080_CONTROL_TC_IRQ_EN;
Linus Walleije8689e62010-09-28 15:57:37 +02001145 }
Linus Walleije8689e62010-09-28 15:57:37 +02001146
Tomasz Figa48924e42013-08-11 19:59:16 +02001147 pl08x_dump_lli(pl08x, llis_va, num_llis);
Linus Walleije8689e62010-09-28 15:57:37 +02001148
1149 return num_llis;
1150}
1151
Linus Walleije8689e62010-09-28 15:57:37 +02001152static void pl08x_free_txd(struct pl08x_driver_data *pl08x,
1153 struct pl08x_txd *txd)
1154{
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301155 struct pl08x_sg *dsg, *_dsg;
1156
Viresh Kumarc1205642011-08-05 15:32:44 +05301157 if (txd->llis_va)
1158 dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus);
Linus Walleije8689e62010-09-28 15:57:37 +02001159
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301160 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
1161 list_del(&dsg->node);
1162 kfree(dsg);
1163 }
1164
Linus Walleije8689e62010-09-28 15:57:37 +02001165 kfree(txd);
1166}
1167
Russell King18536132012-05-26 14:42:23 +01001168static void pl08x_desc_free(struct virt_dma_desc *vd)
1169{
1170 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
1171 struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan);
Russell King18536132012-05-26 14:42:23 +01001172
Russell King - ARM Linux89116bf2013-11-26 16:03:03 -08001173 dma_descriptor_unmap(&vd->tx);
Russell King18536132012-05-26 14:42:23 +01001174 if (!txd->done)
1175 pl08x_release_mux(plchan);
1176
Russell King18536132012-05-26 14:42:23 +01001177 pl08x_free_txd(plchan->host, txd);
Russell King18536132012-05-26 14:42:23 +01001178}
1179
Linus Walleije8689e62010-09-28 15:57:37 +02001180static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x,
1181 struct pl08x_dma_chan *plchan)
1182{
Russell Kingea160562012-05-25 13:10:36 +01001183 LIST_HEAD(head);
Linus Walleije8689e62010-09-28 15:57:37 +02001184
Russell King879f1272012-05-26 14:27:40 +01001185 vchan_get_all_descriptors(&plchan->vc, &head);
Akinobu Mita91998262012-10-28 00:49:31 +09001186 vchan_dma_desc_free_list(&plchan->vc, &head);
Linus Walleije8689e62010-09-28 15:57:37 +02001187}
1188
1189/*
1190 * The DMA ENGINE API
1191 */
1192static int pl08x_alloc_chan_resources(struct dma_chan *chan)
1193{
1194 return 0;
1195}
1196
1197static void pl08x_free_chan_resources(struct dma_chan *chan)
1198{
Russell Kinga0686822012-05-26 17:00:49 +01001199 /* Ensure all queued descriptors are freed */
1200 vchan_free_chan_resources(to_virt_chan(chan));
Linus Walleije8689e62010-09-28 15:57:37 +02001201}
1202
Linus Walleije8689e62010-09-28 15:57:37 +02001203static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt(
1204 struct dma_chan *chan, unsigned long flags)
1205{
1206 struct dma_async_tx_descriptor *retval = NULL;
1207
1208 return retval;
1209}
1210
1211/*
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +00001212 * Code accessing dma_async_is_complete() in a tight loop may give problems.
1213 * If slaves are relying on interrupts to signal completion this function
1214 * must not be called with interrupts disabled.
Linus Walleije8689e62010-09-28 15:57:37 +02001215 */
Viresh Kumar3e27ee82011-08-05 15:32:27 +05301216static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan,
1217 dma_cookie_t cookie, struct dma_tx_state *txstate)
Linus Walleije8689e62010-09-28 15:57:37 +02001218{
1219 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
Russell King06e885b2012-05-26 15:05:52 +01001220 struct virt_dma_desc *vd;
1221 unsigned long flags;
Linus Walleije8689e62010-09-28 15:57:37 +02001222 enum dma_status ret;
Russell King06e885b2012-05-26 15:05:52 +01001223 size_t bytes = 0;
Linus Walleije8689e62010-09-28 15:57:37 +02001224
Russell King - ARM Linux96a2af42012-03-06 22:35:27 +00001225 ret = dma_cookie_status(chan, cookie, txstate);
Vinod Koul0996e892013-10-16 13:33:02 +05301226 if (ret == DMA_COMPLETE)
Linus Walleije8689e62010-09-28 15:57:37 +02001227 return ret;
Linus Walleije8689e62010-09-28 15:57:37 +02001228
1229 /*
Russell King06e885b2012-05-26 15:05:52 +01001230 * There's no point calculating the residue if there's
1231 * no txstate to store the value.
1232 */
1233 if (!txstate) {
1234 if (plchan->state == PL08X_CHAN_PAUSED)
1235 ret = DMA_PAUSED;
1236 return ret;
1237 }
1238
1239 spin_lock_irqsave(&plchan->vc.lock, flags);
1240 ret = dma_cookie_status(chan, cookie, txstate);
Vinod Koul0996e892013-10-16 13:33:02 +05301241 if (ret != DMA_COMPLETE) {
Russell King06e885b2012-05-26 15:05:52 +01001242 vd = vchan_find_desc(&plchan->vc, cookie);
1243 if (vd) {
1244 /* On the issued list, so hasn't been processed yet */
1245 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
1246 struct pl08x_sg *dsg;
1247
1248 list_for_each_entry(dsg, &txd->dsg_list, node)
1249 bytes += dsg->len;
1250 } else {
1251 bytes = pl08x_getbytes_chan(plchan);
1252 }
1253 }
1254 spin_unlock_irqrestore(&plchan->vc.lock, flags);
1255
1256 /*
Linus Walleije8689e62010-09-28 15:57:37 +02001257 * This cookie not complete yet
Russell King - ARM Linux96a2af42012-03-06 22:35:27 +00001258 * Get number of bytes left in the active transactions and queue
Linus Walleije8689e62010-09-28 15:57:37 +02001259 */
Russell King06e885b2012-05-26 15:05:52 +01001260 dma_set_residue(txstate, bytes);
Linus Walleije8689e62010-09-28 15:57:37 +02001261
Russell King06e885b2012-05-26 15:05:52 +01001262 if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS)
1263 ret = DMA_PAUSED;
Linus Walleije8689e62010-09-28 15:57:37 +02001264
1265 /* Whether waiting or running, we're in progress */
Russell King06e885b2012-05-26 15:05:52 +01001266 return ret;
Linus Walleije8689e62010-09-28 15:57:37 +02001267}
1268
1269/* PrimeCell DMA extension */
1270struct burst_table {
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001271 u32 burstwords;
Linus Walleije8689e62010-09-28 15:57:37 +02001272 u32 reg;
1273};
1274
1275static const struct burst_table burst_sizes[] = {
1276 {
1277 .burstwords = 256,
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001278 .reg = PL080_BSIZE_256,
Linus Walleije8689e62010-09-28 15:57:37 +02001279 },
1280 {
1281 .burstwords = 128,
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001282 .reg = PL080_BSIZE_128,
Linus Walleije8689e62010-09-28 15:57:37 +02001283 },
1284 {
1285 .burstwords = 64,
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001286 .reg = PL080_BSIZE_64,
Linus Walleije8689e62010-09-28 15:57:37 +02001287 },
1288 {
1289 .burstwords = 32,
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001290 .reg = PL080_BSIZE_32,
Linus Walleije8689e62010-09-28 15:57:37 +02001291 },
1292 {
1293 .burstwords = 16,
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001294 .reg = PL080_BSIZE_16,
Linus Walleije8689e62010-09-28 15:57:37 +02001295 },
1296 {
1297 .burstwords = 8,
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001298 .reg = PL080_BSIZE_8,
Linus Walleije8689e62010-09-28 15:57:37 +02001299 },
1300 {
1301 .burstwords = 4,
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001302 .reg = PL080_BSIZE_4,
Linus Walleije8689e62010-09-28 15:57:37 +02001303 },
1304 {
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001305 .burstwords = 0,
1306 .reg = PL080_BSIZE_1,
Linus Walleije8689e62010-09-28 15:57:37 +02001307 },
1308};
1309
Russell King - ARM Linux121c8472011-07-21 17:13:48 +01001310/*
1311 * Given the source and destination available bus masks, select which
1312 * will be routed to each port. We try to have source and destination
1313 * on separate ports, but always respect the allowable settings.
1314 */
1315static u32 pl08x_select_bus(u8 src, u8 dst)
1316{
1317 u32 cctl = 0;
1318
1319 if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1)))
1320 cctl |= PL080_CONTROL_DST_AHB2;
1321 if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2)))
1322 cctl |= PL080_CONTROL_SRC_AHB2;
1323
1324 return cctl;
1325}
1326
Russell King - ARM Linuxf14c4262011-07-21 17:12:47 +01001327static u32 pl08x_cctl(u32 cctl)
1328{
1329 cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 |
1330 PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR |
1331 PL080_CONTROL_PROT_MASK);
1332
1333 /* Access the cell in privileged mode, non-bufferable, non-cacheable */
1334 return cctl | PL080_CONTROL_PROT_SYS;
1335}
1336
Russell King - ARM Linuxaa88cda2011-07-21 17:13:28 +01001337static u32 pl08x_width(enum dma_slave_buswidth width)
1338{
1339 switch (width) {
1340 case DMA_SLAVE_BUSWIDTH_1_BYTE:
1341 return PL080_WIDTH_8BIT;
1342 case DMA_SLAVE_BUSWIDTH_2_BYTES:
1343 return PL080_WIDTH_16BIT;
1344 case DMA_SLAVE_BUSWIDTH_4_BYTES:
1345 return PL080_WIDTH_32BIT;
Vinod Koulf32807f2011-07-25 19:22:01 +05301346 default:
1347 return ~0;
Russell King - ARM Linuxaa88cda2011-07-21 17:13:28 +01001348 }
Russell King - ARM Linuxaa88cda2011-07-21 17:13:28 +01001349}
1350
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001351static u32 pl08x_burst(u32 maxburst)
1352{
1353 int i;
1354
1355 for (i = 0; i < ARRAY_SIZE(burst_sizes); i++)
1356 if (burst_sizes[i].burstwords <= maxburst)
1357 break;
1358
1359 return burst_sizes[i].reg;
1360}
1361
Russell King9862ba12012-05-16 11:16:03 +01001362static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan,
1363 enum dma_slave_buswidth addr_width, u32 maxburst)
1364{
1365 u32 width, burst, cctl = 0;
1366
1367 width = pl08x_width(addr_width);
1368 if (width == ~0)
1369 return ~0;
1370
1371 cctl |= width << PL080_CONTROL_SWIDTH_SHIFT;
1372 cctl |= width << PL080_CONTROL_DWIDTH_SHIFT;
1373
1374 /*
1375 * If this channel will only request single transfers, set this
1376 * down to ONE element. Also select one element if no maxburst
1377 * is specified.
1378 */
1379 if (plchan->cd->single)
1380 maxburst = 1;
1381
1382 burst = pl08x_burst(maxburst);
1383 cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT;
1384 cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT;
1385
1386 return pl08x_cctl(cctl);
1387}
1388
Russell King - ARM Linuxf0fd9442011-01-03 22:45:57 +00001389static int dma_set_runtime_config(struct dma_chan *chan,
1390 struct dma_slave_config *config)
Linus Walleije8689e62010-09-28 15:57:37 +02001391{
1392 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
Tomasz Figada1b6c02013-08-11 19:59:17 +02001393 struct pl08x_driver_data *pl08x = plchan->host;
Linus Walleije8689e62010-09-28 15:57:37 +02001394
Russell King - ARM Linuxb7f75862011-01-03 22:46:17 +00001395 if (!plchan->slave)
1396 return -EINVAL;
1397
Russell Kingdc8d5f82012-05-16 12:20:55 +01001398 /* Reject definitely invalid configurations */
1399 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1400 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
Russell King - ARM Linuxf0fd9442011-01-03 22:45:57 +00001401 return -EINVAL;
Linus Walleije8689e62010-09-28 15:57:37 +02001402
Tomasz Figada1b6c02013-08-11 19:59:17 +02001403 if (config->device_fc && pl08x->vd->pl080s) {
1404 dev_err(&pl08x->adev->dev,
1405 "%s: PL080S does not support peripheral flow control\n",
1406 __func__);
1407 return -EINVAL;
1408 }
1409
Russell Kinged91c132012-05-16 11:02:40 +01001410 plchan->cfg = *config;
1411
Russell King - ARM Linuxf0fd9442011-01-03 22:45:57 +00001412 return 0;
Linus Walleije8689e62010-09-28 15:57:37 +02001413}
1414
1415/*
1416 * Slave transactions callback to the slave device to allow
1417 * synchronization of slave DMA signals with the DMAC enable
1418 */
1419static void pl08x_issue_pending(struct dma_chan *chan)
1420{
1421 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
Linus Walleije8689e62010-09-28 15:57:37 +02001422 unsigned long flags;
1423
Russell King083be282012-05-26 14:09:53 +01001424 spin_lock_irqsave(&plchan->vc.lock, flags);
Russell King879f1272012-05-26 14:27:40 +01001425 if (vchan_issue_pending(&plchan->vc)) {
Russell Kinga5a488d2012-05-26 13:54:15 +01001426 if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING)
1427 pl08x_phy_alloc_and_start(plchan);
Linus Walleije8689e62010-09-28 15:57:37 +02001428 }
Russell King083be282012-05-26 14:09:53 +01001429 spin_unlock_irqrestore(&plchan->vc.lock, flags);
Linus Walleije8689e62010-09-28 15:57:37 +02001430}
1431
Russell King879f1272012-05-26 14:27:40 +01001432static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan)
Russell King - ARM Linuxac3cd202011-01-03 22:35:49 +00001433{
Viresh Kumarb201c112011-08-05 15:32:29 +05301434 struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
Russell King - ARM Linuxac3cd202011-01-03 22:35:49 +00001435
1436 if (txd) {
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301437 INIT_LIST_HEAD(&txd->dsg_list);
Russell King - ARM Linux4983a042011-01-03 22:39:33 +00001438
1439 /* Always enable error and terminal interrupts */
1440 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
1441 PL080_CONFIG_TC_IRQ_MASK;
Russell King - ARM Linuxac3cd202011-01-03 22:35:49 +00001442 }
1443 return txd;
1444}
1445
Linus Walleije8689e62010-09-28 15:57:37 +02001446/*
1447 * Initialize a descriptor to be used by memcpy submit
1448 */
1449static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy(
1450 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1451 size_t len, unsigned long flags)
1452{
1453 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1454 struct pl08x_driver_data *pl08x = plchan->host;
1455 struct pl08x_txd *txd;
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301456 struct pl08x_sg *dsg;
Linus Walleije8689e62010-09-28 15:57:37 +02001457 int ret;
1458
Russell King879f1272012-05-26 14:27:40 +01001459 txd = pl08x_get_txd(plchan);
Linus Walleije8689e62010-09-28 15:57:37 +02001460 if (!txd) {
1461 dev_err(&pl08x->adev->dev,
1462 "%s no memory for descriptor\n", __func__);
1463 return NULL;
1464 }
1465
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301466 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
1467 if (!dsg) {
1468 pl08x_free_txd(pl08x, txd);
1469 dev_err(&pl08x->adev->dev, "%s no memory for pl080 sg\n",
1470 __func__);
1471 return NULL;
1472 }
1473 list_add_tail(&dsg->node, &txd->dsg_list);
1474
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301475 dsg->src_addr = src;
1476 dsg->dst_addr = dest;
1477 dsg->len = len;
Linus Walleije8689e62010-09-28 15:57:37 +02001478
1479 /* Set platform data for m2m */
Russell King - ARM Linux4983a042011-01-03 22:39:33 +00001480 txd->ccfg |= PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
Russell Kingdc8d5f82012-05-16 12:20:55 +01001481 txd->cctl = pl08x->pd->memcpy_channel.cctl_memcpy &
Russell King - ARM Linuxc7da9a52011-01-03 22:40:53 +00001482 ~(PL080_CONTROL_DST_AHB2 | PL080_CONTROL_SRC_AHB2);
Russell King - ARM Linux4983a042011-01-03 22:39:33 +00001483
Linus Walleije8689e62010-09-28 15:57:37 +02001484 /* Both to be incremented or the code will break */
Russell King - ARM Linux70b5ed62011-01-03 22:40:13 +00001485 txd->cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR;
Russell King - ARM Linuxc7da9a52011-01-03 22:40:53 +00001486
Russell King - ARM Linuxc7da9a52011-01-03 22:40:53 +00001487 if (pl08x->vd->dualmaster)
Russell King - ARM Linux121c8472011-07-21 17:13:48 +01001488 txd->cctl |= pl08x_select_bus(pl08x->mem_buses,
1489 pl08x->mem_buses);
Linus Walleije8689e62010-09-28 15:57:37 +02001490
Russell Kingaa4afb72012-05-26 15:43:00 +01001491 ret = pl08x_fill_llis_for_desc(plchan->host, txd);
1492 if (!ret) {
1493 pl08x_free_txd(pl08x, txd);
Linus Walleije8689e62010-09-28 15:57:37 +02001494 return NULL;
Russell Kingaa4afb72012-05-26 15:43:00 +01001495 }
Linus Walleije8689e62010-09-28 15:57:37 +02001496
Russell King879f1272012-05-26 14:27:40 +01001497 return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
Linus Walleije8689e62010-09-28 15:57:37 +02001498}
1499
Alban Bedel3b24c202013-08-11 19:59:20 +02001500static struct pl08x_txd *pl08x_init_txd(
1501 struct dma_chan *chan,
1502 enum dma_transfer_direction direction,
1503 dma_addr_t *slave_addr)
Linus Walleije8689e62010-09-28 15:57:37 +02001504{
1505 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1506 struct pl08x_driver_data *pl08x = plchan->host;
1507 struct pl08x_txd *txd;
Russell Kingdc8d5f82012-05-16 12:20:55 +01001508 enum dma_slave_buswidth addr_width;
Viresh Kumar0a235652011-08-05 15:32:42 +05301509 int ret, tmp;
Russell King409ec8d2012-05-16 11:08:43 +01001510 u8 src_buses, dst_buses;
Russell Kingdc8d5f82012-05-16 12:20:55 +01001511 u32 maxburst, cctl;
Linus Walleije8689e62010-09-28 15:57:37 +02001512
Russell King879f1272012-05-26 14:27:40 +01001513 txd = pl08x_get_txd(plchan);
Linus Walleije8689e62010-09-28 15:57:37 +02001514 if (!txd) {
1515 dev_err(&pl08x->adev->dev, "%s no txd\n", __func__);
1516 return NULL;
1517 }
1518
Linus Walleije8689e62010-09-28 15:57:37 +02001519 /*
1520 * Set up addresses, the PrimeCell configured address
1521 * will take precedence since this may configure the
1522 * channel target address dynamically at runtime.
1523 */
Vinod Kouldb8196d2011-10-13 22:34:23 +05301524 if (direction == DMA_MEM_TO_DEV) {
Russell Kingdc8d5f82012-05-16 12:20:55 +01001525 cctl = PL080_CONTROL_SRC_INCR;
Alban Bedel3b24c202013-08-11 19:59:20 +02001526 *slave_addr = plchan->cfg.dst_addr;
Russell Kingdc8d5f82012-05-16 12:20:55 +01001527 addr_width = plchan->cfg.dst_addr_width;
1528 maxburst = plchan->cfg.dst_maxburst;
Russell King409ec8d2012-05-16 11:08:43 +01001529 src_buses = pl08x->mem_buses;
1530 dst_buses = plchan->cd->periph_buses;
Vinod Kouldb8196d2011-10-13 22:34:23 +05301531 } else if (direction == DMA_DEV_TO_MEM) {
Russell Kingdc8d5f82012-05-16 12:20:55 +01001532 cctl = PL080_CONTROL_DST_INCR;
Alban Bedel3b24c202013-08-11 19:59:20 +02001533 *slave_addr = plchan->cfg.src_addr;
Russell Kingdc8d5f82012-05-16 12:20:55 +01001534 addr_width = plchan->cfg.src_addr_width;
1535 maxburst = plchan->cfg.src_maxburst;
Russell King409ec8d2012-05-16 11:08:43 +01001536 src_buses = plchan->cd->periph_buses;
1537 dst_buses = pl08x->mem_buses;
Linus Walleije8689e62010-09-28 15:57:37 +02001538 } else {
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301539 pl08x_free_txd(pl08x, txd);
Linus Walleije8689e62010-09-28 15:57:37 +02001540 dev_err(&pl08x->adev->dev,
1541 "%s direction unsupported\n", __func__);
1542 return NULL;
1543 }
Linus Walleije8689e62010-09-28 15:57:37 +02001544
Russell Kingdc8d5f82012-05-16 12:20:55 +01001545 cctl |= pl08x_get_cctl(plchan, addr_width, maxburst);
Russell King800d6832012-05-16 11:33:31 +01001546 if (cctl == ~0) {
1547 pl08x_free_txd(pl08x, txd);
1548 dev_err(&pl08x->adev->dev,
1549 "DMA slave configuration botched?\n");
1550 return NULL;
1551 }
1552
Russell King409ec8d2012-05-16 11:08:43 +01001553 txd->cctl = cctl | pl08x_select_bus(src_buses, dst_buses);
1554
Russell King95442b22012-05-16 11:05:09 +01001555 if (plchan->cfg.device_fc)
Vinod Kouldb8196d2011-10-13 22:34:23 +05301556 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER :
Viresh Kumar0a235652011-08-05 15:32:42 +05301557 PL080_FLOW_PER2MEM_PER;
1558 else
Vinod Kouldb8196d2011-10-13 22:34:23 +05301559 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER :
Viresh Kumar0a235652011-08-05 15:32:42 +05301560 PL080_FLOW_PER2MEM;
1561
1562 txd->ccfg |= tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT;
1563
Russell Kingc48d4962012-05-25 11:48:51 +01001564 ret = pl08x_request_mux(plchan);
1565 if (ret < 0) {
1566 pl08x_free_txd(pl08x, txd);
1567 dev_dbg(&pl08x->adev->dev,
1568 "unable to mux for transfer on %s due to platform restrictions\n",
1569 plchan->name);
1570 return NULL;
1571 }
1572
1573 dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n",
1574 plchan->signal, plchan->name);
1575
1576 /* Assign the flow control signal to this channel */
1577 if (direction == DMA_MEM_TO_DEV)
1578 txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT;
1579 else
1580 txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT;
1581
Alban Bedel3b24c202013-08-11 19:59:20 +02001582 return txd;
1583}
1584
1585static int pl08x_tx_add_sg(struct pl08x_txd *txd,
1586 enum dma_transfer_direction direction,
1587 dma_addr_t slave_addr,
1588 dma_addr_t buf_addr,
1589 unsigned int len)
1590{
1591 struct pl08x_sg *dsg;
1592
1593 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
1594 if (!dsg)
1595 return -ENOMEM;
1596
1597 list_add_tail(&dsg->node, &txd->dsg_list);
1598
1599 dsg->len = len;
1600 if (direction == DMA_MEM_TO_DEV) {
1601 dsg->src_addr = buf_addr;
1602 dsg->dst_addr = slave_addr;
1603 } else {
1604 dsg->src_addr = slave_addr;
1605 dsg->dst_addr = buf_addr;
1606 }
1607
1608 return 0;
1609}
1610
1611static struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
1612 struct dma_chan *chan, struct scatterlist *sgl,
1613 unsigned int sg_len, enum dma_transfer_direction direction,
1614 unsigned long flags, void *context)
1615{
1616 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1617 struct pl08x_driver_data *pl08x = plchan->host;
1618 struct pl08x_txd *txd;
1619 struct scatterlist *sg;
1620 int ret, tmp;
1621 dma_addr_t slave_addr;
1622
1623 dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n",
1624 __func__, sg_dma_len(sgl), plchan->name);
1625
1626 txd = pl08x_init_txd(chan, direction, &slave_addr);
1627 if (!txd)
1628 return NULL;
1629
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301630 for_each_sg(sgl, sg, sg_len, tmp) {
Alban Bedel3b24c202013-08-11 19:59:20 +02001631 ret = pl08x_tx_add_sg(txd, direction, slave_addr,
1632 sg_dma_address(sg),
1633 sg_dma_len(sg));
1634 if (ret) {
Russell Kingc48d4962012-05-25 11:48:51 +01001635 pl08x_release_mux(plchan);
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301636 pl08x_free_txd(pl08x, txd);
1637 dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n",
1638 __func__);
1639 return NULL;
1640 }
Alban Bedel3b24c202013-08-11 19:59:20 +02001641 }
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301642
Alban Bedel3b24c202013-08-11 19:59:20 +02001643 ret = pl08x_fill_llis_for_desc(plchan->host, txd);
1644 if (!ret) {
1645 pl08x_release_mux(plchan);
1646 pl08x_free_txd(pl08x, txd);
1647 return NULL;
1648 }
1649
1650 return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
1651}
1652
1653static struct dma_async_tx_descriptor *pl08x_prep_dma_cyclic(
1654 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1655 size_t period_len, enum dma_transfer_direction direction,
1656 unsigned long flags, void *context)
1657{
1658 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1659 struct pl08x_driver_data *pl08x = plchan->host;
1660 struct pl08x_txd *txd;
1661 int ret, tmp;
1662 dma_addr_t slave_addr;
1663
1664 dev_dbg(&pl08x->adev->dev,
1665 "%s prepare cyclic transaction of %d/%d bytes %s %s\n",
1666 __func__, period_len, buf_len,
1667 direction == DMA_MEM_TO_DEV ? "to" : "from",
1668 plchan->name);
1669
1670 txd = pl08x_init_txd(chan, direction, &slave_addr);
1671 if (!txd)
1672 return NULL;
1673
1674 txd->cyclic = true;
1675 txd->cctl |= PL080_CONTROL_TC_IRQ_EN;
1676 for (tmp = 0; tmp < buf_len; tmp += period_len) {
1677 ret = pl08x_tx_add_sg(txd, direction, slave_addr,
1678 buf_addr + tmp, period_len);
1679 if (ret) {
1680 pl08x_release_mux(plchan);
1681 pl08x_free_txd(pl08x, txd);
1682 return NULL;
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301683 }
1684 }
1685
Russell Kingaa4afb72012-05-26 15:43:00 +01001686 ret = pl08x_fill_llis_for_desc(plchan->host, txd);
1687 if (!ret) {
1688 pl08x_release_mux(plchan);
1689 pl08x_free_txd(pl08x, txd);
Linus Walleije8689e62010-09-28 15:57:37 +02001690 return NULL;
Russell Kingaa4afb72012-05-26 15:43:00 +01001691 }
Linus Walleije8689e62010-09-28 15:57:37 +02001692
Russell King879f1272012-05-26 14:27:40 +01001693 return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
Linus Walleije8689e62010-09-28 15:57:37 +02001694}
1695
1696static int pl08x_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1697 unsigned long arg)
1698{
1699 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1700 struct pl08x_driver_data *pl08x = plchan->host;
1701 unsigned long flags;
1702 int ret = 0;
1703
1704 /* Controls applicable to inactive channels */
1705 if (cmd == DMA_SLAVE_CONFIG) {
Russell King - ARM Linuxf0fd9442011-01-03 22:45:57 +00001706 return dma_set_runtime_config(chan,
1707 (struct dma_slave_config *)arg);
Linus Walleije8689e62010-09-28 15:57:37 +02001708 }
1709
1710 /*
1711 * Anything succeeds on channels with no physical allocation and
1712 * no queued transfers.
1713 */
Russell King083be282012-05-26 14:09:53 +01001714 spin_lock_irqsave(&plchan->vc.lock, flags);
Linus Walleije8689e62010-09-28 15:57:37 +02001715 if (!plchan->phychan && !plchan->at) {
Russell King083be282012-05-26 14:09:53 +01001716 spin_unlock_irqrestore(&plchan->vc.lock, flags);
Linus Walleije8689e62010-09-28 15:57:37 +02001717 return 0;
1718 }
1719
1720 switch (cmd) {
1721 case DMA_TERMINATE_ALL:
1722 plchan->state = PL08X_CHAN_IDLE;
1723
1724 if (plchan->phychan) {
Linus Walleije8689e62010-09-28 15:57:37 +02001725 /*
1726 * Mark physical channel as free and free any slave
1727 * signal
1728 */
Russell Kinga5a488d2012-05-26 13:54:15 +01001729 pl08x_phy_free(plchan);
Linus Walleije8689e62010-09-28 15:57:37 +02001730 }
Linus Walleije8689e62010-09-28 15:57:37 +02001731 /* Dequeue jobs and free LLIs */
1732 if (plchan->at) {
Russell King18536132012-05-26 14:42:23 +01001733 pl08x_desc_free(&plchan->at->vd);
Linus Walleije8689e62010-09-28 15:57:37 +02001734 plchan->at = NULL;
1735 }
1736 /* Dequeue jobs not yet fired as well */
1737 pl08x_free_txd_list(pl08x, plchan);
1738 break;
1739 case DMA_PAUSE:
1740 pl08x_pause_phy_chan(plchan->phychan);
1741 plchan->state = PL08X_CHAN_PAUSED;
1742 break;
1743 case DMA_RESUME:
1744 pl08x_resume_phy_chan(plchan->phychan);
1745 plchan->state = PL08X_CHAN_RUNNING;
1746 break;
1747 default:
1748 /* Unknown command */
1749 ret = -ENXIO;
1750 break;
1751 }
1752
Russell King083be282012-05-26 14:09:53 +01001753 spin_unlock_irqrestore(&plchan->vc.lock, flags);
Linus Walleije8689e62010-09-28 15:57:37 +02001754
1755 return ret;
1756}
1757
1758bool pl08x_filter_id(struct dma_chan *chan, void *chan_id)
1759{
Russell King - ARM Linux7703eac2011-08-31 09:34:35 +01001760 struct pl08x_dma_chan *plchan;
Linus Walleije8689e62010-09-28 15:57:37 +02001761 char *name = chan_id;
1762
Russell King - ARM Linux7703eac2011-08-31 09:34:35 +01001763 /* Reject channels for devices not bound to this driver */
1764 if (chan->device->dev->driver != &pl08x_amba_driver.drv)
1765 return false;
1766
1767 plchan = to_pl08x_chan(chan);
1768
Linus Walleije8689e62010-09-28 15:57:37 +02001769 /* Check that the channel is not taken! */
1770 if (!strcmp(plchan->name, name))
1771 return true;
1772
1773 return false;
1774}
Sachin Kamat6d05c9f2014-01-23 16:10:07 +05301775EXPORT_SYMBOL_GPL(pl08x_filter_id);
Linus Walleije8689e62010-09-28 15:57:37 +02001776
1777/*
1778 * Just check that the device is there and active
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +00001779 * TODO: turn this bit on/off depending on the number of physical channels
1780 * actually used, if it is zero... well shut it off. That will save some
1781 * power. Cut the clock at the same time.
Linus Walleije8689e62010-09-28 15:57:37 +02001782 */
1783static void pl08x_ensure_on(struct pl08x_driver_data *pl08x)
1784{
Linus Walleijaffa1152012-04-12 09:01:49 +02001785 /* The Nomadik variant does not have the config register */
1786 if (pl08x->vd->nomadik)
1787 return;
Viresh Kumar48a59ef2011-08-05 15:32:34 +05301788 writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG);
Linus Walleije8689e62010-09-28 15:57:37 +02001789}
1790
Linus Walleije8689e62010-09-28 15:57:37 +02001791static irqreturn_t pl08x_irq(int irq, void *dev)
1792{
1793 struct pl08x_driver_data *pl08x = dev;
Viresh Kumar28da2832011-08-05 15:32:36 +05301794 u32 mask = 0, err, tc, i;
Linus Walleije8689e62010-09-28 15:57:37 +02001795
Viresh Kumar28da2832011-08-05 15:32:36 +05301796 /* check & clear - ERR & TC interrupts */
1797 err = readl(pl08x->base + PL080_ERR_STATUS);
1798 if (err) {
1799 dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n",
1800 __func__, err);
1801 writel(err, pl08x->base + PL080_ERR_CLEAR);
Linus Walleije8689e62010-09-28 15:57:37 +02001802 }
Linus Walleijd29bf012012-04-09 22:53:21 +02001803 tc = readl(pl08x->base + PL080_TC_STATUS);
Viresh Kumar28da2832011-08-05 15:32:36 +05301804 if (tc)
1805 writel(tc, pl08x->base + PL080_TC_CLEAR);
1806
1807 if (!err && !tc)
1808 return IRQ_NONE;
1809
Linus Walleije8689e62010-09-28 15:57:37 +02001810 for (i = 0; i < pl08x->vd->channels; i++) {
Viresh Kumar28da2832011-08-05 15:32:36 +05301811 if (((1 << i) & err) || ((1 << i) & tc)) {
Linus Walleije8689e62010-09-28 15:57:37 +02001812 /* Locate physical channel */
1813 struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i];
1814 struct pl08x_dma_chan *plchan = phychan->serving;
Russell Kinga936e792012-05-25 10:51:19 +01001815 struct pl08x_txd *tx;
Linus Walleije8689e62010-09-28 15:57:37 +02001816
Viresh Kumar28da2832011-08-05 15:32:36 +05301817 if (!plchan) {
1818 dev_err(&pl08x->adev->dev,
1819 "%s Error TC interrupt on unused channel: 0x%08x\n",
1820 __func__, i);
1821 continue;
1822 }
1823
Russell King083be282012-05-26 14:09:53 +01001824 spin_lock(&plchan->vc.lock);
Russell Kinga936e792012-05-25 10:51:19 +01001825 tx = plchan->at;
Alban Bedel3b24c202013-08-11 19:59:20 +02001826 if (tx && tx->cyclic) {
1827 vchan_cyclic_callback(&tx->vd);
1828 } else if (tx) {
Russell Kinga936e792012-05-25 10:51:19 +01001829 plchan->at = NULL;
Russell Kingc48d4962012-05-25 11:48:51 +01001830 /*
1831 * This descriptor is done, release its mux
1832 * reservation.
1833 */
1834 pl08x_release_mux(plchan);
Russell King18536132012-05-26 14:42:23 +01001835 tx->done = true;
1836 vchan_cookie_complete(&tx->vd);
Russell Kingc33b6442012-05-25 15:41:13 +01001837
Russell Kinga5a488d2012-05-26 13:54:15 +01001838 /*
1839 * And start the next descriptor (if any),
1840 * otherwise free this channel.
1841 */
Russell King879f1272012-05-26 14:27:40 +01001842 if (vchan_next_desc(&plchan->vc))
Russell Kingc33b6442012-05-25 15:41:13 +01001843 pl08x_start_next_txd(plchan);
Russell Kinga5a488d2012-05-26 13:54:15 +01001844 else
1845 pl08x_phy_free(plchan);
Russell Kinga936e792012-05-25 10:51:19 +01001846 }
Russell King083be282012-05-26 14:09:53 +01001847 spin_unlock(&plchan->vc.lock);
Russell Kinga936e792012-05-25 10:51:19 +01001848
Linus Walleije8689e62010-09-28 15:57:37 +02001849 mask |= (1 << i);
1850 }
1851 }
Linus Walleije8689e62010-09-28 15:57:37 +02001852
1853 return mask ? IRQ_HANDLED : IRQ_NONE;
1854}
1855
Russell King - ARM Linux121c8472011-07-21 17:13:48 +01001856static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan)
1857{
Russell King - ARM Linux121c8472011-07-21 17:13:48 +01001858 chan->slave = true;
1859 chan->name = chan->cd->bus_id;
Russell Kinged91c132012-05-16 11:02:40 +01001860 chan->cfg.src_addr = chan->cd->addr;
1861 chan->cfg.dst_addr = chan->cd->addr;
Russell King - ARM Linux121c8472011-07-21 17:13:48 +01001862}
1863
Linus Walleije8689e62010-09-28 15:57:37 +02001864/*
1865 * Initialise the DMAC memcpy/slave channels.
1866 * Make a local wrapper to hold required data
1867 */
1868static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,
Viresh Kumar3e27ee82011-08-05 15:32:27 +05301869 struct dma_device *dmadev, unsigned int channels, bool slave)
Linus Walleije8689e62010-09-28 15:57:37 +02001870{
1871 struct pl08x_dma_chan *chan;
1872 int i;
1873
1874 INIT_LIST_HEAD(&dmadev->channels);
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +00001875
Linus Walleije8689e62010-09-28 15:57:37 +02001876 /*
1877 * Register as many many memcpy as we have physical channels,
1878 * we won't always be able to use all but the code will have
1879 * to cope with that situation.
1880 */
1881 for (i = 0; i < channels; i++) {
Viresh Kumarb201c112011-08-05 15:32:29 +05301882 chan = kzalloc(sizeof(*chan), GFP_KERNEL);
Linus Walleije8689e62010-09-28 15:57:37 +02001883 if (!chan) {
1884 dev_err(&pl08x->adev->dev,
1885 "%s no memory for channel\n", __func__);
1886 return -ENOMEM;
1887 }
1888
1889 chan->host = pl08x;
1890 chan->state = PL08X_CHAN_IDLE;
Russell Kingad0de2a2012-05-25 11:15:15 +01001891 chan->signal = -1;
Linus Walleije8689e62010-09-28 15:57:37 +02001892
1893 if (slave) {
Linus Walleije8689e62010-09-28 15:57:37 +02001894 chan->cd = &pl08x->pd->slave_channels[i];
Russell King - ARM Linux121c8472011-07-21 17:13:48 +01001895 pl08x_dma_slave_init(chan);
Linus Walleije8689e62010-09-28 15:57:37 +02001896 } else {
1897 chan->cd = &pl08x->pd->memcpy_channel;
1898 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
1899 if (!chan->name) {
1900 kfree(chan);
1901 return -ENOMEM;
1902 }
1903 }
Viresh Kumar175a5e62011-08-05 15:32:32 +05301904 dev_dbg(&pl08x->adev->dev,
Linus Walleije8689e62010-09-28 15:57:37 +02001905 "initialize virtual channel \"%s\"\n",
1906 chan->name);
1907
Russell King18536132012-05-26 14:42:23 +01001908 chan->vc.desc_free = pl08x_desc_free;
Russell King083be282012-05-26 14:09:53 +01001909 vchan_init(&chan->vc, dmadev);
Linus Walleije8689e62010-09-28 15:57:37 +02001910 }
1911 dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n",
1912 i, slave ? "slave" : "memcpy");
1913 return i;
1914}
1915
1916static void pl08x_free_virtual_channels(struct dma_device *dmadev)
1917{
1918 struct pl08x_dma_chan *chan = NULL;
1919 struct pl08x_dma_chan *next;
1920
1921 list_for_each_entry_safe(chan,
Russell King01d8dc62012-05-26 14:04:29 +01001922 next, &dmadev->channels, vc.chan.device_node) {
1923 list_del(&chan->vc.chan.device_node);
Linus Walleije8689e62010-09-28 15:57:37 +02001924 kfree(chan);
1925 }
1926}
1927
1928#ifdef CONFIG_DEBUG_FS
1929static const char *pl08x_state_str(enum pl08x_dma_chan_state state)
1930{
1931 switch (state) {
1932 case PL08X_CHAN_IDLE:
1933 return "idle";
1934 case PL08X_CHAN_RUNNING:
1935 return "running";
1936 case PL08X_CHAN_PAUSED:
1937 return "paused";
1938 case PL08X_CHAN_WAITING:
1939 return "waiting";
1940 default:
1941 break;
1942 }
1943 return "UNKNOWN STATE";
1944}
1945
1946static int pl08x_debugfs_show(struct seq_file *s, void *data)
1947{
1948 struct pl08x_driver_data *pl08x = s->private;
1949 struct pl08x_dma_chan *chan;
1950 struct pl08x_phy_chan *ch;
1951 unsigned long flags;
1952 int i;
1953
1954 seq_printf(s, "PL08x physical channels:\n");
1955 seq_printf(s, "CHANNEL:\tUSER:\n");
1956 seq_printf(s, "--------\t-----\n");
1957 for (i = 0; i < pl08x->vd->channels; i++) {
1958 struct pl08x_dma_chan *virt_chan;
1959
1960 ch = &pl08x->phy_chans[i];
1961
1962 spin_lock_irqsave(&ch->lock, flags);
1963 virt_chan = ch->serving;
1964
Linus Walleijaffa1152012-04-12 09:01:49 +02001965 seq_printf(s, "%d\t\t%s%s\n",
1966 ch->id,
1967 virt_chan ? virt_chan->name : "(none)",
1968 ch->locked ? " LOCKED" : "");
Linus Walleije8689e62010-09-28 15:57:37 +02001969
1970 spin_unlock_irqrestore(&ch->lock, flags);
1971 }
1972
1973 seq_printf(s, "\nPL08x virtual memcpy channels:\n");
1974 seq_printf(s, "CHANNEL:\tSTATE:\n");
1975 seq_printf(s, "--------\t------\n");
Russell King01d8dc62012-05-26 14:04:29 +01001976 list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) {
Russell King - ARM Linux3e2a0372011-01-03 22:32:46 +00001977 seq_printf(s, "%s\t\t%s\n", chan->name,
Linus Walleije8689e62010-09-28 15:57:37 +02001978 pl08x_state_str(chan->state));
1979 }
1980
1981 seq_printf(s, "\nPL08x virtual slave channels:\n");
1982 seq_printf(s, "CHANNEL:\tSTATE:\n");
1983 seq_printf(s, "--------\t------\n");
Russell King01d8dc62012-05-26 14:04:29 +01001984 list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) {
Russell King - ARM Linux3e2a0372011-01-03 22:32:46 +00001985 seq_printf(s, "%s\t\t%s\n", chan->name,
Linus Walleije8689e62010-09-28 15:57:37 +02001986 pl08x_state_str(chan->state));
1987 }
1988
1989 return 0;
1990}
1991
1992static int pl08x_debugfs_open(struct inode *inode, struct file *file)
1993{
1994 return single_open(file, pl08x_debugfs_show, inode->i_private);
1995}
1996
1997static const struct file_operations pl08x_debugfs_operations = {
1998 .open = pl08x_debugfs_open,
1999 .read = seq_read,
2000 .llseek = seq_lseek,
2001 .release = single_release,
2002};
2003
2004static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
2005{
2006 /* Expose a simple debugfs interface to view all clocks */
Viresh Kumar3e27ee82011-08-05 15:32:27 +05302007 (void) debugfs_create_file(dev_name(&pl08x->adev->dev),
2008 S_IFREG | S_IRUGO, NULL, pl08x,
2009 &pl08x_debugfs_operations);
Linus Walleije8689e62010-09-28 15:57:37 +02002010}
2011
2012#else
2013static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
2014{
2015}
2016#endif
2017
Russell Kingaa25afa2011-02-19 15:55:00 +00002018static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)
Linus Walleije8689e62010-09-28 15:57:37 +02002019{
2020 struct pl08x_driver_data *pl08x;
Russell King - ARM Linuxf96ca9ec2011-01-03 22:35:08 +00002021 const struct vendor_data *vd = id->data;
Tomasz Figaba6785f2013-08-11 19:59:15 +02002022 u32 tsfr_size;
Linus Walleije8689e62010-09-28 15:57:37 +02002023 int ret = 0;
2024 int i;
2025
2026 ret = amba_request_regions(adev, NULL);
2027 if (ret)
2028 return ret;
2029
Russell Kingde1a2412013-06-27 10:29:32 +01002030 /* Ensure that we can do DMA */
2031 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32));
2032 if (ret)
2033 goto out_no_pl08x;
2034
Linus Walleije8689e62010-09-28 15:57:37 +02002035 /* Create the driver state holder */
Viresh Kumarb201c112011-08-05 15:32:29 +05302036 pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL);
Linus Walleije8689e62010-09-28 15:57:37 +02002037 if (!pl08x) {
2038 ret = -ENOMEM;
2039 goto out_no_pl08x;
2040 }
2041
2042 /* Initialize memcpy engine */
2043 dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask);
2044 pl08x->memcpy.dev = &adev->dev;
2045 pl08x->memcpy.device_alloc_chan_resources = pl08x_alloc_chan_resources;
2046 pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources;
2047 pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy;
2048 pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;
2049 pl08x->memcpy.device_tx_status = pl08x_dma_tx_status;
2050 pl08x->memcpy.device_issue_pending = pl08x_issue_pending;
2051 pl08x->memcpy.device_control = pl08x_control;
2052
2053 /* Initialize slave engine */
2054 dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask);
Alban Bedel3b24c202013-08-11 19:59:20 +02002055 dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask);
Linus Walleije8689e62010-09-28 15:57:37 +02002056 pl08x->slave.dev = &adev->dev;
2057 pl08x->slave.device_alloc_chan_resources = pl08x_alloc_chan_resources;
2058 pl08x->slave.device_free_chan_resources = pl08x_free_chan_resources;
2059 pl08x->slave.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;
2060 pl08x->slave.device_tx_status = pl08x_dma_tx_status;
2061 pl08x->slave.device_issue_pending = pl08x_issue_pending;
2062 pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg;
Alban Bedel3b24c202013-08-11 19:59:20 +02002063 pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic;
Linus Walleije8689e62010-09-28 15:57:37 +02002064 pl08x->slave.device_control = pl08x_control;
2065
2066 /* Get the platform data */
2067 pl08x->pd = dev_get_platdata(&adev->dev);
2068 if (!pl08x->pd) {
2069 dev_err(&adev->dev, "no platform data supplied\n");
Julia Lawall983d7be2012-08-14 14:58:32 +02002070 ret = -EINVAL;
Linus Walleije8689e62010-09-28 15:57:37 +02002071 goto out_no_platdata;
2072 }
2073
2074 /* Assign useful pointers to the driver state */
2075 pl08x->adev = adev;
2076 pl08x->vd = vd;
2077
Russell King - ARM Linux30749cb2011-01-03 22:41:13 +00002078 /* By default, AHB1 only. If dualmaster, from platform */
2079 pl08x->lli_buses = PL08X_AHB1;
2080 pl08x->mem_buses = PL08X_AHB1;
2081 if (pl08x->vd->dualmaster) {
2082 pl08x->lli_buses = pl08x->pd->lli_buses;
2083 pl08x->mem_buses = pl08x->pd->mem_buses;
2084 }
2085
Tomasz Figada1b6c02013-08-11 19:59:17 +02002086 if (vd->pl080s)
2087 pl08x->lli_words = PL080S_LLI_WORDS;
2088 else
2089 pl08x->lli_words = PL080_LLI_WORDS;
Tomasz Figaba6785f2013-08-11 19:59:15 +02002090 tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32);
2091
Linus Walleije8689e62010-09-28 15:57:37 +02002092 /* A DMA memory pool for LLIs, align on 1-byte boundary */
2093 pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev,
Tomasz Figaba6785f2013-08-11 19:59:15 +02002094 tsfr_size, PL08X_ALIGN, 0);
Linus Walleije8689e62010-09-28 15:57:37 +02002095 if (!pl08x->pool) {
2096 ret = -ENOMEM;
2097 goto out_no_lli_pool;
2098 }
2099
Linus Walleije8689e62010-09-28 15:57:37 +02002100 pl08x->base = ioremap(adev->res.start, resource_size(&adev->res));
2101 if (!pl08x->base) {
2102 ret = -ENOMEM;
2103 goto out_no_ioremap;
2104 }
2105
2106 /* Turn on the PL08x */
2107 pl08x_ensure_on(pl08x);
2108
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +00002109 /* Attach the interrupt handler */
Linus Walleije8689e62010-09-28 15:57:37 +02002110 writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR);
2111 writel(0x000000FF, pl08x->base + PL080_TC_CLEAR);
2112
Michael Opdenacker174b5372013-10-13 07:10:51 +02002113 ret = request_irq(adev->irq[0], pl08x_irq, 0, DRIVER_NAME, pl08x);
Linus Walleije8689e62010-09-28 15:57:37 +02002114 if (ret) {
2115 dev_err(&adev->dev, "%s failed to request interrupt %d\n",
2116 __func__, adev->irq[0]);
2117 goto out_no_irq;
2118 }
2119
2120 /* Initialize physical channels */
Linus Walleijaffa1152012-04-12 09:01:49 +02002121 pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)),
Linus Walleije8689e62010-09-28 15:57:37 +02002122 GFP_KERNEL);
2123 if (!pl08x->phy_chans) {
2124 dev_err(&adev->dev, "%s failed to allocate "
2125 "physical channel holders\n",
2126 __func__);
Julia Lawall983d7be2012-08-14 14:58:32 +02002127 ret = -ENOMEM;
Linus Walleije8689e62010-09-28 15:57:37 +02002128 goto out_no_phychans;
2129 }
2130
2131 for (i = 0; i < vd->channels; i++) {
2132 struct pl08x_phy_chan *ch = &pl08x->phy_chans[i];
2133
2134 ch->id = i;
2135 ch->base = pl08x->base + PL080_Cx_BASE(i);
Tomasz Figad86ccea2013-08-11 19:59:14 +02002136 ch->reg_config = ch->base + vd->config_offset;
Linus Walleije8689e62010-09-28 15:57:37 +02002137 spin_lock_init(&ch->lock);
Linus Walleijaffa1152012-04-12 09:01:49 +02002138
2139 /*
2140 * Nomadik variants can have channels that are locked
2141 * down for the secure world only. Lock up these channels
2142 * by perpetually serving a dummy virtual channel.
2143 */
2144 if (vd->nomadik) {
2145 u32 val;
2146
Tomasz Figad86ccea2013-08-11 19:59:14 +02002147 val = readl(ch->reg_config);
Linus Walleijaffa1152012-04-12 09:01:49 +02002148 if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) {
2149 dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i);
2150 ch->locked = true;
2151 }
2152 }
2153
Viresh Kumar175a5e62011-08-05 15:32:32 +05302154 dev_dbg(&adev->dev, "physical channel %d is %s\n",
2155 i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE");
Linus Walleije8689e62010-09-28 15:57:37 +02002156 }
2157
2158 /* Register as many memcpy channels as there are physical channels */
2159 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy,
2160 pl08x->vd->channels, false);
2161 if (ret <= 0) {
2162 dev_warn(&pl08x->adev->dev,
2163 "%s failed to enumerate memcpy channels - %d\n",
2164 __func__, ret);
2165 goto out_no_memcpy;
2166 }
2167 pl08x->memcpy.chancnt = ret;
2168
2169 /* Register slave channels */
2170 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave,
Viresh Kumar3e27ee82011-08-05 15:32:27 +05302171 pl08x->pd->num_slave_channels, true);
Linus Walleij10804112013-12-14 00:16:23 +01002172 if (ret < 0) {
Linus Walleije8689e62010-09-28 15:57:37 +02002173 dev_warn(&pl08x->adev->dev,
2174 "%s failed to enumerate slave channels - %d\n",
2175 __func__, ret);
2176 goto out_no_slave;
2177 }
2178 pl08x->slave.chancnt = ret;
2179
2180 ret = dma_async_device_register(&pl08x->memcpy);
2181 if (ret) {
2182 dev_warn(&pl08x->adev->dev,
2183 "%s failed to register memcpy as an async device - %d\n",
2184 __func__, ret);
2185 goto out_no_memcpy_reg;
2186 }
2187
2188 ret = dma_async_device_register(&pl08x->slave);
2189 if (ret) {
2190 dev_warn(&pl08x->adev->dev,
2191 "%s failed to register slave as an async device - %d\n",
2192 __func__, ret);
2193 goto out_no_slave_reg;
2194 }
2195
2196 amba_set_drvdata(adev, pl08x);
2197 init_pl08x_debugfs(pl08x);
Tomasz Figada1b6c02013-08-11 19:59:17 +02002198 dev_info(&pl08x->adev->dev, "DMA: PL%03x%s rev%u at 0x%08llx irq %d\n",
2199 amba_part(adev), pl08x->vd->pl080s ? "s" : "", amba_rev(adev),
Russell King - ARM Linuxb05cd8f2011-01-03 22:33:26 +00002200 (unsigned long long)adev->res.start, adev->irq[0]);
Viresh Kumarb7b60182011-08-05 15:32:33 +05302201
Linus Walleije8689e62010-09-28 15:57:37 +02002202 return 0;
2203
2204out_no_slave_reg:
2205 dma_async_device_unregister(&pl08x->memcpy);
2206out_no_memcpy_reg:
2207 pl08x_free_virtual_channels(&pl08x->slave);
2208out_no_slave:
2209 pl08x_free_virtual_channels(&pl08x->memcpy);
2210out_no_memcpy:
2211 kfree(pl08x->phy_chans);
2212out_no_phychans:
2213 free_irq(adev->irq[0], pl08x);
2214out_no_irq:
2215 iounmap(pl08x->base);
2216out_no_ioremap:
2217 dma_pool_destroy(pl08x->pool);
2218out_no_lli_pool:
2219out_no_platdata:
2220 kfree(pl08x);
2221out_no_pl08x:
2222 amba_release_regions(adev);
2223 return ret;
2224}
2225
2226/* PL080 has 8 channels and the PL080 have just 2 */
2227static struct vendor_data vendor_pl080 = {
Tomasz Figad86ccea2013-08-11 19:59:14 +02002228 .config_offset = PL080_CH_CONFIG,
Linus Walleije8689e62010-09-28 15:57:37 +02002229 .channels = 8,
2230 .dualmaster = true,
Tomasz Figa5110e512013-08-11 19:59:18 +02002231 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
Linus Walleije8689e62010-09-28 15:57:37 +02002232};
2233
Linus Walleijaffa1152012-04-12 09:01:49 +02002234static struct vendor_data vendor_nomadik = {
Tomasz Figad86ccea2013-08-11 19:59:14 +02002235 .config_offset = PL080_CH_CONFIG,
Linus Walleijaffa1152012-04-12 09:01:49 +02002236 .channels = 8,
2237 .dualmaster = true,
2238 .nomadik = true,
Tomasz Figa5110e512013-08-11 19:59:18 +02002239 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
Linus Walleijaffa1152012-04-12 09:01:49 +02002240};
2241
Tomasz Figada1b6c02013-08-11 19:59:17 +02002242static struct vendor_data vendor_pl080s = {
2243 .config_offset = PL080S_CH_CONFIG,
2244 .channels = 8,
2245 .pl080s = true,
Tomasz Figa5110e512013-08-11 19:59:18 +02002246 .max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK,
Linus Walleije8689e62010-09-28 15:57:37 +02002247};
2248
2249static struct vendor_data vendor_pl081 = {
Tomasz Figad86ccea2013-08-11 19:59:14 +02002250 .config_offset = PL080_CH_CONFIG,
Linus Walleije8689e62010-09-28 15:57:37 +02002251 .channels = 2,
2252 .dualmaster = false,
Tomasz Figa5110e512013-08-11 19:59:18 +02002253 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
Linus Walleije8689e62010-09-28 15:57:37 +02002254};
2255
2256static struct amba_id pl08x_ids[] = {
Tomasz Figada1b6c02013-08-11 19:59:17 +02002257 /* Samsung PL080S variant */
2258 {
2259 .id = 0x0a141080,
2260 .mask = 0xffffffff,
2261 .data = &vendor_pl080s,
2262 },
Linus Walleije8689e62010-09-28 15:57:37 +02002263 /* PL080 */
2264 {
2265 .id = 0x00041080,
2266 .mask = 0x000fffff,
2267 .data = &vendor_pl080,
2268 },
2269 /* PL081 */
2270 {
2271 .id = 0x00041081,
2272 .mask = 0x000fffff,
2273 .data = &vendor_pl081,
2274 },
2275 /* Nomadik 8815 PL080 variant */
2276 {
Linus Walleijaffa1152012-04-12 09:01:49 +02002277 .id = 0x00280080,
Linus Walleije8689e62010-09-28 15:57:37 +02002278 .mask = 0x00ffffff,
Linus Walleijaffa1152012-04-12 09:01:49 +02002279 .data = &vendor_nomadik,
Linus Walleije8689e62010-09-28 15:57:37 +02002280 },
2281 { 0, 0 },
2282};
2283
Dave Martin037566d2011-10-05 15:15:20 +01002284MODULE_DEVICE_TABLE(amba, pl08x_ids);
2285
Linus Walleije8689e62010-09-28 15:57:37 +02002286static struct amba_driver pl08x_amba_driver = {
2287 .drv.name = DRIVER_NAME,
2288 .id_table = pl08x_ids,
2289 .probe = pl08x_probe,
2290};
2291
2292static int __init pl08x_init(void)
2293{
2294 int retval;
2295 retval = amba_driver_register(&pl08x_amba_driver);
2296 if (retval)
2297 printk(KERN_WARNING DRIVER_NAME
Russell King - ARM Linuxe8b5e112011-01-03 22:30:24 +00002298 "failed to register as an AMBA device (%d)\n",
Linus Walleije8689e62010-09-28 15:57:37 +02002299 retval);
2300 return retval;
2301}
2302subsys_initcall(pl08x_init);