blob: 4a5fd245014e6666721c111b85515b2e1b05f74c [file] [log] [blame]
Linus Walleije8689e62010-09-28 15:57:37 +02001/*
2 * Copyright (c) 2006 ARM Ltd.
3 * Copyright (c) 2010 ST-Ericsson SA
4 *
5 * Author: Peter Pearse <peter.pearse@arm.com>
6 * Author: Linus Walleij <linus.walleij@stericsson.com>
7 *
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU General Public License as published by the Free
10 * Software Foundation; either version 2 of the License, or (at your option)
11 * any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along with
19 * this program; if not, write to the Free Software Foundation, Inc., 59
20 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
21 *
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +000022 * The full GNU General Public License is in this distribution in the file
23 * called COPYING.
Linus Walleije8689e62010-09-28 15:57:37 +020024 *
25 * Documentation: ARM DDI 0196G == PL080
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +000026 * Documentation: ARM DDI 0218E == PL081
Tomasz Figada1b6c02013-08-11 19:59:17 +020027 * Documentation: S3C6410 User's Manual == PL080S
Linus Walleije8689e62010-09-28 15:57:37 +020028 *
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +000029 * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any
30 * channel.
Linus Walleije8689e62010-09-28 15:57:37 +020031 *
32 * The PL080 has 8 channels available for simultaneous use, and the PL081
33 * has only two channels. So on these DMA controllers the number of channels
34 * and the number of incoming DMA signals are two totally different things.
35 * It is usually not possible to theoretically handle all physical signals,
36 * so a multiplexing scheme with possible denial of use is necessary.
37 *
38 * The PL080 has a dual bus master, PL081 has a single master.
39 *
Tomasz Figada1b6c02013-08-11 19:59:17 +020040 * PL080S is a version modified by Samsung and used in S3C64xx SoCs.
41 * It differs in following aspects:
42 * - CH_CONFIG register at different offset,
43 * - separate CH_CONTROL2 register for transfer size,
44 * - bigger maximum transfer size,
45 * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word,
46 * - no support for peripheral flow control.
47 *
Linus Walleije8689e62010-09-28 15:57:37 +020048 * Memory to peripheral transfer may be visualized as
49 * Get data from memory to DMAC
50 * Until no data left
51 * On burst request from peripheral
52 * Destination burst from DMAC to peripheral
53 * Clear burst request
54 * Raise terminal count interrupt
55 *
56 * For peripherals with a FIFO:
57 * Source burst size == half the depth of the peripheral FIFO
58 * Destination burst size == the depth of the peripheral FIFO
59 *
60 * (Bursts are irrelevant for mem to mem transfers - there are no burst
61 * signals, the DMA controller will simply facilitate its AHB master.)
62 *
63 * ASSUMES default (little) endianness for DMA transfers
64 *
Russell King - ARM Linux9dc2c202011-01-03 22:33:06 +000065 * The PL08x has two flow control settings:
66 * - DMAC flow control: the transfer size defines the number of transfers
67 * which occur for the current LLI entry, and the DMAC raises TC at the
68 * end of every LLI entry. Observed behaviour shows the DMAC listening
69 * to both the BREQ and SREQ signals (contrary to documented),
70 * transferring data if either is active. The LBREQ and LSREQ signals
71 * are ignored.
72 *
73 * - Peripheral flow control: the transfer size is ignored (and should be
74 * zero). The data is transferred from the current LLI entry, until
75 * after the final transfer signalled by LBREQ or LSREQ. The DMAC
Tomasz Figada1b6c02013-08-11 19:59:17 +020076 * will then move to the next LLI entry. Unsupported by PL080S.
Linus Walleije8689e62010-09-28 15:57:37 +020077 */
Russell King - ARM Linux730404a2011-01-03 22:34:07 +000078#include <linux/amba/bus.h>
Linus Walleije8689e62010-09-28 15:57:37 +020079#include <linux/amba/pl08x.h>
80#include <linux/debugfs.h>
Viresh Kumar0c38d702011-08-05 15:32:28 +053081#include <linux/delay.h>
82#include <linux/device.h>
83#include <linux/dmaengine.h>
84#include <linux/dmapool.h>
Vinod Koul8516f522011-09-02 16:43:44 +053085#include <linux/dma-mapping.h>
Sachin Kamat6d05c9f2014-01-23 16:10:07 +053086#include <linux/export.h>
Viresh Kumar0c38d702011-08-05 15:32:28 +053087#include <linux/init.h>
88#include <linux/interrupt.h>
89#include <linux/module.h>
Viresh Kumarb7b60182011-08-05 15:32:33 +053090#include <linux/pm_runtime.h>
Linus Walleije8689e62010-09-28 15:57:37 +020091#include <linux/seq_file.h>
Viresh Kumar0c38d702011-08-05 15:32:28 +053092#include <linux/slab.h>
Alessandro Rubini3a95b9f2012-11-24 00:22:56 +000093#include <linux/amba/pl080.h>
Linus Walleije8689e62010-09-28 15:57:37 +020094
Russell King - ARM Linuxd2ebfb32012-03-06 22:34:26 +000095#include "dmaengine.h"
Russell King01d8dc62012-05-26 14:04:29 +010096#include "virt-dma.h"
Russell King - ARM Linuxd2ebfb32012-03-06 22:34:26 +000097
Linus Walleije8689e62010-09-28 15:57:37 +020098#define DRIVER_NAME "pl08xdmac"
99
Russell King - ARM Linux7703eac2011-08-31 09:34:35 +0100100static struct amba_driver pl08x_amba_driver;
Russell Kingb23f2042012-05-16 10:48:44 +0100101struct pl08x_driver_data;
Russell King - ARM Linux7703eac2011-08-31 09:34:35 +0100102
Linus Walleije8689e62010-09-28 15:57:37 +0200103/**
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +0000104 * struct vendor_data - vendor-specific config parameters for PL08x derivatives
Linus Walleije8689e62010-09-28 15:57:37 +0200105 * @channels: the number of channels available in this variant
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +0000106 * @dualmaster: whether this version supports dual AHB masters or not.
Linus Walleijaffa1152012-04-12 09:01:49 +0200107 * @nomadik: whether the channels have Nomadik security extension bits
108 * that need to be checked for permission before use and some registers are
109 * missing
Tomasz Figada1b6c02013-08-11 19:59:17 +0200110 * @pl080s: whether this version is a PL080S, which has separate register and
111 * LLI word for transfer size.
Linus Walleije8689e62010-09-28 15:57:37 +0200112 */
113struct vendor_data {
Tomasz Figad86ccea2013-08-11 19:59:14 +0200114 u8 config_offset;
Linus Walleije8689e62010-09-28 15:57:37 +0200115 u8 channels;
116 bool dualmaster;
Linus Walleijaffa1152012-04-12 09:01:49 +0200117 bool nomadik;
Tomasz Figada1b6c02013-08-11 19:59:17 +0200118 bool pl080s;
Tomasz Figa5110e512013-08-11 19:59:18 +0200119 u32 max_transfer_size;
Linus Walleije8689e62010-09-28 15:57:37 +0200120};
121
122/**
Russell Kingb23f2042012-05-16 10:48:44 +0100123 * struct pl08x_bus_data - information of source or destination
124 * busses for a transfer
125 * @addr: current address
126 * @maxwidth: the maximum width of a transfer on this bus
127 * @buswidth: the width of this bus in bytes: 1, 2 or 4
128 */
129struct pl08x_bus_data {
130 dma_addr_t addr;
131 u8 maxwidth;
132 u8 buswidth;
133};
134
Andre Przywara1c38b282013-08-19 12:19:28 +0200135#define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth)
136
Russell Kingb23f2042012-05-16 10:48:44 +0100137/**
138 * struct pl08x_phy_chan - holder for the physical channels
139 * @id: physical index to this channel
140 * @lock: a lock to use when altering an instance of this struct
Russell Kingb23f2042012-05-16 10:48:44 +0100141 * @serving: the virtual channel currently being served by this physical
142 * channel
Russell Kingad0de2a2012-05-25 11:15:15 +0100143 * @locked: channel unavailable for the system, e.g. dedicated to secure
144 * world
Russell Kingb23f2042012-05-16 10:48:44 +0100145 */
146struct pl08x_phy_chan {
147 unsigned int id;
148 void __iomem *base;
Tomasz Figad86ccea2013-08-11 19:59:14 +0200149 void __iomem *reg_config;
Russell Kingb23f2042012-05-16 10:48:44 +0100150 spinlock_t lock;
Russell Kingb23f2042012-05-16 10:48:44 +0100151 struct pl08x_dma_chan *serving;
Russell Kingad0de2a2012-05-25 11:15:15 +0100152 bool locked;
Russell Kingb23f2042012-05-16 10:48:44 +0100153};
154
155/**
156 * struct pl08x_sg - structure containing data per sg
157 * @src_addr: src address of sg
158 * @dst_addr: dst address of sg
159 * @len: transfer len in bytes
160 * @node: node for txd's dsg_list
161 */
162struct pl08x_sg {
163 dma_addr_t src_addr;
164 dma_addr_t dst_addr;
165 size_t len;
166 struct list_head node;
167};
168
169/**
170 * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor
Russell King01d8dc62012-05-26 14:04:29 +0100171 * @vd: virtual DMA descriptor
Russell Kingb23f2042012-05-16 10:48:44 +0100172 * @dsg_list: list of children sg's
Russell Kingb23f2042012-05-16 10:48:44 +0100173 * @llis_bus: DMA memory address (physical) start for the LLIs
174 * @llis_va: virtual memory address start for the LLIs
175 * @cctl: control reg values for current txd
176 * @ccfg: config reg values for current txd
Russell King18536132012-05-26 14:42:23 +0100177 * @done: this marks completed descriptors, which should not have their
178 * mux released.
Alban Bedel3b24c202013-08-11 19:59:20 +0200179 * @cyclic: indicate cyclic transfers
Russell Kingb23f2042012-05-16 10:48:44 +0100180 */
181struct pl08x_txd {
Russell King01d8dc62012-05-26 14:04:29 +0100182 struct virt_dma_desc vd;
Russell Kingb23f2042012-05-16 10:48:44 +0100183 struct list_head dsg_list;
Russell Kingb23f2042012-05-16 10:48:44 +0100184 dma_addr_t llis_bus;
Tomasz Figaba6785f2013-08-11 19:59:15 +0200185 u32 *llis_va;
Russell Kingb23f2042012-05-16 10:48:44 +0100186 /* Default cctl value for LLIs */
187 u32 cctl;
188 /*
189 * Settings to be put into the physical channel when we
190 * trigger this txd. Other registers are in llis_va[0].
191 */
192 u32 ccfg;
Russell King18536132012-05-26 14:42:23 +0100193 bool done;
Alban Bedel3b24c202013-08-11 19:59:20 +0200194 bool cyclic;
Russell Kingb23f2042012-05-16 10:48:44 +0100195};
196
197/**
198 * struct pl08x_dma_chan_state - holds the PL08x specific virtual channel
199 * states
200 * @PL08X_CHAN_IDLE: the channel is idle
201 * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport
202 * channel and is running a transfer on it
203 * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport
204 * channel, but the transfer is currently paused
205 * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport
206 * channel to become available (only pertains to memcpy channels)
207 */
208enum pl08x_dma_chan_state {
209 PL08X_CHAN_IDLE,
210 PL08X_CHAN_RUNNING,
211 PL08X_CHAN_PAUSED,
212 PL08X_CHAN_WAITING,
213};
214
215/**
216 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel
Russell King01d8dc62012-05-26 14:04:29 +0100217 * @vc: wrappped virtual channel
Russell Kingb23f2042012-05-16 10:48:44 +0100218 * @phychan: the physical channel utilized by this channel, if there is one
Russell Kingb23f2042012-05-16 10:48:44 +0100219 * @name: name of channel
220 * @cd: channel platform data
221 * @runtime_addr: address for RX/TX according to the runtime config
Russell Kingb23f2042012-05-16 10:48:44 +0100222 * @at: active transaction on this channel
223 * @lock: a lock for this channel data
224 * @host: a pointer to the host (internal use)
225 * @state: whether the channel is idle, paused, running etc
226 * @slave: whether this channel is a device (slave) or for memcpy
Russell Kingad0de2a2012-05-25 11:15:15 +0100227 * @signal: the physical DMA request signal which this channel is using
Russell King5e2479b2012-05-25 11:32:45 +0100228 * @mux_use: count of descriptors using this DMA request signal setting
Russell Kingb23f2042012-05-16 10:48:44 +0100229 */
230struct pl08x_dma_chan {
Russell King01d8dc62012-05-26 14:04:29 +0100231 struct virt_dma_chan vc;
Russell Kingb23f2042012-05-16 10:48:44 +0100232 struct pl08x_phy_chan *phychan;
Russell King550ec362012-05-28 10:18:55 +0100233 const char *name;
Russell Kingb23f2042012-05-16 10:48:44 +0100234 const struct pl08x_channel_data *cd;
Russell Kinged91c132012-05-16 11:02:40 +0100235 struct dma_slave_config cfg;
Russell Kingb23f2042012-05-16 10:48:44 +0100236 struct pl08x_txd *at;
Russell Kingb23f2042012-05-16 10:48:44 +0100237 struct pl08x_driver_data *host;
238 enum pl08x_dma_chan_state state;
239 bool slave;
Russell Kingad0de2a2012-05-25 11:15:15 +0100240 int signal;
Russell King5e2479b2012-05-25 11:32:45 +0100241 unsigned mux_use;
Russell Kingb23f2042012-05-16 10:48:44 +0100242};
243
244/**
Linus Walleije8689e62010-09-28 15:57:37 +0200245 * struct pl08x_driver_data - the local state holder for the PL08x
246 * @slave: slave engine for this instance
247 * @memcpy: memcpy engine for this instance
248 * @base: virtual memory base (remapped) for the PL08x
249 * @adev: the corresponding AMBA (PrimeCell) bus entry
250 * @vd: vendor data for this PL08x variant
251 * @pd: platform data passed in from the platform/machine
252 * @phy_chans: array of data for the physical channels
253 * @pool: a pool for the LLI descriptors
Viresh Kumar3e27ee82011-08-05 15:32:27 +0530254 * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI
255 * fetches
Russell King - ARM Linux30749cb2011-01-03 22:41:13 +0000256 * @mem_buses: set to indicate memory transfers on AHB2.
Linus Walleije8689e62010-09-28 15:57:37 +0200257 * @lock: a spinlock for this struct
258 */
259struct pl08x_driver_data {
260 struct dma_device slave;
261 struct dma_device memcpy;
262 void __iomem *base;
263 struct amba_device *adev;
Russell King - ARM Linuxf96ca9ec2011-01-03 22:35:08 +0000264 const struct vendor_data *vd;
Linus Walleije8689e62010-09-28 15:57:37 +0200265 struct pl08x_platform_data *pd;
266 struct pl08x_phy_chan *phy_chans;
267 struct dma_pool *pool;
Russell King - ARM Linux30749cb2011-01-03 22:41:13 +0000268 u8 lli_buses;
269 u8 mem_buses;
Tomasz Figaba6785f2013-08-11 19:59:15 +0200270 u8 lli_words;
Linus Walleije8689e62010-09-28 15:57:37 +0200271};
272
273/*
274 * PL08X specific defines
275 */
276
Tomasz Figaba6785f2013-08-11 19:59:15 +0200277/* The order of words in an LLI. */
278#define PL080_LLI_SRC 0
279#define PL080_LLI_DST 1
280#define PL080_LLI_LLI 2
281#define PL080_LLI_CCTL 3
Tomasz Figada1b6c02013-08-11 19:59:17 +0200282#define PL080S_LLI_CCTL2 4
Linus Walleije8689e62010-09-28 15:57:37 +0200283
Tomasz Figaba6785f2013-08-11 19:59:15 +0200284/* Total words in an LLI. */
285#define PL080_LLI_WORDS 4
Tomasz Figada1b6c02013-08-11 19:59:17 +0200286#define PL080S_LLI_WORDS 8
Tomasz Figaba6785f2013-08-11 19:59:15 +0200287
288/*
289 * Number of LLIs in each LLI buffer allocated for one transfer
290 * (maximum times we call dma_pool_alloc on this pool without freeing)
291 */
292#define MAX_NUM_TSFR_LLIS 512
Linus Walleije8689e62010-09-28 15:57:37 +0200293#define PL08X_ALIGN 8
294
295static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan)
296{
Russell King01d8dc62012-05-26 14:04:29 +0100297 return container_of(chan, struct pl08x_dma_chan, vc.chan);
Linus Walleije8689e62010-09-28 15:57:37 +0200298}
299
Russell King - ARM Linux501e67e2011-01-03 22:44:57 +0000300static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx)
301{
Russell King01d8dc62012-05-26 14:04:29 +0100302 return container_of(tx, struct pl08x_txd, vd.tx);
Russell King - ARM Linux501e67e2011-01-03 22:44:57 +0000303}
304
Linus Walleije8689e62010-09-28 15:57:37 +0200305/*
Russell King6b16c8b2012-05-25 11:10:58 +0100306 * Mux handling.
307 *
308 * This gives us the DMA request input to the PL08x primecell which the
309 * peripheral described by the channel data will be routed to, possibly
310 * via a board/SoC specific external MUX. One important point to note
311 * here is that this does not depend on the physical channel.
312 */
Russell Kingad0de2a2012-05-25 11:15:15 +0100313static int pl08x_request_mux(struct pl08x_dma_chan *plchan)
Russell King6b16c8b2012-05-25 11:10:58 +0100314{
315 const struct pl08x_platform_data *pd = plchan->host->pd;
316 int ret;
317
Mark Brownd7cabee2013-06-19 20:38:28 +0100318 if (plchan->mux_use++ == 0 && pd->get_xfer_signal) {
319 ret = pd->get_xfer_signal(plchan->cd);
Russell King5e2479b2012-05-25 11:32:45 +0100320 if (ret < 0) {
321 plchan->mux_use = 0;
Russell King6b16c8b2012-05-25 11:10:58 +0100322 return ret;
Russell King5e2479b2012-05-25 11:32:45 +0100323 }
Russell King6b16c8b2012-05-25 11:10:58 +0100324
Russell Kingad0de2a2012-05-25 11:15:15 +0100325 plchan->signal = ret;
Russell King6b16c8b2012-05-25 11:10:58 +0100326 }
327 return 0;
328}
329
330static void pl08x_release_mux(struct pl08x_dma_chan *plchan)
331{
332 const struct pl08x_platform_data *pd = plchan->host->pd;
333
Russell King5e2479b2012-05-25 11:32:45 +0100334 if (plchan->signal >= 0) {
335 WARN_ON(plchan->mux_use == 0);
336
Mark Brownd7cabee2013-06-19 20:38:28 +0100337 if (--plchan->mux_use == 0 && pd->put_xfer_signal) {
338 pd->put_xfer_signal(plchan->cd, plchan->signal);
Russell King5e2479b2012-05-25 11:32:45 +0100339 plchan->signal = -1;
340 }
Russell King6b16c8b2012-05-25 11:10:58 +0100341 }
342}
343
344/*
Linus Walleije8689e62010-09-28 15:57:37 +0200345 * Physical channel handling
346 */
347
348/* Whether a certain channel is busy or not */
349static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch)
350{
351 unsigned int val;
352
Tomasz Figad86ccea2013-08-11 19:59:14 +0200353 val = readl(ch->reg_config);
Linus Walleije8689e62010-09-28 15:57:37 +0200354 return val & PL080_CONFIG_ACTIVE;
355}
356
Tomasz Figaba6785f2013-08-11 19:59:15 +0200357static void pl08x_write_lli(struct pl08x_driver_data *pl08x,
358 struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg)
359{
Tomasz Figada1b6c02013-08-11 19:59:17 +0200360 if (pl08x->vd->pl080s)
361 dev_vdbg(&pl08x->adev->dev,
362 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
363 "clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n",
364 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
365 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL],
366 lli[PL080S_LLI_CCTL2], ccfg);
367 else
368 dev_vdbg(&pl08x->adev->dev,
369 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, "
370 "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n",
371 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST],
372 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg);
Tomasz Figaba6785f2013-08-11 19:59:15 +0200373
374 writel_relaxed(lli[PL080_LLI_SRC], phychan->base + PL080_CH_SRC_ADDR);
375 writel_relaxed(lli[PL080_LLI_DST], phychan->base + PL080_CH_DST_ADDR);
376 writel_relaxed(lli[PL080_LLI_LLI], phychan->base + PL080_CH_LLI);
377 writel_relaxed(lli[PL080_LLI_CCTL], phychan->base + PL080_CH_CONTROL);
378
Tomasz Figada1b6c02013-08-11 19:59:17 +0200379 if (pl08x->vd->pl080s)
380 writel_relaxed(lli[PL080S_LLI_CCTL2],
381 phychan->base + PL080S_CH_CONTROL2);
382
Tomasz Figaba6785f2013-08-11 19:59:15 +0200383 writel(ccfg, phychan->reg_config);
384}
385
Linus Walleije8689e62010-09-28 15:57:37 +0200386/*
387 * Set the initial DMA register values i.e. those for the first LLI
Russell King - ARM Linuxe8b5e112011-01-03 22:30:24 +0000388 * The next LLI pointer and the configuration interrupt bit have
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000389 * been set when the LLIs were constructed. Poke them into the hardware
390 * and start the transfer.
Linus Walleije8689e62010-09-28 15:57:37 +0200391 */
Russell Kingeab82532012-05-25 12:32:00 +0100392static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan)
Linus Walleije8689e62010-09-28 15:57:37 +0200393{
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000394 struct pl08x_driver_data *pl08x = plchan->host;
Linus Walleije8689e62010-09-28 15:57:37 +0200395 struct pl08x_phy_chan *phychan = plchan->phychan;
Russell King879f1272012-05-26 14:27:40 +0100396 struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc);
397 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
Russell King - ARM Linux09b3c322011-01-03 22:39:53 +0000398 u32 val;
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000399
Russell King879f1272012-05-26 14:27:40 +0100400 list_del(&txd->vd.node);
Russell Kingeab82532012-05-25 12:32:00 +0100401
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000402 plchan->at = txd;
Linus Walleije8689e62010-09-28 15:57:37 +0200403
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000404 /* Wait for channel inactive */
405 while (pl08x_phy_channel_busy(phychan))
Russell King - ARM Linux19386b322011-01-03 22:36:29 +0000406 cpu_relax();
Linus Walleije8689e62010-09-28 15:57:37 +0200407
Tomasz Figaba6785f2013-08-11 19:59:15 +0200408 pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg);
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000409
410 /* Enable the DMA channel */
411 /* Do not access config register until channel shows as disabled */
412 while (readl(pl08x->base + PL080_EN_CHAN) & (1 << phychan->id))
413 cpu_relax();
414
415 /* Do not access config register until channel shows as inactive */
Tomasz Figad86ccea2013-08-11 19:59:14 +0200416 val = readl(phychan->reg_config);
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000417 while ((val & PL080_CONFIG_ACTIVE) || (val & PL080_CONFIG_ENABLE))
Tomasz Figad86ccea2013-08-11 19:59:14 +0200418 val = readl(phychan->reg_config);
Russell King - ARM Linuxc885bee2011-01-03 22:38:52 +0000419
Tomasz Figad86ccea2013-08-11 19:59:14 +0200420 writel(val | PL080_CONFIG_ENABLE, phychan->reg_config);
Linus Walleije8689e62010-09-28 15:57:37 +0200421}
422
423/*
Russell King - ARM Linux81796612011-01-27 12:37:44 +0000424 * Pause the channel by setting the HALT bit.
Linus Walleije8689e62010-09-28 15:57:37 +0200425 *
Russell King - ARM Linux81796612011-01-27 12:37:44 +0000426 * For M->P transfers, pause the DMAC first and then stop the peripheral -
427 * the FIFO can only drain if the peripheral is still requesting data.
428 * (note: this can still timeout if the DMAC FIFO never drains of data.)
Linus Walleije8689e62010-09-28 15:57:37 +0200429 *
Russell King - ARM Linux81796612011-01-27 12:37:44 +0000430 * For P->M transfers, disable the peripheral first to stop it filling
431 * the DMAC FIFO, and then pause the DMAC.
Linus Walleije8689e62010-09-28 15:57:37 +0200432 */
433static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch)
434{
435 u32 val;
Russell King - ARM Linux81796612011-01-27 12:37:44 +0000436 int timeout;
Linus Walleije8689e62010-09-28 15:57:37 +0200437
438 /* Set the HALT bit and wait for the FIFO to drain */
Tomasz Figad86ccea2013-08-11 19:59:14 +0200439 val = readl(ch->reg_config);
Linus Walleije8689e62010-09-28 15:57:37 +0200440 val |= PL080_CONFIG_HALT;
Tomasz Figad86ccea2013-08-11 19:59:14 +0200441 writel(val, ch->reg_config);
Linus Walleije8689e62010-09-28 15:57:37 +0200442
443 /* Wait for channel inactive */
Russell King - ARM Linux81796612011-01-27 12:37:44 +0000444 for (timeout = 1000; timeout; timeout--) {
445 if (!pl08x_phy_channel_busy(ch))
446 break;
447 udelay(1);
448 }
449 if (pl08x_phy_channel_busy(ch))
450 pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id);
Linus Walleije8689e62010-09-28 15:57:37 +0200451}
452
453static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch)
454{
455 u32 val;
456
457 /* Clear the HALT bit */
Tomasz Figad86ccea2013-08-11 19:59:14 +0200458 val = readl(ch->reg_config);
Linus Walleije8689e62010-09-28 15:57:37 +0200459 val &= ~PL080_CONFIG_HALT;
Tomasz Figad86ccea2013-08-11 19:59:14 +0200460 writel(val, ch->reg_config);
Linus Walleije8689e62010-09-28 15:57:37 +0200461}
462
Russell King - ARM Linuxfb526212011-01-27 12:32:53 +0000463/*
464 * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and
465 * clears any pending interrupt status. This should not be used for
466 * an on-going transfer, but as a method of shutting down a channel
467 * (eg, when it's no longer used) or terminating a transfer.
468 */
469static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x,
470 struct pl08x_phy_chan *ch)
Linus Walleije8689e62010-09-28 15:57:37 +0200471{
Tomasz Figad86ccea2013-08-11 19:59:14 +0200472 u32 val = readl(ch->reg_config);
Linus Walleije8689e62010-09-28 15:57:37 +0200473
Russell King - ARM Linuxfb526212011-01-27 12:32:53 +0000474 val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK |
475 PL080_CONFIG_TC_IRQ_MASK);
Linus Walleije8689e62010-09-28 15:57:37 +0200476
Tomasz Figad86ccea2013-08-11 19:59:14 +0200477 writel(val, ch->reg_config);
Russell King - ARM Linuxfb526212011-01-27 12:32:53 +0000478
479 writel(1 << ch->id, pl08x->base + PL080_ERR_CLEAR);
480 writel(1 << ch->id, pl08x->base + PL080_TC_CLEAR);
Linus Walleije8689e62010-09-28 15:57:37 +0200481}
482
483static inline u32 get_bytes_in_cctl(u32 cctl)
484{
485 /* The source width defines the number of bytes */
486 u32 bytes = cctl & PL080_CONTROL_TRANSFER_SIZE_MASK;
487
Alban Bedelf3287a52013-08-11 19:59:19 +0200488 cctl &= PL080_CONTROL_SWIDTH_MASK;
489
Linus Walleije8689e62010-09-28 15:57:37 +0200490 switch (cctl >> PL080_CONTROL_SWIDTH_SHIFT) {
491 case PL080_WIDTH_8BIT:
492 break;
493 case PL080_WIDTH_16BIT:
494 bytes *= 2;
495 break;
496 case PL080_WIDTH_32BIT:
497 bytes *= 4;
498 break;
499 }
500 return bytes;
501}
502
Tomasz Figada1b6c02013-08-11 19:59:17 +0200503static inline u32 get_bytes_in_cctl_pl080s(u32 cctl, u32 cctl1)
504{
505 /* The source width defines the number of bytes */
506 u32 bytes = cctl1 & PL080S_CONTROL_TRANSFER_SIZE_MASK;
507
Alban Bedelf3287a52013-08-11 19:59:19 +0200508 cctl &= PL080_CONTROL_SWIDTH_MASK;
509
Linus Walleije8689e62010-09-28 15:57:37 +0200510 switch (cctl >> PL080_CONTROL_SWIDTH_SHIFT) {
511 case PL080_WIDTH_8BIT:
512 break;
513 case PL080_WIDTH_16BIT:
514 bytes *= 2;
515 break;
516 case PL080_WIDTH_32BIT:
517 bytes *= 4;
518 break;
519 }
520 return bytes;
521}
522
523/* The channel should be paused when calling this */
524static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan)
525{
Tomasz Figaba6785f2013-08-11 19:59:15 +0200526 struct pl08x_driver_data *pl08x = plchan->host;
527 const u32 *llis_va, *llis_va_limit;
Linus Walleije8689e62010-09-28 15:57:37 +0200528 struct pl08x_phy_chan *ch;
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200529 dma_addr_t llis_bus;
Linus Walleije8689e62010-09-28 15:57:37 +0200530 struct pl08x_txd *txd;
Tomasz Figaba6785f2013-08-11 19:59:15 +0200531 u32 llis_max_words;
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200532 size_t bytes;
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200533 u32 clli;
Linus Walleije8689e62010-09-28 15:57:37 +0200534
Linus Walleije8689e62010-09-28 15:57:37 +0200535 ch = plchan->phychan;
536 txd = plchan->at;
537
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200538 if (!ch || !txd)
539 return 0;
540
Linus Walleije8689e62010-09-28 15:57:37 +0200541 /*
Russell King - ARM Linuxdb9f1362011-01-03 22:38:32 +0000542 * Follow the LLIs to get the number of remaining
543 * bytes in the currently active transaction.
Linus Walleije8689e62010-09-28 15:57:37 +0200544 */
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200545 clli = readl(ch->base + PL080_CH_LLI) & ~PL080_LLI_LM_AHB2;
Linus Walleije8689e62010-09-28 15:57:37 +0200546
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200547 /* First get the remaining bytes in the active transfer */
Tomasz Figada1b6c02013-08-11 19:59:17 +0200548 if (pl08x->vd->pl080s)
549 bytes = get_bytes_in_cctl_pl080s(
550 readl(ch->base + PL080_CH_CONTROL),
551 readl(ch->base + PL080S_CH_CONTROL2));
552 else
Linus Walleije8689e62010-09-28 15:57:37 +0200553 bytes = get_bytes_in_cctl(readl(ch->base + PL080_CH_CONTROL));
554
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200555 if (!clli)
556 return bytes;
Linus Walleije8689e62010-09-28 15:57:37 +0200557
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200558 llis_va = txd->llis_va;
559 llis_bus = txd->llis_bus;
Linus Walleije8689e62010-09-28 15:57:37 +0200560
Tomasz Figaba6785f2013-08-11 19:59:15 +0200561 llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS;
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200562 BUG_ON(clli < llis_bus || clli >= llis_bus +
Tomasz Figaba6785f2013-08-11 19:59:15 +0200563 sizeof(u32) * llis_max_words);
Russell King - ARM Linuxdb9f1362011-01-03 22:38:32 +0000564
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200565 /*
566 * Locate the next LLI - as this is an array,
567 * it's simple maths to find.
568 */
Tomasz Figaba6785f2013-08-11 19:59:15 +0200569 llis_va += (clli - llis_bus) / sizeof(u32);
Russell King - ARM Linuxdb9f1362011-01-03 22:38:32 +0000570
Tomasz Figaba6785f2013-08-11 19:59:15 +0200571 llis_va_limit = llis_va + llis_max_words;
572
573 for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) {
Tomasz Figada1b6c02013-08-11 19:59:17 +0200574 if (pl08x->vd->pl080s)
575 bytes += get_bytes_in_cctl_pl080s(
576 llis_va[PL080_LLI_CCTL],
577 llis_va[PL080S_LLI_CCTL2]);
578 else
579 bytes += get_bytes_in_cctl(llis_va[PL080_LLI_CCTL]);
Linus Walleije8689e62010-09-28 15:57:37 +0200580
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200581 /*
Alban Bedel3b24c202013-08-11 19:59:20 +0200582 * A LLI pointer going backward terminates the LLI list
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200583 */
Alban Bedel3b24c202013-08-11 19:59:20 +0200584 if (llis_va[PL080_LLI_LLI] <= clli)
Tomasz Figa68a7faa2013-08-11 19:59:13 +0200585 break;
Linus Walleije8689e62010-09-28 15:57:37 +0200586 }
587
Linus Walleije8689e62010-09-28 15:57:37 +0200588 return bytes;
589}
590
591/*
592 * Allocate a physical channel for a virtual channel
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +0000593 *
594 * Try to locate a physical channel to be used for this transfer. If all
595 * are taken return NULL and the requester will have to cope by using
596 * some fallback PIO mode or retrying later.
Linus Walleije8689e62010-09-28 15:57:37 +0200597 */
598static struct pl08x_phy_chan *
599pl08x_get_phy_channel(struct pl08x_driver_data *pl08x,
600 struct pl08x_dma_chan *virt_chan)
601{
602 struct pl08x_phy_chan *ch = NULL;
603 unsigned long flags;
604 int i;
605
Linus Walleije8689e62010-09-28 15:57:37 +0200606 for (i = 0; i < pl08x->vd->channels; i++) {
607 ch = &pl08x->phy_chans[i];
608
609 spin_lock_irqsave(&ch->lock, flags);
610
Linus Walleijaffa1152012-04-12 09:01:49 +0200611 if (!ch->locked && !ch->serving) {
Linus Walleije8689e62010-09-28 15:57:37 +0200612 ch->serving = virt_chan;
Linus Walleije8689e62010-09-28 15:57:37 +0200613 spin_unlock_irqrestore(&ch->lock, flags);
614 break;
615 }
616
617 spin_unlock_irqrestore(&ch->lock, flags);
618 }
619
620 if (i == pl08x->vd->channels) {
621 /* No physical channel available, cope with it */
622 return NULL;
623 }
624
625 return ch;
626}
627
Russell Kinga5a488d2012-05-26 13:54:15 +0100628/* Mark the physical channel as free. Note, this write is atomic. */
Linus Walleije8689e62010-09-28 15:57:37 +0200629static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x,
630 struct pl08x_phy_chan *ch)
631{
Linus Walleije8689e62010-09-28 15:57:37 +0200632 ch->serving = NULL;
Russell Kinga5a488d2012-05-26 13:54:15 +0100633}
634
635/*
636 * Try to allocate a physical channel. When successful, assign it to
637 * this virtual channel, and initiate the next descriptor. The
638 * virtual channel lock must be held at this point.
639 */
640static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan)
641{
642 struct pl08x_driver_data *pl08x = plchan->host;
643 struct pl08x_phy_chan *ch;
644
645 ch = pl08x_get_phy_channel(pl08x, plchan);
646 if (!ch) {
647 dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name);
648 plchan->state = PL08X_CHAN_WAITING;
649 return;
650 }
651
652 dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n",
653 ch->id, plchan->name);
654
655 plchan->phychan = ch;
656 plchan->state = PL08X_CHAN_RUNNING;
657 pl08x_start_next_txd(plchan);
658}
659
660static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch,
661 struct pl08x_dma_chan *plchan)
662{
663 struct pl08x_driver_data *pl08x = plchan->host;
664
665 dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n",
666 ch->id, plchan->name);
667
668 /*
669 * We do this without taking the lock; we're really only concerned
670 * about whether this pointer is NULL or not, and we're guaranteed
671 * that this will only be called when it _already_ is non-NULL.
672 */
673 ch->serving = plchan;
674 plchan->phychan = ch;
675 plchan->state = PL08X_CHAN_RUNNING;
676 pl08x_start_next_txd(plchan);
677}
678
679/*
680 * Free a physical DMA channel, potentially reallocating it to another
681 * virtual channel if we have any pending.
682 */
683static void pl08x_phy_free(struct pl08x_dma_chan *plchan)
684{
685 struct pl08x_driver_data *pl08x = plchan->host;
686 struct pl08x_dma_chan *p, *next;
687
688 retry:
689 next = NULL;
690
691 /* Find a waiting virtual channel for the next transfer. */
Russell King01d8dc62012-05-26 14:04:29 +0100692 list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node)
Russell Kinga5a488d2012-05-26 13:54:15 +0100693 if (p->state == PL08X_CHAN_WAITING) {
694 next = p;
695 break;
696 }
697
698 if (!next) {
Russell King01d8dc62012-05-26 14:04:29 +0100699 list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node)
Russell Kinga5a488d2012-05-26 13:54:15 +0100700 if (p->state == PL08X_CHAN_WAITING) {
701 next = p;
702 break;
703 }
704 }
705
706 /* Ensure that the physical channel is stopped */
707 pl08x_terminate_phy_chan(pl08x, plchan->phychan);
708
709 if (next) {
710 bool success;
711
712 /*
713 * Eww. We know this isn't going to deadlock
714 * but lockdep probably doesn't.
715 */
Russell King083be282012-05-26 14:09:53 +0100716 spin_lock(&next->vc.lock);
Russell Kinga5a488d2012-05-26 13:54:15 +0100717 /* Re-check the state now that we have the lock */
718 success = next->state == PL08X_CHAN_WAITING;
719 if (success)
720 pl08x_phy_reassign_start(plchan->phychan, next);
Russell King083be282012-05-26 14:09:53 +0100721 spin_unlock(&next->vc.lock);
Russell Kinga5a488d2012-05-26 13:54:15 +0100722
723 /* If the state changed, try to find another channel */
724 if (!success)
725 goto retry;
726 } else {
727 /* No more jobs, so free up the physical channel */
728 pl08x_put_phy_channel(pl08x, plchan->phychan);
729 }
730
731 plchan->phychan = NULL;
732 plchan->state = PL08X_CHAN_IDLE;
Linus Walleije8689e62010-09-28 15:57:37 +0200733}
734
735/*
736 * LLI handling
737 */
738
739static inline unsigned int pl08x_get_bytes_for_cctl(unsigned int coded)
740{
741 switch (coded) {
742 case PL080_WIDTH_8BIT:
743 return 1;
744 case PL080_WIDTH_16BIT:
745 return 2;
746 case PL080_WIDTH_32BIT:
747 return 4;
748 default:
749 break;
750 }
751 BUG();
752 return 0;
753}
754
755static inline u32 pl08x_cctl_bits(u32 cctl, u8 srcwidth, u8 dstwidth,
Russell King - ARM Linuxcace6582011-01-03 22:37:31 +0000756 size_t tsize)
Linus Walleije8689e62010-09-28 15:57:37 +0200757{
758 u32 retbits = cctl;
759
Russell King - ARM Linuxe8b5e112011-01-03 22:30:24 +0000760 /* Remove all src, dst and transfer size bits */
Linus Walleije8689e62010-09-28 15:57:37 +0200761 retbits &= ~PL080_CONTROL_DWIDTH_MASK;
762 retbits &= ~PL080_CONTROL_SWIDTH_MASK;
763 retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK;
764
765 /* Then set the bits according to the parameters */
766 switch (srcwidth) {
767 case 1:
768 retbits |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT;
769 break;
770 case 2:
771 retbits |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT;
772 break;
773 case 4:
774 retbits |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT;
775 break;
776 default:
777 BUG();
778 break;
779 }
780
781 switch (dstwidth) {
782 case 1:
783 retbits |= PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT;
784 break;
785 case 2:
786 retbits |= PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT;
787 break;
788 case 4:
789 retbits |= PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT;
790 break;
791 default:
792 BUG();
793 break;
794 }
795
Tomasz Figa5110e512013-08-11 19:59:18 +0200796 tsize &= PL080_CONTROL_TRANSFER_SIZE_MASK;
Linus Walleije8689e62010-09-28 15:57:37 +0200797 retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT;
798 return retbits;
799}
800
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000801struct pl08x_lli_build_data {
802 struct pl08x_txd *txd;
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000803 struct pl08x_bus_data srcbus;
804 struct pl08x_bus_data dstbus;
805 size_t remainder;
Russell King - ARM Linux25c94f72011-07-21 17:11:46 +0100806 u32 lli_bus;
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000807};
808
Linus Walleije8689e62010-09-28 15:57:37 +0200809/*
Viresh Kumar0532e6f2011-08-05 15:32:31 +0530810 * Autoselect a master bus to use for the transfer. Slave will be the chosen as
811 * victim in case src & dest are not similarly aligned. i.e. If after aligning
812 * masters address with width requirements of transfer (by sending few byte by
813 * byte data), slave is still not aligned, then its width will be reduced to
814 * BYTE.
815 * - prefers the destination bus if both available
Viresh Kumar036f05f2011-08-05 15:32:41 +0530816 * - prefers bus with fixed address (i.e. peripheral)
Linus Walleije8689e62010-09-28 15:57:37 +0200817 */
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000818static void pl08x_choose_master_bus(struct pl08x_lli_build_data *bd,
819 struct pl08x_bus_data **mbus, struct pl08x_bus_data **sbus, u32 cctl)
Linus Walleije8689e62010-09-28 15:57:37 +0200820{
821 if (!(cctl & PL080_CONTROL_DST_INCR)) {
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000822 *mbus = &bd->dstbus;
823 *sbus = &bd->srcbus;
Viresh Kumar036f05f2011-08-05 15:32:41 +0530824 } else if (!(cctl & PL080_CONTROL_SRC_INCR)) {
825 *mbus = &bd->srcbus;
826 *sbus = &bd->dstbus;
Linus Walleije8689e62010-09-28 15:57:37 +0200827 } else {
Viresh Kumar036f05f2011-08-05 15:32:41 +0530828 if (bd->dstbus.buswidth >= bd->srcbus.buswidth) {
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000829 *mbus = &bd->dstbus;
830 *sbus = &bd->srcbus;
Linus Walleije8689e62010-09-28 15:57:37 +0200831 } else {
Viresh Kumar036f05f2011-08-05 15:32:41 +0530832 *mbus = &bd->srcbus;
833 *sbus = &bd->dstbus;
Linus Walleije8689e62010-09-28 15:57:37 +0200834 }
835 }
836}
837
838/*
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +0000839 * Fills in one LLI for a certain transfer descriptor and advance the counter
Linus Walleije8689e62010-09-28 15:57:37 +0200840 */
Tomasz Figaba6785f2013-08-11 19:59:15 +0200841static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x,
842 struct pl08x_lli_build_data *bd,
Tomasz Figada1b6c02013-08-11 19:59:17 +0200843 int num_llis, int len, u32 cctl, u32 cctl2)
Linus Walleije8689e62010-09-28 15:57:37 +0200844{
Tomasz Figaba6785f2013-08-11 19:59:15 +0200845 u32 offset = num_llis * pl08x->lli_words;
846 u32 *llis_va = bd->txd->llis_va + offset;
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000847 dma_addr_t llis_bus = bd->txd->llis_bus;
Linus Walleije8689e62010-09-28 15:57:37 +0200848
849 BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS);
850
Tomasz Figaba6785f2013-08-11 19:59:15 +0200851 /* Advance the offset to next LLI. */
852 offset += pl08x->lli_words;
853
854 llis_va[PL080_LLI_SRC] = bd->srcbus.addr;
855 llis_va[PL080_LLI_DST] = bd->dstbus.addr;
856 llis_va[PL080_LLI_LLI] = (llis_bus + sizeof(u32) * offset);
857 llis_va[PL080_LLI_LLI] |= bd->lli_bus;
858 llis_va[PL080_LLI_CCTL] = cctl;
Tomasz Figada1b6c02013-08-11 19:59:17 +0200859 if (pl08x->vd->pl080s)
860 llis_va[PL080S_LLI_CCTL2] = cctl2;
Linus Walleije8689e62010-09-28 15:57:37 +0200861
862 if (cctl & PL080_CONTROL_SRC_INCR)
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000863 bd->srcbus.addr += len;
Linus Walleije8689e62010-09-28 15:57:37 +0200864 if (cctl & PL080_CONTROL_DST_INCR)
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000865 bd->dstbus.addr += len;
Linus Walleije8689e62010-09-28 15:57:37 +0200866
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000867 BUG_ON(bd->remainder < len);
Russell King - ARM Linuxcace6582011-01-03 22:37:31 +0000868
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000869 bd->remainder -= len;
Linus Walleije8689e62010-09-28 15:57:37 +0200870}
871
Tomasz Figaba6785f2013-08-11 19:59:15 +0200872static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x,
873 struct pl08x_lli_build_data *bd, u32 *cctl, u32 len,
874 int num_llis, size_t *total_bytes)
Linus Walleije8689e62010-09-28 15:57:37 +0200875{
Viresh Kumar03af5002011-08-05 15:32:39 +0530876 *cctl = pl08x_cctl_bits(*cctl, 1, 1, len);
Tomasz Figada1b6c02013-08-11 19:59:17 +0200877 pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len);
Viresh Kumar03af5002011-08-05 15:32:39 +0530878 (*total_bytes) += len;
Linus Walleije8689e62010-09-28 15:57:37 +0200879}
880
Tomasz Figa48924e42013-08-11 19:59:16 +0200881#ifdef VERBOSE_DEBUG
882static void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
883 const u32 *llis_va, int num_llis)
884{
885 int i;
886
Tomasz Figada1b6c02013-08-11 19:59:17 +0200887 if (pl08x->vd->pl080s) {
Tomasz Figa48924e42013-08-11 19:59:16 +0200888 dev_vdbg(&pl08x->adev->dev,
Tomasz Figada1b6c02013-08-11 19:59:17 +0200889 "%-3s %-9s %-10s %-10s %-10s %-10s %s\n",
890 "lli", "", "csrc", "cdst", "clli", "cctl", "cctl2");
891 for (i = 0; i < num_llis; i++) {
892 dev_vdbg(&pl08x->adev->dev,
893 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
894 i, llis_va, llis_va[PL080_LLI_SRC],
895 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
896 llis_va[PL080_LLI_CCTL],
897 llis_va[PL080S_LLI_CCTL2]);
898 llis_va += pl08x->lli_words;
899 }
900 } else {
901 dev_vdbg(&pl08x->adev->dev,
902 "%-3s %-9s %-10s %-10s %-10s %s\n",
903 "lli", "", "csrc", "cdst", "clli", "cctl");
904 for (i = 0; i < num_llis; i++) {
905 dev_vdbg(&pl08x->adev->dev,
906 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n",
907 i, llis_va, llis_va[PL080_LLI_SRC],
908 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI],
909 llis_va[PL080_LLI_CCTL]);
910 llis_va += pl08x->lli_words;
911 }
Tomasz Figa48924e42013-08-11 19:59:16 +0200912 }
913}
914#else
915static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x,
916 const u32 *llis_va, int num_llis) {}
917#endif
918
Linus Walleije8689e62010-09-28 15:57:37 +0200919/*
920 * This fills in the table of LLIs for the transfer descriptor
921 * Note that we assume we never have to change the burst sizes
922 * Return 0 for error
923 */
924static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,
925 struct pl08x_txd *txd)
926{
Linus Walleije8689e62010-09-28 15:57:37 +0200927 struct pl08x_bus_data *mbus, *sbus;
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000928 struct pl08x_lli_build_data bd;
Linus Walleije8689e62010-09-28 15:57:37 +0200929 int num_llis = 0;
Viresh Kumar03af5002011-08-05 15:32:39 +0530930 u32 cctl, early_bytes = 0;
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530931 size_t max_bytes_per_lli, total_bytes;
Tomasz Figaba6785f2013-08-11 19:59:15 +0200932 u32 *llis_va, *last_lli;
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530933 struct pl08x_sg *dsg;
Linus Walleije8689e62010-09-28 15:57:37 +0200934
Viresh Kumar3e27ee82011-08-05 15:32:27 +0530935 txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus);
Linus Walleije8689e62010-09-28 15:57:37 +0200936 if (!txd->llis_va) {
937 dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__);
938 return 0;
939 }
940
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000941 bd.txd = txd;
Russell King - ARM Linux25c94f72011-07-21 17:11:46 +0100942 bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0;
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530943 cctl = txd->cctl;
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000944
Linus Walleije8689e62010-09-28 15:57:37 +0200945 /* Find maximum width of the source bus */
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000946 bd.srcbus.maxwidth =
Linus Walleije8689e62010-09-28 15:57:37 +0200947 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_SWIDTH_MASK) >>
948 PL080_CONTROL_SWIDTH_SHIFT);
949
950 /* Find maximum width of the destination bus */
Russell King - ARM Linux542361f2011-01-03 22:43:15 +0000951 bd.dstbus.maxwidth =
Linus Walleije8689e62010-09-28 15:57:37 +0200952 pl08x_get_bytes_for_cctl((cctl & PL080_CONTROL_DWIDTH_MASK) >>
953 PL080_CONTROL_DWIDTH_SHIFT);
954
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530955 list_for_each_entry(dsg, &txd->dsg_list, node) {
956 total_bytes = 0;
957 cctl = txd->cctl;
Linus Walleije8689e62010-09-28 15:57:37 +0200958
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530959 bd.srcbus.addr = dsg->src_addr;
960 bd.dstbus.addr = dsg->dst_addr;
961 bd.remainder = dsg->len;
962 bd.srcbus.buswidth = bd.srcbus.maxwidth;
963 bd.dstbus.buswidth = bd.dstbus.maxwidth;
Linus Walleije8689e62010-09-28 15:57:37 +0200964
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530965 pl08x_choose_master_bus(&bd, &mbus, &sbus, cctl);
Linus Walleije8689e62010-09-28 15:57:37 +0200966
Andre Przywarab90ca062013-08-14 14:52:09 +0200967 dev_vdbg(&pl08x->adev->dev,
968 "src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n",
969 (u64)bd.srcbus.addr,
970 cctl & PL080_CONTROL_SRC_INCR ? "+" : "",
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530971 bd.srcbus.buswidth,
Andre Przywarab90ca062013-08-14 14:52:09 +0200972 (u64)bd.dstbus.addr,
973 cctl & PL080_CONTROL_DST_INCR ? "+" : "",
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530974 bd.dstbus.buswidth,
975 bd.remainder);
976 dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n",
977 mbus == &bd.srcbus ? "src" : "dst",
978 sbus == &bd.srcbus ? "src" : "dst");
Russell King - ARM Linuxfc74eb72011-07-21 17:12:06 +0100979
Viresh Kumarb7f69d92011-08-05 15:32:43 +0530980 /*
981 * Zero length is only allowed if all these requirements are
982 * met:
983 * - flow controller is peripheral.
984 * - src.addr is aligned to src.width
985 * - dst.addr is aligned to dst.width
986 *
987 * sg_len == 1 should be true, as there can be two cases here:
988 *
989 * - Memory addresses are contiguous and are not scattered.
990 * Here, Only one sg will be passed by user driver, with
991 * memory address and zero length. We pass this to controller
992 * and after the transfer it will receive the last burst
993 * request from peripheral and so transfer finishes.
994 *
995 * - Memory addresses are scattered and are not contiguous.
996 * Here, Obviously as DMA controller doesn't know when a lli's
997 * transfer gets over, it can't load next lli. So in this
998 * case, there has to be an assumption that only one lli is
999 * supported. Thus, we can't have scattered addresses.
1000 */
1001 if (!bd.remainder) {
1002 u32 fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >>
1003 PL080_CONFIG_FLOW_CONTROL_SHIFT;
1004 if (!((fc >= PL080_FLOW_SRC2DST_DST) &&
Viresh Kumar0a235652011-08-05 15:32:42 +05301005 (fc <= PL080_FLOW_SRC2DST_SRC))) {
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301006 dev_err(&pl08x->adev->dev, "%s sg len can't be zero",
1007 __func__);
1008 return 0;
1009 }
Linus Walleije8689e62010-09-28 15:57:37 +02001010
Andre Przywara1c38b282013-08-19 12:19:28 +02001011 if (!IS_BUS_ALIGNED(&bd.srcbus) ||
1012 !IS_BUS_ALIGNED(&bd.dstbus)) {
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301013 dev_err(&pl08x->adev->dev,
1014 "%s src & dst address must be aligned to src"
1015 " & dst width if peripheral is flow controller",
1016 __func__);
1017 return 0;
1018 }
Linus Walleije8689e62010-09-28 15:57:37 +02001019
Viresh Kumar16a2e7d2011-08-05 15:32:37 +05301020 cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth,
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301021 bd.dstbus.buswidth, 0);
Tomasz Figaba6785f2013-08-11 19:59:15 +02001022 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,
Tomasz Figada1b6c02013-08-11 19:59:17 +02001023 0, cctl, 0);
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301024 break;
Linus Walleije8689e62010-09-28 15:57:37 +02001025 }
1026
1027 /*
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301028 * Send byte by byte for following cases
1029 * - Less than a bus width available
1030 * - until master bus is aligned
Linus Walleije8689e62010-09-28 15:57:37 +02001031 */
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301032 if (bd.remainder < mbus->buswidth)
1033 early_bytes = bd.remainder;
Andre Przywara1c38b282013-08-19 12:19:28 +02001034 else if (!IS_BUS_ALIGNED(mbus)) {
1035 early_bytes = mbus->buswidth -
1036 (mbus->addr & (mbus->buswidth - 1));
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301037 if ((bd.remainder - early_bytes) < mbus->buswidth)
1038 early_bytes = bd.remainder;
Linus Walleije8689e62010-09-28 15:57:37 +02001039 }
Viresh Kumar16a2e7d2011-08-05 15:32:37 +05301040
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301041 if (early_bytes) {
1042 dev_vdbg(&pl08x->adev->dev,
Mark Brown6fc8ae72014-08-01 18:09:48 +01001043 "%s byte width LLIs (remain 0x%08zx)\n",
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301044 __func__, bd.remainder);
Tomasz Figaba6785f2013-08-11 19:59:15 +02001045 prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes,
1046 num_llis++, &total_bytes);
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301047 }
Linus Walleije8689e62010-09-28 15:57:37 +02001048
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301049 if (bd.remainder) {
1050 /*
1051 * Master now aligned
1052 * - if slave is not then we must set its width down
1053 */
Andre Przywara1c38b282013-08-19 12:19:28 +02001054 if (!IS_BUS_ALIGNED(sbus)) {
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301055 dev_dbg(&pl08x->adev->dev,
1056 "%s set down bus width to one byte\n",
1057 __func__);
1058
1059 sbus->buswidth = 1;
1060 }
1061
1062 /*
1063 * Bytes transferred = tsize * src width, not
1064 * MIN(buswidths)
1065 */
1066 max_bytes_per_lli = bd.srcbus.buswidth *
Tomasz Figa5110e512013-08-11 19:59:18 +02001067 pl08x->vd->max_transfer_size;
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301068 dev_vdbg(&pl08x->adev->dev,
1069 "%s max bytes per lli = %zu\n",
1070 __func__, max_bytes_per_lli);
1071
1072 /*
1073 * Make largest possible LLIs until less than one bus
1074 * width left
1075 */
1076 while (bd.remainder > (mbus->buswidth - 1)) {
1077 size_t lli_len, tsize, width;
1078
1079 /*
1080 * If enough left try to send max possible,
1081 * otherwise try to send the remainder
1082 */
1083 lli_len = min(bd.remainder, max_bytes_per_lli);
1084
1085 /*
1086 * Check against maximum bus alignment:
1087 * Calculate actual transfer size in relation to
1088 * bus width an get a maximum remainder of the
1089 * highest bus width - 1
1090 */
1091 width = max(mbus->buswidth, sbus->buswidth);
1092 lli_len = (lli_len / width) * width;
1093 tsize = lli_len / bd.srcbus.buswidth;
1094
1095 dev_vdbg(&pl08x->adev->dev,
1096 "%s fill lli with single lli chunk of "
1097 "size 0x%08zx (remainder 0x%08zx)\n",
1098 __func__, lli_len, bd.remainder);
1099
1100 cctl = pl08x_cctl_bits(cctl, bd.srcbus.buswidth,
1101 bd.dstbus.buswidth, tsize);
Tomasz Figaba6785f2013-08-11 19:59:15 +02001102 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,
Tomasz Figada1b6c02013-08-11 19:59:17 +02001103 lli_len, cctl, tsize);
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301104 total_bytes += lli_len;
1105 }
1106
1107 /*
1108 * Send any odd bytes
1109 */
1110 if (bd.remainder) {
1111 dev_vdbg(&pl08x->adev->dev,
1112 "%s align with boundary, send odd bytes (remain %zu)\n",
1113 __func__, bd.remainder);
Tomasz Figaba6785f2013-08-11 19:59:15 +02001114 prep_byte_width_lli(pl08x, &bd, &cctl,
1115 bd.remainder, num_llis++, &total_bytes);
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301116 }
1117 }
1118
1119 if (total_bytes != dsg->len) {
1120 dev_err(&pl08x->adev->dev,
1121 "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n",
1122 __func__, total_bytes, dsg->len);
1123 return 0;
1124 }
1125
1126 if (num_llis >= MAX_NUM_TSFR_LLIS) {
1127 dev_err(&pl08x->adev->dev,
1128 "%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n",
Tomasz Figaba6785f2013-08-11 19:59:15 +02001129 __func__, MAX_NUM_TSFR_LLIS);
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301130 return 0;
1131 }
Linus Walleije8689e62010-09-28 15:57:37 +02001132 }
Linus Walleije8689e62010-09-28 15:57:37 +02001133
Russell King - ARM Linuxb58b6b52011-01-03 22:34:48 +00001134 llis_va = txd->llis_va;
Tomasz Figaba6785f2013-08-11 19:59:15 +02001135 last_lli = llis_va + (num_llis - 1) * pl08x->lli_words;
Linus Walleije8689e62010-09-28 15:57:37 +02001136
Alban Bedel3b24c202013-08-11 19:59:20 +02001137 if (txd->cyclic) {
1138 /* Link back to the first LLI. */
1139 last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus;
1140 } else {
1141 /* The final LLI terminates the LLI. */
1142 last_lli[PL080_LLI_LLI] = 0;
1143 /* The final LLI element shall also fire an interrupt. */
1144 last_lli[PL080_LLI_CCTL] |= PL080_CONTROL_TC_IRQ_EN;
Linus Walleije8689e62010-09-28 15:57:37 +02001145 }
Linus Walleije8689e62010-09-28 15:57:37 +02001146
Tomasz Figa48924e42013-08-11 19:59:16 +02001147 pl08x_dump_lli(pl08x, llis_va, num_llis);
Linus Walleije8689e62010-09-28 15:57:37 +02001148
1149 return num_llis;
1150}
1151
Linus Walleije8689e62010-09-28 15:57:37 +02001152static void pl08x_free_txd(struct pl08x_driver_data *pl08x,
1153 struct pl08x_txd *txd)
1154{
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301155 struct pl08x_sg *dsg, *_dsg;
1156
Viresh Kumarc1205642011-08-05 15:32:44 +05301157 if (txd->llis_va)
1158 dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus);
Linus Walleije8689e62010-09-28 15:57:37 +02001159
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301160 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
1161 list_del(&dsg->node);
1162 kfree(dsg);
1163 }
1164
Linus Walleije8689e62010-09-28 15:57:37 +02001165 kfree(txd);
1166}
1167
Russell King18536132012-05-26 14:42:23 +01001168static void pl08x_desc_free(struct virt_dma_desc *vd)
1169{
1170 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
1171 struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan);
Russell King18536132012-05-26 14:42:23 +01001172
Russell King - ARM Linux89116bf2013-11-26 16:03:03 -08001173 dma_descriptor_unmap(&vd->tx);
Russell King18536132012-05-26 14:42:23 +01001174 if (!txd->done)
1175 pl08x_release_mux(plchan);
1176
Russell King18536132012-05-26 14:42:23 +01001177 pl08x_free_txd(plchan->host, txd);
Russell King18536132012-05-26 14:42:23 +01001178}
1179
Linus Walleije8689e62010-09-28 15:57:37 +02001180static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x,
1181 struct pl08x_dma_chan *plchan)
1182{
Russell Kingea160562012-05-25 13:10:36 +01001183 LIST_HEAD(head);
Linus Walleije8689e62010-09-28 15:57:37 +02001184
Russell King879f1272012-05-26 14:27:40 +01001185 vchan_get_all_descriptors(&plchan->vc, &head);
Akinobu Mita91998262012-10-28 00:49:31 +09001186 vchan_dma_desc_free_list(&plchan->vc, &head);
Linus Walleije8689e62010-09-28 15:57:37 +02001187}
1188
1189/*
1190 * The DMA ENGINE API
1191 */
1192static int pl08x_alloc_chan_resources(struct dma_chan *chan)
1193{
1194 return 0;
1195}
1196
1197static void pl08x_free_chan_resources(struct dma_chan *chan)
1198{
Russell Kinga0686822012-05-26 17:00:49 +01001199 /* Ensure all queued descriptors are freed */
1200 vchan_free_chan_resources(to_virt_chan(chan));
Linus Walleije8689e62010-09-28 15:57:37 +02001201}
1202
Linus Walleije8689e62010-09-28 15:57:37 +02001203static struct dma_async_tx_descriptor *pl08x_prep_dma_interrupt(
1204 struct dma_chan *chan, unsigned long flags)
1205{
1206 struct dma_async_tx_descriptor *retval = NULL;
1207
1208 return retval;
1209}
1210
1211/*
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +00001212 * Code accessing dma_async_is_complete() in a tight loop may give problems.
1213 * If slaves are relying on interrupts to signal completion this function
1214 * must not be called with interrupts disabled.
Linus Walleije8689e62010-09-28 15:57:37 +02001215 */
Viresh Kumar3e27ee82011-08-05 15:32:27 +05301216static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan,
1217 dma_cookie_t cookie, struct dma_tx_state *txstate)
Linus Walleije8689e62010-09-28 15:57:37 +02001218{
1219 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
Russell King06e885b2012-05-26 15:05:52 +01001220 struct virt_dma_desc *vd;
1221 unsigned long flags;
Linus Walleije8689e62010-09-28 15:57:37 +02001222 enum dma_status ret;
Russell King06e885b2012-05-26 15:05:52 +01001223 size_t bytes = 0;
Linus Walleije8689e62010-09-28 15:57:37 +02001224
Russell King - ARM Linux96a2af42012-03-06 22:35:27 +00001225 ret = dma_cookie_status(chan, cookie, txstate);
Vinod Koul0996e892013-10-16 13:33:02 +05301226 if (ret == DMA_COMPLETE)
Linus Walleije8689e62010-09-28 15:57:37 +02001227 return ret;
Linus Walleije8689e62010-09-28 15:57:37 +02001228
1229 /*
Russell King06e885b2012-05-26 15:05:52 +01001230 * There's no point calculating the residue if there's
1231 * no txstate to store the value.
1232 */
1233 if (!txstate) {
1234 if (plchan->state == PL08X_CHAN_PAUSED)
1235 ret = DMA_PAUSED;
1236 return ret;
1237 }
1238
1239 spin_lock_irqsave(&plchan->vc.lock, flags);
1240 ret = dma_cookie_status(chan, cookie, txstate);
Vinod Koul0996e892013-10-16 13:33:02 +05301241 if (ret != DMA_COMPLETE) {
Russell King06e885b2012-05-26 15:05:52 +01001242 vd = vchan_find_desc(&plchan->vc, cookie);
1243 if (vd) {
1244 /* On the issued list, so hasn't been processed yet */
1245 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx);
1246 struct pl08x_sg *dsg;
1247
1248 list_for_each_entry(dsg, &txd->dsg_list, node)
1249 bytes += dsg->len;
1250 } else {
1251 bytes = pl08x_getbytes_chan(plchan);
1252 }
1253 }
1254 spin_unlock_irqrestore(&plchan->vc.lock, flags);
1255
1256 /*
Linus Walleije8689e62010-09-28 15:57:37 +02001257 * This cookie not complete yet
Russell King - ARM Linux96a2af42012-03-06 22:35:27 +00001258 * Get number of bytes left in the active transactions and queue
Linus Walleije8689e62010-09-28 15:57:37 +02001259 */
Russell King06e885b2012-05-26 15:05:52 +01001260 dma_set_residue(txstate, bytes);
Linus Walleije8689e62010-09-28 15:57:37 +02001261
Russell King06e885b2012-05-26 15:05:52 +01001262 if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS)
1263 ret = DMA_PAUSED;
Linus Walleije8689e62010-09-28 15:57:37 +02001264
1265 /* Whether waiting or running, we're in progress */
Russell King06e885b2012-05-26 15:05:52 +01001266 return ret;
Linus Walleije8689e62010-09-28 15:57:37 +02001267}
1268
1269/* PrimeCell DMA extension */
1270struct burst_table {
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001271 u32 burstwords;
Linus Walleije8689e62010-09-28 15:57:37 +02001272 u32 reg;
1273};
1274
1275static const struct burst_table burst_sizes[] = {
1276 {
1277 .burstwords = 256,
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001278 .reg = PL080_BSIZE_256,
Linus Walleije8689e62010-09-28 15:57:37 +02001279 },
1280 {
1281 .burstwords = 128,
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001282 .reg = PL080_BSIZE_128,
Linus Walleije8689e62010-09-28 15:57:37 +02001283 },
1284 {
1285 .burstwords = 64,
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001286 .reg = PL080_BSIZE_64,
Linus Walleije8689e62010-09-28 15:57:37 +02001287 },
1288 {
1289 .burstwords = 32,
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001290 .reg = PL080_BSIZE_32,
Linus Walleije8689e62010-09-28 15:57:37 +02001291 },
1292 {
1293 .burstwords = 16,
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001294 .reg = PL080_BSIZE_16,
Linus Walleije8689e62010-09-28 15:57:37 +02001295 },
1296 {
1297 .burstwords = 8,
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001298 .reg = PL080_BSIZE_8,
Linus Walleije8689e62010-09-28 15:57:37 +02001299 },
1300 {
1301 .burstwords = 4,
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001302 .reg = PL080_BSIZE_4,
Linus Walleije8689e62010-09-28 15:57:37 +02001303 },
1304 {
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001305 .burstwords = 0,
1306 .reg = PL080_BSIZE_1,
Linus Walleije8689e62010-09-28 15:57:37 +02001307 },
1308};
1309
Russell King - ARM Linux121c8472011-07-21 17:13:48 +01001310/*
1311 * Given the source and destination available bus masks, select which
1312 * will be routed to each port. We try to have source and destination
1313 * on separate ports, but always respect the allowable settings.
1314 */
1315static u32 pl08x_select_bus(u8 src, u8 dst)
1316{
1317 u32 cctl = 0;
1318
1319 if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1)))
1320 cctl |= PL080_CONTROL_DST_AHB2;
1321 if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2)))
1322 cctl |= PL080_CONTROL_SRC_AHB2;
1323
1324 return cctl;
1325}
1326
Russell King - ARM Linuxf14c4262011-07-21 17:12:47 +01001327static u32 pl08x_cctl(u32 cctl)
1328{
1329 cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 |
1330 PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR |
1331 PL080_CONTROL_PROT_MASK);
1332
1333 /* Access the cell in privileged mode, non-bufferable, non-cacheable */
1334 return cctl | PL080_CONTROL_PROT_SYS;
1335}
1336
Russell King - ARM Linuxaa88cda2011-07-21 17:13:28 +01001337static u32 pl08x_width(enum dma_slave_buswidth width)
1338{
1339 switch (width) {
1340 case DMA_SLAVE_BUSWIDTH_1_BYTE:
1341 return PL080_WIDTH_8BIT;
1342 case DMA_SLAVE_BUSWIDTH_2_BYTES:
1343 return PL080_WIDTH_16BIT;
1344 case DMA_SLAVE_BUSWIDTH_4_BYTES:
1345 return PL080_WIDTH_32BIT;
Vinod Koulf32807f2011-07-25 19:22:01 +05301346 default:
1347 return ~0;
Russell King - ARM Linuxaa88cda2011-07-21 17:13:28 +01001348 }
Russell King - ARM Linuxaa88cda2011-07-21 17:13:28 +01001349}
1350
Russell King - ARM Linux760596c62011-07-21 17:14:08 +01001351static u32 pl08x_burst(u32 maxburst)
1352{
1353 int i;
1354
1355 for (i = 0; i < ARRAY_SIZE(burst_sizes); i++)
1356 if (burst_sizes[i].burstwords <= maxburst)
1357 break;
1358
1359 return burst_sizes[i].reg;
1360}
1361
Russell King9862ba12012-05-16 11:16:03 +01001362static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan,
1363 enum dma_slave_buswidth addr_width, u32 maxburst)
1364{
1365 u32 width, burst, cctl = 0;
1366
1367 width = pl08x_width(addr_width);
1368 if (width == ~0)
1369 return ~0;
1370
1371 cctl |= width << PL080_CONTROL_SWIDTH_SHIFT;
1372 cctl |= width << PL080_CONTROL_DWIDTH_SHIFT;
1373
1374 /*
1375 * If this channel will only request single transfers, set this
1376 * down to ONE element. Also select one element if no maxburst
1377 * is specified.
1378 */
1379 if (plchan->cd->single)
1380 maxburst = 1;
1381
1382 burst = pl08x_burst(maxburst);
1383 cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT;
1384 cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT;
1385
1386 return pl08x_cctl(cctl);
1387}
1388
Linus Walleije8689e62010-09-28 15:57:37 +02001389/*
1390 * Slave transactions callback to the slave device to allow
1391 * synchronization of slave DMA signals with the DMAC enable
1392 */
1393static void pl08x_issue_pending(struct dma_chan *chan)
1394{
1395 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
Linus Walleije8689e62010-09-28 15:57:37 +02001396 unsigned long flags;
1397
Russell King083be282012-05-26 14:09:53 +01001398 spin_lock_irqsave(&plchan->vc.lock, flags);
Russell King879f1272012-05-26 14:27:40 +01001399 if (vchan_issue_pending(&plchan->vc)) {
Russell Kinga5a488d2012-05-26 13:54:15 +01001400 if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING)
1401 pl08x_phy_alloc_and_start(plchan);
Linus Walleije8689e62010-09-28 15:57:37 +02001402 }
Russell King083be282012-05-26 14:09:53 +01001403 spin_unlock_irqrestore(&plchan->vc.lock, flags);
Linus Walleije8689e62010-09-28 15:57:37 +02001404}
1405
Russell King879f1272012-05-26 14:27:40 +01001406static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan)
Russell King - ARM Linuxac3cd202011-01-03 22:35:49 +00001407{
Viresh Kumarb201c112011-08-05 15:32:29 +05301408 struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
Russell King - ARM Linuxac3cd202011-01-03 22:35:49 +00001409
1410 if (txd) {
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301411 INIT_LIST_HEAD(&txd->dsg_list);
Russell King - ARM Linux4983a042011-01-03 22:39:33 +00001412
1413 /* Always enable error and terminal interrupts */
1414 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK |
1415 PL080_CONFIG_TC_IRQ_MASK;
Russell King - ARM Linuxac3cd202011-01-03 22:35:49 +00001416 }
1417 return txd;
1418}
1419
Linus Walleije8689e62010-09-28 15:57:37 +02001420/*
1421 * Initialize a descriptor to be used by memcpy submit
1422 */
1423static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy(
1424 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1425 size_t len, unsigned long flags)
1426{
1427 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1428 struct pl08x_driver_data *pl08x = plchan->host;
1429 struct pl08x_txd *txd;
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301430 struct pl08x_sg *dsg;
Linus Walleije8689e62010-09-28 15:57:37 +02001431 int ret;
1432
Russell King879f1272012-05-26 14:27:40 +01001433 txd = pl08x_get_txd(plchan);
Linus Walleije8689e62010-09-28 15:57:37 +02001434 if (!txd) {
1435 dev_err(&pl08x->adev->dev,
1436 "%s no memory for descriptor\n", __func__);
1437 return NULL;
1438 }
1439
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301440 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
1441 if (!dsg) {
1442 pl08x_free_txd(pl08x, txd);
1443 dev_err(&pl08x->adev->dev, "%s no memory for pl080 sg\n",
1444 __func__);
1445 return NULL;
1446 }
1447 list_add_tail(&dsg->node, &txd->dsg_list);
1448
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301449 dsg->src_addr = src;
1450 dsg->dst_addr = dest;
1451 dsg->len = len;
Linus Walleije8689e62010-09-28 15:57:37 +02001452
1453 /* Set platform data for m2m */
Russell King - ARM Linux4983a042011-01-03 22:39:33 +00001454 txd->ccfg |= PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT;
Russell Kingdc8d5f82012-05-16 12:20:55 +01001455 txd->cctl = pl08x->pd->memcpy_channel.cctl_memcpy &
Russell King - ARM Linuxc7da9a52011-01-03 22:40:53 +00001456 ~(PL080_CONTROL_DST_AHB2 | PL080_CONTROL_SRC_AHB2);
Russell King - ARM Linux4983a042011-01-03 22:39:33 +00001457
Linus Walleije8689e62010-09-28 15:57:37 +02001458 /* Both to be incremented or the code will break */
Russell King - ARM Linux70b5ed62011-01-03 22:40:13 +00001459 txd->cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR;
Russell King - ARM Linuxc7da9a52011-01-03 22:40:53 +00001460
Russell King - ARM Linuxc7da9a52011-01-03 22:40:53 +00001461 if (pl08x->vd->dualmaster)
Russell King - ARM Linux121c8472011-07-21 17:13:48 +01001462 txd->cctl |= pl08x_select_bus(pl08x->mem_buses,
1463 pl08x->mem_buses);
Linus Walleije8689e62010-09-28 15:57:37 +02001464
Russell Kingaa4afb72012-05-26 15:43:00 +01001465 ret = pl08x_fill_llis_for_desc(plchan->host, txd);
1466 if (!ret) {
1467 pl08x_free_txd(pl08x, txd);
Linus Walleije8689e62010-09-28 15:57:37 +02001468 return NULL;
Russell Kingaa4afb72012-05-26 15:43:00 +01001469 }
Linus Walleije8689e62010-09-28 15:57:37 +02001470
Russell King879f1272012-05-26 14:27:40 +01001471 return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
Linus Walleije8689e62010-09-28 15:57:37 +02001472}
1473
Alban Bedel3b24c202013-08-11 19:59:20 +02001474static struct pl08x_txd *pl08x_init_txd(
1475 struct dma_chan *chan,
1476 enum dma_transfer_direction direction,
1477 dma_addr_t *slave_addr)
Linus Walleije8689e62010-09-28 15:57:37 +02001478{
1479 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1480 struct pl08x_driver_data *pl08x = plchan->host;
1481 struct pl08x_txd *txd;
Russell Kingdc8d5f82012-05-16 12:20:55 +01001482 enum dma_slave_buswidth addr_width;
Viresh Kumar0a235652011-08-05 15:32:42 +05301483 int ret, tmp;
Russell King409ec8d2012-05-16 11:08:43 +01001484 u8 src_buses, dst_buses;
Russell Kingdc8d5f82012-05-16 12:20:55 +01001485 u32 maxburst, cctl;
Linus Walleije8689e62010-09-28 15:57:37 +02001486
Russell King879f1272012-05-26 14:27:40 +01001487 txd = pl08x_get_txd(plchan);
Linus Walleije8689e62010-09-28 15:57:37 +02001488 if (!txd) {
1489 dev_err(&pl08x->adev->dev, "%s no txd\n", __func__);
1490 return NULL;
1491 }
1492
Linus Walleije8689e62010-09-28 15:57:37 +02001493 /*
1494 * Set up addresses, the PrimeCell configured address
1495 * will take precedence since this may configure the
1496 * channel target address dynamically at runtime.
1497 */
Vinod Kouldb8196d2011-10-13 22:34:23 +05301498 if (direction == DMA_MEM_TO_DEV) {
Russell Kingdc8d5f82012-05-16 12:20:55 +01001499 cctl = PL080_CONTROL_SRC_INCR;
Alban Bedel3b24c202013-08-11 19:59:20 +02001500 *slave_addr = plchan->cfg.dst_addr;
Russell Kingdc8d5f82012-05-16 12:20:55 +01001501 addr_width = plchan->cfg.dst_addr_width;
1502 maxburst = plchan->cfg.dst_maxburst;
Russell King409ec8d2012-05-16 11:08:43 +01001503 src_buses = pl08x->mem_buses;
1504 dst_buses = plchan->cd->periph_buses;
Vinod Kouldb8196d2011-10-13 22:34:23 +05301505 } else if (direction == DMA_DEV_TO_MEM) {
Russell Kingdc8d5f82012-05-16 12:20:55 +01001506 cctl = PL080_CONTROL_DST_INCR;
Alban Bedel3b24c202013-08-11 19:59:20 +02001507 *slave_addr = plchan->cfg.src_addr;
Russell Kingdc8d5f82012-05-16 12:20:55 +01001508 addr_width = plchan->cfg.src_addr_width;
1509 maxburst = plchan->cfg.src_maxburst;
Russell King409ec8d2012-05-16 11:08:43 +01001510 src_buses = plchan->cd->periph_buses;
1511 dst_buses = pl08x->mem_buses;
Linus Walleije8689e62010-09-28 15:57:37 +02001512 } else {
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301513 pl08x_free_txd(pl08x, txd);
Linus Walleije8689e62010-09-28 15:57:37 +02001514 dev_err(&pl08x->adev->dev,
1515 "%s direction unsupported\n", __func__);
1516 return NULL;
1517 }
Linus Walleije8689e62010-09-28 15:57:37 +02001518
Russell Kingdc8d5f82012-05-16 12:20:55 +01001519 cctl |= pl08x_get_cctl(plchan, addr_width, maxburst);
Russell King800d6832012-05-16 11:33:31 +01001520 if (cctl == ~0) {
1521 pl08x_free_txd(pl08x, txd);
1522 dev_err(&pl08x->adev->dev,
1523 "DMA slave configuration botched?\n");
1524 return NULL;
1525 }
1526
Russell King409ec8d2012-05-16 11:08:43 +01001527 txd->cctl = cctl | pl08x_select_bus(src_buses, dst_buses);
1528
Russell King95442b22012-05-16 11:05:09 +01001529 if (plchan->cfg.device_fc)
Vinod Kouldb8196d2011-10-13 22:34:23 +05301530 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER :
Viresh Kumar0a235652011-08-05 15:32:42 +05301531 PL080_FLOW_PER2MEM_PER;
1532 else
Vinod Kouldb8196d2011-10-13 22:34:23 +05301533 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER :
Viresh Kumar0a235652011-08-05 15:32:42 +05301534 PL080_FLOW_PER2MEM;
1535
1536 txd->ccfg |= tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT;
1537
Russell Kingc48d4962012-05-25 11:48:51 +01001538 ret = pl08x_request_mux(plchan);
1539 if (ret < 0) {
1540 pl08x_free_txd(pl08x, txd);
1541 dev_dbg(&pl08x->adev->dev,
1542 "unable to mux for transfer on %s due to platform restrictions\n",
1543 plchan->name);
1544 return NULL;
1545 }
1546
1547 dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n",
1548 plchan->signal, plchan->name);
1549
1550 /* Assign the flow control signal to this channel */
1551 if (direction == DMA_MEM_TO_DEV)
1552 txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT;
1553 else
1554 txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT;
1555
Alban Bedel3b24c202013-08-11 19:59:20 +02001556 return txd;
1557}
1558
1559static int pl08x_tx_add_sg(struct pl08x_txd *txd,
1560 enum dma_transfer_direction direction,
1561 dma_addr_t slave_addr,
1562 dma_addr_t buf_addr,
1563 unsigned int len)
1564{
1565 struct pl08x_sg *dsg;
1566
1567 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT);
1568 if (!dsg)
1569 return -ENOMEM;
1570
1571 list_add_tail(&dsg->node, &txd->dsg_list);
1572
1573 dsg->len = len;
1574 if (direction == DMA_MEM_TO_DEV) {
1575 dsg->src_addr = buf_addr;
1576 dsg->dst_addr = slave_addr;
1577 } else {
1578 dsg->src_addr = slave_addr;
1579 dsg->dst_addr = buf_addr;
1580 }
1581
1582 return 0;
1583}
1584
1585static struct dma_async_tx_descriptor *pl08x_prep_slave_sg(
1586 struct dma_chan *chan, struct scatterlist *sgl,
1587 unsigned int sg_len, enum dma_transfer_direction direction,
1588 unsigned long flags, void *context)
1589{
1590 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1591 struct pl08x_driver_data *pl08x = plchan->host;
1592 struct pl08x_txd *txd;
1593 struct scatterlist *sg;
1594 int ret, tmp;
1595 dma_addr_t slave_addr;
1596
1597 dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n",
1598 __func__, sg_dma_len(sgl), plchan->name);
1599
1600 txd = pl08x_init_txd(chan, direction, &slave_addr);
1601 if (!txd)
1602 return NULL;
1603
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301604 for_each_sg(sgl, sg, sg_len, tmp) {
Alban Bedel3b24c202013-08-11 19:59:20 +02001605 ret = pl08x_tx_add_sg(txd, direction, slave_addr,
1606 sg_dma_address(sg),
1607 sg_dma_len(sg));
1608 if (ret) {
Russell Kingc48d4962012-05-25 11:48:51 +01001609 pl08x_release_mux(plchan);
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301610 pl08x_free_txd(pl08x, txd);
1611 dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n",
1612 __func__);
1613 return NULL;
1614 }
Alban Bedel3b24c202013-08-11 19:59:20 +02001615 }
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301616
Alban Bedel3b24c202013-08-11 19:59:20 +02001617 ret = pl08x_fill_llis_for_desc(plchan->host, txd);
1618 if (!ret) {
1619 pl08x_release_mux(plchan);
1620 pl08x_free_txd(pl08x, txd);
1621 return NULL;
1622 }
1623
1624 return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
1625}
1626
1627static struct dma_async_tx_descriptor *pl08x_prep_dma_cyclic(
1628 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1629 size_t period_len, enum dma_transfer_direction direction,
Laurent Pinchart31c1e5a2014-08-01 12:20:10 +02001630 unsigned long flags)
Alban Bedel3b24c202013-08-11 19:59:20 +02001631{
1632 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1633 struct pl08x_driver_data *pl08x = plchan->host;
1634 struct pl08x_txd *txd;
1635 int ret, tmp;
1636 dma_addr_t slave_addr;
1637
1638 dev_dbg(&pl08x->adev->dev,
Mark Brown6fc8ae72014-08-01 18:09:48 +01001639 "%s prepare cyclic transaction of %zd/%zd bytes %s %s\n",
Alban Bedel3b24c202013-08-11 19:59:20 +02001640 __func__, period_len, buf_len,
1641 direction == DMA_MEM_TO_DEV ? "to" : "from",
1642 plchan->name);
1643
1644 txd = pl08x_init_txd(chan, direction, &slave_addr);
1645 if (!txd)
1646 return NULL;
1647
1648 txd->cyclic = true;
1649 txd->cctl |= PL080_CONTROL_TC_IRQ_EN;
1650 for (tmp = 0; tmp < buf_len; tmp += period_len) {
1651 ret = pl08x_tx_add_sg(txd, direction, slave_addr,
1652 buf_addr + tmp, period_len);
1653 if (ret) {
1654 pl08x_release_mux(plchan);
1655 pl08x_free_txd(pl08x, txd);
1656 return NULL;
Viresh Kumarb7f69d92011-08-05 15:32:43 +05301657 }
1658 }
1659
Russell Kingaa4afb72012-05-26 15:43:00 +01001660 ret = pl08x_fill_llis_for_desc(plchan->host, txd);
1661 if (!ret) {
1662 pl08x_release_mux(plchan);
1663 pl08x_free_txd(pl08x, txd);
Linus Walleije8689e62010-09-28 15:57:37 +02001664 return NULL;
Russell Kingaa4afb72012-05-26 15:43:00 +01001665 }
Linus Walleije8689e62010-09-28 15:57:37 +02001666
Russell King879f1272012-05-26 14:27:40 +01001667 return vchan_tx_prep(&plchan->vc, &txd->vd, flags);
Linus Walleije8689e62010-09-28 15:57:37 +02001668}
1669
Maxime Ripardbcd1b0b2014-11-17 14:42:05 +01001670static int pl08x_config(struct dma_chan *chan,
1671 struct dma_slave_config *config)
1672{
1673 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1674 struct pl08x_driver_data *pl08x = plchan->host;
1675
1676 if (!plchan->slave)
1677 return -EINVAL;
1678
1679 /* Reject definitely invalid configurations */
1680 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
1681 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
1682 return -EINVAL;
1683
1684 if (config->device_fc && pl08x->vd->pl080s) {
1685 dev_err(&pl08x->adev->dev,
1686 "%s: PL080S does not support peripheral flow control\n",
1687 __func__);
1688 return -EINVAL;
1689 }
1690
1691 plchan->cfg = *config;
1692
1693 return 0;
1694}
1695
1696static int pl08x_terminate_all(struct dma_chan *chan)
Linus Walleije8689e62010-09-28 15:57:37 +02001697{
1698 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1699 struct pl08x_driver_data *pl08x = plchan->host;
1700 unsigned long flags;
Linus Walleije8689e62010-09-28 15:57:37 +02001701
Maxime Ripardbcd1b0b2014-11-17 14:42:05 +01001702 spin_lock_irqsave(&plchan->vc.lock, flags);
1703 if (!plchan->phychan && !plchan->at) {
1704 spin_unlock_irqrestore(&plchan->vc.lock, flags);
1705 return 0;
Linus Walleije8689e62010-09-28 15:57:37 +02001706 }
1707
Maxime Ripardbcd1b0b2014-11-17 14:42:05 +01001708 plchan->state = PL08X_CHAN_IDLE;
1709
1710 if (plchan->phychan) {
1711 /*
1712 * Mark physical channel as free and free any slave
1713 * signal
1714 */
1715 pl08x_phy_free(plchan);
1716 }
1717 /* Dequeue jobs and free LLIs */
1718 if (plchan->at) {
1719 pl08x_desc_free(&plchan->at->vd);
1720 plchan->at = NULL;
1721 }
1722 /* Dequeue jobs not yet fired as well */
1723 pl08x_free_txd_list(pl08x, plchan);
1724
1725 spin_unlock_irqrestore(&plchan->vc.lock, flags);
1726
1727 return 0;
1728}
1729
1730static int pl08x_pause(struct dma_chan *chan)
1731{
1732 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1733 unsigned long flags;
1734
Linus Walleije8689e62010-09-28 15:57:37 +02001735 /*
1736 * Anything succeeds on channels with no physical allocation and
1737 * no queued transfers.
1738 */
Russell King083be282012-05-26 14:09:53 +01001739 spin_lock_irqsave(&plchan->vc.lock, flags);
Linus Walleije8689e62010-09-28 15:57:37 +02001740 if (!plchan->phychan && !plchan->at) {
Russell King083be282012-05-26 14:09:53 +01001741 spin_unlock_irqrestore(&plchan->vc.lock, flags);
Linus Walleije8689e62010-09-28 15:57:37 +02001742 return 0;
1743 }
1744
Maxime Ripardbcd1b0b2014-11-17 14:42:05 +01001745 pl08x_pause_phy_chan(plchan->phychan);
1746 plchan->state = PL08X_CHAN_PAUSED;
Linus Walleije8689e62010-09-28 15:57:37 +02001747
Russell King083be282012-05-26 14:09:53 +01001748 spin_unlock_irqrestore(&plchan->vc.lock, flags);
Linus Walleije8689e62010-09-28 15:57:37 +02001749
Maxime Ripardbcd1b0b2014-11-17 14:42:05 +01001750 return 0;
1751}
1752
1753static int pl08x_resume(struct dma_chan *chan)
1754{
1755 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan);
1756 unsigned long flags;
1757
1758 /*
1759 * Anything succeeds on channels with no physical allocation and
1760 * no queued transfers.
1761 */
1762 spin_lock_irqsave(&plchan->vc.lock, flags);
1763 if (!plchan->phychan && !plchan->at) {
1764 spin_unlock_irqrestore(&plchan->vc.lock, flags);
1765 return 0;
1766 }
1767
1768 pl08x_resume_phy_chan(plchan->phychan);
1769 plchan->state = PL08X_CHAN_RUNNING;
1770
1771 spin_unlock_irqrestore(&plchan->vc.lock, flags);
1772
1773 return 0;
Linus Walleije8689e62010-09-28 15:57:37 +02001774}
1775
1776bool pl08x_filter_id(struct dma_chan *chan, void *chan_id)
1777{
Russell King - ARM Linux7703eac2011-08-31 09:34:35 +01001778 struct pl08x_dma_chan *plchan;
Linus Walleije8689e62010-09-28 15:57:37 +02001779 char *name = chan_id;
1780
Russell King - ARM Linux7703eac2011-08-31 09:34:35 +01001781 /* Reject channels for devices not bound to this driver */
1782 if (chan->device->dev->driver != &pl08x_amba_driver.drv)
1783 return false;
1784
1785 plchan = to_pl08x_chan(chan);
1786
Linus Walleije8689e62010-09-28 15:57:37 +02001787 /* Check that the channel is not taken! */
1788 if (!strcmp(plchan->name, name))
1789 return true;
1790
1791 return false;
1792}
Sachin Kamat6d05c9f2014-01-23 16:10:07 +05301793EXPORT_SYMBOL_GPL(pl08x_filter_id);
Linus Walleije8689e62010-09-28 15:57:37 +02001794
1795/*
1796 * Just check that the device is there and active
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +00001797 * TODO: turn this bit on/off depending on the number of physical channels
1798 * actually used, if it is zero... well shut it off. That will save some
1799 * power. Cut the clock at the same time.
Linus Walleije8689e62010-09-28 15:57:37 +02001800 */
1801static void pl08x_ensure_on(struct pl08x_driver_data *pl08x)
1802{
Linus Walleijaffa1152012-04-12 09:01:49 +02001803 /* The Nomadik variant does not have the config register */
1804 if (pl08x->vd->nomadik)
1805 return;
Viresh Kumar48a59ef2011-08-05 15:32:34 +05301806 writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG);
Linus Walleije8689e62010-09-28 15:57:37 +02001807}
1808
Linus Walleije8689e62010-09-28 15:57:37 +02001809static irqreturn_t pl08x_irq(int irq, void *dev)
1810{
1811 struct pl08x_driver_data *pl08x = dev;
Viresh Kumar28da2832011-08-05 15:32:36 +05301812 u32 mask = 0, err, tc, i;
Linus Walleije8689e62010-09-28 15:57:37 +02001813
Viresh Kumar28da2832011-08-05 15:32:36 +05301814 /* check & clear - ERR & TC interrupts */
1815 err = readl(pl08x->base + PL080_ERR_STATUS);
1816 if (err) {
1817 dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n",
1818 __func__, err);
1819 writel(err, pl08x->base + PL080_ERR_CLEAR);
Linus Walleije8689e62010-09-28 15:57:37 +02001820 }
Linus Walleijd29bf012012-04-09 22:53:21 +02001821 tc = readl(pl08x->base + PL080_TC_STATUS);
Viresh Kumar28da2832011-08-05 15:32:36 +05301822 if (tc)
1823 writel(tc, pl08x->base + PL080_TC_CLEAR);
1824
1825 if (!err && !tc)
1826 return IRQ_NONE;
1827
Linus Walleije8689e62010-09-28 15:57:37 +02001828 for (i = 0; i < pl08x->vd->channels; i++) {
Viresh Kumar28da2832011-08-05 15:32:36 +05301829 if (((1 << i) & err) || ((1 << i) & tc)) {
Linus Walleije8689e62010-09-28 15:57:37 +02001830 /* Locate physical channel */
1831 struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i];
1832 struct pl08x_dma_chan *plchan = phychan->serving;
Russell Kinga936e792012-05-25 10:51:19 +01001833 struct pl08x_txd *tx;
Linus Walleije8689e62010-09-28 15:57:37 +02001834
Viresh Kumar28da2832011-08-05 15:32:36 +05301835 if (!plchan) {
1836 dev_err(&pl08x->adev->dev,
1837 "%s Error TC interrupt on unused channel: 0x%08x\n",
1838 __func__, i);
1839 continue;
1840 }
1841
Russell King083be282012-05-26 14:09:53 +01001842 spin_lock(&plchan->vc.lock);
Russell Kinga936e792012-05-25 10:51:19 +01001843 tx = plchan->at;
Alban Bedel3b24c202013-08-11 19:59:20 +02001844 if (tx && tx->cyclic) {
1845 vchan_cyclic_callback(&tx->vd);
1846 } else if (tx) {
Russell Kinga936e792012-05-25 10:51:19 +01001847 plchan->at = NULL;
Russell Kingc48d4962012-05-25 11:48:51 +01001848 /*
1849 * This descriptor is done, release its mux
1850 * reservation.
1851 */
1852 pl08x_release_mux(plchan);
Russell King18536132012-05-26 14:42:23 +01001853 tx->done = true;
1854 vchan_cookie_complete(&tx->vd);
Russell Kingc33b6442012-05-25 15:41:13 +01001855
Russell Kinga5a488d2012-05-26 13:54:15 +01001856 /*
1857 * And start the next descriptor (if any),
1858 * otherwise free this channel.
1859 */
Russell King879f1272012-05-26 14:27:40 +01001860 if (vchan_next_desc(&plchan->vc))
Russell Kingc33b6442012-05-25 15:41:13 +01001861 pl08x_start_next_txd(plchan);
Russell Kinga5a488d2012-05-26 13:54:15 +01001862 else
1863 pl08x_phy_free(plchan);
Russell Kinga936e792012-05-25 10:51:19 +01001864 }
Russell King083be282012-05-26 14:09:53 +01001865 spin_unlock(&plchan->vc.lock);
Russell Kinga936e792012-05-25 10:51:19 +01001866
Linus Walleije8689e62010-09-28 15:57:37 +02001867 mask |= (1 << i);
1868 }
1869 }
Linus Walleije8689e62010-09-28 15:57:37 +02001870
1871 return mask ? IRQ_HANDLED : IRQ_NONE;
1872}
1873
Russell King - ARM Linux121c8472011-07-21 17:13:48 +01001874static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan)
1875{
Russell King - ARM Linux121c8472011-07-21 17:13:48 +01001876 chan->slave = true;
1877 chan->name = chan->cd->bus_id;
Russell Kinged91c132012-05-16 11:02:40 +01001878 chan->cfg.src_addr = chan->cd->addr;
1879 chan->cfg.dst_addr = chan->cd->addr;
Russell King - ARM Linux121c8472011-07-21 17:13:48 +01001880}
1881
Linus Walleije8689e62010-09-28 15:57:37 +02001882/*
1883 * Initialise the DMAC memcpy/slave channels.
1884 * Make a local wrapper to hold required data
1885 */
1886static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,
Viresh Kumar3e27ee82011-08-05 15:32:27 +05301887 struct dma_device *dmadev, unsigned int channels, bool slave)
Linus Walleije8689e62010-09-28 15:57:37 +02001888{
1889 struct pl08x_dma_chan *chan;
1890 int i;
1891
1892 INIT_LIST_HEAD(&dmadev->channels);
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +00001893
Linus Walleije8689e62010-09-28 15:57:37 +02001894 /*
1895 * Register as many many memcpy as we have physical channels,
1896 * we won't always be able to use all but the code will have
1897 * to cope with that situation.
1898 */
1899 for (i = 0; i < channels; i++) {
Viresh Kumarb201c112011-08-05 15:32:29 +05301900 chan = kzalloc(sizeof(*chan), GFP_KERNEL);
Linus Walleije8689e62010-09-28 15:57:37 +02001901 if (!chan) {
1902 dev_err(&pl08x->adev->dev,
1903 "%s no memory for channel\n", __func__);
1904 return -ENOMEM;
1905 }
1906
1907 chan->host = pl08x;
1908 chan->state = PL08X_CHAN_IDLE;
Russell Kingad0de2a2012-05-25 11:15:15 +01001909 chan->signal = -1;
Linus Walleije8689e62010-09-28 15:57:37 +02001910
1911 if (slave) {
Linus Walleije8689e62010-09-28 15:57:37 +02001912 chan->cd = &pl08x->pd->slave_channels[i];
Russell King - ARM Linux121c8472011-07-21 17:13:48 +01001913 pl08x_dma_slave_init(chan);
Linus Walleije8689e62010-09-28 15:57:37 +02001914 } else {
1915 chan->cd = &pl08x->pd->memcpy_channel;
1916 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
1917 if (!chan->name) {
1918 kfree(chan);
1919 return -ENOMEM;
1920 }
1921 }
Viresh Kumar175a5e62011-08-05 15:32:32 +05301922 dev_dbg(&pl08x->adev->dev,
Linus Walleije8689e62010-09-28 15:57:37 +02001923 "initialize virtual channel \"%s\"\n",
1924 chan->name);
1925
Russell King18536132012-05-26 14:42:23 +01001926 chan->vc.desc_free = pl08x_desc_free;
Russell King083be282012-05-26 14:09:53 +01001927 vchan_init(&chan->vc, dmadev);
Linus Walleije8689e62010-09-28 15:57:37 +02001928 }
1929 dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n",
1930 i, slave ? "slave" : "memcpy");
1931 return i;
1932}
1933
1934static void pl08x_free_virtual_channels(struct dma_device *dmadev)
1935{
1936 struct pl08x_dma_chan *chan = NULL;
1937 struct pl08x_dma_chan *next;
1938
1939 list_for_each_entry_safe(chan,
Russell King01d8dc62012-05-26 14:04:29 +01001940 next, &dmadev->channels, vc.chan.device_node) {
1941 list_del(&chan->vc.chan.device_node);
Linus Walleije8689e62010-09-28 15:57:37 +02001942 kfree(chan);
1943 }
1944}
1945
1946#ifdef CONFIG_DEBUG_FS
1947static const char *pl08x_state_str(enum pl08x_dma_chan_state state)
1948{
1949 switch (state) {
1950 case PL08X_CHAN_IDLE:
1951 return "idle";
1952 case PL08X_CHAN_RUNNING:
1953 return "running";
1954 case PL08X_CHAN_PAUSED:
1955 return "paused";
1956 case PL08X_CHAN_WAITING:
1957 return "waiting";
1958 default:
1959 break;
1960 }
1961 return "UNKNOWN STATE";
1962}
1963
1964static int pl08x_debugfs_show(struct seq_file *s, void *data)
1965{
1966 struct pl08x_driver_data *pl08x = s->private;
1967 struct pl08x_dma_chan *chan;
1968 struct pl08x_phy_chan *ch;
1969 unsigned long flags;
1970 int i;
1971
1972 seq_printf(s, "PL08x physical channels:\n");
1973 seq_printf(s, "CHANNEL:\tUSER:\n");
1974 seq_printf(s, "--------\t-----\n");
1975 for (i = 0; i < pl08x->vd->channels; i++) {
1976 struct pl08x_dma_chan *virt_chan;
1977
1978 ch = &pl08x->phy_chans[i];
1979
1980 spin_lock_irqsave(&ch->lock, flags);
1981 virt_chan = ch->serving;
1982
Linus Walleijaffa1152012-04-12 09:01:49 +02001983 seq_printf(s, "%d\t\t%s%s\n",
1984 ch->id,
1985 virt_chan ? virt_chan->name : "(none)",
1986 ch->locked ? " LOCKED" : "");
Linus Walleije8689e62010-09-28 15:57:37 +02001987
1988 spin_unlock_irqrestore(&ch->lock, flags);
1989 }
1990
1991 seq_printf(s, "\nPL08x virtual memcpy channels:\n");
1992 seq_printf(s, "CHANNEL:\tSTATE:\n");
1993 seq_printf(s, "--------\t------\n");
Russell King01d8dc62012-05-26 14:04:29 +01001994 list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) {
Russell King - ARM Linux3e2a0372011-01-03 22:32:46 +00001995 seq_printf(s, "%s\t\t%s\n", chan->name,
Linus Walleije8689e62010-09-28 15:57:37 +02001996 pl08x_state_str(chan->state));
1997 }
1998
1999 seq_printf(s, "\nPL08x virtual slave channels:\n");
2000 seq_printf(s, "CHANNEL:\tSTATE:\n");
2001 seq_printf(s, "--------\t------\n");
Russell King01d8dc62012-05-26 14:04:29 +01002002 list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) {
Russell King - ARM Linux3e2a0372011-01-03 22:32:46 +00002003 seq_printf(s, "%s\t\t%s\n", chan->name,
Linus Walleije8689e62010-09-28 15:57:37 +02002004 pl08x_state_str(chan->state));
2005 }
2006
2007 return 0;
2008}
2009
2010static int pl08x_debugfs_open(struct inode *inode, struct file *file)
2011{
2012 return single_open(file, pl08x_debugfs_show, inode->i_private);
2013}
2014
2015static const struct file_operations pl08x_debugfs_operations = {
2016 .open = pl08x_debugfs_open,
2017 .read = seq_read,
2018 .llseek = seq_lseek,
2019 .release = single_release,
2020};
2021
2022static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
2023{
2024 /* Expose a simple debugfs interface to view all clocks */
Viresh Kumar3e27ee82011-08-05 15:32:27 +05302025 (void) debugfs_create_file(dev_name(&pl08x->adev->dev),
2026 S_IFREG | S_IRUGO, NULL, pl08x,
2027 &pl08x_debugfs_operations);
Linus Walleije8689e62010-09-28 15:57:37 +02002028}
2029
2030#else
2031static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)
2032{
2033}
2034#endif
2035
Russell Kingaa25afa2011-02-19 15:55:00 +00002036static int pl08x_probe(struct amba_device *adev, const struct amba_id *id)
Linus Walleije8689e62010-09-28 15:57:37 +02002037{
2038 struct pl08x_driver_data *pl08x;
Russell King - ARM Linuxf96ca9ec2011-01-03 22:35:08 +00002039 const struct vendor_data *vd = id->data;
Tomasz Figaba6785f2013-08-11 19:59:15 +02002040 u32 tsfr_size;
Linus Walleije8689e62010-09-28 15:57:37 +02002041 int ret = 0;
2042 int i;
2043
2044 ret = amba_request_regions(adev, NULL);
2045 if (ret)
2046 return ret;
2047
Russell Kingde1a2412013-06-27 10:29:32 +01002048 /* Ensure that we can do DMA */
2049 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32));
2050 if (ret)
2051 goto out_no_pl08x;
2052
Linus Walleije8689e62010-09-28 15:57:37 +02002053 /* Create the driver state holder */
Viresh Kumarb201c112011-08-05 15:32:29 +05302054 pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL);
Linus Walleije8689e62010-09-28 15:57:37 +02002055 if (!pl08x) {
2056 ret = -ENOMEM;
2057 goto out_no_pl08x;
2058 }
2059
2060 /* Initialize memcpy engine */
2061 dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask);
2062 pl08x->memcpy.dev = &adev->dev;
2063 pl08x->memcpy.device_alloc_chan_resources = pl08x_alloc_chan_resources;
2064 pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources;
2065 pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy;
2066 pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;
2067 pl08x->memcpy.device_tx_status = pl08x_dma_tx_status;
2068 pl08x->memcpy.device_issue_pending = pl08x_issue_pending;
Maxime Ripardbcd1b0b2014-11-17 14:42:05 +01002069 pl08x->memcpy.device_config = pl08x_config;
2070 pl08x->memcpy.device_pause = pl08x_pause;
2071 pl08x->memcpy.device_resume = pl08x_resume;
2072 pl08x->memcpy.device_terminate_all = pl08x_terminate_all;
Linus Walleije8689e62010-09-28 15:57:37 +02002073
2074 /* Initialize slave engine */
2075 dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask);
Alban Bedel3b24c202013-08-11 19:59:20 +02002076 dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask);
Linus Walleije8689e62010-09-28 15:57:37 +02002077 pl08x->slave.dev = &adev->dev;
2078 pl08x->slave.device_alloc_chan_resources = pl08x_alloc_chan_resources;
2079 pl08x->slave.device_free_chan_resources = pl08x_free_chan_resources;
2080 pl08x->slave.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;
2081 pl08x->slave.device_tx_status = pl08x_dma_tx_status;
2082 pl08x->slave.device_issue_pending = pl08x_issue_pending;
2083 pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg;
Alban Bedel3b24c202013-08-11 19:59:20 +02002084 pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic;
Maxime Ripardbcd1b0b2014-11-17 14:42:05 +01002085 pl08x->slave.device_config = pl08x_config;
2086 pl08x->slave.device_pause = pl08x_pause;
2087 pl08x->slave.device_resume = pl08x_resume;
2088 pl08x->slave.device_terminate_all = pl08x_terminate_all;
Linus Walleije8689e62010-09-28 15:57:37 +02002089
2090 /* Get the platform data */
2091 pl08x->pd = dev_get_platdata(&adev->dev);
2092 if (!pl08x->pd) {
2093 dev_err(&adev->dev, "no platform data supplied\n");
Julia Lawall983d7be2012-08-14 14:58:32 +02002094 ret = -EINVAL;
Linus Walleije8689e62010-09-28 15:57:37 +02002095 goto out_no_platdata;
2096 }
2097
2098 /* Assign useful pointers to the driver state */
2099 pl08x->adev = adev;
2100 pl08x->vd = vd;
2101
Russell King - ARM Linux30749cb2011-01-03 22:41:13 +00002102 /* By default, AHB1 only. If dualmaster, from platform */
2103 pl08x->lli_buses = PL08X_AHB1;
2104 pl08x->mem_buses = PL08X_AHB1;
2105 if (pl08x->vd->dualmaster) {
2106 pl08x->lli_buses = pl08x->pd->lli_buses;
2107 pl08x->mem_buses = pl08x->pd->mem_buses;
2108 }
2109
Tomasz Figada1b6c02013-08-11 19:59:17 +02002110 if (vd->pl080s)
2111 pl08x->lli_words = PL080S_LLI_WORDS;
2112 else
2113 pl08x->lli_words = PL080_LLI_WORDS;
Tomasz Figaba6785f2013-08-11 19:59:15 +02002114 tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32);
2115
Linus Walleije8689e62010-09-28 15:57:37 +02002116 /* A DMA memory pool for LLIs, align on 1-byte boundary */
2117 pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev,
Tomasz Figaba6785f2013-08-11 19:59:15 +02002118 tsfr_size, PL08X_ALIGN, 0);
Linus Walleije8689e62010-09-28 15:57:37 +02002119 if (!pl08x->pool) {
2120 ret = -ENOMEM;
2121 goto out_no_lli_pool;
2122 }
2123
Linus Walleije8689e62010-09-28 15:57:37 +02002124 pl08x->base = ioremap(adev->res.start, resource_size(&adev->res));
2125 if (!pl08x->base) {
2126 ret = -ENOMEM;
2127 goto out_no_ioremap;
2128 }
2129
2130 /* Turn on the PL08x */
2131 pl08x_ensure_on(pl08x);
2132
Russell King - ARM Linux94ae8522011-01-16 20:18:05 +00002133 /* Attach the interrupt handler */
Linus Walleije8689e62010-09-28 15:57:37 +02002134 writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR);
2135 writel(0x000000FF, pl08x->base + PL080_TC_CLEAR);
2136
Michael Opdenacker174b5372013-10-13 07:10:51 +02002137 ret = request_irq(adev->irq[0], pl08x_irq, 0, DRIVER_NAME, pl08x);
Linus Walleije8689e62010-09-28 15:57:37 +02002138 if (ret) {
2139 dev_err(&adev->dev, "%s failed to request interrupt %d\n",
2140 __func__, adev->irq[0]);
2141 goto out_no_irq;
2142 }
2143
2144 /* Initialize physical channels */
Linus Walleijaffa1152012-04-12 09:01:49 +02002145 pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)),
Linus Walleije8689e62010-09-28 15:57:37 +02002146 GFP_KERNEL);
2147 if (!pl08x->phy_chans) {
2148 dev_err(&adev->dev, "%s failed to allocate "
2149 "physical channel holders\n",
2150 __func__);
Julia Lawall983d7be2012-08-14 14:58:32 +02002151 ret = -ENOMEM;
Linus Walleije8689e62010-09-28 15:57:37 +02002152 goto out_no_phychans;
2153 }
2154
2155 for (i = 0; i < vd->channels; i++) {
2156 struct pl08x_phy_chan *ch = &pl08x->phy_chans[i];
2157
2158 ch->id = i;
2159 ch->base = pl08x->base + PL080_Cx_BASE(i);
Tomasz Figad86ccea2013-08-11 19:59:14 +02002160 ch->reg_config = ch->base + vd->config_offset;
Linus Walleije8689e62010-09-28 15:57:37 +02002161 spin_lock_init(&ch->lock);
Linus Walleijaffa1152012-04-12 09:01:49 +02002162
2163 /*
2164 * Nomadik variants can have channels that are locked
2165 * down for the secure world only. Lock up these channels
2166 * by perpetually serving a dummy virtual channel.
2167 */
2168 if (vd->nomadik) {
2169 u32 val;
2170
Tomasz Figad86ccea2013-08-11 19:59:14 +02002171 val = readl(ch->reg_config);
Linus Walleijaffa1152012-04-12 09:01:49 +02002172 if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) {
2173 dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i);
2174 ch->locked = true;
2175 }
2176 }
2177
Viresh Kumar175a5e62011-08-05 15:32:32 +05302178 dev_dbg(&adev->dev, "physical channel %d is %s\n",
2179 i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE");
Linus Walleije8689e62010-09-28 15:57:37 +02002180 }
2181
2182 /* Register as many memcpy channels as there are physical channels */
2183 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy,
2184 pl08x->vd->channels, false);
2185 if (ret <= 0) {
2186 dev_warn(&pl08x->adev->dev,
2187 "%s failed to enumerate memcpy channels - %d\n",
2188 __func__, ret);
2189 goto out_no_memcpy;
2190 }
Linus Walleije8689e62010-09-28 15:57:37 +02002191
2192 /* Register slave channels */
2193 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave,
Viresh Kumar3e27ee82011-08-05 15:32:27 +05302194 pl08x->pd->num_slave_channels, true);
Linus Walleij10804112013-12-14 00:16:23 +01002195 if (ret < 0) {
Linus Walleije8689e62010-09-28 15:57:37 +02002196 dev_warn(&pl08x->adev->dev,
2197 "%s failed to enumerate slave channels - %d\n",
2198 __func__, ret);
2199 goto out_no_slave;
2200 }
Linus Walleije8689e62010-09-28 15:57:37 +02002201
2202 ret = dma_async_device_register(&pl08x->memcpy);
2203 if (ret) {
2204 dev_warn(&pl08x->adev->dev,
2205 "%s failed to register memcpy as an async device - %d\n",
2206 __func__, ret);
2207 goto out_no_memcpy_reg;
2208 }
2209
2210 ret = dma_async_device_register(&pl08x->slave);
2211 if (ret) {
2212 dev_warn(&pl08x->adev->dev,
2213 "%s failed to register slave as an async device - %d\n",
2214 __func__, ret);
2215 goto out_no_slave_reg;
2216 }
2217
2218 amba_set_drvdata(adev, pl08x);
2219 init_pl08x_debugfs(pl08x);
Tomasz Figada1b6c02013-08-11 19:59:17 +02002220 dev_info(&pl08x->adev->dev, "DMA: PL%03x%s rev%u at 0x%08llx irq %d\n",
2221 amba_part(adev), pl08x->vd->pl080s ? "s" : "", amba_rev(adev),
Russell King - ARM Linuxb05cd8f2011-01-03 22:33:26 +00002222 (unsigned long long)adev->res.start, adev->irq[0]);
Viresh Kumarb7b60182011-08-05 15:32:33 +05302223
Linus Walleije8689e62010-09-28 15:57:37 +02002224 return 0;
2225
2226out_no_slave_reg:
2227 dma_async_device_unregister(&pl08x->memcpy);
2228out_no_memcpy_reg:
2229 pl08x_free_virtual_channels(&pl08x->slave);
2230out_no_slave:
2231 pl08x_free_virtual_channels(&pl08x->memcpy);
2232out_no_memcpy:
2233 kfree(pl08x->phy_chans);
2234out_no_phychans:
2235 free_irq(adev->irq[0], pl08x);
2236out_no_irq:
2237 iounmap(pl08x->base);
2238out_no_ioremap:
2239 dma_pool_destroy(pl08x->pool);
2240out_no_lli_pool:
2241out_no_platdata:
2242 kfree(pl08x);
2243out_no_pl08x:
2244 amba_release_regions(adev);
2245 return ret;
2246}
2247
2248/* PL080 has 8 channels and the PL080 have just 2 */
2249static struct vendor_data vendor_pl080 = {
Tomasz Figad86ccea2013-08-11 19:59:14 +02002250 .config_offset = PL080_CH_CONFIG,
Linus Walleije8689e62010-09-28 15:57:37 +02002251 .channels = 8,
2252 .dualmaster = true,
Tomasz Figa5110e512013-08-11 19:59:18 +02002253 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
Linus Walleije8689e62010-09-28 15:57:37 +02002254};
2255
Linus Walleijaffa1152012-04-12 09:01:49 +02002256static struct vendor_data vendor_nomadik = {
Tomasz Figad86ccea2013-08-11 19:59:14 +02002257 .config_offset = PL080_CH_CONFIG,
Linus Walleijaffa1152012-04-12 09:01:49 +02002258 .channels = 8,
2259 .dualmaster = true,
2260 .nomadik = true,
Tomasz Figa5110e512013-08-11 19:59:18 +02002261 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
Linus Walleijaffa1152012-04-12 09:01:49 +02002262};
2263
Tomasz Figada1b6c02013-08-11 19:59:17 +02002264static struct vendor_data vendor_pl080s = {
2265 .config_offset = PL080S_CH_CONFIG,
2266 .channels = 8,
2267 .pl080s = true,
Tomasz Figa5110e512013-08-11 19:59:18 +02002268 .max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK,
Linus Walleije8689e62010-09-28 15:57:37 +02002269};
2270
2271static struct vendor_data vendor_pl081 = {
Tomasz Figad86ccea2013-08-11 19:59:14 +02002272 .config_offset = PL080_CH_CONFIG,
Linus Walleije8689e62010-09-28 15:57:37 +02002273 .channels = 2,
2274 .dualmaster = false,
Tomasz Figa5110e512013-08-11 19:59:18 +02002275 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK,
Linus Walleije8689e62010-09-28 15:57:37 +02002276};
2277
2278static struct amba_id pl08x_ids[] = {
Tomasz Figada1b6c02013-08-11 19:59:17 +02002279 /* Samsung PL080S variant */
2280 {
2281 .id = 0x0a141080,
2282 .mask = 0xffffffff,
2283 .data = &vendor_pl080s,
2284 },
Linus Walleije8689e62010-09-28 15:57:37 +02002285 /* PL080 */
2286 {
2287 .id = 0x00041080,
2288 .mask = 0x000fffff,
2289 .data = &vendor_pl080,
2290 },
2291 /* PL081 */
2292 {
2293 .id = 0x00041081,
2294 .mask = 0x000fffff,
2295 .data = &vendor_pl081,
2296 },
2297 /* Nomadik 8815 PL080 variant */
2298 {
Linus Walleijaffa1152012-04-12 09:01:49 +02002299 .id = 0x00280080,
Linus Walleije8689e62010-09-28 15:57:37 +02002300 .mask = 0x00ffffff,
Linus Walleijaffa1152012-04-12 09:01:49 +02002301 .data = &vendor_nomadik,
Linus Walleije8689e62010-09-28 15:57:37 +02002302 },
2303 { 0, 0 },
2304};
2305
Dave Martin037566d2011-10-05 15:15:20 +01002306MODULE_DEVICE_TABLE(amba, pl08x_ids);
2307
Linus Walleije8689e62010-09-28 15:57:37 +02002308static struct amba_driver pl08x_amba_driver = {
2309 .drv.name = DRIVER_NAME,
2310 .id_table = pl08x_ids,
2311 .probe = pl08x_probe,
2312};
2313
2314static int __init pl08x_init(void)
2315{
2316 int retval;
2317 retval = amba_driver_register(&pl08x_amba_driver);
2318 if (retval)
2319 printk(KERN_WARNING DRIVER_NAME
Russell King - ARM Linuxe8b5e112011-01-03 22:30:24 +00002320 "failed to register as an AMBA device (%d)\n",
Linus Walleije8689e62010-09-28 15:57:37 +02002321 retval);
2322 return retval;
2323}
2324subsys_initcall(pl08x_init);