blob: af955de035f42dd4d6a96ea97c11f16295b44e7b [file] [log] [blame]
Linus Walleij8d318a52010-03-30 15:33:42 +02001/*
Per Forlind49278e2010-12-20 18:31:38 +01002 * Copyright (C) Ericsson AB 2007-2008
3 * Copyright (C) ST-Ericsson SA 2008-2010
Per Forlin661385f2010-10-06 09:05:28 +00004 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson
Jonas Aaberg767a9672010-08-09 12:08:34 +00005 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson
Linus Walleij8d318a52010-03-30 15:33:42 +02006 * License terms: GNU General Public License (GPL) version 2
Linus Walleij8d318a52010-03-30 15:33:42 +02007 */
8
9#include <linux/kernel.h>
10#include <linux/slab.h>
11#include <linux/dmaengine.h>
12#include <linux/platform_device.h>
13#include <linux/clk.h>
14#include <linux/delay.h>
Jonas Aaberg698e4732010-08-09 12:08:56 +000015#include <linux/err.h>
Linus Walleij8d318a52010-03-30 15:33:42 +020016
17#include <plat/ste_dma40.h>
18
19#include "ste_dma40_ll.h"
20
21#define D40_NAME "dma40"
22
23#define D40_PHY_CHAN -1
24
25/* For masking out/in 2 bit channel positions */
26#define D40_CHAN_POS(chan) (2 * (chan / 2))
27#define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan))
28
29/* Maximum iterations taken before giving up suspending a channel */
30#define D40_SUSPEND_MAX_IT 500
31
Linus Walleij508849a2010-06-20 21:26:07 +000032/* Hardware requirement on LCLA alignment */
33#define LCLA_ALIGNMENT 0x40000
Jonas Aaberg698e4732010-08-09 12:08:56 +000034
35/* Max number of links per event group */
36#define D40_LCLA_LINK_PER_EVENT_GRP 128
37#define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP
38
Linus Walleij508849a2010-06-20 21:26:07 +000039/* Attempts before giving up to trying to get pages that are aligned */
40#define MAX_LCLA_ALLOC_ATTEMPTS 256
41
42/* Bit markings for allocation map */
Linus Walleij8d318a52010-03-30 15:33:42 +020043#define D40_ALLOC_FREE (1 << 31)
44#define D40_ALLOC_PHY (1 << 30)
45#define D40_ALLOC_LOG_FREE 0
46
Linus Walleij8d318a52010-03-30 15:33:42 +020047/* Hardware designer of the block */
Jonas Aaberg3ae02672010-08-09 12:08:18 +000048#define D40_HW_DESIGNER 0x8
Linus Walleij8d318a52010-03-30 15:33:42 +020049
50/**
51 * enum 40_command - The different commands and/or statuses.
52 *
53 * @D40_DMA_STOP: DMA channel command STOP or status STOPPED,
54 * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN.
55 * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible.
56 * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED.
57 */
58enum d40_command {
59 D40_DMA_STOP = 0,
60 D40_DMA_RUN = 1,
61 D40_DMA_SUSPEND_REQ = 2,
62 D40_DMA_SUSPENDED = 3
63};
64
65/**
66 * struct d40_lli_pool - Structure for keeping LLIs in memory
67 *
68 * @base: Pointer to memory area when the pre_alloc_lli's are not large
69 * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if
70 * pre_alloc_lli is used.
Rabin Vincentb00f9382011-01-25 11:18:15 +010071 * @dma_addr: DMA address, if mapped
Linus Walleij8d318a52010-03-30 15:33:42 +020072 * @size: The size in bytes of the memory at base or the size of pre_alloc_lli.
73 * @pre_alloc_lli: Pre allocated area for the most common case of transfers,
74 * one buffer to one buffer.
75 */
76struct d40_lli_pool {
77 void *base;
Linus Walleij508849a2010-06-20 21:26:07 +000078 int size;
Rabin Vincentb00f9382011-01-25 11:18:15 +010079 dma_addr_t dma_addr;
Linus Walleij8d318a52010-03-30 15:33:42 +020080 /* Space for dst and src, plus an extra for padding */
Linus Walleij508849a2010-06-20 21:26:07 +000081 u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)];
Linus Walleij8d318a52010-03-30 15:33:42 +020082};
83
84/**
85 * struct d40_desc - A descriptor is one DMA job.
86 *
87 * @lli_phy: LLI settings for physical channel. Both src and dst=
88 * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if
89 * lli_len equals one.
90 * @lli_log: Same as above but for logical channels.
91 * @lli_pool: The pool with two entries pre-allocated.
Per Friden941b77a2010-06-20 21:24:45 +000092 * @lli_len: Number of llis of current descriptor.
Jonas Aaberg698e4732010-08-09 12:08:56 +000093 * @lli_current: Number of transfered llis.
94 * @lcla_alloc: Number of LCLA entries allocated.
Linus Walleij8d318a52010-03-30 15:33:42 +020095 * @txd: DMA engine struct. Used for among other things for communication
96 * during a transfer.
97 * @node: List entry.
Linus Walleij8d318a52010-03-30 15:33:42 +020098 * @is_in_client_list: true if the client owns this descriptor.
Jonas Aabergaa182ae2010-08-09 12:08:26 +000099 * the previous one.
Linus Walleij8d318a52010-03-30 15:33:42 +0200100 *
101 * This descriptor is used for both logical and physical transfers.
102 */
Linus Walleij8d318a52010-03-30 15:33:42 +0200103struct d40_desc {
104 /* LLI physical */
105 struct d40_phy_lli_bidir lli_phy;
106 /* LLI logical */
107 struct d40_log_lli_bidir lli_log;
108
109 struct d40_lli_pool lli_pool;
Per Friden941b77a2010-06-20 21:24:45 +0000110 int lli_len;
Jonas Aaberg698e4732010-08-09 12:08:56 +0000111 int lli_current;
112 int lcla_alloc;
Linus Walleij8d318a52010-03-30 15:33:42 +0200113
114 struct dma_async_tx_descriptor txd;
115 struct list_head node;
116
Linus Walleij8d318a52010-03-30 15:33:42 +0200117 bool is_in_client_list;
Rabin Vincent0c842b52011-01-25 11:18:35 +0100118 bool cyclic;
Linus Walleij8d318a52010-03-30 15:33:42 +0200119};
120
121/**
122 * struct d40_lcla_pool - LCLA pool settings and data.
123 *
Linus Walleij508849a2010-06-20 21:26:07 +0000124 * @base: The virtual address of LCLA. 18 bit aligned.
125 * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used.
126 * This pointer is only there for clean-up on error.
127 * @pages: The number of pages needed for all physical channels.
128 * Only used later for clean-up on error
Linus Walleij8d318a52010-03-30 15:33:42 +0200129 * @lock: Lock to protect the content in this struct.
Jonas Aaberg698e4732010-08-09 12:08:56 +0000130 * @alloc_map: big map over which LCLA entry is own by which job.
Linus Walleij8d318a52010-03-30 15:33:42 +0200131 */
132struct d40_lcla_pool {
133 void *base;
Rabin Vincent026cbc42011-01-25 11:18:14 +0100134 dma_addr_t dma_addr;
Linus Walleij508849a2010-06-20 21:26:07 +0000135 void *base_unaligned;
136 int pages;
Linus Walleij8d318a52010-03-30 15:33:42 +0200137 spinlock_t lock;
Jonas Aaberg698e4732010-08-09 12:08:56 +0000138 struct d40_desc **alloc_map;
Linus Walleij8d318a52010-03-30 15:33:42 +0200139};
140
141/**
142 * struct d40_phy_res - struct for handling eventlines mapped to physical
143 * channels.
144 *
145 * @lock: A lock protection this entity.
146 * @num: The physical channel number of this entity.
147 * @allocated_src: Bit mapped to show which src event line's are mapped to
148 * this physical channel. Can also be free or physically allocated.
149 * @allocated_dst: Same as for src but is dst.
150 * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as
Jonas Aaberg767a9672010-08-09 12:08:34 +0000151 * event line number.
Linus Walleij8d318a52010-03-30 15:33:42 +0200152 */
153struct d40_phy_res {
154 spinlock_t lock;
155 int num;
156 u32 allocated_src;
157 u32 allocated_dst;
158};
159
160struct d40_base;
161
162/**
163 * struct d40_chan - Struct that describes a channel.
164 *
165 * @lock: A spinlock to protect this struct.
166 * @log_num: The logical number, if any of this channel.
167 * @completed: Starts with 1, after first interrupt it is set to dma engine's
168 * current cookie.
169 * @pending_tx: The number of pending transfers. Used between interrupt handler
170 * and tasklet.
171 * @busy: Set to true when transfer is ongoing on this channel.
Jonas Aaberg2a614342010-06-20 21:25:24 +0000172 * @phy_chan: Pointer to physical channel which this instance runs on. If this
173 * point is NULL, then the channel is not allocated.
Linus Walleij8d318a52010-03-30 15:33:42 +0200174 * @chan: DMA engine handle.
175 * @tasklet: Tasklet that gets scheduled from interrupt context to complete a
176 * transfer and call client callback.
177 * @client: Cliented owned descriptor list.
178 * @active: Active descriptor.
179 * @queue: Queued jobs.
Linus Walleij8d318a52010-03-30 15:33:42 +0200180 * @dma_cfg: The client configuration of this dma channel.
Rabin Vincentce2ca122010-10-12 13:00:49 +0000181 * @configured: whether the dma_cfg configuration is valid
Linus Walleij8d318a52010-03-30 15:33:42 +0200182 * @base: Pointer to the device instance struct.
183 * @src_def_cfg: Default cfg register setting for src.
184 * @dst_def_cfg: Default cfg register setting for dst.
185 * @log_def: Default logical channel settings.
186 * @lcla: Space for one dst src pair for logical channel transfers.
187 * @lcpa: Pointer to dst and src lcpa settings.
188 *
189 * This struct can either "be" a logical or a physical channel.
190 */
191struct d40_chan {
192 spinlock_t lock;
193 int log_num;
194 /* ID of the most recent completed transfer */
195 int completed;
196 int pending_tx;
197 bool busy;
198 struct d40_phy_res *phy_chan;
199 struct dma_chan chan;
200 struct tasklet_struct tasklet;
201 struct list_head client;
202 struct list_head active;
203 struct list_head queue;
Linus Walleij8d318a52010-03-30 15:33:42 +0200204 struct stedma40_chan_cfg dma_cfg;
Rabin Vincentce2ca122010-10-12 13:00:49 +0000205 bool configured;
Linus Walleij8d318a52010-03-30 15:33:42 +0200206 struct d40_base *base;
207 /* Default register configurations */
208 u32 src_def_cfg;
209 u32 dst_def_cfg;
210 struct d40_def_lcsp log_def;
Linus Walleij8d318a52010-03-30 15:33:42 +0200211 struct d40_log_lli_full *lcpa;
Linus Walleij95e14002010-08-04 13:37:45 +0200212 /* Runtime reconfiguration */
213 dma_addr_t runtime_addr;
214 enum dma_data_direction runtime_direction;
Linus Walleij8d318a52010-03-30 15:33:42 +0200215};
216
217/**
218 * struct d40_base - The big global struct, one for each probe'd instance.
219 *
220 * @interrupt_lock: Lock used to make sure one interrupt is handle a time.
221 * @execmd_lock: Lock for execute command usage since several channels share
222 * the same physical register.
223 * @dev: The device structure.
224 * @virtbase: The virtual base address of the DMA's register.
Linus Walleijf4185592010-06-22 18:06:42 -0700225 * @rev: silicon revision detected.
Linus Walleij8d318a52010-03-30 15:33:42 +0200226 * @clk: Pointer to the DMA clock structure.
227 * @phy_start: Physical memory start of the DMA registers.
228 * @phy_size: Size of the DMA register map.
229 * @irq: The IRQ number.
230 * @num_phy_chans: The number of physical channels. Read from HW. This
231 * is the number of available channels for this driver, not counting "Secure
232 * mode" allocated physical channels.
233 * @num_log_chans: The number of logical channels. Calculated from
234 * num_phy_chans.
235 * @dma_both: dma_device channels that can do both memcpy and slave transfers.
236 * @dma_slave: dma_device channels that can do only do slave transfers.
237 * @dma_memcpy: dma_device channels that can do only do memcpy transfers.
Linus Walleij8d318a52010-03-30 15:33:42 +0200238 * @log_chans: Room for all possible logical channels in system.
239 * @lookup_log_chans: Used to map interrupt number to logical channel. Points
240 * to log_chans entries.
241 * @lookup_phy_chans: Used to map interrupt number to physical channel. Points
242 * to phy_chans entries.
243 * @plat_data: Pointer to provided platform_data which is the driver
244 * configuration.
245 * @phy_res: Vector containing all physical channels.
246 * @lcla_pool: lcla pool settings and data.
247 * @lcpa_base: The virtual mapped address of LCPA.
248 * @phy_lcpa: The physical address of the LCPA.
249 * @lcpa_size: The size of the LCPA area.
Jonas Aabergc675b1b2010-06-20 21:25:08 +0000250 * @desc_slab: cache for descriptors.
Linus Walleij8d318a52010-03-30 15:33:42 +0200251 */
252struct d40_base {
253 spinlock_t interrupt_lock;
254 spinlock_t execmd_lock;
255 struct device *dev;
256 void __iomem *virtbase;
Linus Walleijf4185592010-06-22 18:06:42 -0700257 u8 rev:4;
Linus Walleij8d318a52010-03-30 15:33:42 +0200258 struct clk *clk;
259 phys_addr_t phy_start;
260 resource_size_t phy_size;
261 int irq;
262 int num_phy_chans;
263 int num_log_chans;
264 struct dma_device dma_both;
265 struct dma_device dma_slave;
266 struct dma_device dma_memcpy;
267 struct d40_chan *phy_chans;
268 struct d40_chan *log_chans;
269 struct d40_chan **lookup_log_chans;
270 struct d40_chan **lookup_phy_chans;
271 struct stedma40_platform_data *plat_data;
272 /* Physical half channels */
273 struct d40_phy_res *phy_res;
274 struct d40_lcla_pool lcla_pool;
275 void *lcpa_base;
276 dma_addr_t phy_lcpa;
277 resource_size_t lcpa_size;
Jonas Aabergc675b1b2010-06-20 21:25:08 +0000278 struct kmem_cache *desc_slab;
Linus Walleij8d318a52010-03-30 15:33:42 +0200279};
280
281/**
282 * struct d40_interrupt_lookup - lookup table for interrupt handler
283 *
284 * @src: Interrupt mask register.
285 * @clr: Interrupt clear register.
286 * @is_error: true if this is an error interrupt.
287 * @offset: start delta in the lookup_log_chans in d40_base. If equals to
288 * D40_PHY_CHAN, the lookup_phy_chans shall be used instead.
289 */
290struct d40_interrupt_lookup {
291 u32 src;
292 u32 clr;
293 bool is_error;
294 int offset;
295};
296
297/**
298 * struct d40_reg_val - simple lookup struct
299 *
300 * @reg: The register.
301 * @val: The value that belongs to the register in reg.
302 */
303struct d40_reg_val {
304 unsigned int reg;
305 unsigned int val;
306};
307
Rabin Vincent262d2912011-01-25 11:18:05 +0100308static struct device *chan2dev(struct d40_chan *d40c)
309{
310 return &d40c->chan.dev->device;
311}
312
Rabin Vincent724a8572011-01-25 11:18:08 +0100313static bool chan_is_physical(struct d40_chan *chan)
314{
315 return chan->log_num == D40_PHY_CHAN;
316}
317
318static bool chan_is_logical(struct d40_chan *chan)
319{
320 return !chan_is_physical(chan);
321}
322
Rabin Vincent8ca84682011-01-25 11:18:07 +0100323static void __iomem *chan_base(struct d40_chan *chan)
324{
325 return chan->base->virtbase + D40_DREG_PCBASE +
326 chan->phy_chan->num * D40_DREG_PCDELTA;
327}
328
Rabin Vincent6db5a8b2011-01-25 11:18:09 +0100329#define d40_err(dev, format, arg...) \
330 dev_err(dev, "[%s] " format, __func__, ## arg)
331
332#define chan_err(d40c, format, arg...) \
333 d40_err(chan2dev(d40c), format, ## arg)
334
Rabin Vincentb00f9382011-01-25 11:18:15 +0100335static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d,
Rabin Vincentdbd88782011-01-25 11:18:19 +0100336 int lli_len)
Linus Walleij8d318a52010-03-30 15:33:42 +0200337{
Rabin Vincentdbd88782011-01-25 11:18:19 +0100338 bool is_log = chan_is_logical(d40c);
Linus Walleij8d318a52010-03-30 15:33:42 +0200339 u32 align;
340 void *base;
341
342 if (is_log)
343 align = sizeof(struct d40_log_lli);
344 else
345 align = sizeof(struct d40_phy_lli);
346
347 if (lli_len == 1) {
348 base = d40d->lli_pool.pre_alloc_lli;
349 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli);
350 d40d->lli_pool.base = NULL;
351 } else {
Rabin Vincent594ece42011-01-25 11:18:12 +0100352 d40d->lli_pool.size = lli_len * 2 * align;
Linus Walleij8d318a52010-03-30 15:33:42 +0200353
354 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT);
355 d40d->lli_pool.base = base;
356
357 if (d40d->lli_pool.base == NULL)
358 return -ENOMEM;
359 }
360
361 if (is_log) {
Rabin Vincentd924aba2011-01-25 11:18:16 +0100362 d40d->lli_log.src = PTR_ALIGN(base, align);
Rabin Vincent594ece42011-01-25 11:18:12 +0100363 d40d->lli_log.dst = d40d->lli_log.src + lli_len;
Rabin Vincentb00f9382011-01-25 11:18:15 +0100364
365 d40d->lli_pool.dma_addr = 0;
Linus Walleij8d318a52010-03-30 15:33:42 +0200366 } else {
Rabin Vincentd924aba2011-01-25 11:18:16 +0100367 d40d->lli_phy.src = PTR_ALIGN(base, align);
Rabin Vincent594ece42011-01-25 11:18:12 +0100368 d40d->lli_phy.dst = d40d->lli_phy.src + lli_len;
Rabin Vincentb00f9382011-01-25 11:18:15 +0100369
370 d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev,
371 d40d->lli_phy.src,
372 d40d->lli_pool.size,
373 DMA_TO_DEVICE);
374
375 if (dma_mapping_error(d40c->base->dev,
376 d40d->lli_pool.dma_addr)) {
377 kfree(d40d->lli_pool.base);
378 d40d->lli_pool.base = NULL;
379 d40d->lli_pool.dma_addr = 0;
380 return -ENOMEM;
381 }
Linus Walleij8d318a52010-03-30 15:33:42 +0200382 }
383
384 return 0;
385}
386
Rabin Vincentb00f9382011-01-25 11:18:15 +0100387static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d)
Linus Walleij8d318a52010-03-30 15:33:42 +0200388{
Rabin Vincentb00f9382011-01-25 11:18:15 +0100389 if (d40d->lli_pool.dma_addr)
390 dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr,
391 d40d->lli_pool.size, DMA_TO_DEVICE);
392
Linus Walleij8d318a52010-03-30 15:33:42 +0200393 kfree(d40d->lli_pool.base);
394 d40d->lli_pool.base = NULL;
395 d40d->lli_pool.size = 0;
396 d40d->lli_log.src = NULL;
397 d40d->lli_log.dst = NULL;
398 d40d->lli_phy.src = NULL;
399 d40d->lli_phy.dst = NULL;
Linus Walleij8d318a52010-03-30 15:33:42 +0200400}
401
Jonas Aaberg698e4732010-08-09 12:08:56 +0000402static int d40_lcla_alloc_one(struct d40_chan *d40c,
403 struct d40_desc *d40d)
404{
405 unsigned long flags;
406 int i;
407 int ret = -EINVAL;
408 int p;
409
410 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
411
412 p = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP;
413
414 /*
415 * Allocate both src and dst at the same time, therefore the half
416 * start on 1 since 0 can't be used since zero is used as end marker.
417 */
418 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
419 if (!d40c->base->lcla_pool.alloc_map[p + i]) {
420 d40c->base->lcla_pool.alloc_map[p + i] = d40d;
421 d40d->lcla_alloc++;
422 ret = i;
423 break;
424 }
425 }
426
427 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
428
429 return ret;
430}
431
432static int d40_lcla_free_all(struct d40_chan *d40c,
433 struct d40_desc *d40d)
434{
435 unsigned long flags;
436 int i;
437 int ret = -EINVAL;
438
Rabin Vincent724a8572011-01-25 11:18:08 +0100439 if (chan_is_physical(d40c))
Jonas Aaberg698e4732010-08-09 12:08:56 +0000440 return 0;
441
442 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags);
443
444 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) {
445 if (d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
446 D40_LCLA_LINK_PER_EVENT_GRP + i] == d40d) {
447 d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num *
448 D40_LCLA_LINK_PER_EVENT_GRP + i] = NULL;
449 d40d->lcla_alloc--;
450 if (d40d->lcla_alloc == 0) {
451 ret = 0;
452 break;
453 }
454 }
455 }
456
457 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags);
458
459 return ret;
460
461}
462
Linus Walleij8d318a52010-03-30 15:33:42 +0200463static void d40_desc_remove(struct d40_desc *d40d)
464{
465 list_del(&d40d->node);
466}
467
468static struct d40_desc *d40_desc_get(struct d40_chan *d40c)
469{
Rabin Vincenta2c15fa2010-10-06 08:20:37 +0000470 struct d40_desc *desc = NULL;
Linus Walleij8d318a52010-03-30 15:33:42 +0200471
472 if (!list_empty(&d40c->client)) {
Rabin Vincenta2c15fa2010-10-06 08:20:37 +0000473 struct d40_desc *d;
474 struct d40_desc *_d;
475
Linus Walleij8d318a52010-03-30 15:33:42 +0200476 list_for_each_entry_safe(d, _d, &d40c->client, node)
477 if (async_tx_test_ack(&d->txd)) {
Rabin Vincentb00f9382011-01-25 11:18:15 +0100478 d40_pool_lli_free(d40c, d);
Linus Walleij8d318a52010-03-30 15:33:42 +0200479 d40_desc_remove(d);
Rabin Vincenta2c15fa2010-10-06 08:20:37 +0000480 desc = d;
481 memset(desc, 0, sizeof(*desc));
Jonas Aabergc675b1b2010-06-20 21:25:08 +0000482 break;
Linus Walleij8d318a52010-03-30 15:33:42 +0200483 }
Linus Walleij8d318a52010-03-30 15:33:42 +0200484 }
Rabin Vincenta2c15fa2010-10-06 08:20:37 +0000485
486 if (!desc)
487 desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT);
488
489 if (desc)
490 INIT_LIST_HEAD(&desc->node);
491
492 return desc;
Linus Walleij8d318a52010-03-30 15:33:42 +0200493}
494
495static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d)
496{
Jonas Aaberg698e4732010-08-09 12:08:56 +0000497
Rabin Vincentb00f9382011-01-25 11:18:15 +0100498 d40_pool_lli_free(d40c, d40d);
Jonas Aaberg698e4732010-08-09 12:08:56 +0000499 d40_lcla_free_all(d40c, d40d);
Jonas Aabergc675b1b2010-06-20 21:25:08 +0000500 kmem_cache_free(d40c->base->desc_slab, d40d);
Linus Walleij8d318a52010-03-30 15:33:42 +0200501}
502
503static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc)
504{
505 list_add_tail(&desc->node, &d40c->active);
506}
507
Rabin Vincent1c4b0922011-01-25 11:18:24 +0100508static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc)
509{
510 struct d40_phy_lli *lli_dst = desc->lli_phy.dst;
511 struct d40_phy_lli *lli_src = desc->lli_phy.src;
512 void __iomem *base = chan_base(chan);
513
514 writel(lli_src->reg_cfg, base + D40_CHAN_REG_SSCFG);
515 writel(lli_src->reg_elt, base + D40_CHAN_REG_SSELT);
516 writel(lli_src->reg_ptr, base + D40_CHAN_REG_SSPTR);
517 writel(lli_src->reg_lnk, base + D40_CHAN_REG_SSLNK);
518
519 writel(lli_dst->reg_cfg, base + D40_CHAN_REG_SDCFG);
520 writel(lli_dst->reg_elt, base + D40_CHAN_REG_SDELT);
521 writel(lli_dst->reg_ptr, base + D40_CHAN_REG_SDPTR);
522 writel(lli_dst->reg_lnk, base + D40_CHAN_REG_SDLNK);
523}
524
Rabin Vincente65889c2011-01-25 11:18:31 +0100525static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc)
526{
527 struct d40_lcla_pool *pool = &chan->base->lcla_pool;
528 struct d40_log_lli_bidir *lli = &desc->lli_log;
529 int lli_current = desc->lli_current;
530 int lli_len = desc->lli_len;
Rabin Vincent0c842b52011-01-25 11:18:35 +0100531 bool cyclic = desc->cyclic;
Rabin Vincente65889c2011-01-25 11:18:31 +0100532 int curr_lcla = -EINVAL;
Rabin Vincent0c842b52011-01-25 11:18:35 +0100533 int first_lcla = 0;
534 bool linkback;
Rabin Vincente65889c2011-01-25 11:18:31 +0100535
Rabin Vincent0c842b52011-01-25 11:18:35 +0100536 /*
537 * We may have partially running cyclic transfers, in case we did't get
538 * enough LCLA entries.
539 */
540 linkback = cyclic && lli_current == 0;
541
542 /*
543 * For linkback, we need one LCLA even with only one link, because we
544 * can't link back to the one in LCPA space
545 */
546 if (linkback || (lli_len - lli_current > 1)) {
Rabin Vincente65889c2011-01-25 11:18:31 +0100547 curr_lcla = d40_lcla_alloc_one(chan, desc);
Rabin Vincent0c842b52011-01-25 11:18:35 +0100548 first_lcla = curr_lcla;
549 }
Rabin Vincente65889c2011-01-25 11:18:31 +0100550
Rabin Vincent0c842b52011-01-25 11:18:35 +0100551 /*
552 * For linkback, we normally load the LCPA in the loop since we need to
553 * link it to the second LCLA and not the first. However, if we
554 * couldn't even get a first LCLA, then we have to run in LCPA and
555 * reload manually.
556 */
557 if (!linkback || curr_lcla == -EINVAL) {
558 unsigned int flags = 0;
Rabin Vincente65889c2011-01-25 11:18:31 +0100559
Rabin Vincent0c842b52011-01-25 11:18:35 +0100560 if (curr_lcla == -EINVAL)
561 flags |= LLI_TERM_INT;
562
563 d40_log_lli_lcpa_write(chan->lcpa,
564 &lli->dst[lli_current],
565 &lli->src[lli_current],
566 curr_lcla,
567 flags);
568 lli_current++;
569 }
Rabin Vincent6045f0b2011-01-25 11:18:32 +0100570
571 if (curr_lcla < 0)
572 goto out;
573
Rabin Vincente65889c2011-01-25 11:18:31 +0100574 for (; lli_current < lli_len; lli_current++) {
575 unsigned int lcla_offset = chan->phy_chan->num * 1024 +
576 8 * curr_lcla * 2;
577 struct d40_log_lli *lcla = pool->base + lcla_offset;
Rabin Vincent0c842b52011-01-25 11:18:35 +0100578 unsigned int flags = 0;
Rabin Vincente65889c2011-01-25 11:18:31 +0100579 int next_lcla;
580
581 if (lli_current + 1 < lli_len)
582 next_lcla = d40_lcla_alloc_one(chan, desc);
583 else
Rabin Vincent0c842b52011-01-25 11:18:35 +0100584 next_lcla = linkback ? first_lcla : -EINVAL;
Rabin Vincente65889c2011-01-25 11:18:31 +0100585
Rabin Vincent0c842b52011-01-25 11:18:35 +0100586 if (cyclic || next_lcla == -EINVAL)
587 flags |= LLI_TERM_INT;
588
589 if (linkback && curr_lcla == first_lcla) {
590 /* First link goes in both LCPA and LCLA */
591 d40_log_lli_lcpa_write(chan->lcpa,
592 &lli->dst[lli_current],
593 &lli->src[lli_current],
594 next_lcla, flags);
595 }
596
597 /*
598 * One unused LCLA in the cyclic case if the very first
599 * next_lcla fails...
600 */
Rabin Vincente65889c2011-01-25 11:18:31 +0100601 d40_log_lli_lcla_write(lcla,
602 &lli->dst[lli_current],
603 &lli->src[lli_current],
Rabin Vincent0c842b52011-01-25 11:18:35 +0100604 next_lcla, flags);
Rabin Vincente65889c2011-01-25 11:18:31 +0100605
606 dma_sync_single_range_for_device(chan->base->dev,
607 pool->dma_addr, lcla_offset,
608 2 * sizeof(struct d40_log_lli),
609 DMA_TO_DEVICE);
610
611 curr_lcla = next_lcla;
612
Rabin Vincent0c842b52011-01-25 11:18:35 +0100613 if (curr_lcla == -EINVAL || curr_lcla == first_lcla) {
Rabin Vincente65889c2011-01-25 11:18:31 +0100614 lli_current++;
615 break;
616 }
617 }
618
Rabin Vincent6045f0b2011-01-25 11:18:32 +0100619out:
Rabin Vincente65889c2011-01-25 11:18:31 +0100620 desc->lli_current = lli_current;
621}
622
Jonas Aaberg698e4732010-08-09 12:08:56 +0000623static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d)
624{
Rabin Vincent724a8572011-01-25 11:18:08 +0100625 if (chan_is_physical(d40c)) {
Rabin Vincent1c4b0922011-01-25 11:18:24 +0100626 d40_phy_lli_load(d40c, d40d);
Jonas Aaberg698e4732010-08-09 12:08:56 +0000627 d40d->lli_current = d40d->lli_len;
Rabin Vincente65889c2011-01-25 11:18:31 +0100628 } else
629 d40_log_lli_to_lcxa(d40c, d40d);
Jonas Aaberg698e4732010-08-09 12:08:56 +0000630}
631
Linus Walleij8d318a52010-03-30 15:33:42 +0200632static struct d40_desc *d40_first_active_get(struct d40_chan *d40c)
633{
634 struct d40_desc *d;
635
636 if (list_empty(&d40c->active))
637 return NULL;
638
639 d = list_first_entry(&d40c->active,
640 struct d40_desc,
641 node);
642 return d;
643}
644
645static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc)
646{
647 list_add_tail(&desc->node, &d40c->queue);
648}
649
650static struct d40_desc *d40_first_queued(struct d40_chan *d40c)
651{
652 struct d40_desc *d;
653
654 if (list_empty(&d40c->queue))
655 return NULL;
656
657 d = list_first_entry(&d40c->queue,
658 struct d40_desc,
659 node);
660 return d;
661}
662
Per Forlind49278e2010-12-20 18:31:38 +0100663static int d40_psize_2_burst_size(bool is_log, int psize)
664{
665 if (is_log) {
666 if (psize == STEDMA40_PSIZE_LOG_1)
667 return 1;
668 } else {
669 if (psize == STEDMA40_PSIZE_PHY_1)
670 return 1;
671 }
Linus Walleij8d318a52010-03-30 15:33:42 +0200672
Per Forlind49278e2010-12-20 18:31:38 +0100673 return 2 << psize;
674}
675
676/*
677 * The dma only supports transmitting packages up to
678 * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of
679 * dma elements required to send the entire sg list
680 */
681static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2)
682{
683 int dmalen;
684 u32 max_w = max(data_width1, data_width2);
685 u32 min_w = min(data_width1, data_width2);
686 u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w);
687
688 if (seg_max > STEDMA40_MAX_SEG_SIZE)
689 seg_max -= (1 << max_w);
690
691 if (!IS_ALIGNED(size, 1 << max_w))
692 return -EINVAL;
693
694 if (size <= seg_max)
695 dmalen = 1;
696 else {
697 dmalen = size / seg_max;
698 if (dmalen * seg_max < size)
699 dmalen++;
700 }
701 return dmalen;
702}
703
704static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len,
705 u32 data_width1, u32 data_width2)
706{
707 struct scatterlist *sg;
708 int i;
709 int len = 0;
710 int ret;
711
712 for_each_sg(sgl, sg, sg_len, i) {
713 ret = d40_size_2_dmalen(sg_dma_len(sg),
714 data_width1, data_width2);
715 if (ret < 0)
716 return ret;
717 len += ret;
718 }
719 return len;
720}
721
722/* Support functions for logical channels */
Linus Walleij8d318a52010-03-30 15:33:42 +0200723
724static int d40_channel_execute_command(struct d40_chan *d40c,
725 enum d40_command command)
726{
Jonas Aaberg767a9672010-08-09 12:08:34 +0000727 u32 status;
728 int i;
Linus Walleij8d318a52010-03-30 15:33:42 +0200729 void __iomem *active_reg;
730 int ret = 0;
731 unsigned long flags;
Jonas Aaberg1d392a72010-06-20 21:26:01 +0000732 u32 wmask;
Linus Walleij8d318a52010-03-30 15:33:42 +0200733
734 spin_lock_irqsave(&d40c->base->execmd_lock, flags);
735
736 if (d40c->phy_chan->num % 2 == 0)
737 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
738 else
739 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
740
741 if (command == D40_DMA_SUSPEND_REQ) {
742 status = (readl(active_reg) &
743 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
744 D40_CHAN_POS(d40c->phy_chan->num);
745
746 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
747 goto done;
748 }
749
Jonas Aaberg1d392a72010-06-20 21:26:01 +0000750 wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num));
751 writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)),
752 active_reg);
Linus Walleij8d318a52010-03-30 15:33:42 +0200753
754 if (command == D40_DMA_SUSPEND_REQ) {
755
756 for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) {
757 status = (readl(active_reg) &
758 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
759 D40_CHAN_POS(d40c->phy_chan->num);
760
761 cpu_relax();
762 /*
763 * Reduce the number of bus accesses while
764 * waiting for the DMA to suspend.
765 */
766 udelay(3);
767
768 if (status == D40_DMA_STOP ||
769 status == D40_DMA_SUSPENDED)
770 break;
771 }
772
773 if (i == D40_SUSPEND_MAX_IT) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +0100774 chan_err(d40c,
775 "unable to suspend the chl %d (log: %d) status %x\n",
776 d40c->phy_chan->num, d40c->log_num,
Linus Walleij8d318a52010-03-30 15:33:42 +0200777 status);
778 dump_stack();
779 ret = -EBUSY;
780 }
781
782 }
783done:
784 spin_unlock_irqrestore(&d40c->base->execmd_lock, flags);
785 return ret;
786}
787
788static void d40_term_all(struct d40_chan *d40c)
789{
790 struct d40_desc *d40d;
Linus Walleij8d318a52010-03-30 15:33:42 +0200791
792 /* Release active descriptors */
793 while ((d40d = d40_first_active_get(d40c))) {
794 d40_desc_remove(d40d);
Linus Walleij8d318a52010-03-30 15:33:42 +0200795 d40_desc_free(d40c, d40d);
796 }
797
798 /* Release queued descriptors waiting for transfer */
799 while ((d40d = d40_first_queued(d40c))) {
800 d40_desc_remove(d40d);
Linus Walleij8d318a52010-03-30 15:33:42 +0200801 d40_desc_free(d40c, d40d);
802 }
803
Linus Walleij8d318a52010-03-30 15:33:42 +0200804
805 d40c->pending_tx = 0;
806 d40c->busy = false;
807}
808
Rabin Vincent262d2912011-01-25 11:18:05 +0100809static void __d40_config_set_event(struct d40_chan *d40c, bool enable,
810 u32 event, int reg)
811{
Rabin Vincent8ca84682011-01-25 11:18:07 +0100812 void __iomem *addr = chan_base(d40c) + reg;
Rabin Vincent262d2912011-01-25 11:18:05 +0100813 int tries;
814
815 if (!enable) {
816 writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
817 | ~D40_EVENTLINE_MASK(event), addr);
818 return;
819 }
820
821 /*
822 * The hardware sometimes doesn't register the enable when src and dst
823 * event lines are active on the same logical channel. Retry to ensure
824 * it does. Usually only one retry is sufficient.
825 */
826 tries = 100;
827 while (--tries) {
828 writel((D40_ACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event))
829 | ~D40_EVENTLINE_MASK(event), addr);
830
831 if (readl(addr) & D40_EVENTLINE_MASK(event))
832 break;
833 }
834
835 if (tries != 99)
836 dev_dbg(chan2dev(d40c),
837 "[%s] workaround enable S%cLNK (%d tries)\n",
838 __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D',
839 100 - tries);
840
841 WARN_ON(!tries);
842}
843
Linus Walleij8d318a52010-03-30 15:33:42 +0200844static void d40_config_set_event(struct d40_chan *d40c, bool do_enable)
845{
Linus Walleij8d318a52010-03-30 15:33:42 +0200846 unsigned long flags;
847
Linus Walleij8d318a52010-03-30 15:33:42 +0200848 spin_lock_irqsave(&d40c->phy_chan->lock, flags);
849
850 /* Enable event line connected to device (or memcpy) */
851 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
852 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) {
853 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
854
Rabin Vincent262d2912011-01-25 11:18:05 +0100855 __d40_config_set_event(d40c, do_enable, event,
856 D40_CHAN_REG_SSLNK);
Linus Walleij8d318a52010-03-30 15:33:42 +0200857 }
Rabin Vincent262d2912011-01-25 11:18:05 +0100858
Linus Walleij8d318a52010-03-30 15:33:42 +0200859 if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) {
860 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
861
Rabin Vincent262d2912011-01-25 11:18:05 +0100862 __d40_config_set_event(d40c, do_enable, event,
863 D40_CHAN_REG_SDLNK);
Linus Walleij8d318a52010-03-30 15:33:42 +0200864 }
865
866 spin_unlock_irqrestore(&d40c->phy_chan->lock, flags);
867}
868
Jonas Aaberga5ebca42010-05-18 00:41:09 +0200869static u32 d40_chan_has_events(struct d40_chan *d40c)
Linus Walleij8d318a52010-03-30 15:33:42 +0200870{
Rabin Vincent8ca84682011-01-25 11:18:07 +0100871 void __iomem *chanbase = chan_base(d40c);
Jonas Aabergbe8cb7d2010-08-09 12:07:44 +0000872 u32 val;
Linus Walleij8d318a52010-03-30 15:33:42 +0200873
Rabin Vincent8ca84682011-01-25 11:18:07 +0100874 val = readl(chanbase + D40_CHAN_REG_SSLNK);
875 val |= readl(chanbase + D40_CHAN_REG_SDLNK);
Linus Walleij8d318a52010-03-30 15:33:42 +0200876
Jonas Aaberga5ebca42010-05-18 00:41:09 +0200877 return val;
Linus Walleij8d318a52010-03-30 15:33:42 +0200878}
879
Rabin Vincent20a5b6d2010-10-12 13:00:52 +0000880static u32 d40_get_prmo(struct d40_chan *d40c)
881{
882 static const unsigned int phy_map[] = {
883 [STEDMA40_PCHAN_BASIC_MODE]
884 = D40_DREG_PRMO_PCHAN_BASIC,
885 [STEDMA40_PCHAN_MODULO_MODE]
886 = D40_DREG_PRMO_PCHAN_MODULO,
887 [STEDMA40_PCHAN_DOUBLE_DST_MODE]
888 = D40_DREG_PRMO_PCHAN_DOUBLE_DST,
889 };
890 static const unsigned int log_map[] = {
891 [STEDMA40_LCHAN_SRC_PHY_DST_LOG]
892 = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG,
893 [STEDMA40_LCHAN_SRC_LOG_DST_PHY]
894 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY,
895 [STEDMA40_LCHAN_SRC_LOG_DST_LOG]
896 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG,
897 };
898
Rabin Vincent724a8572011-01-25 11:18:08 +0100899 if (chan_is_physical(d40c))
Rabin Vincent20a5b6d2010-10-12 13:00:52 +0000900 return phy_map[d40c->dma_cfg.mode_opt];
901 else
902 return log_map[d40c->dma_cfg.mode_opt];
903}
904
Jonas Aabergb55912c2010-08-09 12:08:02 +0000905static void d40_config_write(struct d40_chan *d40c)
Linus Walleij8d318a52010-03-30 15:33:42 +0200906{
907 u32 addr_base;
908 u32 var;
Linus Walleij8d318a52010-03-30 15:33:42 +0200909
910 /* Odd addresses are even addresses + 4 */
911 addr_base = (d40c->phy_chan->num % 2) * 4;
912 /* Setup channel mode to logical or physical */
Rabin Vincent724a8572011-01-25 11:18:08 +0100913 var = ((u32)(chan_is_logical(d40c)) + 1) <<
Linus Walleij8d318a52010-03-30 15:33:42 +0200914 D40_CHAN_POS(d40c->phy_chan->num);
915 writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base);
916
917 /* Setup operational mode option register */
Rabin Vincent20a5b6d2010-10-12 13:00:52 +0000918 var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num);
Linus Walleij8d318a52010-03-30 15:33:42 +0200919
920 writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base);
921
Rabin Vincent724a8572011-01-25 11:18:08 +0100922 if (chan_is_logical(d40c)) {
Rabin Vincent8ca84682011-01-25 11:18:07 +0100923 int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS)
924 & D40_SREG_ELEM_LOG_LIDX_MASK;
925 void __iomem *chanbase = chan_base(d40c);
926
Linus Walleij8d318a52010-03-30 15:33:42 +0200927 /* Set default config for CFG reg */
Rabin Vincent8ca84682011-01-25 11:18:07 +0100928 writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG);
929 writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG);
Linus Walleij8d318a52010-03-30 15:33:42 +0200930
Jonas Aabergb55912c2010-08-09 12:08:02 +0000931 /* Set LIDX for lcla */
Rabin Vincent8ca84682011-01-25 11:18:07 +0100932 writel(lidx, chanbase + D40_CHAN_REG_SSELT);
933 writel(lidx, chanbase + D40_CHAN_REG_SDELT);
Linus Walleij8d318a52010-03-30 15:33:42 +0200934 }
Linus Walleij8d318a52010-03-30 15:33:42 +0200935}
936
Jonas Aabergaa182ae2010-08-09 12:08:26 +0000937static u32 d40_residue(struct d40_chan *d40c)
938{
939 u32 num_elt;
940
Rabin Vincent724a8572011-01-25 11:18:08 +0100941 if (chan_is_logical(d40c))
Jonas Aabergaa182ae2010-08-09 12:08:26 +0000942 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK)
943 >> D40_MEM_LCSP2_ECNT_POS;
Rabin Vincent8ca84682011-01-25 11:18:07 +0100944 else {
945 u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT);
946 num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK)
947 >> D40_SREG_ELEM_PHY_ECNT_POS;
948 }
949
Jonas Aabergaa182ae2010-08-09 12:08:26 +0000950 return num_elt * (1 << d40c->dma_cfg.dst_info.data_width);
951}
952
953static bool d40_tx_is_linked(struct d40_chan *d40c)
954{
955 bool is_link;
956
Rabin Vincent724a8572011-01-25 11:18:08 +0100957 if (chan_is_logical(d40c))
Jonas Aabergaa182ae2010-08-09 12:08:26 +0000958 is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK;
959 else
Rabin Vincent8ca84682011-01-25 11:18:07 +0100960 is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK)
961 & D40_SREG_LNK_PHYS_LNK_MASK;
962
Jonas Aabergaa182ae2010-08-09 12:08:26 +0000963 return is_link;
964}
965
Rabin Vincent86eb5fb2011-01-25 11:18:34 +0100966static int d40_pause(struct d40_chan *d40c)
Jonas Aabergaa182ae2010-08-09 12:08:26 +0000967{
Jonas Aabergaa182ae2010-08-09 12:08:26 +0000968 int res = 0;
969 unsigned long flags;
970
Jonas Aaberg3ac012a2010-08-09 12:09:12 +0000971 if (!d40c->busy)
972 return 0;
973
Jonas Aabergaa182ae2010-08-09 12:08:26 +0000974 spin_lock_irqsave(&d40c->lock, flags);
975
976 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
977 if (res == 0) {
Rabin Vincent724a8572011-01-25 11:18:08 +0100978 if (chan_is_logical(d40c)) {
Jonas Aabergaa182ae2010-08-09 12:08:26 +0000979 d40_config_set_event(d40c, false);
980 /* Resume the other logical channels if any */
981 if (d40_chan_has_events(d40c))
982 res = d40_channel_execute_command(d40c,
983 D40_DMA_RUN);
984 }
985 }
986
987 spin_unlock_irqrestore(&d40c->lock, flags);
988 return res;
989}
990
Rabin Vincent86eb5fb2011-01-25 11:18:34 +0100991static int d40_resume(struct d40_chan *d40c)
Jonas Aabergaa182ae2010-08-09 12:08:26 +0000992{
Jonas Aabergaa182ae2010-08-09 12:08:26 +0000993 int res = 0;
994 unsigned long flags;
995
Jonas Aaberg3ac012a2010-08-09 12:09:12 +0000996 if (!d40c->busy)
997 return 0;
998
Jonas Aabergaa182ae2010-08-09 12:08:26 +0000999 spin_lock_irqsave(&d40c->lock, flags);
1000
1001 if (d40c->base->rev == 0)
Rabin Vincent724a8572011-01-25 11:18:08 +01001002 if (chan_is_logical(d40c)) {
Jonas Aabergaa182ae2010-08-09 12:08:26 +00001003 res = d40_channel_execute_command(d40c,
1004 D40_DMA_SUSPEND_REQ);
1005 goto no_suspend;
1006 }
1007
1008 /* If bytes left to transfer or linked tx resume job */
1009 if (d40_residue(d40c) || d40_tx_is_linked(d40c)) {
1010
Rabin Vincent724a8572011-01-25 11:18:08 +01001011 if (chan_is_logical(d40c))
Jonas Aabergaa182ae2010-08-09 12:08:26 +00001012 d40_config_set_event(d40c, true);
1013
1014 res = d40_channel_execute_command(d40c, D40_DMA_RUN);
1015 }
1016
1017no_suspend:
1018 spin_unlock_irqrestore(&d40c->lock, flags);
1019 return res;
1020}
1021
Rabin Vincent86eb5fb2011-01-25 11:18:34 +01001022static int d40_terminate_all(struct d40_chan *chan)
1023{
1024 unsigned long flags;
1025 int ret = 0;
1026
1027 ret = d40_pause(chan);
1028 if (!ret && chan_is_physical(chan))
1029 ret = d40_channel_execute_command(chan, D40_DMA_STOP);
1030
1031 spin_lock_irqsave(&chan->lock, flags);
1032 d40_term_all(chan);
1033 spin_unlock_irqrestore(&chan->lock, flags);
1034
1035 return ret;
1036}
1037
Linus Walleij8d318a52010-03-30 15:33:42 +02001038static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx)
1039{
1040 struct d40_chan *d40c = container_of(tx->chan,
1041 struct d40_chan,
1042 chan);
1043 struct d40_desc *d40d = container_of(tx, struct d40_desc, txd);
1044 unsigned long flags;
1045
1046 spin_lock_irqsave(&d40c->lock, flags);
1047
Jonas Aabergaa182ae2010-08-09 12:08:26 +00001048 d40c->chan.cookie++;
1049
1050 if (d40c->chan.cookie < 0)
1051 d40c->chan.cookie = 1;
1052
1053 d40d->txd.cookie = d40c->chan.cookie;
1054
Linus Walleij8d318a52010-03-30 15:33:42 +02001055 d40_desc_queue(d40c, d40d);
1056
1057 spin_unlock_irqrestore(&d40c->lock, flags);
1058
1059 return tx->cookie;
1060}
1061
1062static int d40_start(struct d40_chan *d40c)
1063{
Linus Walleijf4185592010-06-22 18:06:42 -07001064 if (d40c->base->rev == 0) {
1065 int err;
1066
Rabin Vincent724a8572011-01-25 11:18:08 +01001067 if (chan_is_logical(d40c)) {
Linus Walleijf4185592010-06-22 18:06:42 -07001068 err = d40_channel_execute_command(d40c,
1069 D40_DMA_SUSPEND_REQ);
1070 if (err)
1071 return err;
1072 }
1073 }
1074
Rabin Vincent724a8572011-01-25 11:18:08 +01001075 if (chan_is_logical(d40c))
Linus Walleij8d318a52010-03-30 15:33:42 +02001076 d40_config_set_event(d40c, true);
Linus Walleij8d318a52010-03-30 15:33:42 +02001077
Jonas Aaberg0c322692010-06-20 21:25:46 +00001078 return d40_channel_execute_command(d40c, D40_DMA_RUN);
Linus Walleij8d318a52010-03-30 15:33:42 +02001079}
1080
1081static struct d40_desc *d40_queue_start(struct d40_chan *d40c)
1082{
1083 struct d40_desc *d40d;
1084 int err;
1085
1086 /* Start queued jobs, if any */
1087 d40d = d40_first_queued(d40c);
1088
1089 if (d40d != NULL) {
1090 d40c->busy = true;
1091
1092 /* Remove from queue */
1093 d40_desc_remove(d40d);
1094
1095 /* Add to active queue */
1096 d40_desc_submit(d40c, d40d);
1097
Rabin Vincent7d83a852011-01-25 11:18:06 +01001098 /* Initiate DMA job */
1099 d40_desc_load(d40c, d40d);
Jonas Aaberg698e4732010-08-09 12:08:56 +00001100
Rabin Vincent7d83a852011-01-25 11:18:06 +01001101 /* Start dma job */
1102 err = d40_start(d40c);
Linus Walleij8d318a52010-03-30 15:33:42 +02001103
Rabin Vincent7d83a852011-01-25 11:18:06 +01001104 if (err)
1105 return NULL;
Linus Walleij8d318a52010-03-30 15:33:42 +02001106 }
1107
1108 return d40d;
1109}
1110
1111/* called from interrupt context */
1112static void dma_tc_handle(struct d40_chan *d40c)
1113{
1114 struct d40_desc *d40d;
1115
Linus Walleij8d318a52010-03-30 15:33:42 +02001116 /* Get first active entry from list */
1117 d40d = d40_first_active_get(d40c);
1118
1119 if (d40d == NULL)
1120 return;
1121
Rabin Vincent0c842b52011-01-25 11:18:35 +01001122 if (d40d->cyclic) {
1123 /*
1124 * If this was a paritially loaded list, we need to reloaded
1125 * it, and only when the list is completed. We need to check
1126 * for done because the interrupt will hit for every link, and
1127 * not just the last one.
1128 */
1129 if (d40d->lli_current < d40d->lli_len
1130 && !d40_tx_is_linked(d40c)
1131 && !d40_residue(d40c)) {
1132 d40_lcla_free_all(d40c, d40d);
1133 d40_desc_load(d40c, d40d);
1134 (void) d40_start(d40c);
Linus Walleij8d318a52010-03-30 15:33:42 +02001135
Rabin Vincent0c842b52011-01-25 11:18:35 +01001136 if (d40d->lli_current == d40d->lli_len)
1137 d40d->lli_current = 0;
1138 }
1139 } else {
1140 d40_lcla_free_all(d40c, d40d);
1141
1142 if (d40d->lli_current < d40d->lli_len) {
1143 d40_desc_load(d40c, d40d);
1144 /* Start dma job */
1145 (void) d40_start(d40c);
1146 return;
1147 }
1148
1149 if (d40_queue_start(d40c) == NULL)
1150 d40c->busy = false;
Linus Walleij8d318a52010-03-30 15:33:42 +02001151 }
1152
Linus Walleij8d318a52010-03-30 15:33:42 +02001153 d40c->pending_tx++;
1154 tasklet_schedule(&d40c->tasklet);
1155
1156}
1157
1158static void dma_tasklet(unsigned long data)
1159{
1160 struct d40_chan *d40c = (struct d40_chan *) data;
Jonas Aaberg767a9672010-08-09 12:08:34 +00001161 struct d40_desc *d40d;
Linus Walleij8d318a52010-03-30 15:33:42 +02001162 unsigned long flags;
1163 dma_async_tx_callback callback;
1164 void *callback_param;
1165
1166 spin_lock_irqsave(&d40c->lock, flags);
1167
1168 /* Get first active entry from list */
Jonas Aaberg767a9672010-08-09 12:08:34 +00001169 d40d = d40_first_active_get(d40c);
Jonas Aaberg767a9672010-08-09 12:08:34 +00001170 if (d40d == NULL)
Linus Walleij8d318a52010-03-30 15:33:42 +02001171 goto err;
1172
Rabin Vincent0c842b52011-01-25 11:18:35 +01001173 if (!d40d->cyclic)
1174 d40c->completed = d40d->txd.cookie;
Linus Walleij8d318a52010-03-30 15:33:42 +02001175
1176 /*
1177 * If terminating a channel pending_tx is set to zero.
1178 * This prevents any finished active jobs to return to the client.
1179 */
1180 if (d40c->pending_tx == 0) {
1181 spin_unlock_irqrestore(&d40c->lock, flags);
1182 return;
1183 }
1184
1185 /* Callback to client */
Jonas Aaberg767a9672010-08-09 12:08:34 +00001186 callback = d40d->txd.callback;
1187 callback_param = d40d->txd.callback_param;
Linus Walleij8d318a52010-03-30 15:33:42 +02001188
Rabin Vincent0c842b52011-01-25 11:18:35 +01001189 if (!d40d->cyclic) {
1190 if (async_tx_test_ack(&d40d->txd)) {
1191 d40_pool_lli_free(d40c, d40d);
Jonas Aaberg767a9672010-08-09 12:08:34 +00001192 d40_desc_remove(d40d);
Rabin Vincent0c842b52011-01-25 11:18:35 +01001193 d40_desc_free(d40c, d40d);
1194 } else {
1195 if (!d40d->is_in_client_list) {
1196 d40_desc_remove(d40d);
1197 d40_lcla_free_all(d40c, d40d);
1198 list_add_tail(&d40d->node, &d40c->client);
1199 d40d->is_in_client_list = true;
1200 }
Linus Walleij8d318a52010-03-30 15:33:42 +02001201 }
1202 }
1203
1204 d40c->pending_tx--;
1205
1206 if (d40c->pending_tx)
1207 tasklet_schedule(&d40c->tasklet);
1208
1209 spin_unlock_irqrestore(&d40c->lock, flags);
1210
Jonas Aaberg767a9672010-08-09 12:08:34 +00001211 if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT))
Linus Walleij8d318a52010-03-30 15:33:42 +02001212 callback(callback_param);
1213
1214 return;
1215
1216 err:
1217 /* Rescue manouver if receiving double interrupts */
1218 if (d40c->pending_tx > 0)
1219 d40c->pending_tx--;
1220 spin_unlock_irqrestore(&d40c->lock, flags);
1221}
1222
1223static irqreturn_t d40_handle_interrupt(int irq, void *data)
1224{
1225 static const struct d40_interrupt_lookup il[] = {
1226 {D40_DREG_LCTIS0, D40_DREG_LCICR0, false, 0},
1227 {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32},
1228 {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64},
1229 {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96},
1230 {D40_DREG_LCEIS0, D40_DREG_LCICR0, true, 0},
1231 {D40_DREG_LCEIS1, D40_DREG_LCICR1, true, 32},
1232 {D40_DREG_LCEIS2, D40_DREG_LCICR2, true, 64},
1233 {D40_DREG_LCEIS3, D40_DREG_LCICR3, true, 96},
1234 {D40_DREG_PCTIS, D40_DREG_PCICR, false, D40_PHY_CHAN},
1235 {D40_DREG_PCEIS, D40_DREG_PCICR, true, D40_PHY_CHAN},
1236 };
1237
1238 int i;
1239 u32 regs[ARRAY_SIZE(il)];
Linus Walleij8d318a52010-03-30 15:33:42 +02001240 u32 idx;
1241 u32 row;
1242 long chan = -1;
1243 struct d40_chan *d40c;
1244 unsigned long flags;
1245 struct d40_base *base = data;
1246
1247 spin_lock_irqsave(&base->interrupt_lock, flags);
1248
1249 /* Read interrupt status of both logical and physical channels */
1250 for (i = 0; i < ARRAY_SIZE(il); i++)
1251 regs[i] = readl(base->virtbase + il[i].src);
1252
1253 for (;;) {
1254
1255 chan = find_next_bit((unsigned long *)regs,
1256 BITS_PER_LONG * ARRAY_SIZE(il), chan + 1);
1257
1258 /* No more set bits found? */
1259 if (chan == BITS_PER_LONG * ARRAY_SIZE(il))
1260 break;
1261
1262 row = chan / BITS_PER_LONG;
1263 idx = chan & (BITS_PER_LONG - 1);
1264
1265 /* ACK interrupt */
Jonas Aaberg1b003482010-08-09 12:07:54 +00001266 writel(1 << idx, base->virtbase + il[row].clr);
Linus Walleij8d318a52010-03-30 15:33:42 +02001267
1268 if (il[row].offset == D40_PHY_CHAN)
1269 d40c = base->lookup_phy_chans[idx];
1270 else
1271 d40c = base->lookup_log_chans[il[row].offset + idx];
1272 spin_lock(&d40c->lock);
1273
1274 if (!il[row].is_error)
1275 dma_tc_handle(d40c);
1276 else
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001277 d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n",
1278 chan, il[row].offset, idx);
Linus Walleij8d318a52010-03-30 15:33:42 +02001279
1280 spin_unlock(&d40c->lock);
1281 }
1282
1283 spin_unlock_irqrestore(&base->interrupt_lock, flags);
1284
1285 return IRQ_HANDLED;
1286}
1287
Linus Walleij8d318a52010-03-30 15:33:42 +02001288static int d40_validate_conf(struct d40_chan *d40c,
1289 struct stedma40_chan_cfg *conf)
1290{
1291 int res = 0;
1292 u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type);
1293 u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type);
Rabin Vincent38bdbf02010-10-12 13:00:51 +00001294 bool is_log = conf->mode == STEDMA40_MODE_LOGICAL;
Linus Walleij8d318a52010-03-30 15:33:42 +02001295
Linus Walleij0747c7ba2010-08-09 12:07:36 +00001296 if (!conf->dir) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001297 chan_err(d40c, "Invalid direction.\n");
Linus Walleij0747c7ba2010-08-09 12:07:36 +00001298 res = -EINVAL;
1299 }
1300
1301 if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY &&
1302 d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 &&
1303 d40c->runtime_addr == 0) {
1304
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001305 chan_err(d40c, "Invalid TX channel address (%d)\n",
1306 conf->dst_dev_type);
Linus Walleij0747c7ba2010-08-09 12:07:36 +00001307 res = -EINVAL;
1308 }
1309
1310 if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY &&
1311 d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 &&
1312 d40c->runtime_addr == 0) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001313 chan_err(d40c, "Invalid RX channel address (%d)\n",
1314 conf->src_dev_type);
Linus Walleij0747c7ba2010-08-09 12:07:36 +00001315 res = -EINVAL;
1316 }
1317
1318 if (conf->dir == STEDMA40_MEM_TO_PERIPH &&
Linus Walleij8d318a52010-03-30 15:33:42 +02001319 dst_event_group == STEDMA40_DEV_DST_MEMORY) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001320 chan_err(d40c, "Invalid dst\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02001321 res = -EINVAL;
1322 }
1323
Linus Walleij0747c7ba2010-08-09 12:07:36 +00001324 if (conf->dir == STEDMA40_PERIPH_TO_MEM &&
Linus Walleij8d318a52010-03-30 15:33:42 +02001325 src_event_group == STEDMA40_DEV_SRC_MEMORY) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001326 chan_err(d40c, "Invalid src\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02001327 res = -EINVAL;
1328 }
1329
1330 if (src_event_group == STEDMA40_DEV_SRC_MEMORY &&
1331 dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001332 chan_err(d40c, "No event line\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02001333 res = -EINVAL;
1334 }
1335
1336 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH &&
1337 (src_event_group != dst_event_group)) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001338 chan_err(d40c, "Invalid event group\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02001339 res = -EINVAL;
1340 }
1341
1342 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) {
1343 /*
1344 * DMAC HW supports it. Will be added to this driver,
1345 * in case any dma client requires it.
1346 */
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001347 chan_err(d40c, "periph to periph not supported\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02001348 res = -EINVAL;
1349 }
1350
Per Forlind49278e2010-12-20 18:31:38 +01001351 if (d40_psize_2_burst_size(is_log, conf->src_info.psize) *
1352 (1 << conf->src_info.data_width) !=
1353 d40_psize_2_burst_size(is_log, conf->dst_info.psize) *
1354 (1 << conf->dst_info.data_width)) {
1355 /*
1356 * The DMAC hardware only supports
1357 * src (burst x width) == dst (burst x width)
1358 */
1359
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001360 chan_err(d40c, "src (burst x width) != dst (burst x width)\n");
Per Forlind49278e2010-12-20 18:31:38 +01001361 res = -EINVAL;
1362 }
1363
Linus Walleij8d318a52010-03-30 15:33:42 +02001364 return res;
1365}
1366
1367static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src,
Marcin Mielczarczyk4aed79b2010-05-18 00:41:21 +02001368 int log_event_line, bool is_log)
Linus Walleij8d318a52010-03-30 15:33:42 +02001369{
1370 unsigned long flags;
1371 spin_lock_irqsave(&phy->lock, flags);
Marcin Mielczarczyk4aed79b2010-05-18 00:41:21 +02001372 if (!is_log) {
Linus Walleij8d318a52010-03-30 15:33:42 +02001373 /* Physical interrupts are masked per physical full channel */
1374 if (phy->allocated_src == D40_ALLOC_FREE &&
1375 phy->allocated_dst == D40_ALLOC_FREE) {
1376 phy->allocated_dst = D40_ALLOC_PHY;
1377 phy->allocated_src = D40_ALLOC_PHY;
1378 goto found;
1379 } else
1380 goto not_found;
1381 }
1382
1383 /* Logical channel */
1384 if (is_src) {
1385 if (phy->allocated_src == D40_ALLOC_PHY)
1386 goto not_found;
1387
1388 if (phy->allocated_src == D40_ALLOC_FREE)
1389 phy->allocated_src = D40_ALLOC_LOG_FREE;
1390
1391 if (!(phy->allocated_src & (1 << log_event_line))) {
1392 phy->allocated_src |= 1 << log_event_line;
1393 goto found;
1394 } else
1395 goto not_found;
1396 } else {
1397 if (phy->allocated_dst == D40_ALLOC_PHY)
1398 goto not_found;
1399
1400 if (phy->allocated_dst == D40_ALLOC_FREE)
1401 phy->allocated_dst = D40_ALLOC_LOG_FREE;
1402
1403 if (!(phy->allocated_dst & (1 << log_event_line))) {
1404 phy->allocated_dst |= 1 << log_event_line;
1405 goto found;
1406 } else
1407 goto not_found;
1408 }
1409
1410not_found:
1411 spin_unlock_irqrestore(&phy->lock, flags);
1412 return false;
1413found:
1414 spin_unlock_irqrestore(&phy->lock, flags);
1415 return true;
1416}
1417
1418static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src,
1419 int log_event_line)
1420{
1421 unsigned long flags;
1422 bool is_free = false;
1423
1424 spin_lock_irqsave(&phy->lock, flags);
1425 if (!log_event_line) {
Linus Walleij8d318a52010-03-30 15:33:42 +02001426 phy->allocated_dst = D40_ALLOC_FREE;
1427 phy->allocated_src = D40_ALLOC_FREE;
1428 is_free = true;
1429 goto out;
1430 }
1431
1432 /* Logical channel */
1433 if (is_src) {
1434 phy->allocated_src &= ~(1 << log_event_line);
1435 if (phy->allocated_src == D40_ALLOC_LOG_FREE)
1436 phy->allocated_src = D40_ALLOC_FREE;
1437 } else {
1438 phy->allocated_dst &= ~(1 << log_event_line);
1439 if (phy->allocated_dst == D40_ALLOC_LOG_FREE)
1440 phy->allocated_dst = D40_ALLOC_FREE;
1441 }
1442
1443 is_free = ((phy->allocated_src | phy->allocated_dst) ==
1444 D40_ALLOC_FREE);
1445
1446out:
1447 spin_unlock_irqrestore(&phy->lock, flags);
1448
1449 return is_free;
1450}
1451
1452static int d40_allocate_channel(struct d40_chan *d40c)
1453{
1454 int dev_type;
1455 int event_group;
1456 int event_line;
1457 struct d40_phy_res *phys;
1458 int i;
1459 int j;
1460 int log_num;
1461 bool is_src;
Rabin Vincent38bdbf02010-10-12 13:00:51 +00001462 bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL;
Linus Walleij8d318a52010-03-30 15:33:42 +02001463
1464 phys = d40c->base->phy_res;
1465
1466 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1467 dev_type = d40c->dma_cfg.src_dev_type;
1468 log_num = 2 * dev_type;
1469 is_src = true;
1470 } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1471 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1472 /* dst event lines are used for logical memcpy */
1473 dev_type = d40c->dma_cfg.dst_dev_type;
1474 log_num = 2 * dev_type + 1;
1475 is_src = false;
1476 } else
1477 return -EINVAL;
1478
1479 event_group = D40_TYPE_TO_GROUP(dev_type);
1480 event_line = D40_TYPE_TO_EVENT(dev_type);
1481
1482 if (!is_log) {
1483 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1484 /* Find physical half channel */
1485 for (i = 0; i < d40c->base->num_phy_chans; i++) {
1486
Marcin Mielczarczyk4aed79b2010-05-18 00:41:21 +02001487 if (d40_alloc_mask_set(&phys[i], is_src,
1488 0, is_log))
Linus Walleij8d318a52010-03-30 15:33:42 +02001489 goto found_phy;
1490 }
1491 } else
1492 for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1493 int phy_num = j + event_group * 2;
1494 for (i = phy_num; i < phy_num + 2; i++) {
Linus Walleij508849a2010-06-20 21:26:07 +00001495 if (d40_alloc_mask_set(&phys[i],
1496 is_src,
1497 0,
1498 is_log))
Linus Walleij8d318a52010-03-30 15:33:42 +02001499 goto found_phy;
1500 }
1501 }
1502 return -EINVAL;
1503found_phy:
1504 d40c->phy_chan = &phys[i];
1505 d40c->log_num = D40_PHY_CHAN;
1506 goto out;
1507 }
1508 if (dev_type == -1)
1509 return -EINVAL;
1510
1511 /* Find logical channel */
1512 for (j = 0; j < d40c->base->num_phy_chans; j += 8) {
1513 int phy_num = j + event_group * 2;
1514 /*
1515 * Spread logical channels across all available physical rather
1516 * than pack every logical channel at the first available phy
1517 * channels.
1518 */
1519 if (is_src) {
1520 for (i = phy_num; i < phy_num + 2; i++) {
1521 if (d40_alloc_mask_set(&phys[i], is_src,
Marcin Mielczarczyk4aed79b2010-05-18 00:41:21 +02001522 event_line, is_log))
Linus Walleij8d318a52010-03-30 15:33:42 +02001523 goto found_log;
1524 }
1525 } else {
1526 for (i = phy_num + 1; i >= phy_num; i--) {
1527 if (d40_alloc_mask_set(&phys[i], is_src,
Marcin Mielczarczyk4aed79b2010-05-18 00:41:21 +02001528 event_line, is_log))
Linus Walleij8d318a52010-03-30 15:33:42 +02001529 goto found_log;
1530 }
1531 }
1532 }
1533 return -EINVAL;
1534
1535found_log:
1536 d40c->phy_chan = &phys[i];
1537 d40c->log_num = log_num;
1538out:
1539
1540 if (is_log)
1541 d40c->base->lookup_log_chans[d40c->log_num] = d40c;
1542 else
1543 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c;
1544
1545 return 0;
1546
1547}
1548
Linus Walleij8d318a52010-03-30 15:33:42 +02001549static int d40_config_memcpy(struct d40_chan *d40c)
1550{
1551 dma_cap_mask_t cap = d40c->chan.device->cap_mask;
1552
1553 if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) {
1554 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log;
1555 d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY;
1556 d40c->dma_cfg.dst_dev_type = d40c->base->plat_data->
1557 memcpy[d40c->chan.chan_id];
1558
1559 } else if (dma_has_cap(DMA_MEMCPY, cap) &&
1560 dma_has_cap(DMA_SLAVE, cap)) {
1561 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy;
1562 } else {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001563 chan_err(d40c, "No memcpy\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02001564 return -EINVAL;
1565 }
1566
1567 return 0;
1568}
1569
1570
1571static int d40_free_dma(struct d40_chan *d40c)
1572{
1573
1574 int res = 0;
Jonas Aabergd181b3a2010-06-20 21:26:38 +00001575 u32 event;
Linus Walleij8d318a52010-03-30 15:33:42 +02001576 struct d40_phy_res *phy = d40c->phy_chan;
1577 bool is_src;
Per Fridena8be8622010-06-20 21:24:59 +00001578 struct d40_desc *d;
1579 struct d40_desc *_d;
1580
Linus Walleij8d318a52010-03-30 15:33:42 +02001581
1582 /* Terminate all queued and active transfers */
1583 d40_term_all(d40c);
1584
Per Fridena8be8622010-06-20 21:24:59 +00001585 /* Release client owned descriptors */
1586 if (!list_empty(&d40c->client))
1587 list_for_each_entry_safe(d, _d, &d40c->client, node) {
Rabin Vincentb00f9382011-01-25 11:18:15 +01001588 d40_pool_lli_free(d40c, d);
Per Fridena8be8622010-06-20 21:24:59 +00001589 d40_desc_remove(d);
Per Fridena8be8622010-06-20 21:24:59 +00001590 d40_desc_free(d40c, d);
1591 }
1592
Linus Walleij8d318a52010-03-30 15:33:42 +02001593 if (phy == NULL) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001594 chan_err(d40c, "phy == null\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02001595 return -EINVAL;
1596 }
1597
1598 if (phy->allocated_src == D40_ALLOC_FREE &&
1599 phy->allocated_dst == D40_ALLOC_FREE) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001600 chan_err(d40c, "channel already free\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02001601 return -EINVAL;
1602 }
1603
Linus Walleij8d318a52010-03-30 15:33:42 +02001604 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
1605 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
1606 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
Linus Walleij8d318a52010-03-30 15:33:42 +02001607 is_src = false;
1608 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
1609 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
Linus Walleij8d318a52010-03-30 15:33:42 +02001610 is_src = true;
1611 } else {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001612 chan_err(d40c, "Unknown direction\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02001613 return -EINVAL;
1614 }
1615
Jonas Aabergd181b3a2010-06-20 21:26:38 +00001616 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ);
1617 if (res) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001618 chan_err(d40c, "suspend failed\n");
Jonas Aabergd181b3a2010-06-20 21:26:38 +00001619 return res;
1620 }
Linus Walleij8d318a52010-03-30 15:33:42 +02001621
Rabin Vincent724a8572011-01-25 11:18:08 +01001622 if (chan_is_logical(d40c)) {
Jonas Aabergd181b3a2010-06-20 21:26:38 +00001623 /* Release logical channel, deactivate the event line */
1624
1625 d40_config_set_event(d40c, false);
Linus Walleij8d318a52010-03-30 15:33:42 +02001626 d40c->base->lookup_log_chans[d40c->log_num] = NULL;
1627
1628 /*
1629 * Check if there are more logical allocation
1630 * on this phy channel.
1631 */
1632 if (!d40_alloc_mask_free(phy, is_src, event)) {
1633 /* Resume the other logical channels if any */
1634 if (d40_chan_has_events(d40c)) {
1635 res = d40_channel_execute_command(d40c,
1636 D40_DMA_RUN);
1637 if (res) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001638 chan_err(d40c,
1639 "Executing RUN command\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02001640 return res;
1641 }
1642 }
1643 return 0;
1644 }
Jonas Aabergd181b3a2010-06-20 21:26:38 +00001645 } else {
1646 (void) d40_alloc_mask_free(phy, is_src, 0);
1647 }
Linus Walleij8d318a52010-03-30 15:33:42 +02001648
1649 /* Release physical channel */
1650 res = d40_channel_execute_command(d40c, D40_DMA_STOP);
1651 if (res) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001652 chan_err(d40c, "Failed to stop channel\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02001653 return res;
1654 }
1655 d40c->phy_chan = NULL;
Rabin Vincentce2ca122010-10-12 13:00:49 +00001656 d40c->configured = false;
Linus Walleij8d318a52010-03-30 15:33:42 +02001657 d40c->base->lookup_phy_chans[phy->num] = NULL;
1658
1659 return 0;
Linus Walleij8d318a52010-03-30 15:33:42 +02001660}
1661
Jonas Aaberga5ebca42010-05-18 00:41:09 +02001662static bool d40_is_paused(struct d40_chan *d40c)
1663{
Rabin Vincent8ca84682011-01-25 11:18:07 +01001664 void __iomem *chanbase = chan_base(d40c);
Jonas Aaberga5ebca42010-05-18 00:41:09 +02001665 bool is_paused = false;
1666 unsigned long flags;
1667 void __iomem *active_reg;
1668 u32 status;
1669 u32 event;
Jonas Aaberga5ebca42010-05-18 00:41:09 +02001670
1671 spin_lock_irqsave(&d40c->lock, flags);
1672
Rabin Vincent724a8572011-01-25 11:18:08 +01001673 if (chan_is_physical(d40c)) {
Jonas Aaberga5ebca42010-05-18 00:41:09 +02001674 if (d40c->phy_chan->num % 2 == 0)
1675 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE;
1676 else
1677 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO;
1678
1679 status = (readl(active_reg) &
1680 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >>
1681 D40_CHAN_POS(d40c->phy_chan->num);
1682 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP)
1683 is_paused = true;
1684
1685 goto _exit;
1686 }
1687
Jonas Aaberga5ebca42010-05-18 00:41:09 +02001688 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH ||
Jonas Aaberg9dbfbd35c2010-08-09 12:08:41 +00001689 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) {
Jonas Aaberga5ebca42010-05-18 00:41:09 +02001690 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type);
Rabin Vincent8ca84682011-01-25 11:18:07 +01001691 status = readl(chanbase + D40_CHAN_REG_SDLNK);
Jonas Aaberg9dbfbd35c2010-08-09 12:08:41 +00001692 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) {
Jonas Aaberga5ebca42010-05-18 00:41:09 +02001693 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type);
Rabin Vincent8ca84682011-01-25 11:18:07 +01001694 status = readl(chanbase + D40_CHAN_REG_SSLNK);
Jonas Aaberg9dbfbd35c2010-08-09 12:08:41 +00001695 } else {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001696 chan_err(d40c, "Unknown direction\n");
Jonas Aaberga5ebca42010-05-18 00:41:09 +02001697 goto _exit;
1698 }
Jonas Aaberg9dbfbd35c2010-08-09 12:08:41 +00001699
Jonas Aaberga5ebca42010-05-18 00:41:09 +02001700 status = (status & D40_EVENTLINE_MASK(event)) >>
1701 D40_EVENTLINE_POS(event);
1702
1703 if (status != D40_DMA_RUN)
1704 is_paused = true;
Jonas Aaberga5ebca42010-05-18 00:41:09 +02001705_exit:
1706 spin_unlock_irqrestore(&d40c->lock, flags);
1707 return is_paused;
1708
1709}
1710
1711
Linus Walleij8d318a52010-03-30 15:33:42 +02001712static u32 stedma40_residue(struct dma_chan *chan)
1713{
1714 struct d40_chan *d40c =
1715 container_of(chan, struct d40_chan, chan);
1716 u32 bytes_left;
1717 unsigned long flags;
1718
1719 spin_lock_irqsave(&d40c->lock, flags);
1720 bytes_left = d40_residue(d40c);
1721 spin_unlock_irqrestore(&d40c->lock, flags);
1722
1723 return bytes_left;
1724}
1725
Rabin Vincent3e3a0762011-01-25 11:18:21 +01001726static int
1727d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc,
1728 struct scatterlist *sg_src, struct scatterlist *sg_dst,
Rabin Vincent822c5672011-01-25 11:18:28 +01001729 unsigned int sg_len, dma_addr_t src_dev_addr,
1730 dma_addr_t dst_dev_addr)
Rabin Vincent3e3a0762011-01-25 11:18:21 +01001731{
1732 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1733 struct stedma40_half_channel_info *src_info = &cfg->src_info;
1734 struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
Rabin Vincent5ed04b82011-01-25 11:18:26 +01001735 int ret;
Rabin Vincent3e3a0762011-01-25 11:18:21 +01001736
Rabin Vincent5ed04b82011-01-25 11:18:26 +01001737 ret = d40_log_sg_to_lli(sg_src, sg_len,
1738 src_dev_addr,
1739 desc->lli_log.src,
1740 chan->log_def.lcsp1,
1741 src_info->data_width,
1742 dst_info->data_width);
Rabin Vincent3e3a0762011-01-25 11:18:21 +01001743
Rabin Vincent5ed04b82011-01-25 11:18:26 +01001744 ret = d40_log_sg_to_lli(sg_dst, sg_len,
1745 dst_dev_addr,
1746 desc->lli_log.dst,
1747 chan->log_def.lcsp3,
1748 dst_info->data_width,
1749 src_info->data_width);
Rabin Vincent3e3a0762011-01-25 11:18:21 +01001750
Rabin Vincent5ed04b82011-01-25 11:18:26 +01001751 return ret < 0 ? ret : 0;
Rabin Vincent3e3a0762011-01-25 11:18:21 +01001752}
1753
1754static int
1755d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc,
1756 struct scatterlist *sg_src, struct scatterlist *sg_dst,
Rabin Vincent822c5672011-01-25 11:18:28 +01001757 unsigned int sg_len, dma_addr_t src_dev_addr,
1758 dma_addr_t dst_dev_addr)
Rabin Vincent3e3a0762011-01-25 11:18:21 +01001759{
Rabin Vincent3e3a0762011-01-25 11:18:21 +01001760 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1761 struct stedma40_half_channel_info *src_info = &cfg->src_info;
1762 struct stedma40_half_channel_info *dst_info = &cfg->dst_info;
Rabin Vincent0c842b52011-01-25 11:18:35 +01001763 unsigned long flags = 0;
Rabin Vincent3e3a0762011-01-25 11:18:21 +01001764 int ret;
1765
Rabin Vincent0c842b52011-01-25 11:18:35 +01001766 if (desc->cyclic)
1767 flags |= LLI_CYCLIC | LLI_TERM_INT;
1768
Rabin Vincent3e3a0762011-01-25 11:18:21 +01001769 ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr,
1770 desc->lli_phy.src,
1771 virt_to_phys(desc->lli_phy.src),
1772 chan->src_def_cfg,
Rabin Vincent0c842b52011-01-25 11:18:35 +01001773 src_info, dst_info, flags);
Rabin Vincent3e3a0762011-01-25 11:18:21 +01001774
1775 ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr,
1776 desc->lli_phy.dst,
1777 virt_to_phys(desc->lli_phy.dst),
1778 chan->dst_def_cfg,
Rabin Vincent0c842b52011-01-25 11:18:35 +01001779 dst_info, src_info, flags);
Rabin Vincent3e3a0762011-01-25 11:18:21 +01001780
1781 dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr,
1782 desc->lli_pool.size, DMA_TO_DEVICE);
1783
1784 return ret < 0 ? ret : 0;
1785}
1786
1787
Rabin Vincent5f811582011-01-25 11:18:18 +01001788static struct d40_desc *
1789d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg,
1790 unsigned int sg_len, unsigned long dma_flags)
1791{
1792 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1793 struct d40_desc *desc;
Rabin Vincentdbd88782011-01-25 11:18:19 +01001794 int ret;
Rabin Vincent5f811582011-01-25 11:18:18 +01001795
1796 desc = d40_desc_get(chan);
1797 if (!desc)
1798 return NULL;
1799
1800 desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width,
1801 cfg->dst_info.data_width);
1802 if (desc->lli_len < 0) {
1803 chan_err(chan, "Unaligned size\n");
Rabin Vincentdbd88782011-01-25 11:18:19 +01001804 goto err;
Rabin Vincent5f811582011-01-25 11:18:18 +01001805 }
1806
Rabin Vincentdbd88782011-01-25 11:18:19 +01001807 ret = d40_pool_lli_alloc(chan, desc, desc->lli_len);
1808 if (ret < 0) {
1809 chan_err(chan, "Could not allocate lli\n");
1810 goto err;
1811 }
1812
1813
Rabin Vincent5f811582011-01-25 11:18:18 +01001814 desc->lli_current = 0;
1815 desc->txd.flags = dma_flags;
1816 desc->txd.tx_submit = d40_tx_submit;
1817
1818 dma_async_tx_descriptor_init(&desc->txd, &chan->chan);
1819
1820 return desc;
Rabin Vincentdbd88782011-01-25 11:18:19 +01001821
1822err:
1823 d40_desc_free(chan, desc);
1824 return NULL;
Rabin Vincent5f811582011-01-25 11:18:18 +01001825}
1826
Rabin Vincentcade1d32011-01-25 11:18:23 +01001827static dma_addr_t
1828d40_get_dev_addr(struct d40_chan *chan, enum dma_data_direction direction)
Linus Walleij8d318a52010-03-30 15:33:42 +02001829{
Rabin Vincentcade1d32011-01-25 11:18:23 +01001830 struct stedma40_platform_data *plat = chan->base->plat_data;
1831 struct stedma40_chan_cfg *cfg = &chan->dma_cfg;
1832 dma_addr_t addr;
Linus Walleij8d318a52010-03-30 15:33:42 +02001833
Rabin Vincentcade1d32011-01-25 11:18:23 +01001834 if (chan->runtime_addr)
1835 return chan->runtime_addr;
1836
1837 if (direction == DMA_FROM_DEVICE)
1838 addr = plat->dev_rx[cfg->src_dev_type];
1839 else if (direction == DMA_TO_DEVICE)
1840 addr = plat->dev_tx[cfg->dst_dev_type];
1841
1842 return addr;
1843}
1844
1845static struct dma_async_tx_descriptor *
1846d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src,
1847 struct scatterlist *sg_dst, unsigned int sg_len,
1848 enum dma_data_direction direction, unsigned long dma_flags)
1849{
1850 struct d40_chan *chan = container_of(dchan, struct d40_chan, chan);
Rabin Vincent822c5672011-01-25 11:18:28 +01001851 dma_addr_t src_dev_addr = 0;
1852 dma_addr_t dst_dev_addr = 0;
Rabin Vincentcade1d32011-01-25 11:18:23 +01001853 struct d40_desc *desc;
1854 unsigned long flags;
1855 int ret;
1856
1857 if (!chan->phy_chan) {
1858 chan_err(chan, "Cannot prepare unallocated channel\n");
1859 return NULL;
Jonas Aaberg0d0f6b82010-06-20 21:25:31 +00001860 }
1861
Rabin Vincent0c842b52011-01-25 11:18:35 +01001862
Rabin Vincentcade1d32011-01-25 11:18:23 +01001863 spin_lock_irqsave(&chan->lock, flags);
Linus Walleij8d318a52010-03-30 15:33:42 +02001864
Rabin Vincentcade1d32011-01-25 11:18:23 +01001865 desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags);
1866 if (desc == NULL)
Linus Walleij8d318a52010-03-30 15:33:42 +02001867 goto err;
1868
Rabin Vincent0c842b52011-01-25 11:18:35 +01001869 if (sg_next(&sg_src[sg_len - 1]) == sg_src)
1870 desc->cyclic = true;
1871
Rabin Vincent822c5672011-01-25 11:18:28 +01001872 if (direction != DMA_NONE) {
1873 dma_addr_t dev_addr = d40_get_dev_addr(chan, direction);
1874
1875 if (direction == DMA_FROM_DEVICE)
1876 src_dev_addr = dev_addr;
1877 else if (direction == DMA_TO_DEVICE)
1878 dst_dev_addr = dev_addr;
1879 }
Rabin Vincentcade1d32011-01-25 11:18:23 +01001880
1881 if (chan_is_logical(chan))
1882 ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst,
Rabin Vincent822c5672011-01-25 11:18:28 +01001883 sg_len, src_dev_addr, dst_dev_addr);
Rabin Vincentcade1d32011-01-25 11:18:23 +01001884 else
1885 ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst,
Rabin Vincent822c5672011-01-25 11:18:28 +01001886 sg_len, src_dev_addr, dst_dev_addr);
Rabin Vincentcade1d32011-01-25 11:18:23 +01001887
1888 if (ret) {
1889 chan_err(chan, "Failed to prepare %s sg job: %d\n",
1890 chan_is_logical(chan) ? "log" : "phy", ret);
1891 goto err;
Linus Walleij8d318a52010-03-30 15:33:42 +02001892 }
1893
Rabin Vincentcade1d32011-01-25 11:18:23 +01001894 spin_unlock_irqrestore(&chan->lock, flags);
Linus Walleij8d318a52010-03-30 15:33:42 +02001895
Rabin Vincentcade1d32011-01-25 11:18:23 +01001896 return &desc->txd;
1897
Linus Walleij8d318a52010-03-30 15:33:42 +02001898err:
Rabin Vincentcade1d32011-01-25 11:18:23 +01001899 if (desc)
1900 d40_desc_free(chan, desc);
1901 spin_unlock_irqrestore(&chan->lock, flags);
Linus Walleij8d318a52010-03-30 15:33:42 +02001902 return NULL;
1903}
Linus Walleij8d318a52010-03-30 15:33:42 +02001904
1905bool stedma40_filter(struct dma_chan *chan, void *data)
1906{
1907 struct stedma40_chan_cfg *info = data;
1908 struct d40_chan *d40c =
1909 container_of(chan, struct d40_chan, chan);
1910 int err;
1911
1912 if (data) {
1913 err = d40_validate_conf(d40c, info);
1914 if (!err)
1915 d40c->dma_cfg = *info;
1916 } else
1917 err = d40_config_memcpy(d40c);
1918
Rabin Vincentce2ca122010-10-12 13:00:49 +00001919 if (!err)
1920 d40c->configured = true;
1921
Linus Walleij8d318a52010-03-30 15:33:42 +02001922 return err == 0;
1923}
1924EXPORT_SYMBOL(stedma40_filter);
1925
Rabin Vincentac2c0a32011-01-25 11:18:11 +01001926static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src)
1927{
1928 bool realtime = d40c->dma_cfg.realtime;
1929 bool highprio = d40c->dma_cfg.high_priority;
1930 u32 prioreg = highprio ? D40_DREG_PSEG1 : D40_DREG_PCEG1;
1931 u32 rtreg = realtime ? D40_DREG_RSEG1 : D40_DREG_RCEG1;
1932 u32 event = D40_TYPE_TO_EVENT(dev_type);
1933 u32 group = D40_TYPE_TO_GROUP(dev_type);
1934 u32 bit = 1 << event;
1935
1936 /* Destination event lines are stored in the upper halfword */
1937 if (!src)
1938 bit <<= 16;
1939
1940 writel(bit, d40c->base->virtbase + prioreg + group * 4);
1941 writel(bit, d40c->base->virtbase + rtreg + group * 4);
1942}
1943
1944static void d40_set_prio_realtime(struct d40_chan *d40c)
1945{
1946 if (d40c->base->rev < 3)
1947 return;
1948
1949 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) ||
1950 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
1951 __d40_set_prio_rt(d40c, d40c->dma_cfg.src_dev_type, true);
1952
1953 if ((d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH) ||
1954 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH))
1955 __d40_set_prio_rt(d40c, d40c->dma_cfg.dst_dev_type, false);
1956}
1957
Linus Walleij8d318a52010-03-30 15:33:42 +02001958/* DMA ENGINE functions */
1959static int d40_alloc_chan_resources(struct dma_chan *chan)
1960{
1961 int err;
1962 unsigned long flags;
1963 struct d40_chan *d40c =
1964 container_of(chan, struct d40_chan, chan);
Linus Walleijef1872e2010-06-20 21:24:52 +00001965 bool is_free_phy;
Linus Walleij8d318a52010-03-30 15:33:42 +02001966 spin_lock_irqsave(&d40c->lock, flags);
1967
1968 d40c->completed = chan->cookie = 1;
1969
Rabin Vincentce2ca122010-10-12 13:00:49 +00001970 /* If no dma configuration is set use default configuration (memcpy) */
1971 if (!d40c->configured) {
Linus Walleij8d318a52010-03-30 15:33:42 +02001972 err = d40_config_memcpy(d40c);
Jonas Aabergff0b12b2010-06-20 21:25:15 +00001973 if (err) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001974 chan_err(d40c, "Failed to configure memcpy channel\n");
Jonas Aabergff0b12b2010-06-20 21:25:15 +00001975 goto fail;
1976 }
Linus Walleij8d318a52010-03-30 15:33:42 +02001977 }
Linus Walleijef1872e2010-06-20 21:24:52 +00001978 is_free_phy = (d40c->phy_chan == NULL);
Linus Walleij8d318a52010-03-30 15:33:42 +02001979
1980 err = d40_allocate_channel(d40c);
1981 if (err) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01001982 chan_err(d40c, "Failed to allocate channel\n");
Jonas Aabergff0b12b2010-06-20 21:25:15 +00001983 goto fail;
Linus Walleij8d318a52010-03-30 15:33:42 +02001984 }
1985
Linus Walleijef1872e2010-06-20 21:24:52 +00001986 /* Fill in basic CFG register values */
1987 d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg,
Rabin Vincent724a8572011-01-25 11:18:08 +01001988 &d40c->dst_def_cfg, chan_is_logical(d40c));
Linus Walleijef1872e2010-06-20 21:24:52 +00001989
Rabin Vincentac2c0a32011-01-25 11:18:11 +01001990 d40_set_prio_realtime(d40c);
1991
Rabin Vincent724a8572011-01-25 11:18:08 +01001992 if (chan_is_logical(d40c)) {
Linus Walleijef1872e2010-06-20 21:24:52 +00001993 d40_log_cfg(&d40c->dma_cfg,
1994 &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
1995
1996 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM)
1997 d40c->lcpa = d40c->base->lcpa_base +
1998 d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE;
1999 else
2000 d40c->lcpa = d40c->base->lcpa_base +
2001 d40c->dma_cfg.dst_dev_type *
2002 D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA;
2003 }
2004
2005 /*
2006 * Only write channel configuration to the DMA if the physical
2007 * resource is free. In case of multiple logical channels
2008 * on the same physical resource, only the first write is necessary.
2009 */
Jonas Aabergb55912c2010-08-09 12:08:02 +00002010 if (is_free_phy)
2011 d40_config_write(d40c);
Jonas Aabergff0b12b2010-06-20 21:25:15 +00002012fail:
Linus Walleij8d318a52010-03-30 15:33:42 +02002013 spin_unlock_irqrestore(&d40c->lock, flags);
Jonas Aabergff0b12b2010-06-20 21:25:15 +00002014 return err;
Linus Walleij8d318a52010-03-30 15:33:42 +02002015}
2016
2017static void d40_free_chan_resources(struct dma_chan *chan)
2018{
2019 struct d40_chan *d40c =
2020 container_of(chan, struct d40_chan, chan);
2021 int err;
2022 unsigned long flags;
2023
Jonas Aaberg0d0f6b82010-06-20 21:25:31 +00002024 if (d40c->phy_chan == NULL) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002025 chan_err(d40c, "Cannot free unallocated channel\n");
Jonas Aaberg0d0f6b82010-06-20 21:25:31 +00002026 return;
2027 }
2028
2029
Linus Walleij8d318a52010-03-30 15:33:42 +02002030 spin_lock_irqsave(&d40c->lock, flags);
2031
2032 err = d40_free_dma(d40c);
2033
2034 if (err)
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002035 chan_err(d40c, "Failed to free channel\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02002036 spin_unlock_irqrestore(&d40c->lock, flags);
2037}
2038
2039static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan,
2040 dma_addr_t dst,
2041 dma_addr_t src,
2042 size_t size,
Jonas Aaberg2a614342010-06-20 21:25:24 +00002043 unsigned long dma_flags)
Linus Walleij8d318a52010-03-30 15:33:42 +02002044{
Rabin Vincent95944c62011-01-25 11:18:17 +01002045 struct scatterlist dst_sg;
2046 struct scatterlist src_sg;
Linus Walleij8d318a52010-03-30 15:33:42 +02002047
Rabin Vincent95944c62011-01-25 11:18:17 +01002048 sg_init_table(&dst_sg, 1);
2049 sg_init_table(&src_sg, 1);
Jonas Aaberg0d0f6b82010-06-20 21:25:31 +00002050
Rabin Vincent95944c62011-01-25 11:18:17 +01002051 sg_dma_address(&dst_sg) = dst;
2052 sg_dma_address(&src_sg) = src;
Linus Walleij8d318a52010-03-30 15:33:42 +02002053
Rabin Vincent95944c62011-01-25 11:18:17 +01002054 sg_dma_len(&dst_sg) = size;
2055 sg_dma_len(&src_sg) = size;
Linus Walleij8d318a52010-03-30 15:33:42 +02002056
Rabin Vincentcade1d32011-01-25 11:18:23 +01002057 return d40_prep_sg(chan, &src_sg, &dst_sg, 1, DMA_NONE, dma_flags);
Linus Walleij8d318a52010-03-30 15:33:42 +02002058}
2059
Ira Snyder0d688662010-09-30 11:46:47 +00002060static struct dma_async_tx_descriptor *
Rabin Vincentcade1d32011-01-25 11:18:23 +01002061d40_prep_memcpy_sg(struct dma_chan *chan,
2062 struct scatterlist *dst_sg, unsigned int dst_nents,
2063 struct scatterlist *src_sg, unsigned int src_nents,
2064 unsigned long dma_flags)
Ira Snyder0d688662010-09-30 11:46:47 +00002065{
2066 if (dst_nents != src_nents)
2067 return NULL;
2068
Rabin Vincentcade1d32011-01-25 11:18:23 +01002069 return d40_prep_sg(chan, src_sg, dst_sg, src_nents, DMA_NONE, dma_flags);
Rabin Vincent00ac0342011-01-25 11:18:20 +01002070}
2071
Linus Walleij8d318a52010-03-30 15:33:42 +02002072static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan,
2073 struct scatterlist *sgl,
2074 unsigned int sg_len,
2075 enum dma_data_direction direction,
Jonas Aaberg2a614342010-06-20 21:25:24 +00002076 unsigned long dma_flags)
Linus Walleij8d318a52010-03-30 15:33:42 +02002077{
Rabin Vincent00ac0342011-01-25 11:18:20 +01002078 if (direction != DMA_FROM_DEVICE && direction != DMA_TO_DEVICE)
2079 return NULL;
2080
Rabin Vincentcade1d32011-01-25 11:18:23 +01002081 return d40_prep_sg(chan, sgl, sgl, sg_len, direction, dma_flags);
Linus Walleij8d318a52010-03-30 15:33:42 +02002082}
2083
Rabin Vincent0c842b52011-01-25 11:18:35 +01002084static struct dma_async_tx_descriptor *
2085dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
2086 size_t buf_len, size_t period_len,
2087 enum dma_data_direction direction)
2088{
2089 unsigned int periods = buf_len / period_len;
2090 struct dma_async_tx_descriptor *txd;
2091 struct scatterlist *sg;
2092 int i;
2093
2094 sg = kcalloc(periods + 1, sizeof(struct scatterlist), GFP_KERNEL);
2095 for (i = 0; i < periods; i++) {
2096 sg_dma_address(&sg[i]) = dma_addr;
2097 sg_dma_len(&sg[i]) = period_len;
2098 dma_addr += period_len;
2099 }
2100
2101 sg[periods].offset = 0;
2102 sg[periods].length = 0;
2103 sg[periods].page_link =
2104 ((unsigned long)sg | 0x01) & ~0x02;
2105
2106 txd = d40_prep_sg(chan, sg, sg, periods, direction,
2107 DMA_PREP_INTERRUPT);
2108
2109 kfree(sg);
2110
2111 return txd;
2112}
2113
Linus Walleij8d318a52010-03-30 15:33:42 +02002114static enum dma_status d40_tx_status(struct dma_chan *chan,
2115 dma_cookie_t cookie,
2116 struct dma_tx_state *txstate)
2117{
2118 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2119 dma_cookie_t last_used;
2120 dma_cookie_t last_complete;
2121 int ret;
2122
Jonas Aaberg0d0f6b82010-06-20 21:25:31 +00002123 if (d40c->phy_chan == NULL) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002124 chan_err(d40c, "Cannot read status of unallocated channel\n");
Jonas Aaberg0d0f6b82010-06-20 21:25:31 +00002125 return -EINVAL;
2126 }
2127
Linus Walleij8d318a52010-03-30 15:33:42 +02002128 last_complete = d40c->completed;
2129 last_used = chan->cookie;
2130
Jonas Aaberga5ebca42010-05-18 00:41:09 +02002131 if (d40_is_paused(d40c))
2132 ret = DMA_PAUSED;
2133 else
2134 ret = dma_async_is_complete(cookie, last_complete, last_used);
Linus Walleij8d318a52010-03-30 15:33:42 +02002135
Jonas Aaberga5ebca42010-05-18 00:41:09 +02002136 dma_set_tx_state(txstate, last_complete, last_used,
2137 stedma40_residue(chan));
Linus Walleij8d318a52010-03-30 15:33:42 +02002138
2139 return ret;
2140}
2141
2142static void d40_issue_pending(struct dma_chan *chan)
2143{
2144 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2145 unsigned long flags;
2146
Jonas Aaberg0d0f6b82010-06-20 21:25:31 +00002147 if (d40c->phy_chan == NULL) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002148 chan_err(d40c, "Channel is not allocated!\n");
Jonas Aaberg0d0f6b82010-06-20 21:25:31 +00002149 return;
2150 }
2151
Linus Walleij8d318a52010-03-30 15:33:42 +02002152 spin_lock_irqsave(&d40c->lock, flags);
2153
2154 /* Busy means that pending jobs are already being processed */
2155 if (!d40c->busy)
2156 (void) d40_queue_start(d40c);
2157
2158 spin_unlock_irqrestore(&d40c->lock, flags);
2159}
2160
Linus Walleij95e14002010-08-04 13:37:45 +02002161/* Runtime reconfiguration extension */
2162static void d40_set_runtime_config(struct dma_chan *chan,
2163 struct dma_slave_config *config)
2164{
2165 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2166 struct stedma40_chan_cfg *cfg = &d40c->dma_cfg;
2167 enum dma_slave_buswidth config_addr_width;
2168 dma_addr_t config_addr;
2169 u32 config_maxburst;
2170 enum stedma40_periph_data_width addr_width;
2171 int psize;
2172
2173 if (config->direction == DMA_FROM_DEVICE) {
2174 dma_addr_t dev_addr_rx =
2175 d40c->base->plat_data->dev_rx[cfg->src_dev_type];
2176
2177 config_addr = config->src_addr;
2178 if (dev_addr_rx)
2179 dev_dbg(d40c->base->dev,
2180 "channel has a pre-wired RX address %08x "
2181 "overriding with %08x\n",
2182 dev_addr_rx, config_addr);
2183 if (cfg->dir != STEDMA40_PERIPH_TO_MEM)
2184 dev_dbg(d40c->base->dev,
2185 "channel was not configured for peripheral "
2186 "to memory transfer (%d) overriding\n",
2187 cfg->dir);
2188 cfg->dir = STEDMA40_PERIPH_TO_MEM;
2189
2190 config_addr_width = config->src_addr_width;
2191 config_maxburst = config->src_maxburst;
2192
2193 } else if (config->direction == DMA_TO_DEVICE) {
2194 dma_addr_t dev_addr_tx =
2195 d40c->base->plat_data->dev_tx[cfg->dst_dev_type];
2196
2197 config_addr = config->dst_addr;
2198 if (dev_addr_tx)
2199 dev_dbg(d40c->base->dev,
2200 "channel has a pre-wired TX address %08x "
2201 "overriding with %08x\n",
2202 dev_addr_tx, config_addr);
2203 if (cfg->dir != STEDMA40_MEM_TO_PERIPH)
2204 dev_dbg(d40c->base->dev,
2205 "channel was not configured for memory "
2206 "to peripheral transfer (%d) overriding\n",
2207 cfg->dir);
2208 cfg->dir = STEDMA40_MEM_TO_PERIPH;
2209
2210 config_addr_width = config->dst_addr_width;
2211 config_maxburst = config->dst_maxburst;
2212
2213 } else {
2214 dev_err(d40c->base->dev,
2215 "unrecognized channel direction %d\n",
2216 config->direction);
2217 return;
2218 }
2219
2220 switch (config_addr_width) {
2221 case DMA_SLAVE_BUSWIDTH_1_BYTE:
2222 addr_width = STEDMA40_BYTE_WIDTH;
2223 break;
2224 case DMA_SLAVE_BUSWIDTH_2_BYTES:
2225 addr_width = STEDMA40_HALFWORD_WIDTH;
2226 break;
2227 case DMA_SLAVE_BUSWIDTH_4_BYTES:
2228 addr_width = STEDMA40_WORD_WIDTH;
2229 break;
2230 case DMA_SLAVE_BUSWIDTH_8_BYTES:
2231 addr_width = STEDMA40_DOUBLEWORD_WIDTH;
2232 break;
2233 default:
2234 dev_err(d40c->base->dev,
2235 "illegal peripheral address width "
2236 "requested (%d)\n",
2237 config->src_addr_width);
2238 return;
2239 }
2240
Rabin Vincent724a8572011-01-25 11:18:08 +01002241 if (chan_is_logical(d40c)) {
Per Forlina59670a2010-10-06 09:05:27 +00002242 if (config_maxburst >= 16)
2243 psize = STEDMA40_PSIZE_LOG_16;
2244 else if (config_maxburst >= 8)
2245 psize = STEDMA40_PSIZE_LOG_8;
2246 else if (config_maxburst >= 4)
2247 psize = STEDMA40_PSIZE_LOG_4;
2248 else
2249 psize = STEDMA40_PSIZE_LOG_1;
2250 } else {
2251 if (config_maxburst >= 16)
2252 psize = STEDMA40_PSIZE_PHY_16;
2253 else if (config_maxburst >= 8)
2254 psize = STEDMA40_PSIZE_PHY_8;
2255 else if (config_maxburst >= 4)
2256 psize = STEDMA40_PSIZE_PHY_4;
Per Forlind49278e2010-12-20 18:31:38 +01002257 else if (config_maxburst >= 2)
2258 psize = STEDMA40_PSIZE_PHY_2;
Per Forlina59670a2010-10-06 09:05:27 +00002259 else
2260 psize = STEDMA40_PSIZE_PHY_1;
2261 }
Linus Walleij95e14002010-08-04 13:37:45 +02002262
2263 /* Set up all the endpoint configs */
2264 cfg->src_info.data_width = addr_width;
2265 cfg->src_info.psize = psize;
Rabin Vincent51f5d742010-10-12 13:00:54 +00002266 cfg->src_info.big_endian = false;
Linus Walleij95e14002010-08-04 13:37:45 +02002267 cfg->src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2268 cfg->dst_info.data_width = addr_width;
2269 cfg->dst_info.psize = psize;
Rabin Vincent51f5d742010-10-12 13:00:54 +00002270 cfg->dst_info.big_endian = false;
Linus Walleij95e14002010-08-04 13:37:45 +02002271 cfg->dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL;
2272
Per Forlina59670a2010-10-06 09:05:27 +00002273 /* Fill in register values */
Rabin Vincent724a8572011-01-25 11:18:08 +01002274 if (chan_is_logical(d40c))
Per Forlina59670a2010-10-06 09:05:27 +00002275 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3);
2276 else
2277 d40_phy_cfg(cfg, &d40c->src_def_cfg,
2278 &d40c->dst_def_cfg, false);
2279
Linus Walleij95e14002010-08-04 13:37:45 +02002280 /* These settings will take precedence later */
2281 d40c->runtime_addr = config_addr;
2282 d40c->runtime_direction = config->direction;
2283 dev_dbg(d40c->base->dev,
2284 "configured channel %s for %s, data width %d, "
2285 "maxburst %d bytes, LE, no flow control\n",
2286 dma_chan_name(chan),
2287 (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX",
2288 config_addr_width,
2289 config_maxburst);
2290}
2291
Linus Walleij05827632010-05-17 16:30:42 -07002292static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
2293 unsigned long arg)
Linus Walleij8d318a52010-03-30 15:33:42 +02002294{
Linus Walleij8d318a52010-03-30 15:33:42 +02002295 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan);
2296
Jonas Aaberg0d0f6b82010-06-20 21:25:31 +00002297 if (d40c->phy_chan == NULL) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002298 chan_err(d40c, "Channel is not allocated!\n");
Jonas Aaberg0d0f6b82010-06-20 21:25:31 +00002299 return -EINVAL;
2300 }
2301
Linus Walleij8d318a52010-03-30 15:33:42 +02002302 switch (cmd) {
2303 case DMA_TERMINATE_ALL:
Rabin Vincent86eb5fb2011-01-25 11:18:34 +01002304 return d40_terminate_all(d40c);
Linus Walleij8d318a52010-03-30 15:33:42 +02002305 case DMA_PAUSE:
Rabin Vincent86eb5fb2011-01-25 11:18:34 +01002306 return d40_pause(d40c);
Linus Walleij8d318a52010-03-30 15:33:42 +02002307 case DMA_RESUME:
Rabin Vincent86eb5fb2011-01-25 11:18:34 +01002308 return d40_resume(d40c);
Linus Walleij95e14002010-08-04 13:37:45 +02002309 case DMA_SLAVE_CONFIG:
2310 d40_set_runtime_config(chan,
2311 (struct dma_slave_config *) arg);
2312 return 0;
2313 default:
2314 break;
Linus Walleij8d318a52010-03-30 15:33:42 +02002315 }
2316
2317 /* Other commands are unimplemented */
2318 return -ENXIO;
2319}
2320
2321/* Initialization functions */
2322
2323static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma,
2324 struct d40_chan *chans, int offset,
2325 int num_chans)
2326{
2327 int i = 0;
2328 struct d40_chan *d40c;
2329
2330 INIT_LIST_HEAD(&dma->channels);
2331
2332 for (i = offset; i < offset + num_chans; i++) {
2333 d40c = &chans[i];
2334 d40c->base = base;
2335 d40c->chan.device = dma;
2336
Linus Walleij8d318a52010-03-30 15:33:42 +02002337 spin_lock_init(&d40c->lock);
2338
2339 d40c->log_num = D40_PHY_CHAN;
2340
Linus Walleij8d318a52010-03-30 15:33:42 +02002341 INIT_LIST_HEAD(&d40c->active);
2342 INIT_LIST_HEAD(&d40c->queue);
2343 INIT_LIST_HEAD(&d40c->client);
2344
Linus Walleij8d318a52010-03-30 15:33:42 +02002345 tasklet_init(&d40c->tasklet, dma_tasklet,
2346 (unsigned long) d40c);
2347
2348 list_add_tail(&d40c->chan.device_node,
2349 &dma->channels);
2350 }
2351}
2352
Rabin Vincent7ad74a72011-01-25 11:18:33 +01002353static void d40_ops_init(struct d40_base *base, struct dma_device *dev)
2354{
2355 if (dma_has_cap(DMA_SLAVE, dev->cap_mask))
2356 dev->device_prep_slave_sg = d40_prep_slave_sg;
2357
2358 if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) {
2359 dev->device_prep_dma_memcpy = d40_prep_memcpy;
2360
2361 /*
2362 * This controller can only access address at even
2363 * 32bit boundaries, i.e. 2^2
2364 */
2365 dev->copy_align = 2;
2366 }
2367
2368 if (dma_has_cap(DMA_SG, dev->cap_mask))
2369 dev->device_prep_dma_sg = d40_prep_memcpy_sg;
2370
Rabin Vincent0c842b52011-01-25 11:18:35 +01002371 if (dma_has_cap(DMA_CYCLIC, dev->cap_mask))
2372 dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic;
2373
Rabin Vincent7ad74a72011-01-25 11:18:33 +01002374 dev->device_alloc_chan_resources = d40_alloc_chan_resources;
2375 dev->device_free_chan_resources = d40_free_chan_resources;
2376 dev->device_issue_pending = d40_issue_pending;
2377 dev->device_tx_status = d40_tx_status;
2378 dev->device_control = d40_control;
2379 dev->dev = base->dev;
2380}
2381
Linus Walleij8d318a52010-03-30 15:33:42 +02002382static int __init d40_dmaengine_init(struct d40_base *base,
2383 int num_reserved_chans)
2384{
2385 int err ;
2386
2387 d40_chan_init(base, &base->dma_slave, base->log_chans,
2388 0, base->num_log_chans);
2389
2390 dma_cap_zero(base->dma_slave.cap_mask);
2391 dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask);
Rabin Vincent0c842b52011-01-25 11:18:35 +01002392 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
Linus Walleij8d318a52010-03-30 15:33:42 +02002393
Rabin Vincent7ad74a72011-01-25 11:18:33 +01002394 d40_ops_init(base, &base->dma_slave);
Linus Walleij8d318a52010-03-30 15:33:42 +02002395
2396 err = dma_async_device_register(&base->dma_slave);
2397
2398 if (err) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002399 d40_err(base->dev, "Failed to register slave channels\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02002400 goto failure1;
2401 }
2402
2403 d40_chan_init(base, &base->dma_memcpy, base->log_chans,
2404 base->num_log_chans, base->plat_data->memcpy_len);
2405
2406 dma_cap_zero(base->dma_memcpy.cap_mask);
2407 dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask);
Rabin Vincent7ad74a72011-01-25 11:18:33 +01002408 dma_cap_set(DMA_SG, base->dma_memcpy.cap_mask);
Linus Walleij8d318a52010-03-30 15:33:42 +02002409
Rabin Vincent7ad74a72011-01-25 11:18:33 +01002410 d40_ops_init(base, &base->dma_memcpy);
Linus Walleij8d318a52010-03-30 15:33:42 +02002411
2412 err = dma_async_device_register(&base->dma_memcpy);
2413
2414 if (err) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002415 d40_err(base->dev,
2416 "Failed to regsiter memcpy only channels\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02002417 goto failure2;
2418 }
2419
2420 d40_chan_init(base, &base->dma_both, base->phy_chans,
2421 0, num_reserved_chans);
2422
2423 dma_cap_zero(base->dma_both.cap_mask);
2424 dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask);
2425 dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask);
Rabin Vincent7ad74a72011-01-25 11:18:33 +01002426 dma_cap_set(DMA_SG, base->dma_both.cap_mask);
Rabin Vincent0c842b52011-01-25 11:18:35 +01002427 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask);
Linus Walleij8d318a52010-03-30 15:33:42 +02002428
Rabin Vincent7ad74a72011-01-25 11:18:33 +01002429 d40_ops_init(base, &base->dma_both);
Linus Walleij8d318a52010-03-30 15:33:42 +02002430 err = dma_async_device_register(&base->dma_both);
2431
2432 if (err) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002433 d40_err(base->dev,
2434 "Failed to register logical and physical capable channels\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02002435 goto failure3;
2436 }
2437 return 0;
2438failure3:
2439 dma_async_device_unregister(&base->dma_memcpy);
2440failure2:
2441 dma_async_device_unregister(&base->dma_slave);
2442failure1:
2443 return err;
2444}
2445
2446/* Initialization functions. */
2447
2448static int __init d40_phy_res_init(struct d40_base *base)
2449{
2450 int i;
2451 int num_phy_chans_avail = 0;
2452 u32 val[2];
2453 int odd_even_bit = -2;
2454
2455 val[0] = readl(base->virtbase + D40_DREG_PRSME);
2456 val[1] = readl(base->virtbase + D40_DREG_PRSMO);
2457
2458 for (i = 0; i < base->num_phy_chans; i++) {
2459 base->phy_res[i].num = i;
2460 odd_even_bit += 2 * ((i % 2) == 0);
2461 if (((val[i % 2] >> odd_even_bit) & 3) == 1) {
2462 /* Mark security only channels as occupied */
2463 base->phy_res[i].allocated_src = D40_ALLOC_PHY;
2464 base->phy_res[i].allocated_dst = D40_ALLOC_PHY;
2465 } else {
2466 base->phy_res[i].allocated_src = D40_ALLOC_FREE;
2467 base->phy_res[i].allocated_dst = D40_ALLOC_FREE;
2468 num_phy_chans_avail++;
2469 }
2470 spin_lock_init(&base->phy_res[i].lock);
2471 }
Jonas Aaberg6b7acd82010-06-20 21:26:59 +00002472
2473 /* Mark disabled channels as occupied */
2474 for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) {
Rabin Vincentf57b4072010-10-06 08:20:35 +00002475 int chan = base->plat_data->disabled_channels[i];
2476
2477 base->phy_res[chan].allocated_src = D40_ALLOC_PHY;
2478 base->phy_res[chan].allocated_dst = D40_ALLOC_PHY;
2479 num_phy_chans_avail--;
Jonas Aaberg6b7acd82010-06-20 21:26:59 +00002480 }
2481
Linus Walleij8d318a52010-03-30 15:33:42 +02002482 dev_info(base->dev, "%d of %d physical DMA channels available\n",
2483 num_phy_chans_avail, base->num_phy_chans);
2484
2485 /* Verify settings extended vs standard */
2486 val[0] = readl(base->virtbase + D40_DREG_PRTYP);
2487
2488 for (i = 0; i < base->num_phy_chans; i++) {
2489
2490 if (base->phy_res[i].allocated_src == D40_ALLOC_FREE &&
2491 (val[0] & 0x3) != 1)
2492 dev_info(base->dev,
2493 "[%s] INFO: channel %d is misconfigured (%d)\n",
2494 __func__, i, val[0] & 0x3);
2495
2496 val[0] = val[0] >> 2;
2497 }
2498
2499 return num_phy_chans_avail;
2500}
2501
2502static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev)
2503{
2504 static const struct d40_reg_val dma_id_regs[] = {
2505 /* Peripheral Id */
2506 { .reg = D40_DREG_PERIPHID0, .val = 0x0040},
2507 { .reg = D40_DREG_PERIPHID1, .val = 0x0000},
2508 /*
2509 * D40_DREG_PERIPHID2 Depends on HW revision:
Rabin Vincent4d594902011-01-25 11:18:10 +01002510 * DB8500ed has 0x0008,
Linus Walleij8d318a52010-03-30 15:33:42 +02002511 * ? has 0x0018,
Rabin Vincent4d594902011-01-25 11:18:10 +01002512 * DB8500v1 has 0x0028
2513 * DB8500v2 has 0x0038
Linus Walleij8d318a52010-03-30 15:33:42 +02002514 */
2515 { .reg = D40_DREG_PERIPHID3, .val = 0x0000},
2516
2517 /* PCell Id */
2518 { .reg = D40_DREG_CELLID0, .val = 0x000d},
2519 { .reg = D40_DREG_CELLID1, .val = 0x00f0},
2520 { .reg = D40_DREG_CELLID2, .val = 0x0005},
2521 { .reg = D40_DREG_CELLID3, .val = 0x00b1}
2522 };
2523 struct stedma40_platform_data *plat_data;
2524 struct clk *clk = NULL;
2525 void __iomem *virtbase = NULL;
2526 struct resource *res = NULL;
2527 struct d40_base *base = NULL;
2528 int num_log_chans = 0;
2529 int num_phy_chans;
2530 int i;
Linus Walleijf4185592010-06-22 18:06:42 -07002531 u32 val;
Jonas Aaberg3ae02672010-08-09 12:08:18 +00002532 u32 rev;
Linus Walleij8d318a52010-03-30 15:33:42 +02002533
2534 clk = clk_get(&pdev->dev, NULL);
2535
2536 if (IS_ERR(clk)) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002537 d40_err(&pdev->dev, "No matching clock found\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02002538 goto failure;
2539 }
2540
2541 clk_enable(clk);
2542
2543 /* Get IO for DMAC base address */
2544 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
2545 if (!res)
2546 goto failure;
2547
2548 if (request_mem_region(res->start, resource_size(res),
2549 D40_NAME " I/O base") == NULL)
2550 goto failure;
2551
2552 virtbase = ioremap(res->start, resource_size(res));
2553 if (!virtbase)
2554 goto failure;
2555
2556 /* HW version check */
2557 for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) {
2558 if (dma_id_regs[i].val !=
2559 readl(virtbase + dma_id_regs[i].reg)) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002560 d40_err(&pdev->dev,
2561 "Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n",
Linus Walleij8d318a52010-03-30 15:33:42 +02002562 dma_id_regs[i].val,
2563 dma_id_regs[i].reg,
2564 readl(virtbase + dma_id_regs[i].reg));
2565 goto failure;
2566 }
2567 }
2568
Jonas Aaberg3ae02672010-08-09 12:08:18 +00002569 /* Get silicon revision and designer */
Linus Walleijf4185592010-06-22 18:06:42 -07002570 val = readl(virtbase + D40_DREG_PERIPHID2);
Linus Walleij8d318a52010-03-30 15:33:42 +02002571
Jonas Aaberg3ae02672010-08-09 12:08:18 +00002572 if ((val & D40_DREG_PERIPHID2_DESIGNER_MASK) !=
2573 D40_HW_DESIGNER) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002574 d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n",
2575 val & D40_DREG_PERIPHID2_DESIGNER_MASK,
Jonas Aaberg3ae02672010-08-09 12:08:18 +00002576 D40_HW_DESIGNER);
Linus Walleij8d318a52010-03-30 15:33:42 +02002577 goto failure;
2578 }
2579
Jonas Aaberg3ae02672010-08-09 12:08:18 +00002580 rev = (val & D40_DREG_PERIPHID2_REV_MASK) >>
2581 D40_DREG_PERIPHID2_REV_POS;
2582
Linus Walleij8d318a52010-03-30 15:33:42 +02002583 /* The number of physical channels on this HW */
2584 num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4;
2585
2586 dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n",
Jonas Aaberg3ae02672010-08-09 12:08:18 +00002587 rev, res->start);
Linus Walleij8d318a52010-03-30 15:33:42 +02002588
2589 plat_data = pdev->dev.platform_data;
2590
2591 /* Count the number of logical channels in use */
2592 for (i = 0; i < plat_data->dev_len; i++)
2593 if (plat_data->dev_rx[i] != 0)
2594 num_log_chans++;
2595
2596 for (i = 0; i < plat_data->dev_len; i++)
2597 if (plat_data->dev_tx[i] != 0)
2598 num_log_chans++;
2599
2600 base = kzalloc(ALIGN(sizeof(struct d40_base), 4) +
2601 (num_phy_chans + num_log_chans + plat_data->memcpy_len) *
2602 sizeof(struct d40_chan), GFP_KERNEL);
2603
2604 if (base == NULL) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002605 d40_err(&pdev->dev, "Out of memory\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02002606 goto failure;
2607 }
2608
Jonas Aaberg3ae02672010-08-09 12:08:18 +00002609 base->rev = rev;
Linus Walleij8d318a52010-03-30 15:33:42 +02002610 base->clk = clk;
2611 base->num_phy_chans = num_phy_chans;
2612 base->num_log_chans = num_log_chans;
2613 base->phy_start = res->start;
2614 base->phy_size = resource_size(res);
2615 base->virtbase = virtbase;
2616 base->plat_data = plat_data;
2617 base->dev = &pdev->dev;
2618 base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4);
2619 base->log_chans = &base->phy_chans[num_phy_chans];
2620
2621 base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res),
2622 GFP_KERNEL);
2623 if (!base->phy_res)
2624 goto failure;
2625
2626 base->lookup_phy_chans = kzalloc(num_phy_chans *
2627 sizeof(struct d40_chan *),
2628 GFP_KERNEL);
2629 if (!base->lookup_phy_chans)
2630 goto failure;
2631
2632 if (num_log_chans + plat_data->memcpy_len) {
2633 /*
2634 * The max number of logical channels are event lines for all
2635 * src devices and dst devices
2636 */
2637 base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 *
2638 sizeof(struct d40_chan *),
2639 GFP_KERNEL);
2640 if (!base->lookup_log_chans)
2641 goto failure;
2642 }
Jonas Aaberg698e4732010-08-09 12:08:56 +00002643
2644 base->lcla_pool.alloc_map = kzalloc(num_phy_chans *
2645 sizeof(struct d40_desc *) *
2646 D40_LCLA_LINK_PER_EVENT_GRP,
Linus Walleij8d318a52010-03-30 15:33:42 +02002647 GFP_KERNEL);
2648 if (!base->lcla_pool.alloc_map)
2649 goto failure;
2650
Jonas Aabergc675b1b2010-06-20 21:25:08 +00002651 base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc),
2652 0, SLAB_HWCACHE_ALIGN,
2653 NULL);
2654 if (base->desc_slab == NULL)
2655 goto failure;
2656
Linus Walleij8d318a52010-03-30 15:33:42 +02002657 return base;
2658
2659failure:
Rabin Vincentc6134c92010-10-06 08:20:36 +00002660 if (!IS_ERR(clk)) {
Linus Walleij8d318a52010-03-30 15:33:42 +02002661 clk_disable(clk);
2662 clk_put(clk);
2663 }
2664 if (virtbase)
2665 iounmap(virtbase);
2666 if (res)
2667 release_mem_region(res->start,
2668 resource_size(res));
2669 if (virtbase)
2670 iounmap(virtbase);
2671
2672 if (base) {
2673 kfree(base->lcla_pool.alloc_map);
2674 kfree(base->lookup_log_chans);
2675 kfree(base->lookup_phy_chans);
2676 kfree(base->phy_res);
2677 kfree(base);
2678 }
2679
2680 return NULL;
2681}
2682
2683static void __init d40_hw_init(struct d40_base *base)
2684{
2685
2686 static const struct d40_reg_val dma_init_reg[] = {
2687 /* Clock every part of the DMA block from start */
2688 { .reg = D40_DREG_GCC, .val = 0x0000ff01},
2689
2690 /* Interrupts on all logical channels */
2691 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF},
2692 { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF},
2693 { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF},
2694 { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF},
2695 { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF},
2696 { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF},
2697 { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF},
2698 { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF},
2699 { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF},
2700 { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF},
2701 { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF},
2702 { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF}
2703 };
2704 int i;
2705 u32 prmseo[2] = {0, 0};
2706 u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF};
2707 u32 pcmis = 0;
2708 u32 pcicr = 0;
2709
2710 for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++)
2711 writel(dma_init_reg[i].val,
2712 base->virtbase + dma_init_reg[i].reg);
2713
2714 /* Configure all our dma channels to default settings */
2715 for (i = 0; i < base->num_phy_chans; i++) {
2716
2717 activeo[i % 2] = activeo[i % 2] << 2;
2718
2719 if (base->phy_res[base->num_phy_chans - i - 1].allocated_src
2720 == D40_ALLOC_PHY) {
2721 activeo[i % 2] |= 3;
2722 continue;
2723 }
2724
2725 /* Enable interrupt # */
2726 pcmis = (pcmis << 1) | 1;
2727
2728 /* Clear interrupt # */
2729 pcicr = (pcicr << 1) | 1;
2730
2731 /* Set channel to physical mode */
2732 prmseo[i % 2] = prmseo[i % 2] << 2;
2733 prmseo[i % 2] |= 1;
2734
2735 }
2736
2737 writel(prmseo[1], base->virtbase + D40_DREG_PRMSE);
2738 writel(prmseo[0], base->virtbase + D40_DREG_PRMSO);
2739 writel(activeo[1], base->virtbase + D40_DREG_ACTIVE);
2740 writel(activeo[0], base->virtbase + D40_DREG_ACTIVO);
2741
2742 /* Write which interrupt to enable */
2743 writel(pcmis, base->virtbase + D40_DREG_PCMIS);
2744
2745 /* Write which interrupt to clear */
2746 writel(pcicr, base->virtbase + D40_DREG_PCICR);
2747
2748}
2749
Linus Walleij508849a2010-06-20 21:26:07 +00002750static int __init d40_lcla_allocate(struct d40_base *base)
2751{
Rabin Vincent026cbc42011-01-25 11:18:14 +01002752 struct d40_lcla_pool *pool = &base->lcla_pool;
Linus Walleij508849a2010-06-20 21:26:07 +00002753 unsigned long *page_list;
2754 int i, j;
2755 int ret = 0;
2756
2757 /*
2758 * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned,
2759 * To full fill this hardware requirement without wasting 256 kb
2760 * we allocate pages until we get an aligned one.
2761 */
2762 page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS,
2763 GFP_KERNEL);
2764
2765 if (!page_list) {
2766 ret = -ENOMEM;
2767 goto failure;
2768 }
2769
2770 /* Calculating how many pages that are required */
2771 base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE;
2772
2773 for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) {
2774 page_list[i] = __get_free_pages(GFP_KERNEL,
2775 base->lcla_pool.pages);
2776 if (!page_list[i]) {
2777
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002778 d40_err(base->dev, "Failed to allocate %d pages.\n",
2779 base->lcla_pool.pages);
Linus Walleij508849a2010-06-20 21:26:07 +00002780
2781 for (j = 0; j < i; j++)
2782 free_pages(page_list[j], base->lcla_pool.pages);
2783 goto failure;
2784 }
2785
2786 if ((virt_to_phys((void *)page_list[i]) &
2787 (LCLA_ALIGNMENT - 1)) == 0)
2788 break;
2789 }
2790
2791 for (j = 0; j < i; j++)
2792 free_pages(page_list[j], base->lcla_pool.pages);
2793
2794 if (i < MAX_LCLA_ALLOC_ATTEMPTS) {
2795 base->lcla_pool.base = (void *)page_list[i];
2796 } else {
Jonas Aaberg767a9672010-08-09 12:08:34 +00002797 /*
2798 * After many attempts and no succees with finding the correct
2799 * alignment, try with allocating a big buffer.
2800 */
Linus Walleij508849a2010-06-20 21:26:07 +00002801 dev_warn(base->dev,
2802 "[%s] Failed to get %d pages @ 18 bit align.\n",
2803 __func__, base->lcla_pool.pages);
2804 base->lcla_pool.base_unaligned = kmalloc(SZ_1K *
2805 base->num_phy_chans +
2806 LCLA_ALIGNMENT,
2807 GFP_KERNEL);
2808 if (!base->lcla_pool.base_unaligned) {
2809 ret = -ENOMEM;
2810 goto failure;
2811 }
2812
2813 base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned,
2814 LCLA_ALIGNMENT);
2815 }
2816
Rabin Vincent026cbc42011-01-25 11:18:14 +01002817 pool->dma_addr = dma_map_single(base->dev, pool->base,
2818 SZ_1K * base->num_phy_chans,
2819 DMA_TO_DEVICE);
2820 if (dma_mapping_error(base->dev, pool->dma_addr)) {
2821 pool->dma_addr = 0;
2822 ret = -ENOMEM;
2823 goto failure;
2824 }
2825
Linus Walleij508849a2010-06-20 21:26:07 +00002826 writel(virt_to_phys(base->lcla_pool.base),
2827 base->virtbase + D40_DREG_LCLA);
2828failure:
2829 kfree(page_list);
2830 return ret;
2831}
2832
Linus Walleij8d318a52010-03-30 15:33:42 +02002833static int __init d40_probe(struct platform_device *pdev)
2834{
2835 int err;
2836 int ret = -ENOENT;
2837 struct d40_base *base;
2838 struct resource *res = NULL;
2839 int num_reserved_chans;
2840 u32 val;
2841
2842 base = d40_hw_detect_init(pdev);
2843
2844 if (!base)
2845 goto failure;
2846
2847 num_reserved_chans = d40_phy_res_init(base);
2848
2849 platform_set_drvdata(pdev, base);
2850
2851 spin_lock_init(&base->interrupt_lock);
2852 spin_lock_init(&base->execmd_lock);
2853
2854 /* Get IO for logical channel parameter address */
2855 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa");
2856 if (!res) {
2857 ret = -ENOENT;
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002858 d40_err(&pdev->dev, "No \"lcpa\" memory resource\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02002859 goto failure;
2860 }
2861 base->lcpa_size = resource_size(res);
2862 base->phy_lcpa = res->start;
2863
2864 if (request_mem_region(res->start, resource_size(res),
2865 D40_NAME " I/O lcpa") == NULL) {
2866 ret = -EBUSY;
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002867 d40_err(&pdev->dev,
2868 "Failed to request LCPA region 0x%x-0x%x\n",
2869 res->start, res->end);
Linus Walleij8d318a52010-03-30 15:33:42 +02002870 goto failure;
2871 }
2872
2873 /* We make use of ESRAM memory for this. */
2874 val = readl(base->virtbase + D40_DREG_LCPA);
2875 if (res->start != val && val != 0) {
2876 dev_warn(&pdev->dev,
2877 "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n",
2878 __func__, val, res->start);
2879 } else
2880 writel(res->start, base->virtbase + D40_DREG_LCPA);
2881
2882 base->lcpa_base = ioremap(res->start, resource_size(res));
2883 if (!base->lcpa_base) {
2884 ret = -ENOMEM;
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002885 d40_err(&pdev->dev, "Failed to ioremap LCPA region\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02002886 goto failure;
2887 }
Linus Walleij508849a2010-06-20 21:26:07 +00002888
2889 ret = d40_lcla_allocate(base);
2890 if (ret) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002891 d40_err(&pdev->dev, "Failed to allocate LCLA area\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02002892 goto failure;
2893 }
2894
Linus Walleij8d318a52010-03-30 15:33:42 +02002895 spin_lock_init(&base->lcla_pool.lock);
2896
Linus Walleij8d318a52010-03-30 15:33:42 +02002897 base->irq = platform_get_irq(pdev, 0);
2898
2899 ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base);
Linus Walleij8d318a52010-03-30 15:33:42 +02002900 if (ret) {
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002901 d40_err(&pdev->dev, "No IRQ defined\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02002902 goto failure;
2903 }
2904
2905 err = d40_dmaengine_init(base, num_reserved_chans);
2906 if (err)
2907 goto failure;
2908
2909 d40_hw_init(base);
2910
2911 dev_info(base->dev, "initialized\n");
2912 return 0;
2913
2914failure:
2915 if (base) {
Jonas Aabergc675b1b2010-06-20 21:25:08 +00002916 if (base->desc_slab)
2917 kmem_cache_destroy(base->desc_slab);
Linus Walleij8d318a52010-03-30 15:33:42 +02002918 if (base->virtbase)
2919 iounmap(base->virtbase);
Rabin Vincent026cbc42011-01-25 11:18:14 +01002920
2921 if (base->lcla_pool.dma_addr)
2922 dma_unmap_single(base->dev, base->lcla_pool.dma_addr,
2923 SZ_1K * base->num_phy_chans,
2924 DMA_TO_DEVICE);
2925
Linus Walleij508849a2010-06-20 21:26:07 +00002926 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base)
2927 free_pages((unsigned long)base->lcla_pool.base,
2928 base->lcla_pool.pages);
Jonas Aaberg767a9672010-08-09 12:08:34 +00002929
2930 kfree(base->lcla_pool.base_unaligned);
2931
Linus Walleij8d318a52010-03-30 15:33:42 +02002932 if (base->phy_lcpa)
2933 release_mem_region(base->phy_lcpa,
2934 base->lcpa_size);
2935 if (base->phy_start)
2936 release_mem_region(base->phy_start,
2937 base->phy_size);
2938 if (base->clk) {
2939 clk_disable(base->clk);
2940 clk_put(base->clk);
2941 }
2942
2943 kfree(base->lcla_pool.alloc_map);
2944 kfree(base->lookup_log_chans);
2945 kfree(base->lookup_phy_chans);
2946 kfree(base->phy_res);
2947 kfree(base);
2948 }
2949
Rabin Vincent6db5a8b2011-01-25 11:18:09 +01002950 d40_err(&pdev->dev, "probe failed\n");
Linus Walleij8d318a52010-03-30 15:33:42 +02002951 return ret;
2952}
2953
2954static struct platform_driver d40_driver = {
2955 .driver = {
2956 .owner = THIS_MODULE,
2957 .name = D40_NAME,
2958 },
2959};
2960
Rabin Vincentcb9ab2d2011-01-25 11:18:04 +01002961static int __init stedma40_init(void)
Linus Walleij8d318a52010-03-30 15:33:42 +02002962{
2963 return platform_driver_probe(&d40_driver, d40_probe);
2964}
2965arch_initcall(stedma40_init);