blob: 14091f878f80a5ca7cd0843e70284d890ad49f0c [file] [log] [blame]
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001/*
2 * offload engine driver for the Marvell XOR engine
3 * Copyright (C) 2007, 2008, Marvell International Ltd.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
Saeed Bisharaff7b0472008-07-08 11:58:36 -070013 */
14
15#include <linux/init.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090016#include <linux/slab.h>
Saeed Bisharaff7b0472008-07-08 11:58:36 -070017#include <linux/delay.h>
18#include <linux/dma-mapping.h>
19#include <linux/spinlock.h>
20#include <linux/interrupt.h>
Lior Amsalem6f166312015-05-26 15:07:34 +020021#include <linux/of_device.h>
Saeed Bisharaff7b0472008-07-08 11:58:36 -070022#include <linux/platform_device.h>
23#include <linux/memory.h>
Andrew Lunnc5101822012-02-19 13:30:26 +010024#include <linux/clk.h>
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +010025#include <linux/of.h>
26#include <linux/of_irq.h>
27#include <linux/irqdomain.h>
Thomas Petazzoni77757292015-07-08 16:28:19 +020028#include <linux/cpumask.h>
Arnd Bergmannc02cecb2012-08-24 15:21:54 +020029#include <linux/platform_data/dma-mv_xor.h>
Russell King - ARM Linuxd2ebfb32012-03-06 22:34:26 +000030
31#include "dmaengine.h"
Saeed Bisharaff7b0472008-07-08 11:58:36 -070032#include "mv_xor.h"
33
Lior Amsalem6f166312015-05-26 15:07:34 +020034enum mv_xor_mode {
35 XOR_MODE_IN_REG,
36 XOR_MODE_IN_DESC,
37};
38
Saeed Bisharaff7b0472008-07-08 11:58:36 -070039static void mv_xor_issue_pending(struct dma_chan *chan);
40
41#define to_mv_xor_chan(chan) \
Thomas Petazzoni98817b92012-11-15 14:57:44 +010042 container_of(chan, struct mv_xor_chan, dmachan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -070043
44#define to_mv_xor_slot(tx) \
45 container_of(tx, struct mv_xor_desc_slot, async_tx)
46
Thomas Petazzonic98c1782012-11-15 14:17:18 +010047#define mv_chan_to_devp(chan) \
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +010048 ((chan)->dmadev.dev)
Thomas Petazzonic98c1782012-11-15 14:17:18 +010049
Lior Amsalemdfc97662014-08-27 10:52:51 -030050static void mv_desc_init(struct mv_xor_desc_slot *desc,
Lior Amsalemba87d132014-08-27 10:52:53 -030051 dma_addr_t addr, u32 byte_count,
52 enum dma_ctrl_flags flags)
Saeed Bisharaff7b0472008-07-08 11:58:36 -070053{
54 struct mv_xor_desc *hw_desc = desc->hw_desc;
55
Ezequiel Garcia0e7488e2014-08-27 10:52:52 -030056 hw_desc->status = XOR_DESC_DMA_OWNED;
Saeed Bisharaff7b0472008-07-08 11:58:36 -070057 hw_desc->phy_next_desc = 0;
Lior Amsalemba87d132014-08-27 10:52:53 -030058 /* Enable end-of-descriptor interrupts only for DMA_PREP_INTERRUPT */
59 hw_desc->desc_command = (flags & DMA_PREP_INTERRUPT) ?
60 XOR_DESC_EOD_INT_EN : 0;
Lior Amsalemdfc97662014-08-27 10:52:51 -030061 hw_desc->phy_dest_addr = addr;
Saeed Bisharaff7b0472008-07-08 11:58:36 -070062 hw_desc->byte_count = byte_count;
63}
64
Lior Amsalem6f166312015-05-26 15:07:34 +020065static void mv_desc_set_mode(struct mv_xor_desc_slot *desc)
66{
67 struct mv_xor_desc *hw_desc = desc->hw_desc;
68
69 switch (desc->type) {
70 case DMA_XOR:
71 case DMA_INTERRUPT:
72 hw_desc->desc_command |= XOR_DESC_OPERATION_XOR;
73 break;
74 case DMA_MEMCPY:
75 hw_desc->desc_command |= XOR_DESC_OPERATION_MEMCPY;
76 break;
77 default:
78 BUG();
79 return;
80 }
81}
82
Saeed Bisharaff7b0472008-07-08 11:58:36 -070083static void mv_desc_set_next_desc(struct mv_xor_desc_slot *desc,
84 u32 next_desc_addr)
85{
86 struct mv_xor_desc *hw_desc = desc->hw_desc;
87 BUG_ON(hw_desc->phy_next_desc);
88 hw_desc->phy_next_desc = next_desc_addr;
89}
90
Saeed Bisharaff7b0472008-07-08 11:58:36 -070091static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc,
92 int index, dma_addr_t addr)
93{
94 struct mv_xor_desc *hw_desc = desc->hw_desc;
Thomas Petazzonie03bc652013-07-29 17:42:14 +020095 hw_desc->phy_src_addr[mv_phy_src_idx(index)] = addr;
Saeed Bisharaff7b0472008-07-08 11:58:36 -070096 if (desc->type == DMA_XOR)
97 hw_desc->desc_command |= (1 << index);
98}
99
100static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan)
101{
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200102 return readl_relaxed(XOR_CURR_DESC(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700103}
104
105static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan,
106 u32 next_desc_addr)
107{
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200108 writel_relaxed(next_desc_addr, XOR_NEXT_DESC(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700109}
110
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700111static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan)
112{
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200113 u32 val = readl_relaxed(XOR_INTR_MASK(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700114 val |= XOR_INTR_MASK_VALUE << (chan->idx * 16);
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200115 writel_relaxed(val, XOR_INTR_MASK(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700116}
117
118static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan)
119{
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200120 u32 intr_cause = readl_relaxed(XOR_INTR_CAUSE(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700121 intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF;
122 return intr_cause;
123}
124
Maxime Ripard0951e722015-05-26 15:07:33 +0200125static void mv_chan_clear_eoc_cause(struct mv_xor_chan *chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700126{
Lior Amsalemba87d132014-08-27 10:52:53 -0300127 u32 val;
128
129 val = XOR_INT_END_OF_DESC | XOR_INT_END_OF_CHAIN | XOR_INT_STOPPED;
130 val = ~(val << (chan->idx * 16));
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100131 dev_dbg(mv_chan_to_devp(chan), "%s, val 0x%08x\n", __func__, val);
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200132 writel_relaxed(val, XOR_INTR_CAUSE(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700133}
134
Maxime Ripard0951e722015-05-26 15:07:33 +0200135static void mv_chan_clear_err_status(struct mv_xor_chan *chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700136{
137 u32 val = 0xFFFF0000 >> (chan->idx * 16);
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200138 writel_relaxed(val, XOR_INTR_CAUSE(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700139}
140
Maxime Ripard0951e722015-05-26 15:07:33 +0200141static void mv_chan_set_mode(struct mv_xor_chan *chan,
Thomas Petazzoni81aafb32015-12-22 11:43:28 +0100142 u32 op_mode)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700143{
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200144 u32 config = readl_relaxed(XOR_CONFIG(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700145
Lior Amsalem6f166312015-05-26 15:07:34 +0200146 config &= ~0x7;
147 config |= op_mode;
148
Thomas Petazzonie03bc652013-07-29 17:42:14 +0200149#if defined(__BIG_ENDIAN)
150 config |= XOR_DESCRIPTOR_SWAP;
151#else
152 config &= ~XOR_DESCRIPTOR_SWAP;
153#endif
154
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200155 writel_relaxed(config, XOR_CONFIG(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700156}
157
158static void mv_chan_activate(struct mv_xor_chan *chan)
159{
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100160 dev_dbg(mv_chan_to_devp(chan), " activate chan.\n");
Ezequiel Garcia5a9a55b2014-05-21 14:02:35 -0700161
162 /* writel ensures all descriptors are flushed before activation */
163 writel(BIT(0), XOR_ACTIVATION(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700164}
165
166static char mv_chan_is_busy(struct mv_xor_chan *chan)
167{
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200168 u32 state = readl_relaxed(XOR_ACTIVATION(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700169
170 state = (state >> 4) & 0x3;
171
172 return (state == 1) ? 1 : 0;
173}
174
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700175/*
Maxime Ripard0951e722015-05-26 15:07:33 +0200176 * mv_chan_start_new_chain - program the engine to operate on new
177 * chain headed by sw_desc
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700178 * Caller must hold &mv_chan->lock while calling this function
179 */
Maxime Ripard0951e722015-05-26 15:07:33 +0200180static void mv_chan_start_new_chain(struct mv_xor_chan *mv_chan,
181 struct mv_xor_desc_slot *sw_desc)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700182{
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100183 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: sw_desc %p\n",
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700184 __func__, __LINE__, sw_desc);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700185
Bartlomiej Zolnierkiewicz48a9db42013-07-03 15:05:06 -0700186 /* set the hardware chain */
187 mv_chan_set_next_descriptor(mv_chan, sw_desc->async_tx.phys);
188
Lior Amsalemdfc97662014-08-27 10:52:51 -0300189 mv_chan->pending++;
Thomas Petazzoni98817b92012-11-15 14:57:44 +0100190 mv_xor_issue_pending(&mv_chan->dmachan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700191}
192
193static dma_cookie_t
Maxime Ripard0951e722015-05-26 15:07:33 +0200194mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc,
195 struct mv_xor_chan *mv_chan,
196 dma_cookie_t cookie)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700197{
198 BUG_ON(desc->async_tx.cookie < 0);
199
200 if (desc->async_tx.cookie > 0) {
201 cookie = desc->async_tx.cookie;
202
203 /* call the callback (must not sleep or submit new
204 * operations to this channel)
205 */
206 if (desc->async_tx.callback)
207 desc->async_tx.callback(
208 desc->async_tx.callback_param);
209
Dan Williamsd38a8c62013-10-18 19:35:23 +0200210 dma_descriptor_unmap(&desc->async_tx);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700211 }
212
213 /* run dependent operations */
Dan Williams07f22112009-01-05 17:14:31 -0700214 dma_run_dependencies(&desc->async_tx);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700215
216 return cookie;
217}
218
219static int
Maxime Ripard0951e722015-05-26 15:07:33 +0200220mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700221{
222 struct mv_xor_desc_slot *iter, *_iter;
223
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100224 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700225 list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200226 node) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700227
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200228 if (async_tx_test_ack(&iter->async_tx))
229 list_move_tail(&iter->node, &mv_chan->free_slots);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700230 }
231 return 0;
232}
233
234static int
Maxime Ripard0951e722015-05-26 15:07:33 +0200235mv_desc_clean_slot(struct mv_xor_desc_slot *desc,
236 struct mv_xor_chan *mv_chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700237{
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100238 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: desc %p flags %d\n",
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700239 __func__, __LINE__, desc, desc->async_tx.flags);
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200240
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700241 /* the client is allowed to attach dependent operations
242 * until 'ack' is set
243 */
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200244 if (!async_tx_test_ack(&desc->async_tx))
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700245 /* move this slot to the completed_slots */
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200246 list_move_tail(&desc->node, &mv_chan->completed_slots);
247 else
248 list_move_tail(&desc->node, &mv_chan->free_slots);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700249
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700250 return 0;
251}
252
Ezequiel Garciafbeec992014-03-07 16:46:47 -0300253/* This function must be called with the mv_xor_chan spinlock held */
Maxime Ripard0951e722015-05-26 15:07:33 +0200254static void mv_chan_slot_cleanup(struct mv_xor_chan *mv_chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700255{
256 struct mv_xor_desc_slot *iter, *_iter;
257 dma_cookie_t cookie = 0;
258 int busy = mv_chan_is_busy(mv_chan);
259 u32 current_desc = mv_chan_get_current_desc(mv_chan);
Lior Amsalem91362912015-05-26 15:07:32 +0200260 int current_cleaned = 0;
261 struct mv_xor_desc *hw_desc;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700262
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100263 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
264 dev_dbg(mv_chan_to_devp(mv_chan), "current_desc %x\n", current_desc);
Maxime Ripard0951e722015-05-26 15:07:33 +0200265 mv_chan_clean_completed_slots(mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700266
267 /* free completed slots from the chain starting with
268 * the oldest descriptor
269 */
270
271 list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200272 node) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700273
Lior Amsalem91362912015-05-26 15:07:32 +0200274 /* clean finished descriptors */
275 hw_desc = iter->hw_desc;
276 if (hw_desc->status & XOR_DESC_SUCCESS) {
Maxime Ripard0951e722015-05-26 15:07:33 +0200277 cookie = mv_desc_run_tx_complete_actions(iter, mv_chan,
278 cookie);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700279
Lior Amsalem91362912015-05-26 15:07:32 +0200280 /* done processing desc, clean slot */
Maxime Ripard0951e722015-05-26 15:07:33 +0200281 mv_desc_clean_slot(iter, mv_chan);
Lior Amsalem91362912015-05-26 15:07:32 +0200282
283 /* break if we did cleaned the current */
284 if (iter->async_tx.phys == current_desc) {
285 current_cleaned = 1;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700286 break;
Lior Amsalem91362912015-05-26 15:07:32 +0200287 }
288 } else {
289 if (iter->async_tx.phys == current_desc) {
290 current_cleaned = 0;
291 break;
292 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700293 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700294 }
295
296 if ((busy == 0) && !list_empty(&mv_chan->chain)) {
Lior Amsalem91362912015-05-26 15:07:32 +0200297 if (current_cleaned) {
298 /*
299 * current descriptor cleaned and removed, run
300 * from list head
301 */
302 iter = list_entry(mv_chan->chain.next,
303 struct mv_xor_desc_slot,
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200304 node);
Maxime Ripard0951e722015-05-26 15:07:33 +0200305 mv_chan_start_new_chain(mv_chan, iter);
Lior Amsalem91362912015-05-26 15:07:32 +0200306 } else {
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200307 if (!list_is_last(&iter->node, &mv_chan->chain)) {
Lior Amsalem91362912015-05-26 15:07:32 +0200308 /*
309 * descriptors are still waiting after
310 * current, trigger them
311 */
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200312 iter = list_entry(iter->node.next,
Lior Amsalem91362912015-05-26 15:07:32 +0200313 struct mv_xor_desc_slot,
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200314 node);
Maxime Ripard0951e722015-05-26 15:07:33 +0200315 mv_chan_start_new_chain(mv_chan, iter);
Lior Amsalem91362912015-05-26 15:07:32 +0200316 } else {
317 /*
318 * some descriptors are still waiting
319 * to be cleaned
320 */
321 tasklet_schedule(&mv_chan->irq_tasklet);
322 }
323 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700324 }
325
326 if (cookie > 0)
Thomas Petazzoni98817b92012-11-15 14:57:44 +0100327 mv_chan->dmachan.completed_cookie = cookie;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700328}
329
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700330static void mv_xor_tasklet(unsigned long data)
331{
332 struct mv_xor_chan *chan = (struct mv_xor_chan *) data;
Ezequiel Garciae43147a2014-03-07 16:46:46 -0300333
334 spin_lock_bh(&chan->lock);
Maxime Ripard0951e722015-05-26 15:07:33 +0200335 mv_chan_slot_cleanup(chan);
Ezequiel Garciae43147a2014-03-07 16:46:46 -0300336 spin_unlock_bh(&chan->lock);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700337}
338
339static struct mv_xor_desc_slot *
Maxime Ripard0951e722015-05-26 15:07:33 +0200340mv_chan_alloc_slot(struct mv_xor_chan *mv_chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700341{
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200342 struct mv_xor_desc_slot *iter;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700343
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200344 spin_lock_bh(&mv_chan->lock);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700345
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200346 if (!list_empty(&mv_chan->free_slots)) {
347 iter = list_first_entry(&mv_chan->free_slots,
348 struct mv_xor_desc_slot,
349 node);
Lior Amsalemdfc97662014-08-27 10:52:51 -0300350
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200351 list_move_tail(&iter->node, &mv_chan->allocated_slots);
352
353 spin_unlock_bh(&mv_chan->lock);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700354
Lior Amsalemdfc97662014-08-27 10:52:51 -0300355 /* pre-ack descriptor */
356 async_tx_ack(&iter->async_tx);
Lior Amsalemdfc97662014-08-27 10:52:51 -0300357 iter->async_tx.cookie = -EBUSY;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700358
Lior Amsalemdfc97662014-08-27 10:52:51 -0300359 return iter;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700360
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700361 }
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200362
363 spin_unlock_bh(&mv_chan->lock);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700364
365 /* try to free some slots if the allocation fails */
366 tasklet_schedule(&mv_chan->irq_tasklet);
367
368 return NULL;
369}
370
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700371/************************ DMA engine API functions ****************************/
372static dma_cookie_t
373mv_xor_tx_submit(struct dma_async_tx_descriptor *tx)
374{
375 struct mv_xor_desc_slot *sw_desc = to_mv_xor_slot(tx);
376 struct mv_xor_chan *mv_chan = to_mv_xor_chan(tx->chan);
Lior Amsalemdfc97662014-08-27 10:52:51 -0300377 struct mv_xor_desc_slot *old_chain_tail;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700378 dma_cookie_t cookie;
379 int new_hw_chain = 1;
380
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100381 dev_dbg(mv_chan_to_devp(mv_chan),
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700382 "%s sw_desc %p: async_tx %p\n",
383 __func__, sw_desc, &sw_desc->async_tx);
384
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700385 spin_lock_bh(&mv_chan->lock);
Russell King - ARM Linux884485e2012-03-06 22:34:46 +0000386 cookie = dma_cookie_assign(tx);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700387
388 if (list_empty(&mv_chan->chain))
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200389 list_move_tail(&sw_desc->node, &mv_chan->chain);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700390 else {
391 new_hw_chain = 0;
392
393 old_chain_tail = list_entry(mv_chan->chain.prev,
394 struct mv_xor_desc_slot,
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200395 node);
396 list_move_tail(&sw_desc->node, &mv_chan->chain);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700397
Olof Johansson31fd8f52014-02-03 17:13:23 -0800398 dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %pa\n",
399 &old_chain_tail->async_tx.phys);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700400
401 /* fix up the hardware chain */
Lior Amsalemdfc97662014-08-27 10:52:51 -0300402 mv_desc_set_next_desc(old_chain_tail, sw_desc->async_tx.phys);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700403
404 /* if the channel is not busy */
405 if (!mv_chan_is_busy(mv_chan)) {
406 u32 current_desc = mv_chan_get_current_desc(mv_chan);
407 /*
408 * and the curren desc is the end of the chain before
409 * the append, then we need to start the channel
410 */
411 if (current_desc == old_chain_tail->async_tx.phys)
412 new_hw_chain = 1;
413 }
414 }
415
416 if (new_hw_chain)
Maxime Ripard0951e722015-05-26 15:07:33 +0200417 mv_chan_start_new_chain(mv_chan, sw_desc);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700418
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700419 spin_unlock_bh(&mv_chan->lock);
420
421 return cookie;
422}
423
424/* returns the number of allocated descriptors */
Dan Williamsaa1e6f12009-01-06 11:38:17 -0700425static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700426{
Olof Johansson31fd8f52014-02-03 17:13:23 -0800427 void *virt_desc;
428 dma_addr_t dma_desc;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700429 int idx;
430 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
431 struct mv_xor_desc_slot *slot = NULL;
Thomas Petazzonib503fa02012-11-15 15:55:30 +0100432 int num_descs_in_pool = MV_XOR_POOL_SIZE/MV_XOR_SLOT_SIZE;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700433
434 /* Allocate descriptor slots */
435 idx = mv_chan->slots_allocated;
436 while (idx < num_descs_in_pool) {
437 slot = kzalloc(sizeof(*slot), GFP_KERNEL);
438 if (!slot) {
Ezequiel Garciab8291dd2014-08-27 10:52:49 -0300439 dev_info(mv_chan_to_devp(mv_chan),
440 "channel only initialized %d descriptor slots",
441 idx);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700442 break;
443 }
Olof Johansson31fd8f52014-02-03 17:13:23 -0800444 virt_desc = mv_chan->dma_desc_pool_virt;
445 slot->hw_desc = virt_desc + idx * MV_XOR_SLOT_SIZE;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700446
447 dma_async_tx_descriptor_init(&slot->async_tx, chan);
448 slot->async_tx.tx_submit = mv_xor_tx_submit;
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200449 INIT_LIST_HEAD(&slot->node);
Olof Johansson31fd8f52014-02-03 17:13:23 -0800450 dma_desc = mv_chan->dma_desc_pool;
451 slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700452 slot->idx = idx++;
453
454 spin_lock_bh(&mv_chan->lock);
455 mv_chan->slots_allocated = idx;
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200456 list_add_tail(&slot->node, &mv_chan->free_slots);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700457 spin_unlock_bh(&mv_chan->lock);
458 }
459
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100460 dev_dbg(mv_chan_to_devp(mv_chan),
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200461 "allocated %d descriptor slots\n",
462 mv_chan->slots_allocated);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700463
464 return mv_chan->slots_allocated ? : -ENOMEM;
465}
466
467static struct dma_async_tx_descriptor *
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700468mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
469 unsigned int src_cnt, size_t len, unsigned long flags)
470{
471 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
Lior Amsalemdfc97662014-08-27 10:52:51 -0300472 struct mv_xor_desc_slot *sw_desc;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700473
474 if (unlikely(len < MV_XOR_MIN_BYTE_COUNT))
475 return NULL;
476
Coly Li7912d302011-03-27 01:26:53 +0800477 BUG_ON(len > MV_XOR_MAX_BYTE_COUNT);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700478
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100479 dev_dbg(mv_chan_to_devp(mv_chan),
Olof Johansson31fd8f52014-02-03 17:13:23 -0800480 "%s src_cnt: %d len: %u dest %pad flags: %ld\n",
481 __func__, src_cnt, len, &dest, flags);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700482
Maxime Ripard0951e722015-05-26 15:07:33 +0200483 sw_desc = mv_chan_alloc_slot(mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700484 if (sw_desc) {
485 sw_desc->type = DMA_XOR;
486 sw_desc->async_tx.flags = flags;
Lior Amsalemba87d132014-08-27 10:52:53 -0300487 mv_desc_init(sw_desc, dest, len, flags);
Lior Amsalem6f166312015-05-26 15:07:34 +0200488 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
489 mv_desc_set_mode(sw_desc);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700490 while (src_cnt--)
Lior Amsalemdfc97662014-08-27 10:52:51 -0300491 mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700492 }
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200493
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100494 dev_dbg(mv_chan_to_devp(mv_chan),
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700495 "%s sw_desc %p async_tx %p \n",
496 __func__, sw_desc, &sw_desc->async_tx);
497 return sw_desc ? &sw_desc->async_tx : NULL;
498}
499
Lior Amsalem3e4f52e2014-08-27 10:52:50 -0300500static struct dma_async_tx_descriptor *
501mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
502 size_t len, unsigned long flags)
503{
504 /*
505 * A MEMCPY operation is identical to an XOR operation with only
506 * a single source address.
507 */
508 return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
509}
510
Lior Amsalem22843542014-08-27 10:52:55 -0300511static struct dma_async_tx_descriptor *
512mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
513{
514 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
515 dma_addr_t src, dest;
516 size_t len;
517
518 src = mv_chan->dummy_src_addr;
519 dest = mv_chan->dummy_dst_addr;
520 len = MV_XOR_MIN_BYTE_COUNT;
521
522 /*
523 * We implement the DMA_INTERRUPT operation as a minimum sized
524 * XOR operation with a single dummy source address.
525 */
526 return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
527}
528
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700529static void mv_xor_free_chan_resources(struct dma_chan *chan)
530{
531 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
532 struct mv_xor_desc_slot *iter, *_iter;
533 int in_use_descs = 0;
534
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700535 spin_lock_bh(&mv_chan->lock);
Ezequiel Garciae43147a2014-03-07 16:46:46 -0300536
Maxime Ripard0951e722015-05-26 15:07:33 +0200537 mv_chan_slot_cleanup(mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700538
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700539 list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200540 node) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700541 in_use_descs++;
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200542 list_move_tail(&iter->node, &mv_chan->free_slots);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700543 }
544 list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200545 node) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700546 in_use_descs++;
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200547 list_move_tail(&iter->node, &mv_chan->free_slots);
548 }
549 list_for_each_entry_safe(iter, _iter, &mv_chan->allocated_slots,
550 node) {
551 in_use_descs++;
552 list_move_tail(&iter->node, &mv_chan->free_slots);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700553 }
554 list_for_each_entry_safe_reverse(
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200555 iter, _iter, &mv_chan->free_slots, node) {
556 list_del(&iter->node);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700557 kfree(iter);
558 mv_chan->slots_allocated--;
559 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700560
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100561 dev_dbg(mv_chan_to_devp(mv_chan), "%s slots_allocated %d\n",
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700562 __func__, mv_chan->slots_allocated);
563 spin_unlock_bh(&mv_chan->lock);
564
565 if (in_use_descs)
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100566 dev_err(mv_chan_to_devp(mv_chan),
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700567 "freeing %d in use descriptors!\n", in_use_descs);
568}
569
570/**
Linus Walleij07934482010-03-26 16:50:49 -0700571 * mv_xor_status - poll the status of an XOR transaction
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700572 * @chan: XOR channel handle
573 * @cookie: XOR transaction identifier
Linus Walleij07934482010-03-26 16:50:49 -0700574 * @txstate: XOR transactions state holder (or NULL)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700575 */
Linus Walleij07934482010-03-26 16:50:49 -0700576static enum dma_status mv_xor_status(struct dma_chan *chan,
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700577 dma_cookie_t cookie,
Linus Walleij07934482010-03-26 16:50:49 -0700578 struct dma_tx_state *txstate)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700579{
580 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700581 enum dma_status ret;
582
Russell King - ARM Linux96a2af42012-03-06 22:35:27 +0000583 ret = dma_cookie_status(chan, cookie, txstate);
Ezequiel Garcia890766d2014-03-07 16:46:45 -0300584 if (ret == DMA_COMPLETE)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700585 return ret;
Ezequiel Garciae43147a2014-03-07 16:46:46 -0300586
587 spin_lock_bh(&mv_chan->lock);
Maxime Ripard0951e722015-05-26 15:07:33 +0200588 mv_chan_slot_cleanup(mv_chan);
Ezequiel Garciae43147a2014-03-07 16:46:46 -0300589 spin_unlock_bh(&mv_chan->lock);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700590
Russell King - ARM Linux96a2af42012-03-06 22:35:27 +0000591 return dma_cookie_status(chan, cookie, txstate);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700592}
593
Maxime Ripard0951e722015-05-26 15:07:33 +0200594static void mv_chan_dump_regs(struct mv_xor_chan *chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700595{
596 u32 val;
597
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200598 val = readl_relaxed(XOR_CONFIG(chan));
Joe Perches1ba151c2012-10-28 01:05:44 -0700599 dev_err(mv_chan_to_devp(chan), "config 0x%08x\n", val);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700600
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200601 val = readl_relaxed(XOR_ACTIVATION(chan));
Joe Perches1ba151c2012-10-28 01:05:44 -0700602 dev_err(mv_chan_to_devp(chan), "activation 0x%08x\n", val);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700603
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200604 val = readl_relaxed(XOR_INTR_CAUSE(chan));
Joe Perches1ba151c2012-10-28 01:05:44 -0700605 dev_err(mv_chan_to_devp(chan), "intr cause 0x%08x\n", val);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700606
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200607 val = readl_relaxed(XOR_INTR_MASK(chan));
Joe Perches1ba151c2012-10-28 01:05:44 -0700608 dev_err(mv_chan_to_devp(chan), "intr mask 0x%08x\n", val);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700609
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200610 val = readl_relaxed(XOR_ERROR_CAUSE(chan));
Joe Perches1ba151c2012-10-28 01:05:44 -0700611 dev_err(mv_chan_to_devp(chan), "error cause 0x%08x\n", val);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700612
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200613 val = readl_relaxed(XOR_ERROR_ADDR(chan));
Joe Perches1ba151c2012-10-28 01:05:44 -0700614 dev_err(mv_chan_to_devp(chan), "error addr 0x%08x\n", val);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700615}
616
Maxime Ripard0951e722015-05-26 15:07:33 +0200617static void mv_chan_err_interrupt_handler(struct mv_xor_chan *chan,
618 u32 intr_cause)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700619{
Ezequiel Garcia0e7488e2014-08-27 10:52:52 -0300620 if (intr_cause & XOR_INT_ERR_DECODE) {
621 dev_dbg(mv_chan_to_devp(chan), "ignoring address decode error\n");
622 return;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700623 }
624
Ezequiel Garcia0e7488e2014-08-27 10:52:52 -0300625 dev_err(mv_chan_to_devp(chan), "error on chan %d. intr cause 0x%08x\n",
Thomas Petazzonia3fc74b2012-11-15 12:50:27 +0100626 chan->idx, intr_cause);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700627
Maxime Ripard0951e722015-05-26 15:07:33 +0200628 mv_chan_dump_regs(chan);
Ezequiel Garcia0e7488e2014-08-27 10:52:52 -0300629 WARN_ON(1);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700630}
631
632static irqreturn_t mv_xor_interrupt_handler(int irq, void *data)
633{
634 struct mv_xor_chan *chan = data;
635 u32 intr_cause = mv_chan_get_intr_cause(chan);
636
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100637 dev_dbg(mv_chan_to_devp(chan), "intr cause %x\n", intr_cause);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700638
Ezequiel Garcia0e7488e2014-08-27 10:52:52 -0300639 if (intr_cause & XOR_INTR_ERRORS)
Maxime Ripard0951e722015-05-26 15:07:33 +0200640 mv_chan_err_interrupt_handler(chan, intr_cause);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700641
642 tasklet_schedule(&chan->irq_tasklet);
643
Maxime Ripard0951e722015-05-26 15:07:33 +0200644 mv_chan_clear_eoc_cause(chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700645
646 return IRQ_HANDLED;
647}
648
649static void mv_xor_issue_pending(struct dma_chan *chan)
650{
651 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
652
653 if (mv_chan->pending >= MV_XOR_THRESHOLD) {
654 mv_chan->pending = 0;
655 mv_chan_activate(mv_chan);
656 }
657}
658
659/*
660 * Perform a transaction to verify the HW works.
661 */
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700662
Maxime Ripard0951e722015-05-26 15:07:33 +0200663static int mv_chan_memcpy_self_test(struct mv_xor_chan *mv_chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700664{
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300665 int i, ret;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700666 void *src, *dest;
667 dma_addr_t src_dma, dest_dma;
668 struct dma_chan *dma_chan;
669 dma_cookie_t cookie;
670 struct dma_async_tx_descriptor *tx;
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300671 struct dmaengine_unmap_data *unmap;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700672 int err = 0;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700673
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300674 src = kmalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700675 if (!src)
676 return -ENOMEM;
677
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300678 dest = kzalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700679 if (!dest) {
680 kfree(src);
681 return -ENOMEM;
682 }
683
684 /* Fill in src buffer */
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300685 for (i = 0; i < PAGE_SIZE; i++)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700686 ((u8 *) src)[i] = (u8)i;
687
Thomas Petazzoni275cc0c2012-11-15 15:09:42 +0100688 dma_chan = &mv_chan->dmachan;
Dan Williamsaa1e6f12009-01-06 11:38:17 -0700689 if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700690 err = -ENODEV;
691 goto out;
692 }
693
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300694 unmap = dmaengine_get_unmap_data(dma_chan->device->dev, 2, GFP_KERNEL);
695 if (!unmap) {
696 err = -ENOMEM;
697 goto free_resources;
698 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700699
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300700 src_dma = dma_map_page(dma_chan->device->dev, virt_to_page(src), 0,
701 PAGE_SIZE, DMA_TO_DEVICE);
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300702 unmap->addr[0] = src_dma;
703
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300704 ret = dma_mapping_error(dma_chan->device->dev, src_dma);
705 if (ret) {
706 err = -ENOMEM;
707 goto free_resources;
708 }
709 unmap->to_cnt = 1;
710
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300711 dest_dma = dma_map_page(dma_chan->device->dev, virt_to_page(dest), 0,
712 PAGE_SIZE, DMA_FROM_DEVICE);
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300713 unmap->addr[1] = dest_dma;
714
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300715 ret = dma_mapping_error(dma_chan->device->dev, dest_dma);
716 if (ret) {
717 err = -ENOMEM;
718 goto free_resources;
719 }
720 unmap->from_cnt = 1;
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300721 unmap->len = PAGE_SIZE;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700722
723 tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300724 PAGE_SIZE, 0);
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300725 if (!tx) {
726 dev_err(dma_chan->device->dev,
727 "Self-test cannot prepare operation, disabling\n");
728 err = -ENODEV;
729 goto free_resources;
730 }
731
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700732 cookie = mv_xor_tx_submit(tx);
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300733 if (dma_submit_error(cookie)) {
734 dev_err(dma_chan->device->dev,
735 "Self-test submit error, disabling\n");
736 err = -ENODEV;
737 goto free_resources;
738 }
739
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700740 mv_xor_issue_pending(dma_chan);
741 async_tx_ack(tx);
742 msleep(1);
743
Linus Walleij07934482010-03-26 16:50:49 -0700744 if (mv_xor_status(dma_chan, cookie, NULL) !=
Vinod Koulb3efb8f2013-10-16 20:51:04 +0530745 DMA_COMPLETE) {
Thomas Petazzonia3fc74b2012-11-15 12:50:27 +0100746 dev_err(dma_chan->device->dev,
747 "Self-test copy timed out, disabling\n");
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700748 err = -ENODEV;
749 goto free_resources;
750 }
751
Thomas Petazzonic35064c2012-11-15 13:01:59 +0100752 dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300753 PAGE_SIZE, DMA_FROM_DEVICE);
754 if (memcmp(src, dest, PAGE_SIZE)) {
Thomas Petazzonia3fc74b2012-11-15 12:50:27 +0100755 dev_err(dma_chan->device->dev,
756 "Self-test copy failed compare, disabling\n");
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700757 err = -ENODEV;
758 goto free_resources;
759 }
760
761free_resources:
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300762 dmaengine_unmap_put(unmap);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700763 mv_xor_free_chan_resources(dma_chan);
764out:
765 kfree(src);
766 kfree(dest);
767 return err;
768}
769
770#define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */
Bill Pemberton463a1f82012-11-19 13:22:55 -0500771static int
Maxime Ripard0951e722015-05-26 15:07:33 +0200772mv_chan_xor_self_test(struct mv_xor_chan *mv_chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700773{
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300774 int i, src_idx, ret;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700775 struct page *dest;
776 struct page *xor_srcs[MV_XOR_NUM_SRC_TEST];
777 dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST];
778 dma_addr_t dest_dma;
779 struct dma_async_tx_descriptor *tx;
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300780 struct dmaengine_unmap_data *unmap;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700781 struct dma_chan *dma_chan;
782 dma_cookie_t cookie;
783 u8 cmp_byte = 0;
784 u32 cmp_word;
785 int err = 0;
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300786 int src_count = MV_XOR_NUM_SRC_TEST;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700787
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300788 for (src_idx = 0; src_idx < src_count; src_idx++) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700789 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
Roel Kluina09b09a2009-02-25 13:56:21 +0100790 if (!xor_srcs[src_idx]) {
791 while (src_idx--)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700792 __free_page(xor_srcs[src_idx]);
Roel Kluina09b09a2009-02-25 13:56:21 +0100793 return -ENOMEM;
794 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700795 }
796
797 dest = alloc_page(GFP_KERNEL);
Roel Kluina09b09a2009-02-25 13:56:21 +0100798 if (!dest) {
799 while (src_idx--)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700800 __free_page(xor_srcs[src_idx]);
Roel Kluina09b09a2009-02-25 13:56:21 +0100801 return -ENOMEM;
802 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700803
804 /* Fill in src buffers */
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300805 for (src_idx = 0; src_idx < src_count; src_idx++) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700806 u8 *ptr = page_address(xor_srcs[src_idx]);
807 for (i = 0; i < PAGE_SIZE; i++)
808 ptr[i] = (1 << src_idx);
809 }
810
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300811 for (src_idx = 0; src_idx < src_count; src_idx++)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700812 cmp_byte ^= (u8) (1 << src_idx);
813
814 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
815 (cmp_byte << 8) | cmp_byte;
816
817 memset(page_address(dest), 0, PAGE_SIZE);
818
Thomas Petazzoni275cc0c2012-11-15 15:09:42 +0100819 dma_chan = &mv_chan->dmachan;
Dan Williamsaa1e6f12009-01-06 11:38:17 -0700820 if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700821 err = -ENODEV;
822 goto out;
823 }
824
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300825 unmap = dmaengine_get_unmap_data(dma_chan->device->dev, src_count + 1,
826 GFP_KERNEL);
827 if (!unmap) {
828 err = -ENOMEM;
829 goto free_resources;
830 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700831
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300832 /* test xor */
833 for (i = 0; i < src_count; i++) {
834 unmap->addr[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
835 0, PAGE_SIZE, DMA_TO_DEVICE);
836 dma_srcs[i] = unmap->addr[i];
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300837 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[i]);
838 if (ret) {
839 err = -ENOMEM;
840 goto free_resources;
841 }
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300842 unmap->to_cnt++;
843 }
844
845 unmap->addr[src_count] = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE,
846 DMA_FROM_DEVICE);
847 dest_dma = unmap->addr[src_count];
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300848 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[src_count]);
849 if (ret) {
850 err = -ENOMEM;
851 goto free_resources;
852 }
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300853 unmap->from_cnt = 1;
854 unmap->len = PAGE_SIZE;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700855
856 tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300857 src_count, PAGE_SIZE, 0);
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300858 if (!tx) {
859 dev_err(dma_chan->device->dev,
860 "Self-test cannot prepare operation, disabling\n");
861 err = -ENODEV;
862 goto free_resources;
863 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700864
865 cookie = mv_xor_tx_submit(tx);
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300866 if (dma_submit_error(cookie)) {
867 dev_err(dma_chan->device->dev,
868 "Self-test submit error, disabling\n");
869 err = -ENODEV;
870 goto free_resources;
871 }
872
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700873 mv_xor_issue_pending(dma_chan);
874 async_tx_ack(tx);
875 msleep(8);
876
Linus Walleij07934482010-03-26 16:50:49 -0700877 if (mv_xor_status(dma_chan, cookie, NULL) !=
Vinod Koulb3efb8f2013-10-16 20:51:04 +0530878 DMA_COMPLETE) {
Thomas Petazzonia3fc74b2012-11-15 12:50:27 +0100879 dev_err(dma_chan->device->dev,
880 "Self-test xor timed out, disabling\n");
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700881 err = -ENODEV;
882 goto free_resources;
883 }
884
Thomas Petazzonic35064c2012-11-15 13:01:59 +0100885 dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700886 PAGE_SIZE, DMA_FROM_DEVICE);
887 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
888 u32 *ptr = page_address(dest);
889 if (ptr[i] != cmp_word) {
Thomas Petazzonia3fc74b2012-11-15 12:50:27 +0100890 dev_err(dma_chan->device->dev,
Joe Perches1ba151c2012-10-28 01:05:44 -0700891 "Self-test xor failed compare, disabling. index %d, data %x, expected %x\n",
892 i, ptr[i], cmp_word);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700893 err = -ENODEV;
894 goto free_resources;
895 }
896 }
897
898free_resources:
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300899 dmaengine_unmap_put(unmap);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700900 mv_xor_free_chan_resources(dma_chan);
901out:
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300902 src_idx = src_count;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700903 while (src_idx--)
904 __free_page(xor_srcs[src_idx]);
905 __free_page(dest);
906 return err;
907}
908
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100909static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700910{
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700911 struct dma_chan *chan, *_chan;
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100912 struct device *dev = mv_chan->dmadev.dev;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700913
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100914 dma_async_device_unregister(&mv_chan->dmadev);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700915
Thomas Petazzonib503fa02012-11-15 15:55:30 +0100916 dma_free_coherent(dev, MV_XOR_POOL_SIZE,
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100917 mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
Lior Amsalem22843542014-08-27 10:52:55 -0300918 dma_unmap_single(dev, mv_chan->dummy_src_addr,
919 MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
920 dma_unmap_single(dev, mv_chan->dummy_dst_addr,
921 MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700922
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100923 list_for_each_entry_safe(chan, _chan, &mv_chan->dmadev.channels,
Thomas Petazzonia6b4a9d2012-10-29 16:45:46 +0100924 device_node) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700925 list_del(&chan->device_node);
926 }
927
Thomas Petazzoni88eb92c2012-11-15 16:11:18 +0100928 free_irq(mv_chan->irq, mv_chan);
929
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700930 return 0;
931}
932
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100933static struct mv_xor_chan *
Thomas Petazzoni297eedb2012-11-15 15:29:53 +0100934mv_xor_channel_add(struct mv_xor_device *xordev,
Thomas Petazzonia6b4a9d2012-10-29 16:45:46 +0100935 struct platform_device *pdev,
Lior Amsalem6f166312015-05-26 15:07:34 +0200936 int idx, dma_cap_mask_t cap_mask, int irq, int op_in_desc)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700937{
938 int ret = 0;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700939 struct mv_xor_chan *mv_chan;
940 struct dma_device *dma_dev;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700941
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100942 mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL);
Sachin Kamata5776592013-09-02 13:54:20 +0530943 if (!mv_chan)
944 return ERR_PTR(-ENOMEM);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700945
Thomas Petazzoni9aedbdb2012-11-15 15:36:37 +0100946 mv_chan->idx = idx;
Thomas Petazzoni88eb92c2012-11-15 16:11:18 +0100947 mv_chan->irq = irq;
Lior Amsalem6f166312015-05-26 15:07:34 +0200948 mv_chan->op_in_desc = op_in_desc;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700949
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100950 dma_dev = &mv_chan->dmadev;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700951
Lior Amsalem22843542014-08-27 10:52:55 -0300952 /*
953 * These source and destination dummy buffers are used to implement
954 * a DMA_INTERRUPT operation as a minimum-sized XOR operation.
955 * Hence, we only need to map the buffers at initialization-time.
956 */
957 mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev,
958 mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
959 mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev,
960 mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
961
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700962 /* allocate coherent memory for hardware descriptors
963 * note: writecombine gives slightly better performance, but
964 * requires that we explicitly flush the writes
965 */
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100966 mv_chan->dma_desc_pool_virt =
Thomas Petazzonib503fa02012-11-15 15:55:30 +0100967 dma_alloc_writecombine(&pdev->dev, MV_XOR_POOL_SIZE,
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100968 &mv_chan->dma_desc_pool, GFP_KERNEL);
969 if (!mv_chan->dma_desc_pool_virt)
Thomas Petazzonia6b4a9d2012-10-29 16:45:46 +0100970 return ERR_PTR(-ENOMEM);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700971
972 /* discover transaction capabilites from the platform data */
Thomas Petazzonia6b4a9d2012-10-29 16:45:46 +0100973 dma_dev->cap_mask = cap_mask;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700974
975 INIT_LIST_HEAD(&dma_dev->channels);
976
977 /* set base routines */
978 dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources;
979 dma_dev->device_free_chan_resources = mv_xor_free_chan_resources;
Linus Walleij07934482010-03-26 16:50:49 -0700980 dma_dev->device_tx_status = mv_xor_status;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700981 dma_dev->device_issue_pending = mv_xor_issue_pending;
982 dma_dev->dev = &pdev->dev;
983
984 /* set prep routines based on capability */
Lior Amsalem22843542014-08-27 10:52:55 -0300985 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
986 dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700987 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
988 dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700989 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
Joe Perchesc0198942009-06-28 09:26:21 -0700990 dma_dev->max_xor = 8;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700991 dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor;
992 }
993
Thomas Petazzoni297eedb2012-11-15 15:29:53 +0100994 mv_chan->mmr_base = xordev->xor_base;
Ezequiel Garcia82a14022013-10-30 12:01:43 -0300995 mv_chan->mmr_high_base = xordev->xor_high_base;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700996 tasklet_init(&mv_chan->irq_tasklet, mv_xor_tasklet, (unsigned long)
997 mv_chan);
998
999 /* clear errors before enabling interrupts */
Maxime Ripard0951e722015-05-26 15:07:33 +02001000 mv_chan_clear_err_status(mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001001
Thomas Petazzoni2d0a0742012-11-22 18:19:09 +01001002 ret = request_irq(mv_chan->irq, mv_xor_interrupt_handler,
1003 0, dev_name(&pdev->dev), mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001004 if (ret)
1005 goto err_free_dma;
1006
1007 mv_chan_unmask_interrupts(mv_chan);
1008
Lior Amsalem6f166312015-05-26 15:07:34 +02001009 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
Thomas Petazzoni81aafb32015-12-22 11:43:28 +01001010 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_IN_DESC);
Lior Amsalem6f166312015-05-26 15:07:34 +02001011 else
Thomas Petazzoni81aafb32015-12-22 11:43:28 +01001012 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_XOR);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001013
1014 spin_lock_init(&mv_chan->lock);
1015 INIT_LIST_HEAD(&mv_chan->chain);
1016 INIT_LIST_HEAD(&mv_chan->completed_slots);
Lior Amsalemfbea28a2015-05-26 15:07:36 +02001017 INIT_LIST_HEAD(&mv_chan->free_slots);
1018 INIT_LIST_HEAD(&mv_chan->allocated_slots);
Thomas Petazzoni98817b92012-11-15 14:57:44 +01001019 mv_chan->dmachan.device = dma_dev;
1020 dma_cookie_init(&mv_chan->dmachan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001021
Thomas Petazzoni98817b92012-11-15 14:57:44 +01001022 list_add_tail(&mv_chan->dmachan.device_node, &dma_dev->channels);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001023
1024 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
Maxime Ripard0951e722015-05-26 15:07:33 +02001025 ret = mv_chan_memcpy_self_test(mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001026 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1027 if (ret)
Thomas Petazzoni2d0a0742012-11-22 18:19:09 +01001028 goto err_free_irq;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001029 }
1030
1031 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
Maxime Ripard0951e722015-05-26 15:07:33 +02001032 ret = mv_chan_xor_self_test(mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001033 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1034 if (ret)
Thomas Petazzoni2d0a0742012-11-22 18:19:09 +01001035 goto err_free_irq;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001036 }
1037
Lior Amsalem6f166312015-05-26 15:07:34 +02001038 dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n",
1039 mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode",
Joe Perches1ba151c2012-10-28 01:05:44 -07001040 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
Joe Perches1ba151c2012-10-28 01:05:44 -07001041 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1042 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001043
1044 dma_async_device_register(dma_dev);
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +01001045 return mv_chan;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001046
Thomas Petazzoni2d0a0742012-11-22 18:19:09 +01001047err_free_irq:
1048 free_irq(mv_chan->irq, mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001049 err_free_dma:
Thomas Petazzonib503fa02012-11-15 15:55:30 +01001050 dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE,
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +01001051 mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
Thomas Petazzonia6b4a9d2012-10-29 16:45:46 +01001052 return ERR_PTR(ret);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001053}
1054
1055static void
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001056mv_xor_conf_mbus_windows(struct mv_xor_device *xordev,
Andrew Lunn63a93322011-12-07 21:48:07 +01001057 const struct mbus_dram_target_info *dram)
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001058{
Ezequiel Garcia82a14022013-10-30 12:01:43 -03001059 void __iomem *base = xordev->xor_high_base;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001060 u32 win_enable = 0;
1061 int i;
1062
1063 for (i = 0; i < 8; i++) {
1064 writel(0, base + WINDOW_BASE(i));
1065 writel(0, base + WINDOW_SIZE(i));
1066 if (i < 4)
1067 writel(0, base + WINDOW_REMAP_HIGH(i));
1068 }
1069
1070 for (i = 0; i < dram->num_cs; i++) {
Andrew Lunn63a93322011-12-07 21:48:07 +01001071 const struct mbus_dram_window *cs = dram->cs + i;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001072
1073 writel((cs->base & 0xffff0000) |
1074 (cs->mbus_attr << 8) |
1075 dram->mbus_dram_target_id, base + WINDOW_BASE(i));
1076 writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i));
1077
1078 win_enable |= (1 << i);
1079 win_enable |= 3 << (16 + (2 * i));
1080 }
1081
1082 writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1083 writel(win_enable, base + WINDOW_BAR_ENABLE(1));
Thomas Petazzonic4b4b732012-11-22 18:16:37 +01001084 writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1085 writel(0, base + WINDOW_OVERRIDE_CTRL(1));
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001086}
1087
Thomas Petazzoni8b648432015-12-22 11:43:29 +01001088/*
1089 * Since this XOR driver is basically used only for RAID5, we don't
1090 * need to care about synchronizing ->suspend with DMA activity,
1091 * because the DMA engine will naturally be quiet due to the block
1092 * devices being suspended.
1093 */
1094static int mv_xor_suspend(struct platform_device *pdev, pm_message_t state)
1095{
1096 struct mv_xor_device *xordev = platform_get_drvdata(pdev);
1097 int i;
1098
1099 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1100 struct mv_xor_chan *mv_chan = xordev->channels[i];
1101
1102 if (!mv_chan)
1103 continue;
1104
1105 mv_chan->saved_config_reg =
1106 readl_relaxed(XOR_CONFIG(mv_chan));
1107 mv_chan->saved_int_mask_reg =
1108 readl_relaxed(XOR_INTR_MASK(mv_chan));
1109 }
1110
1111 return 0;
1112}
1113
1114static int mv_xor_resume(struct platform_device *dev)
1115{
1116 struct mv_xor_device *xordev = platform_get_drvdata(dev);
1117 const struct mbus_dram_target_info *dram;
1118 int i;
1119
1120 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1121 struct mv_xor_chan *mv_chan = xordev->channels[i];
1122
1123 if (!mv_chan)
1124 continue;
1125
1126 writel_relaxed(mv_chan->saved_config_reg,
1127 XOR_CONFIG(mv_chan));
1128 writel_relaxed(mv_chan->saved_int_mask_reg,
1129 XOR_INTR_MASK(mv_chan));
1130 }
1131
1132 dram = mv_mbus_dram_info();
1133 if (dram)
1134 mv_xor_conf_mbus_windows(xordev, dram);
1135
1136 return 0;
1137}
1138
Lior Amsalem6f166312015-05-26 15:07:34 +02001139static const struct of_device_id mv_xor_dt_ids[] = {
1140 { .compatible = "marvell,orion-xor", .data = (void *)XOR_MODE_IN_REG },
1141 { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_MODE_IN_DESC },
1142 {},
1143};
Lior Amsalem6f166312015-05-26 15:07:34 +02001144
Thomas Petazzoni77757292015-07-08 16:28:19 +02001145static unsigned int mv_xor_engine_count;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001146
Linus Torvaldsc2714332012-12-14 14:54:26 -08001147static int mv_xor_probe(struct platform_device *pdev)
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001148{
Andrew Lunn63a93322011-12-07 21:48:07 +01001149 const struct mbus_dram_target_info *dram;
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001150 struct mv_xor_device *xordev;
Jingoo Hand4adcc02013-07-30 17:09:11 +09001151 struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001152 struct resource *res;
Thomas Petazzoni77757292015-07-08 16:28:19 +02001153 unsigned int max_engines, max_channels;
Thomas Petazzoni60d151f2012-10-29 16:54:49 +01001154 int i, ret;
Lior Amsalem6f166312015-05-26 15:07:34 +02001155 int op_in_desc;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001156
Joe Perches1ba151c2012-10-28 01:05:44 -07001157 dev_notice(&pdev->dev, "Marvell shared XOR driver\n");
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001158
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001159 xordev = devm_kzalloc(&pdev->dev, sizeof(*xordev), GFP_KERNEL);
1160 if (!xordev)
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001161 return -ENOMEM;
1162
1163 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1164 if (!res)
1165 return -ENODEV;
1166
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001167 xordev->xor_base = devm_ioremap(&pdev->dev, res->start,
1168 resource_size(res));
1169 if (!xordev->xor_base)
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001170 return -EBUSY;
1171
1172 res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1173 if (!res)
1174 return -ENODEV;
1175
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001176 xordev->xor_high_base = devm_ioremap(&pdev->dev, res->start,
1177 resource_size(res));
1178 if (!xordev->xor_high_base)
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001179 return -EBUSY;
1180
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001181 platform_set_drvdata(pdev, xordev);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001182
1183 /*
1184 * (Re-)program MBUS remapping windows if we are asked to.
1185 */
Andrew Lunn63a93322011-12-07 21:48:07 +01001186 dram = mv_mbus_dram_info();
1187 if (dram)
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001188 mv_xor_conf_mbus_windows(xordev, dram);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001189
Andrew Lunnc5101822012-02-19 13:30:26 +01001190 /* Not all platforms can gate the clock, so it is not
1191 * an error if the clock does not exists.
1192 */
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001193 xordev->clk = clk_get(&pdev->dev, NULL);
1194 if (!IS_ERR(xordev->clk))
1195 clk_prepare_enable(xordev->clk);
Andrew Lunnc5101822012-02-19 13:30:26 +01001196
Thomas Petazzoni77757292015-07-08 16:28:19 +02001197 /*
1198 * We don't want to have more than one channel per CPU in
1199 * order for async_tx to perform well. So we limit the number
1200 * of engines and channels so that we take into account this
1201 * constraint. Note that we also want to use channels from
1202 * separate engines when possible.
1203 */
1204 max_engines = num_present_cpus();
1205 max_channels = min_t(unsigned int,
1206 MV_XOR_MAX_CHANNELS,
1207 DIV_ROUND_UP(num_present_cpus(), 2));
1208
1209 if (mv_xor_engine_count >= max_engines)
1210 return 0;
1211
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001212 if (pdev->dev.of_node) {
1213 struct device_node *np;
1214 int i = 0;
Lior Amsalem6f166312015-05-26 15:07:34 +02001215 const struct of_device_id *of_id =
1216 of_match_device(mv_xor_dt_ids,
1217 &pdev->dev);
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001218
1219 for_each_child_of_node(pdev->dev.of_node, np) {
Russell King0be82532013-12-12 23:59:08 +00001220 struct mv_xor_chan *chan;
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001221 dma_cap_mask_t cap_mask;
1222 int irq;
Lior Amsalem6f166312015-05-26 15:07:34 +02001223 op_in_desc = (int)of_id->data;
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001224
Thomas Petazzoni77757292015-07-08 16:28:19 +02001225 if (i >= max_channels)
1226 continue;
1227
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001228 dma_cap_zero(cap_mask);
Thomas Petazzoni6d8f7ab2015-07-08 16:28:16 +02001229 dma_cap_set(DMA_MEMCPY, cap_mask);
1230 dma_cap_set(DMA_XOR, cap_mask);
1231 dma_cap_set(DMA_INTERRUPT, cap_mask);
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001232
1233 irq = irq_of_parse_and_map(np, 0);
Thomas Petazzonif8eb9e72012-11-22 18:22:12 +01001234 if (!irq) {
1235 ret = -ENODEV;
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001236 goto err_channel_add;
1237 }
1238
Russell King0be82532013-12-12 23:59:08 +00001239 chan = mv_xor_channel_add(xordev, pdev, i,
Lior Amsalem6f166312015-05-26 15:07:34 +02001240 cap_mask, irq, op_in_desc);
Russell King0be82532013-12-12 23:59:08 +00001241 if (IS_ERR(chan)) {
1242 ret = PTR_ERR(chan);
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001243 irq_dispose_mapping(irq);
1244 goto err_channel_add;
1245 }
1246
Russell King0be82532013-12-12 23:59:08 +00001247 xordev->channels[i] = chan;
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001248 i++;
1249 }
1250 } else if (pdata && pdata->channels) {
Thomas Petazzoni77757292015-07-08 16:28:19 +02001251 for (i = 0; i < max_channels; i++) {
Thomas Petazzonie39f6ec2012-10-30 11:56:26 +01001252 struct mv_xor_channel_data *cd;
Russell King0be82532013-12-12 23:59:08 +00001253 struct mv_xor_chan *chan;
Thomas Petazzoni60d151f2012-10-29 16:54:49 +01001254 int irq;
1255
1256 cd = &pdata->channels[i];
1257 if (!cd) {
1258 ret = -ENODEV;
1259 goto err_channel_add;
1260 }
1261
1262 irq = platform_get_irq(pdev, i);
1263 if (irq < 0) {
1264 ret = irq;
1265 goto err_channel_add;
1266 }
1267
Russell King0be82532013-12-12 23:59:08 +00001268 chan = mv_xor_channel_add(xordev, pdev, i,
Lior Amsalem6f166312015-05-26 15:07:34 +02001269 cd->cap_mask, irq,
1270 XOR_MODE_IN_REG);
Russell King0be82532013-12-12 23:59:08 +00001271 if (IS_ERR(chan)) {
1272 ret = PTR_ERR(chan);
Thomas Petazzoni60d151f2012-10-29 16:54:49 +01001273 goto err_channel_add;
1274 }
Russell King0be82532013-12-12 23:59:08 +00001275
1276 xordev->channels[i] = chan;
Thomas Petazzoni60d151f2012-10-29 16:54:49 +01001277 }
1278 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001279
1280 return 0;
Thomas Petazzoni60d151f2012-10-29 16:54:49 +01001281
1282err_channel_add:
1283 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++)
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001284 if (xordev->channels[i]) {
Thomas Petazzoniab6e4392013-01-06 11:10:43 +01001285 mv_xor_channel_remove(xordev->channels[i]);
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001286 if (pdev->dev.of_node)
1287 irq_dispose_mapping(xordev->channels[i]->irq);
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001288 }
Thomas Petazzoni60d151f2012-10-29 16:54:49 +01001289
Thomas Petazzonidab92062013-01-06 11:10:44 +01001290 if (!IS_ERR(xordev->clk)) {
1291 clk_disable_unprepare(xordev->clk);
1292 clk_put(xordev->clk);
1293 }
1294
Thomas Petazzoni60d151f2012-10-29 16:54:49 +01001295 return ret;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001296}
1297
Thomas Petazzoni61971652012-10-30 12:05:40 +01001298static struct platform_driver mv_xor_driver = {
1299 .probe = mv_xor_probe,
Thomas Petazzoni8b648432015-12-22 11:43:29 +01001300 .suspend = mv_xor_suspend,
1301 .resume = mv_xor_resume,
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001302 .driver = {
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001303 .name = MV_XOR_NAME,
1304 .of_match_table = of_match_ptr(mv_xor_dt_ids),
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001305 },
1306};
1307
1308
1309static int __init mv_xor_init(void)
1310{
Thomas Petazzoni61971652012-10-30 12:05:40 +01001311 return platform_driver_register(&mv_xor_driver);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001312}
Paul Gortmaker25cf68d2015-08-21 16:27:49 -04001313device_initcall(mv_xor_init);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001314
Paul Gortmaker25cf68d2015-08-21 16:27:49 -04001315/*
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001316MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>");
1317MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine");
1318MODULE_LICENSE("GPL");
Paul Gortmaker25cf68d2015-08-21 16:27:49 -04001319*/