blob: 25d1dadcddd1ddf22cd63e6d91e0b95901d34bbe [file] [log] [blame]
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001/*
2 * offload engine driver for the Marvell XOR engine
3 * Copyright (C) 2007, 2008, Marvell International Ltd.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
Saeed Bisharaff7b0472008-07-08 11:58:36 -070013 */
14
15#include <linux/init.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090016#include <linux/slab.h>
Saeed Bisharaff7b0472008-07-08 11:58:36 -070017#include <linux/delay.h>
18#include <linux/dma-mapping.h>
19#include <linux/spinlock.h>
20#include <linux/interrupt.h>
Lior Amsalem6f166312015-05-26 15:07:34 +020021#include <linux/of_device.h>
Saeed Bisharaff7b0472008-07-08 11:58:36 -070022#include <linux/platform_device.h>
23#include <linux/memory.h>
Andrew Lunnc5101822012-02-19 13:30:26 +010024#include <linux/clk.h>
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +010025#include <linux/of.h>
26#include <linux/of_irq.h>
27#include <linux/irqdomain.h>
Thomas Petazzoni77757292015-07-08 16:28:19 +020028#include <linux/cpumask.h>
Arnd Bergmannc02cecb2012-08-24 15:21:54 +020029#include <linux/platform_data/dma-mv_xor.h>
Russell King - ARM Linuxd2ebfb32012-03-06 22:34:26 +000030
31#include "dmaengine.h"
Saeed Bisharaff7b0472008-07-08 11:58:36 -070032#include "mv_xor.h"
33
Gregory CLEMENTdd130c62016-04-29 09:49:06 +020034enum mv_xor_type {
35 XOR_ORION,
36 XOR_ARMADA_38X,
Marcin Wojtasac5f0f32016-04-29 09:49:07 +020037 XOR_ARMADA_37XX,
Gregory CLEMENTdd130c62016-04-29 09:49:06 +020038};
39
Lior Amsalem6f166312015-05-26 15:07:34 +020040enum mv_xor_mode {
41 XOR_MODE_IN_REG,
42 XOR_MODE_IN_DESC,
43};
44
Saeed Bisharaff7b0472008-07-08 11:58:36 -070045static void mv_xor_issue_pending(struct dma_chan *chan);
46
47#define to_mv_xor_chan(chan) \
Thomas Petazzoni98817b92012-11-15 14:57:44 +010048 container_of(chan, struct mv_xor_chan, dmachan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -070049
50#define to_mv_xor_slot(tx) \
51 container_of(tx, struct mv_xor_desc_slot, async_tx)
52
Thomas Petazzonic98c1782012-11-15 14:17:18 +010053#define mv_chan_to_devp(chan) \
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +010054 ((chan)->dmadev.dev)
Thomas Petazzonic98c1782012-11-15 14:17:18 +010055
Lior Amsalemdfc97662014-08-27 10:52:51 -030056static void mv_desc_init(struct mv_xor_desc_slot *desc,
Lior Amsalemba87d132014-08-27 10:52:53 -030057 dma_addr_t addr, u32 byte_count,
58 enum dma_ctrl_flags flags)
Saeed Bisharaff7b0472008-07-08 11:58:36 -070059{
60 struct mv_xor_desc *hw_desc = desc->hw_desc;
61
Ezequiel Garcia0e7488e2014-08-27 10:52:52 -030062 hw_desc->status = XOR_DESC_DMA_OWNED;
Saeed Bisharaff7b0472008-07-08 11:58:36 -070063 hw_desc->phy_next_desc = 0;
Lior Amsalemba87d132014-08-27 10:52:53 -030064 /* Enable end-of-descriptor interrupts only for DMA_PREP_INTERRUPT */
65 hw_desc->desc_command = (flags & DMA_PREP_INTERRUPT) ?
66 XOR_DESC_EOD_INT_EN : 0;
Lior Amsalemdfc97662014-08-27 10:52:51 -030067 hw_desc->phy_dest_addr = addr;
Saeed Bisharaff7b0472008-07-08 11:58:36 -070068 hw_desc->byte_count = byte_count;
69}
70
Lior Amsalem6f166312015-05-26 15:07:34 +020071static void mv_desc_set_mode(struct mv_xor_desc_slot *desc)
72{
73 struct mv_xor_desc *hw_desc = desc->hw_desc;
74
75 switch (desc->type) {
76 case DMA_XOR:
77 case DMA_INTERRUPT:
78 hw_desc->desc_command |= XOR_DESC_OPERATION_XOR;
79 break;
80 case DMA_MEMCPY:
81 hw_desc->desc_command |= XOR_DESC_OPERATION_MEMCPY;
82 break;
83 default:
84 BUG();
85 return;
86 }
87}
88
Saeed Bisharaff7b0472008-07-08 11:58:36 -070089static void mv_desc_set_next_desc(struct mv_xor_desc_slot *desc,
90 u32 next_desc_addr)
91{
92 struct mv_xor_desc *hw_desc = desc->hw_desc;
93 BUG_ON(hw_desc->phy_next_desc);
94 hw_desc->phy_next_desc = next_desc_addr;
95}
96
Saeed Bisharaff7b0472008-07-08 11:58:36 -070097static void mv_desc_set_src_addr(struct mv_xor_desc_slot *desc,
98 int index, dma_addr_t addr)
99{
100 struct mv_xor_desc *hw_desc = desc->hw_desc;
Thomas Petazzonie03bc652013-07-29 17:42:14 +0200101 hw_desc->phy_src_addr[mv_phy_src_idx(index)] = addr;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700102 if (desc->type == DMA_XOR)
103 hw_desc->desc_command |= (1 << index);
104}
105
106static u32 mv_chan_get_current_desc(struct mv_xor_chan *chan)
107{
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200108 return readl_relaxed(XOR_CURR_DESC(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700109}
110
111static void mv_chan_set_next_descriptor(struct mv_xor_chan *chan,
112 u32 next_desc_addr)
113{
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200114 writel_relaxed(next_desc_addr, XOR_NEXT_DESC(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700115}
116
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700117static void mv_chan_unmask_interrupts(struct mv_xor_chan *chan)
118{
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200119 u32 val = readl_relaxed(XOR_INTR_MASK(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700120 val |= XOR_INTR_MASK_VALUE << (chan->idx * 16);
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200121 writel_relaxed(val, XOR_INTR_MASK(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700122}
123
124static u32 mv_chan_get_intr_cause(struct mv_xor_chan *chan)
125{
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200126 u32 intr_cause = readl_relaxed(XOR_INTR_CAUSE(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700127 intr_cause = (intr_cause >> (chan->idx * 16)) & 0xFFFF;
128 return intr_cause;
129}
130
Maxime Ripard0951e722015-05-26 15:07:33 +0200131static void mv_chan_clear_eoc_cause(struct mv_xor_chan *chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700132{
Lior Amsalemba87d132014-08-27 10:52:53 -0300133 u32 val;
134
135 val = XOR_INT_END_OF_DESC | XOR_INT_END_OF_CHAIN | XOR_INT_STOPPED;
136 val = ~(val << (chan->idx * 16));
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100137 dev_dbg(mv_chan_to_devp(chan), "%s, val 0x%08x\n", __func__, val);
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200138 writel_relaxed(val, XOR_INTR_CAUSE(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700139}
140
Maxime Ripard0951e722015-05-26 15:07:33 +0200141static void mv_chan_clear_err_status(struct mv_xor_chan *chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700142{
143 u32 val = 0xFFFF0000 >> (chan->idx * 16);
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200144 writel_relaxed(val, XOR_INTR_CAUSE(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700145}
146
Maxime Ripard0951e722015-05-26 15:07:33 +0200147static void mv_chan_set_mode(struct mv_xor_chan *chan,
Thomas Petazzoni81aafb32015-12-22 11:43:28 +0100148 u32 op_mode)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700149{
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200150 u32 config = readl_relaxed(XOR_CONFIG(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700151
Lior Amsalem6f166312015-05-26 15:07:34 +0200152 config &= ~0x7;
153 config |= op_mode;
154
Thomas Petazzonie03bc652013-07-29 17:42:14 +0200155#if defined(__BIG_ENDIAN)
156 config |= XOR_DESCRIPTOR_SWAP;
157#else
158 config &= ~XOR_DESCRIPTOR_SWAP;
159#endif
160
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200161 writel_relaxed(config, XOR_CONFIG(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700162}
163
164static void mv_chan_activate(struct mv_xor_chan *chan)
165{
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100166 dev_dbg(mv_chan_to_devp(chan), " activate chan.\n");
Ezequiel Garcia5a9a55b2014-05-21 14:02:35 -0700167
168 /* writel ensures all descriptors are flushed before activation */
169 writel(BIT(0), XOR_ACTIVATION(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700170}
171
172static char mv_chan_is_busy(struct mv_xor_chan *chan)
173{
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200174 u32 state = readl_relaxed(XOR_ACTIVATION(chan));
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700175
176 state = (state >> 4) & 0x3;
177
178 return (state == 1) ? 1 : 0;
179}
180
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700181/*
Maxime Ripard0951e722015-05-26 15:07:33 +0200182 * mv_chan_start_new_chain - program the engine to operate on new
183 * chain headed by sw_desc
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700184 * Caller must hold &mv_chan->lock while calling this function
185 */
Maxime Ripard0951e722015-05-26 15:07:33 +0200186static void mv_chan_start_new_chain(struct mv_xor_chan *mv_chan,
187 struct mv_xor_desc_slot *sw_desc)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700188{
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100189 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: sw_desc %p\n",
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700190 __func__, __LINE__, sw_desc);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700191
Bartlomiej Zolnierkiewicz48a9db42013-07-03 15:05:06 -0700192 /* set the hardware chain */
193 mv_chan_set_next_descriptor(mv_chan, sw_desc->async_tx.phys);
194
Lior Amsalemdfc97662014-08-27 10:52:51 -0300195 mv_chan->pending++;
Thomas Petazzoni98817b92012-11-15 14:57:44 +0100196 mv_xor_issue_pending(&mv_chan->dmachan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700197}
198
199static dma_cookie_t
Maxime Ripard0951e722015-05-26 15:07:33 +0200200mv_desc_run_tx_complete_actions(struct mv_xor_desc_slot *desc,
201 struct mv_xor_chan *mv_chan,
202 dma_cookie_t cookie)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700203{
204 BUG_ON(desc->async_tx.cookie < 0);
205
206 if (desc->async_tx.cookie > 0) {
207 cookie = desc->async_tx.cookie;
208
209 /* call the callback (must not sleep or submit new
210 * operations to this channel)
211 */
212 if (desc->async_tx.callback)
213 desc->async_tx.callback(
214 desc->async_tx.callback_param);
215
Dan Williamsd38a8c62013-10-18 19:35:23 +0200216 dma_descriptor_unmap(&desc->async_tx);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700217 }
218
219 /* run dependent operations */
Dan Williams07f22112009-01-05 17:14:31 -0700220 dma_run_dependencies(&desc->async_tx);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700221
222 return cookie;
223}
224
225static int
Maxime Ripard0951e722015-05-26 15:07:33 +0200226mv_chan_clean_completed_slots(struct mv_xor_chan *mv_chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700227{
228 struct mv_xor_desc_slot *iter, *_iter;
229
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100230 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700231 list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200232 node) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700233
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200234 if (async_tx_test_ack(&iter->async_tx))
235 list_move_tail(&iter->node, &mv_chan->free_slots);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700236 }
237 return 0;
238}
239
240static int
Maxime Ripard0951e722015-05-26 15:07:33 +0200241mv_desc_clean_slot(struct mv_xor_desc_slot *desc,
242 struct mv_xor_chan *mv_chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700243{
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100244 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d: desc %p flags %d\n",
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700245 __func__, __LINE__, desc, desc->async_tx.flags);
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200246
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700247 /* the client is allowed to attach dependent operations
248 * until 'ack' is set
249 */
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200250 if (!async_tx_test_ack(&desc->async_tx))
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700251 /* move this slot to the completed_slots */
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200252 list_move_tail(&desc->node, &mv_chan->completed_slots);
253 else
254 list_move_tail(&desc->node, &mv_chan->free_slots);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700255
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700256 return 0;
257}
258
Ezequiel Garciafbeec992014-03-07 16:46:47 -0300259/* This function must be called with the mv_xor_chan spinlock held */
Maxime Ripard0951e722015-05-26 15:07:33 +0200260static void mv_chan_slot_cleanup(struct mv_xor_chan *mv_chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700261{
262 struct mv_xor_desc_slot *iter, *_iter;
263 dma_cookie_t cookie = 0;
264 int busy = mv_chan_is_busy(mv_chan);
265 u32 current_desc = mv_chan_get_current_desc(mv_chan);
Lior Amsalem91362912015-05-26 15:07:32 +0200266 int current_cleaned = 0;
267 struct mv_xor_desc *hw_desc;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700268
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100269 dev_dbg(mv_chan_to_devp(mv_chan), "%s %d\n", __func__, __LINE__);
270 dev_dbg(mv_chan_to_devp(mv_chan), "current_desc %x\n", current_desc);
Maxime Ripard0951e722015-05-26 15:07:33 +0200271 mv_chan_clean_completed_slots(mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700272
273 /* free completed slots from the chain starting with
274 * the oldest descriptor
275 */
276
277 list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200278 node) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700279
Lior Amsalem91362912015-05-26 15:07:32 +0200280 /* clean finished descriptors */
281 hw_desc = iter->hw_desc;
282 if (hw_desc->status & XOR_DESC_SUCCESS) {
Maxime Ripard0951e722015-05-26 15:07:33 +0200283 cookie = mv_desc_run_tx_complete_actions(iter, mv_chan,
284 cookie);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700285
Lior Amsalem91362912015-05-26 15:07:32 +0200286 /* done processing desc, clean slot */
Maxime Ripard0951e722015-05-26 15:07:33 +0200287 mv_desc_clean_slot(iter, mv_chan);
Lior Amsalem91362912015-05-26 15:07:32 +0200288
289 /* break if we did cleaned the current */
290 if (iter->async_tx.phys == current_desc) {
291 current_cleaned = 1;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700292 break;
Lior Amsalem91362912015-05-26 15:07:32 +0200293 }
294 } else {
295 if (iter->async_tx.phys == current_desc) {
296 current_cleaned = 0;
297 break;
298 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700299 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700300 }
301
302 if ((busy == 0) && !list_empty(&mv_chan->chain)) {
Lior Amsalem91362912015-05-26 15:07:32 +0200303 if (current_cleaned) {
304 /*
305 * current descriptor cleaned and removed, run
306 * from list head
307 */
308 iter = list_entry(mv_chan->chain.next,
309 struct mv_xor_desc_slot,
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200310 node);
Maxime Ripard0951e722015-05-26 15:07:33 +0200311 mv_chan_start_new_chain(mv_chan, iter);
Lior Amsalem91362912015-05-26 15:07:32 +0200312 } else {
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200313 if (!list_is_last(&iter->node, &mv_chan->chain)) {
Lior Amsalem91362912015-05-26 15:07:32 +0200314 /*
315 * descriptors are still waiting after
316 * current, trigger them
317 */
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200318 iter = list_entry(iter->node.next,
Lior Amsalem91362912015-05-26 15:07:32 +0200319 struct mv_xor_desc_slot,
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200320 node);
Maxime Ripard0951e722015-05-26 15:07:33 +0200321 mv_chan_start_new_chain(mv_chan, iter);
Lior Amsalem91362912015-05-26 15:07:32 +0200322 } else {
323 /*
324 * some descriptors are still waiting
325 * to be cleaned
326 */
327 tasklet_schedule(&mv_chan->irq_tasklet);
328 }
329 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700330 }
331
332 if (cookie > 0)
Thomas Petazzoni98817b92012-11-15 14:57:44 +0100333 mv_chan->dmachan.completed_cookie = cookie;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700334}
335
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700336static void mv_xor_tasklet(unsigned long data)
337{
338 struct mv_xor_chan *chan = (struct mv_xor_chan *) data;
Ezequiel Garciae43147a2014-03-07 16:46:46 -0300339
340 spin_lock_bh(&chan->lock);
Maxime Ripard0951e722015-05-26 15:07:33 +0200341 mv_chan_slot_cleanup(chan);
Ezequiel Garciae43147a2014-03-07 16:46:46 -0300342 spin_unlock_bh(&chan->lock);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700343}
344
345static struct mv_xor_desc_slot *
Maxime Ripard0951e722015-05-26 15:07:33 +0200346mv_chan_alloc_slot(struct mv_xor_chan *mv_chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700347{
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200348 struct mv_xor_desc_slot *iter;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700349
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200350 spin_lock_bh(&mv_chan->lock);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700351
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200352 if (!list_empty(&mv_chan->free_slots)) {
353 iter = list_first_entry(&mv_chan->free_slots,
354 struct mv_xor_desc_slot,
355 node);
Lior Amsalemdfc97662014-08-27 10:52:51 -0300356
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200357 list_move_tail(&iter->node, &mv_chan->allocated_slots);
358
359 spin_unlock_bh(&mv_chan->lock);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700360
Lior Amsalemdfc97662014-08-27 10:52:51 -0300361 /* pre-ack descriptor */
362 async_tx_ack(&iter->async_tx);
Lior Amsalemdfc97662014-08-27 10:52:51 -0300363 iter->async_tx.cookie = -EBUSY;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700364
Lior Amsalemdfc97662014-08-27 10:52:51 -0300365 return iter;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700366
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700367 }
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200368
369 spin_unlock_bh(&mv_chan->lock);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700370
371 /* try to free some slots if the allocation fails */
372 tasklet_schedule(&mv_chan->irq_tasklet);
373
374 return NULL;
375}
376
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700377/************************ DMA engine API functions ****************************/
378static dma_cookie_t
379mv_xor_tx_submit(struct dma_async_tx_descriptor *tx)
380{
381 struct mv_xor_desc_slot *sw_desc = to_mv_xor_slot(tx);
382 struct mv_xor_chan *mv_chan = to_mv_xor_chan(tx->chan);
Lior Amsalemdfc97662014-08-27 10:52:51 -0300383 struct mv_xor_desc_slot *old_chain_tail;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700384 dma_cookie_t cookie;
385 int new_hw_chain = 1;
386
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100387 dev_dbg(mv_chan_to_devp(mv_chan),
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700388 "%s sw_desc %p: async_tx %p\n",
389 __func__, sw_desc, &sw_desc->async_tx);
390
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700391 spin_lock_bh(&mv_chan->lock);
Russell King - ARM Linux884485e2012-03-06 22:34:46 +0000392 cookie = dma_cookie_assign(tx);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700393
394 if (list_empty(&mv_chan->chain))
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200395 list_move_tail(&sw_desc->node, &mv_chan->chain);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700396 else {
397 new_hw_chain = 0;
398
399 old_chain_tail = list_entry(mv_chan->chain.prev,
400 struct mv_xor_desc_slot,
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200401 node);
402 list_move_tail(&sw_desc->node, &mv_chan->chain);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700403
Olof Johansson31fd8f52014-02-03 17:13:23 -0800404 dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %pa\n",
405 &old_chain_tail->async_tx.phys);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700406
407 /* fix up the hardware chain */
Lior Amsalemdfc97662014-08-27 10:52:51 -0300408 mv_desc_set_next_desc(old_chain_tail, sw_desc->async_tx.phys);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700409
410 /* if the channel is not busy */
411 if (!mv_chan_is_busy(mv_chan)) {
412 u32 current_desc = mv_chan_get_current_desc(mv_chan);
413 /*
414 * and the curren desc is the end of the chain before
415 * the append, then we need to start the channel
416 */
417 if (current_desc == old_chain_tail->async_tx.phys)
418 new_hw_chain = 1;
419 }
420 }
421
422 if (new_hw_chain)
Maxime Ripard0951e722015-05-26 15:07:33 +0200423 mv_chan_start_new_chain(mv_chan, sw_desc);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700424
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700425 spin_unlock_bh(&mv_chan->lock);
426
427 return cookie;
428}
429
430/* returns the number of allocated descriptors */
Dan Williamsaa1e6f12009-01-06 11:38:17 -0700431static int mv_xor_alloc_chan_resources(struct dma_chan *chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700432{
Olof Johansson31fd8f52014-02-03 17:13:23 -0800433 void *virt_desc;
434 dma_addr_t dma_desc;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700435 int idx;
436 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
437 struct mv_xor_desc_slot *slot = NULL;
Thomas Petazzonib503fa02012-11-15 15:55:30 +0100438 int num_descs_in_pool = MV_XOR_POOL_SIZE/MV_XOR_SLOT_SIZE;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700439
440 /* Allocate descriptor slots */
441 idx = mv_chan->slots_allocated;
442 while (idx < num_descs_in_pool) {
443 slot = kzalloc(sizeof(*slot), GFP_KERNEL);
444 if (!slot) {
Ezequiel Garciab8291dd2014-08-27 10:52:49 -0300445 dev_info(mv_chan_to_devp(mv_chan),
446 "channel only initialized %d descriptor slots",
447 idx);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700448 break;
449 }
Olof Johansson31fd8f52014-02-03 17:13:23 -0800450 virt_desc = mv_chan->dma_desc_pool_virt;
451 slot->hw_desc = virt_desc + idx * MV_XOR_SLOT_SIZE;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700452
453 dma_async_tx_descriptor_init(&slot->async_tx, chan);
454 slot->async_tx.tx_submit = mv_xor_tx_submit;
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200455 INIT_LIST_HEAD(&slot->node);
Olof Johansson31fd8f52014-02-03 17:13:23 -0800456 dma_desc = mv_chan->dma_desc_pool;
457 slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700458 slot->idx = idx++;
459
460 spin_lock_bh(&mv_chan->lock);
461 mv_chan->slots_allocated = idx;
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200462 list_add_tail(&slot->node, &mv_chan->free_slots);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700463 spin_unlock_bh(&mv_chan->lock);
464 }
465
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100466 dev_dbg(mv_chan_to_devp(mv_chan),
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200467 "allocated %d descriptor slots\n",
468 mv_chan->slots_allocated);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700469
470 return mv_chan->slots_allocated ? : -ENOMEM;
471}
472
473static struct dma_async_tx_descriptor *
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700474mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
475 unsigned int src_cnt, size_t len, unsigned long flags)
476{
477 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
Lior Amsalemdfc97662014-08-27 10:52:51 -0300478 struct mv_xor_desc_slot *sw_desc;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700479
480 if (unlikely(len < MV_XOR_MIN_BYTE_COUNT))
481 return NULL;
482
Coly Li7912d302011-03-27 01:26:53 +0800483 BUG_ON(len > MV_XOR_MAX_BYTE_COUNT);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700484
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100485 dev_dbg(mv_chan_to_devp(mv_chan),
Gregory CLEMENTbc822e12016-04-29 09:49:05 +0200486 "%s src_cnt: %d len: %zu dest %pad flags: %ld\n",
Olof Johansson31fd8f52014-02-03 17:13:23 -0800487 __func__, src_cnt, len, &dest, flags);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700488
Maxime Ripard0951e722015-05-26 15:07:33 +0200489 sw_desc = mv_chan_alloc_slot(mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700490 if (sw_desc) {
491 sw_desc->type = DMA_XOR;
492 sw_desc->async_tx.flags = flags;
Lior Amsalemba87d132014-08-27 10:52:53 -0300493 mv_desc_init(sw_desc, dest, len, flags);
Lior Amsalem6f166312015-05-26 15:07:34 +0200494 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
495 mv_desc_set_mode(sw_desc);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700496 while (src_cnt--)
Lior Amsalemdfc97662014-08-27 10:52:51 -0300497 mv_desc_set_src_addr(sw_desc, src_cnt, src[src_cnt]);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700498 }
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200499
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100500 dev_dbg(mv_chan_to_devp(mv_chan),
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700501 "%s sw_desc %p async_tx %p \n",
502 __func__, sw_desc, &sw_desc->async_tx);
503 return sw_desc ? &sw_desc->async_tx : NULL;
504}
505
Lior Amsalem3e4f52e2014-08-27 10:52:50 -0300506static struct dma_async_tx_descriptor *
507mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
508 size_t len, unsigned long flags)
509{
510 /*
511 * A MEMCPY operation is identical to an XOR operation with only
512 * a single source address.
513 */
514 return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
515}
516
Lior Amsalem22843542014-08-27 10:52:55 -0300517static struct dma_async_tx_descriptor *
518mv_xor_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags)
519{
520 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
521 dma_addr_t src, dest;
522 size_t len;
523
524 src = mv_chan->dummy_src_addr;
525 dest = mv_chan->dummy_dst_addr;
526 len = MV_XOR_MIN_BYTE_COUNT;
527
528 /*
529 * We implement the DMA_INTERRUPT operation as a minimum sized
530 * XOR operation with a single dummy source address.
531 */
532 return mv_xor_prep_dma_xor(chan, dest, &src, 1, len, flags);
533}
534
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700535static void mv_xor_free_chan_resources(struct dma_chan *chan)
536{
537 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
538 struct mv_xor_desc_slot *iter, *_iter;
539 int in_use_descs = 0;
540
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700541 spin_lock_bh(&mv_chan->lock);
Ezequiel Garciae43147a2014-03-07 16:46:46 -0300542
Maxime Ripard0951e722015-05-26 15:07:33 +0200543 mv_chan_slot_cleanup(mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700544
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700545 list_for_each_entry_safe(iter, _iter, &mv_chan->chain,
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200546 node) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700547 in_use_descs++;
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200548 list_move_tail(&iter->node, &mv_chan->free_slots);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700549 }
550 list_for_each_entry_safe(iter, _iter, &mv_chan->completed_slots,
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200551 node) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700552 in_use_descs++;
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200553 list_move_tail(&iter->node, &mv_chan->free_slots);
554 }
555 list_for_each_entry_safe(iter, _iter, &mv_chan->allocated_slots,
556 node) {
557 in_use_descs++;
558 list_move_tail(&iter->node, &mv_chan->free_slots);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700559 }
560 list_for_each_entry_safe_reverse(
Lior Amsalemfbea28a2015-05-26 15:07:36 +0200561 iter, _iter, &mv_chan->free_slots, node) {
562 list_del(&iter->node);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700563 kfree(iter);
564 mv_chan->slots_allocated--;
565 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700566
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100567 dev_dbg(mv_chan_to_devp(mv_chan), "%s slots_allocated %d\n",
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700568 __func__, mv_chan->slots_allocated);
569 spin_unlock_bh(&mv_chan->lock);
570
571 if (in_use_descs)
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100572 dev_err(mv_chan_to_devp(mv_chan),
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700573 "freeing %d in use descriptors!\n", in_use_descs);
574}
575
576/**
Linus Walleij07934482010-03-26 16:50:49 -0700577 * mv_xor_status - poll the status of an XOR transaction
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700578 * @chan: XOR channel handle
579 * @cookie: XOR transaction identifier
Linus Walleij07934482010-03-26 16:50:49 -0700580 * @txstate: XOR transactions state holder (or NULL)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700581 */
Linus Walleij07934482010-03-26 16:50:49 -0700582static enum dma_status mv_xor_status(struct dma_chan *chan,
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700583 dma_cookie_t cookie,
Linus Walleij07934482010-03-26 16:50:49 -0700584 struct dma_tx_state *txstate)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700585{
586 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700587 enum dma_status ret;
588
Russell King - ARM Linux96a2af42012-03-06 22:35:27 +0000589 ret = dma_cookie_status(chan, cookie, txstate);
Ezequiel Garcia890766d2014-03-07 16:46:45 -0300590 if (ret == DMA_COMPLETE)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700591 return ret;
Ezequiel Garciae43147a2014-03-07 16:46:46 -0300592
593 spin_lock_bh(&mv_chan->lock);
Maxime Ripard0951e722015-05-26 15:07:33 +0200594 mv_chan_slot_cleanup(mv_chan);
Ezequiel Garciae43147a2014-03-07 16:46:46 -0300595 spin_unlock_bh(&mv_chan->lock);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700596
Russell King - ARM Linux96a2af42012-03-06 22:35:27 +0000597 return dma_cookie_status(chan, cookie, txstate);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700598}
599
Maxime Ripard0951e722015-05-26 15:07:33 +0200600static void mv_chan_dump_regs(struct mv_xor_chan *chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700601{
602 u32 val;
603
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200604 val = readl_relaxed(XOR_CONFIG(chan));
Joe Perches1ba151c2012-10-28 01:05:44 -0700605 dev_err(mv_chan_to_devp(chan), "config 0x%08x\n", val);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700606
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200607 val = readl_relaxed(XOR_ACTIVATION(chan));
Joe Perches1ba151c2012-10-28 01:05:44 -0700608 dev_err(mv_chan_to_devp(chan), "activation 0x%08x\n", val);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700609
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200610 val = readl_relaxed(XOR_INTR_CAUSE(chan));
Joe Perches1ba151c2012-10-28 01:05:44 -0700611 dev_err(mv_chan_to_devp(chan), "intr cause 0x%08x\n", val);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700612
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200613 val = readl_relaxed(XOR_INTR_MASK(chan));
Joe Perches1ba151c2012-10-28 01:05:44 -0700614 dev_err(mv_chan_to_devp(chan), "intr mask 0x%08x\n", val);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700615
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200616 val = readl_relaxed(XOR_ERROR_CAUSE(chan));
Joe Perches1ba151c2012-10-28 01:05:44 -0700617 dev_err(mv_chan_to_devp(chan), "error cause 0x%08x\n", val);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700618
Thomas Petazzoni5733c382013-07-29 17:42:13 +0200619 val = readl_relaxed(XOR_ERROR_ADDR(chan));
Joe Perches1ba151c2012-10-28 01:05:44 -0700620 dev_err(mv_chan_to_devp(chan), "error addr 0x%08x\n", val);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700621}
622
Maxime Ripard0951e722015-05-26 15:07:33 +0200623static void mv_chan_err_interrupt_handler(struct mv_xor_chan *chan,
624 u32 intr_cause)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700625{
Ezequiel Garcia0e7488e2014-08-27 10:52:52 -0300626 if (intr_cause & XOR_INT_ERR_DECODE) {
627 dev_dbg(mv_chan_to_devp(chan), "ignoring address decode error\n");
628 return;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700629 }
630
Ezequiel Garcia0e7488e2014-08-27 10:52:52 -0300631 dev_err(mv_chan_to_devp(chan), "error on chan %d. intr cause 0x%08x\n",
Thomas Petazzonia3fc74b2012-11-15 12:50:27 +0100632 chan->idx, intr_cause);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700633
Maxime Ripard0951e722015-05-26 15:07:33 +0200634 mv_chan_dump_regs(chan);
Ezequiel Garcia0e7488e2014-08-27 10:52:52 -0300635 WARN_ON(1);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700636}
637
638static irqreturn_t mv_xor_interrupt_handler(int irq, void *data)
639{
640 struct mv_xor_chan *chan = data;
641 u32 intr_cause = mv_chan_get_intr_cause(chan);
642
Thomas Petazzonic98c1782012-11-15 14:17:18 +0100643 dev_dbg(mv_chan_to_devp(chan), "intr cause %x\n", intr_cause);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700644
Ezequiel Garcia0e7488e2014-08-27 10:52:52 -0300645 if (intr_cause & XOR_INTR_ERRORS)
Maxime Ripard0951e722015-05-26 15:07:33 +0200646 mv_chan_err_interrupt_handler(chan, intr_cause);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700647
648 tasklet_schedule(&chan->irq_tasklet);
649
Maxime Ripard0951e722015-05-26 15:07:33 +0200650 mv_chan_clear_eoc_cause(chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700651
652 return IRQ_HANDLED;
653}
654
655static void mv_xor_issue_pending(struct dma_chan *chan)
656{
657 struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan);
658
659 if (mv_chan->pending >= MV_XOR_THRESHOLD) {
660 mv_chan->pending = 0;
661 mv_chan_activate(mv_chan);
662 }
663}
664
665/*
666 * Perform a transaction to verify the HW works.
667 */
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700668
Maxime Ripard0951e722015-05-26 15:07:33 +0200669static int mv_chan_memcpy_self_test(struct mv_xor_chan *mv_chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700670{
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300671 int i, ret;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700672 void *src, *dest;
673 dma_addr_t src_dma, dest_dma;
674 struct dma_chan *dma_chan;
675 dma_cookie_t cookie;
676 struct dma_async_tx_descriptor *tx;
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300677 struct dmaengine_unmap_data *unmap;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700678 int err = 0;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700679
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300680 src = kmalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700681 if (!src)
682 return -ENOMEM;
683
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300684 dest = kzalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700685 if (!dest) {
686 kfree(src);
687 return -ENOMEM;
688 }
689
690 /* Fill in src buffer */
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300691 for (i = 0; i < PAGE_SIZE; i++)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700692 ((u8 *) src)[i] = (u8)i;
693
Thomas Petazzoni275cc0c2012-11-15 15:09:42 +0100694 dma_chan = &mv_chan->dmachan;
Dan Williamsaa1e6f12009-01-06 11:38:17 -0700695 if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700696 err = -ENODEV;
697 goto out;
698 }
699
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300700 unmap = dmaengine_get_unmap_data(dma_chan->device->dev, 2, GFP_KERNEL);
701 if (!unmap) {
702 err = -ENOMEM;
703 goto free_resources;
704 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700705
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300706 src_dma = dma_map_page(dma_chan->device->dev, virt_to_page(src), 0,
707 PAGE_SIZE, DMA_TO_DEVICE);
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300708 unmap->addr[0] = src_dma;
709
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300710 ret = dma_mapping_error(dma_chan->device->dev, src_dma);
711 if (ret) {
712 err = -ENOMEM;
713 goto free_resources;
714 }
715 unmap->to_cnt = 1;
716
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300717 dest_dma = dma_map_page(dma_chan->device->dev, virt_to_page(dest), 0,
718 PAGE_SIZE, DMA_FROM_DEVICE);
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300719 unmap->addr[1] = dest_dma;
720
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300721 ret = dma_mapping_error(dma_chan->device->dev, dest_dma);
722 if (ret) {
723 err = -ENOMEM;
724 goto free_resources;
725 }
726 unmap->from_cnt = 1;
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300727 unmap->len = PAGE_SIZE;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700728
729 tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma,
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300730 PAGE_SIZE, 0);
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300731 if (!tx) {
732 dev_err(dma_chan->device->dev,
733 "Self-test cannot prepare operation, disabling\n");
734 err = -ENODEV;
735 goto free_resources;
736 }
737
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700738 cookie = mv_xor_tx_submit(tx);
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300739 if (dma_submit_error(cookie)) {
740 dev_err(dma_chan->device->dev,
741 "Self-test submit error, disabling\n");
742 err = -ENODEV;
743 goto free_resources;
744 }
745
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700746 mv_xor_issue_pending(dma_chan);
747 async_tx_ack(tx);
748 msleep(1);
749
Linus Walleij07934482010-03-26 16:50:49 -0700750 if (mv_xor_status(dma_chan, cookie, NULL) !=
Vinod Koulb3efb8f2013-10-16 20:51:04 +0530751 DMA_COMPLETE) {
Thomas Petazzonia3fc74b2012-11-15 12:50:27 +0100752 dev_err(dma_chan->device->dev,
753 "Self-test copy timed out, disabling\n");
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700754 err = -ENODEV;
755 goto free_resources;
756 }
757
Thomas Petazzonic35064c2012-11-15 13:01:59 +0100758 dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300759 PAGE_SIZE, DMA_FROM_DEVICE);
760 if (memcmp(src, dest, PAGE_SIZE)) {
Thomas Petazzonia3fc74b2012-11-15 12:50:27 +0100761 dev_err(dma_chan->device->dev,
762 "Self-test copy failed compare, disabling\n");
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700763 err = -ENODEV;
764 goto free_resources;
765 }
766
767free_resources:
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300768 dmaengine_unmap_put(unmap);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700769 mv_xor_free_chan_resources(dma_chan);
770out:
771 kfree(src);
772 kfree(dest);
773 return err;
774}
775
776#define MV_XOR_NUM_SRC_TEST 4 /* must be <= 15 */
Bill Pemberton463a1f82012-11-19 13:22:55 -0500777static int
Maxime Ripard0951e722015-05-26 15:07:33 +0200778mv_chan_xor_self_test(struct mv_xor_chan *mv_chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700779{
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300780 int i, src_idx, ret;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700781 struct page *dest;
782 struct page *xor_srcs[MV_XOR_NUM_SRC_TEST];
783 dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST];
784 dma_addr_t dest_dma;
785 struct dma_async_tx_descriptor *tx;
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300786 struct dmaengine_unmap_data *unmap;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700787 struct dma_chan *dma_chan;
788 dma_cookie_t cookie;
789 u8 cmp_byte = 0;
790 u32 cmp_word;
791 int err = 0;
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300792 int src_count = MV_XOR_NUM_SRC_TEST;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700793
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300794 for (src_idx = 0; src_idx < src_count; src_idx++) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700795 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
Roel Kluina09b09a2009-02-25 13:56:21 +0100796 if (!xor_srcs[src_idx]) {
797 while (src_idx--)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700798 __free_page(xor_srcs[src_idx]);
Roel Kluina09b09a2009-02-25 13:56:21 +0100799 return -ENOMEM;
800 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700801 }
802
803 dest = alloc_page(GFP_KERNEL);
Roel Kluina09b09a2009-02-25 13:56:21 +0100804 if (!dest) {
805 while (src_idx--)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700806 __free_page(xor_srcs[src_idx]);
Roel Kluina09b09a2009-02-25 13:56:21 +0100807 return -ENOMEM;
808 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700809
810 /* Fill in src buffers */
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300811 for (src_idx = 0; src_idx < src_count; src_idx++) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700812 u8 *ptr = page_address(xor_srcs[src_idx]);
813 for (i = 0; i < PAGE_SIZE; i++)
814 ptr[i] = (1 << src_idx);
815 }
816
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300817 for (src_idx = 0; src_idx < src_count; src_idx++)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700818 cmp_byte ^= (u8) (1 << src_idx);
819
820 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
821 (cmp_byte << 8) | cmp_byte;
822
823 memset(page_address(dest), 0, PAGE_SIZE);
824
Thomas Petazzoni275cc0c2012-11-15 15:09:42 +0100825 dma_chan = &mv_chan->dmachan;
Dan Williamsaa1e6f12009-01-06 11:38:17 -0700826 if (mv_xor_alloc_chan_resources(dma_chan) < 1) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700827 err = -ENODEV;
828 goto out;
829 }
830
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300831 unmap = dmaengine_get_unmap_data(dma_chan->device->dev, src_count + 1,
832 GFP_KERNEL);
833 if (!unmap) {
834 err = -ENOMEM;
835 goto free_resources;
836 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700837
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300838 /* test xor */
839 for (i = 0; i < src_count; i++) {
840 unmap->addr[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i],
841 0, PAGE_SIZE, DMA_TO_DEVICE);
842 dma_srcs[i] = unmap->addr[i];
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300843 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[i]);
844 if (ret) {
845 err = -ENOMEM;
846 goto free_resources;
847 }
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300848 unmap->to_cnt++;
849 }
850
851 unmap->addr[src_count] = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE,
852 DMA_FROM_DEVICE);
853 dest_dma = unmap->addr[src_count];
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300854 ret = dma_mapping_error(dma_chan->device->dev, unmap->addr[src_count]);
855 if (ret) {
856 err = -ENOMEM;
857 goto free_resources;
858 }
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300859 unmap->from_cnt = 1;
860 unmap->len = PAGE_SIZE;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700861
862 tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300863 src_count, PAGE_SIZE, 0);
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300864 if (!tx) {
865 dev_err(dma_chan->device->dev,
866 "Self-test cannot prepare operation, disabling\n");
867 err = -ENODEV;
868 goto free_resources;
869 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700870
871 cookie = mv_xor_tx_submit(tx);
Ezequiel Garciab8c01d22013-12-10 09:32:37 -0300872 if (dma_submit_error(cookie)) {
873 dev_err(dma_chan->device->dev,
874 "Self-test submit error, disabling\n");
875 err = -ENODEV;
876 goto free_resources;
877 }
878
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700879 mv_xor_issue_pending(dma_chan);
880 async_tx_ack(tx);
881 msleep(8);
882
Linus Walleij07934482010-03-26 16:50:49 -0700883 if (mv_xor_status(dma_chan, cookie, NULL) !=
Vinod Koulb3efb8f2013-10-16 20:51:04 +0530884 DMA_COMPLETE) {
Thomas Petazzonia3fc74b2012-11-15 12:50:27 +0100885 dev_err(dma_chan->device->dev,
886 "Self-test xor timed out, disabling\n");
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700887 err = -ENODEV;
888 goto free_resources;
889 }
890
Thomas Petazzonic35064c2012-11-15 13:01:59 +0100891 dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma,
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700892 PAGE_SIZE, DMA_FROM_DEVICE);
893 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
894 u32 *ptr = page_address(dest);
895 if (ptr[i] != cmp_word) {
Thomas Petazzonia3fc74b2012-11-15 12:50:27 +0100896 dev_err(dma_chan->device->dev,
Joe Perches1ba151c2012-10-28 01:05:44 -0700897 "Self-test xor failed compare, disabling. index %d, data %x, expected %x\n",
898 i, ptr[i], cmp_word);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700899 err = -ENODEV;
900 goto free_resources;
901 }
902 }
903
904free_resources:
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300905 dmaengine_unmap_put(unmap);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700906 mv_xor_free_chan_resources(dma_chan);
907out:
Ezequiel Garciad16695a2013-12-10 09:32:36 -0300908 src_idx = src_count;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700909 while (src_idx--)
910 __free_page(xor_srcs[src_idx]);
911 __free_page(dest);
912 return err;
913}
914
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100915static int mv_xor_channel_remove(struct mv_xor_chan *mv_chan)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700916{
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700917 struct dma_chan *chan, *_chan;
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100918 struct device *dev = mv_chan->dmadev.dev;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700919
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100920 dma_async_device_unregister(&mv_chan->dmadev);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700921
Thomas Petazzonib503fa02012-11-15 15:55:30 +0100922 dma_free_coherent(dev, MV_XOR_POOL_SIZE,
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100923 mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
Lior Amsalem22843542014-08-27 10:52:55 -0300924 dma_unmap_single(dev, mv_chan->dummy_src_addr,
925 MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
926 dma_unmap_single(dev, mv_chan->dummy_dst_addr,
927 MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700928
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100929 list_for_each_entry_safe(chan, _chan, &mv_chan->dmadev.channels,
Thomas Petazzonia6b4a9d2012-10-29 16:45:46 +0100930 device_node) {
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700931 list_del(&chan->device_node);
932 }
933
Thomas Petazzoni88eb92c2012-11-15 16:11:18 +0100934 free_irq(mv_chan->irq, mv_chan);
935
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700936 return 0;
937}
938
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100939static struct mv_xor_chan *
Thomas Petazzoni297eedb2012-11-15 15:29:53 +0100940mv_xor_channel_add(struct mv_xor_device *xordev,
Thomas Petazzonia6b4a9d2012-10-29 16:45:46 +0100941 struct platform_device *pdev,
Gregory CLEMENTdd130c62016-04-29 09:49:06 +0200942 int idx, dma_cap_mask_t cap_mask, int irq)
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700943{
944 int ret = 0;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700945 struct mv_xor_chan *mv_chan;
946 struct dma_device *dma_dev;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700947
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100948 mv_chan = devm_kzalloc(&pdev->dev, sizeof(*mv_chan), GFP_KERNEL);
Sachin Kamata5776592013-09-02 13:54:20 +0530949 if (!mv_chan)
950 return ERR_PTR(-ENOMEM);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700951
Thomas Petazzoni9aedbdb2012-11-15 15:36:37 +0100952 mv_chan->idx = idx;
Thomas Petazzoni88eb92c2012-11-15 16:11:18 +0100953 mv_chan->irq = irq;
Gregory CLEMENTdd130c62016-04-29 09:49:06 +0200954 if (xordev->xor_type == XOR_ORION)
955 mv_chan->op_in_desc = XOR_MODE_IN_REG;
956 else
957 mv_chan->op_in_desc = XOR_MODE_IN_DESC;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700958
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100959 dma_dev = &mv_chan->dmadev;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700960
Lior Amsalem22843542014-08-27 10:52:55 -0300961 /*
962 * These source and destination dummy buffers are used to implement
963 * a DMA_INTERRUPT operation as a minimum-sized XOR operation.
964 * Hence, we only need to map the buffers at initialization-time.
965 */
966 mv_chan->dummy_src_addr = dma_map_single(dma_dev->dev,
967 mv_chan->dummy_src, MV_XOR_MIN_BYTE_COUNT, DMA_FROM_DEVICE);
968 mv_chan->dummy_dst_addr = dma_map_single(dma_dev->dev,
969 mv_chan->dummy_dst, MV_XOR_MIN_BYTE_COUNT, DMA_TO_DEVICE);
970
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700971 /* allocate coherent memory for hardware descriptors
972 * note: writecombine gives slightly better performance, but
973 * requires that we explicitly flush the writes
974 */
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100975 mv_chan->dma_desc_pool_virt =
Luis R. Rodriguezf6e45662016-01-22 18:34:22 -0800976 dma_alloc_wc(&pdev->dev, MV_XOR_POOL_SIZE, &mv_chan->dma_desc_pool,
977 GFP_KERNEL);
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +0100978 if (!mv_chan->dma_desc_pool_virt)
Thomas Petazzonia6b4a9d2012-10-29 16:45:46 +0100979 return ERR_PTR(-ENOMEM);
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700980
981 /* discover transaction capabilites from the platform data */
Thomas Petazzonia6b4a9d2012-10-29 16:45:46 +0100982 dma_dev->cap_mask = cap_mask;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700983
984 INIT_LIST_HEAD(&dma_dev->channels);
985
986 /* set base routines */
987 dma_dev->device_alloc_chan_resources = mv_xor_alloc_chan_resources;
988 dma_dev->device_free_chan_resources = mv_xor_free_chan_resources;
Linus Walleij07934482010-03-26 16:50:49 -0700989 dma_dev->device_tx_status = mv_xor_status;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700990 dma_dev->device_issue_pending = mv_xor_issue_pending;
991 dma_dev->dev = &pdev->dev;
992
993 /* set prep routines based on capability */
Lior Amsalem22843542014-08-27 10:52:55 -0300994 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
995 dma_dev->device_prep_dma_interrupt = mv_xor_prep_dma_interrupt;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700996 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
997 dma_dev->device_prep_dma_memcpy = mv_xor_prep_dma_memcpy;
Saeed Bisharaff7b0472008-07-08 11:58:36 -0700998 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
Joe Perchesc0198942009-06-28 09:26:21 -0700999 dma_dev->max_xor = 8;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001000 dma_dev->device_prep_dma_xor = mv_xor_prep_dma_xor;
1001 }
1002
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001003 mv_chan->mmr_base = xordev->xor_base;
Ezequiel Garcia82a14022013-10-30 12:01:43 -03001004 mv_chan->mmr_high_base = xordev->xor_high_base;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001005 tasklet_init(&mv_chan->irq_tasklet, mv_xor_tasklet, (unsigned long)
1006 mv_chan);
1007
1008 /* clear errors before enabling interrupts */
Maxime Ripard0951e722015-05-26 15:07:33 +02001009 mv_chan_clear_err_status(mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001010
Thomas Petazzoni2d0a0742012-11-22 18:19:09 +01001011 ret = request_irq(mv_chan->irq, mv_xor_interrupt_handler,
1012 0, dev_name(&pdev->dev), mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001013 if (ret)
1014 goto err_free_dma;
1015
1016 mv_chan_unmask_interrupts(mv_chan);
1017
Lior Amsalem6f166312015-05-26 15:07:34 +02001018 if (mv_chan->op_in_desc == XOR_MODE_IN_DESC)
Thomas Petazzoni81aafb32015-12-22 11:43:28 +01001019 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_IN_DESC);
Lior Amsalem6f166312015-05-26 15:07:34 +02001020 else
Thomas Petazzoni81aafb32015-12-22 11:43:28 +01001021 mv_chan_set_mode(mv_chan, XOR_OPERATION_MODE_XOR);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001022
1023 spin_lock_init(&mv_chan->lock);
1024 INIT_LIST_HEAD(&mv_chan->chain);
1025 INIT_LIST_HEAD(&mv_chan->completed_slots);
Lior Amsalemfbea28a2015-05-26 15:07:36 +02001026 INIT_LIST_HEAD(&mv_chan->free_slots);
1027 INIT_LIST_HEAD(&mv_chan->allocated_slots);
Thomas Petazzoni98817b92012-11-15 14:57:44 +01001028 mv_chan->dmachan.device = dma_dev;
1029 dma_cookie_init(&mv_chan->dmachan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001030
Thomas Petazzoni98817b92012-11-15 14:57:44 +01001031 list_add_tail(&mv_chan->dmachan.device_node, &dma_dev->channels);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001032
1033 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
Maxime Ripard0951e722015-05-26 15:07:33 +02001034 ret = mv_chan_memcpy_self_test(mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001035 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1036 if (ret)
Thomas Petazzoni2d0a0742012-11-22 18:19:09 +01001037 goto err_free_irq;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001038 }
1039
1040 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
Maxime Ripard0951e722015-05-26 15:07:33 +02001041 ret = mv_chan_xor_self_test(mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001042 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1043 if (ret)
Thomas Petazzoni2d0a0742012-11-22 18:19:09 +01001044 goto err_free_irq;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001045 }
1046
Lior Amsalem6f166312015-05-26 15:07:34 +02001047 dev_info(&pdev->dev, "Marvell XOR (%s): ( %s%s%s)\n",
1048 mv_chan->op_in_desc ? "Descriptor Mode" : "Registers Mode",
Joe Perches1ba151c2012-10-28 01:05:44 -07001049 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
Joe Perches1ba151c2012-10-28 01:05:44 -07001050 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1051 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001052
1053 dma_async_device_register(dma_dev);
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +01001054 return mv_chan;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001055
Thomas Petazzoni2d0a0742012-11-22 18:19:09 +01001056err_free_irq:
1057 free_irq(mv_chan->irq, mv_chan);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001058 err_free_dma:
Thomas Petazzonib503fa02012-11-15 15:55:30 +01001059 dma_free_coherent(&pdev->dev, MV_XOR_POOL_SIZE,
Thomas Petazzoni1ef48a22012-11-15 15:17:05 +01001060 mv_chan->dma_desc_pool_virt, mv_chan->dma_desc_pool);
Thomas Petazzonia6b4a9d2012-10-29 16:45:46 +01001061 return ERR_PTR(ret);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001062}
1063
1064static void
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001065mv_xor_conf_mbus_windows(struct mv_xor_device *xordev,
Andrew Lunn63a93322011-12-07 21:48:07 +01001066 const struct mbus_dram_target_info *dram)
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001067{
Ezequiel Garcia82a14022013-10-30 12:01:43 -03001068 void __iomem *base = xordev->xor_high_base;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001069 u32 win_enable = 0;
1070 int i;
1071
1072 for (i = 0; i < 8; i++) {
1073 writel(0, base + WINDOW_BASE(i));
1074 writel(0, base + WINDOW_SIZE(i));
1075 if (i < 4)
1076 writel(0, base + WINDOW_REMAP_HIGH(i));
1077 }
1078
1079 for (i = 0; i < dram->num_cs; i++) {
Andrew Lunn63a93322011-12-07 21:48:07 +01001080 const struct mbus_dram_window *cs = dram->cs + i;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001081
1082 writel((cs->base & 0xffff0000) |
1083 (cs->mbus_attr << 8) |
1084 dram->mbus_dram_target_id, base + WINDOW_BASE(i));
1085 writel((cs->size - 1) & 0xffff0000, base + WINDOW_SIZE(i));
1086
1087 win_enable |= (1 << i);
1088 win_enable |= 3 << (16 + (2 * i));
1089 }
1090
1091 writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1092 writel(win_enable, base + WINDOW_BAR_ENABLE(1));
Thomas Petazzonic4b4b732012-11-22 18:16:37 +01001093 writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1094 writel(0, base + WINDOW_OVERRIDE_CTRL(1));
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001095}
1096
Marcin Wojtasac5f0f32016-04-29 09:49:07 +02001097static void
1098mv_xor_conf_mbus_windows_a3700(struct mv_xor_device *xordev)
1099{
1100 void __iomem *base = xordev->xor_high_base;
1101 u32 win_enable = 0;
1102 int i;
1103
1104 for (i = 0; i < 8; i++) {
1105 writel(0, base + WINDOW_BASE(i));
1106 writel(0, base + WINDOW_SIZE(i));
1107 if (i < 4)
1108 writel(0, base + WINDOW_REMAP_HIGH(i));
1109 }
1110 /*
1111 * For Armada3700 open default 4GB Mbus window. The dram
1112 * related configuration are done at AXIS level.
1113 */
1114 writel(0xffff0000, base + WINDOW_SIZE(0));
1115 win_enable |= 1;
1116 win_enable |= 3 << 16;
1117
1118 writel(win_enable, base + WINDOW_BAR_ENABLE(0));
1119 writel(win_enable, base + WINDOW_BAR_ENABLE(1));
1120 writel(0, base + WINDOW_OVERRIDE_CTRL(0));
1121 writel(0, base + WINDOW_OVERRIDE_CTRL(1));
1122}
1123
Thomas Petazzoni8b648432015-12-22 11:43:29 +01001124/*
1125 * Since this XOR driver is basically used only for RAID5, we don't
1126 * need to care about synchronizing ->suspend with DMA activity,
1127 * because the DMA engine will naturally be quiet due to the block
1128 * devices being suspended.
1129 */
1130static int mv_xor_suspend(struct platform_device *pdev, pm_message_t state)
1131{
1132 struct mv_xor_device *xordev = platform_get_drvdata(pdev);
1133 int i;
1134
1135 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1136 struct mv_xor_chan *mv_chan = xordev->channels[i];
1137
1138 if (!mv_chan)
1139 continue;
1140
1141 mv_chan->saved_config_reg =
1142 readl_relaxed(XOR_CONFIG(mv_chan));
1143 mv_chan->saved_int_mask_reg =
1144 readl_relaxed(XOR_INTR_MASK(mv_chan));
1145 }
1146
1147 return 0;
1148}
1149
1150static int mv_xor_resume(struct platform_device *dev)
1151{
1152 struct mv_xor_device *xordev = platform_get_drvdata(dev);
1153 const struct mbus_dram_target_info *dram;
1154 int i;
1155
1156 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) {
1157 struct mv_xor_chan *mv_chan = xordev->channels[i];
1158
1159 if (!mv_chan)
1160 continue;
1161
1162 writel_relaxed(mv_chan->saved_config_reg,
1163 XOR_CONFIG(mv_chan));
1164 writel_relaxed(mv_chan->saved_int_mask_reg,
1165 XOR_INTR_MASK(mv_chan));
1166 }
1167
Marcin Wojtasac5f0f32016-04-29 09:49:07 +02001168 if (xordev->xor_type == XOR_ARMADA_37XX) {
1169 mv_xor_conf_mbus_windows_a3700(xordev);
1170 return 0;
1171 }
1172
Thomas Petazzoni8b648432015-12-22 11:43:29 +01001173 dram = mv_mbus_dram_info();
1174 if (dram)
1175 mv_xor_conf_mbus_windows(xordev, dram);
1176
1177 return 0;
1178}
1179
Lior Amsalem6f166312015-05-26 15:07:34 +02001180static const struct of_device_id mv_xor_dt_ids[] = {
Gregory CLEMENTdd130c62016-04-29 09:49:06 +02001181 { .compatible = "marvell,orion-xor", .data = (void *)XOR_ORION },
1182 { .compatible = "marvell,armada-380-xor", .data = (void *)XOR_ARMADA_38X },
Marcin Wojtasac5f0f32016-04-29 09:49:07 +02001183 { .compatible = "marvell,armada-3700-xor", .data = (void *)XOR_ARMADA_37XX },
Lior Amsalem6f166312015-05-26 15:07:34 +02001184 {},
1185};
Lior Amsalem6f166312015-05-26 15:07:34 +02001186
Thomas Petazzoni77757292015-07-08 16:28:19 +02001187static unsigned int mv_xor_engine_count;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001188
Linus Torvaldsc2714332012-12-14 14:54:26 -08001189static int mv_xor_probe(struct platform_device *pdev)
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001190{
Andrew Lunn63a93322011-12-07 21:48:07 +01001191 const struct mbus_dram_target_info *dram;
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001192 struct mv_xor_device *xordev;
Jingoo Hand4adcc02013-07-30 17:09:11 +09001193 struct mv_xor_platform_data *pdata = dev_get_platdata(&pdev->dev);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001194 struct resource *res;
Thomas Petazzoni77757292015-07-08 16:28:19 +02001195 unsigned int max_engines, max_channels;
Thomas Petazzoni60d151f2012-10-29 16:54:49 +01001196 int i, ret;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001197
Joe Perches1ba151c2012-10-28 01:05:44 -07001198 dev_notice(&pdev->dev, "Marvell shared XOR driver\n");
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001199
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001200 xordev = devm_kzalloc(&pdev->dev, sizeof(*xordev), GFP_KERNEL);
1201 if (!xordev)
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001202 return -ENOMEM;
1203
1204 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1205 if (!res)
1206 return -ENODEV;
1207
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001208 xordev->xor_base = devm_ioremap(&pdev->dev, res->start,
1209 resource_size(res));
1210 if (!xordev->xor_base)
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001211 return -EBUSY;
1212
1213 res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1214 if (!res)
1215 return -ENODEV;
1216
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001217 xordev->xor_high_base = devm_ioremap(&pdev->dev, res->start,
1218 resource_size(res));
1219 if (!xordev->xor_high_base)
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001220 return -EBUSY;
1221
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001222 platform_set_drvdata(pdev, xordev);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001223
Gregory CLEMENTdd130c62016-04-29 09:49:06 +02001224
1225 /*
1226 * We need to know which type of XOR device we use before
1227 * setting up. In non-dt case it can only be the legacy one.
1228 */
1229 xordev->xor_type = XOR_ORION;
1230 if (pdev->dev.of_node) {
1231 const struct of_device_id *of_id =
1232 of_match_device(mv_xor_dt_ids,
1233 &pdev->dev);
1234
1235 xordev->xor_type = (uintptr_t)of_id->data;
1236 }
1237
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001238 /*
1239 * (Re-)program MBUS remapping windows if we are asked to.
1240 */
Marcin Wojtasac5f0f32016-04-29 09:49:07 +02001241 if (xordev->xor_type == XOR_ARMADA_37XX) {
1242 mv_xor_conf_mbus_windows_a3700(xordev);
1243 } else {
1244 dram = mv_mbus_dram_info();
1245 if (dram)
1246 mv_xor_conf_mbus_windows(xordev, dram);
1247 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001248
Andrew Lunnc5101822012-02-19 13:30:26 +01001249 /* Not all platforms can gate the clock, so it is not
1250 * an error if the clock does not exists.
1251 */
Thomas Petazzoni297eedb2012-11-15 15:29:53 +01001252 xordev->clk = clk_get(&pdev->dev, NULL);
1253 if (!IS_ERR(xordev->clk))
1254 clk_prepare_enable(xordev->clk);
Andrew Lunnc5101822012-02-19 13:30:26 +01001255
Thomas Petazzoni77757292015-07-08 16:28:19 +02001256 /*
1257 * We don't want to have more than one channel per CPU in
1258 * order for async_tx to perform well. So we limit the number
1259 * of engines and channels so that we take into account this
1260 * constraint. Note that we also want to use channels from
Marcin Wojtasac5f0f32016-04-29 09:49:07 +02001261 * separate engines when possible. For dual-CPU Armada 3700
1262 * SoC with single XOR engine allow using its both channels.
Thomas Petazzoni77757292015-07-08 16:28:19 +02001263 */
1264 max_engines = num_present_cpus();
Marcin Wojtasac5f0f32016-04-29 09:49:07 +02001265 if (xordev->xor_type == XOR_ARMADA_37XX)
1266 max_channels = num_present_cpus();
1267 else
1268 max_channels = min_t(unsigned int,
1269 MV_XOR_MAX_CHANNELS,
1270 DIV_ROUND_UP(num_present_cpus(), 2));
Thomas Petazzoni77757292015-07-08 16:28:19 +02001271
1272 if (mv_xor_engine_count >= max_engines)
1273 return 0;
1274
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001275 if (pdev->dev.of_node) {
1276 struct device_node *np;
1277 int i = 0;
1278
1279 for_each_child_of_node(pdev->dev.of_node, np) {
Russell King0be82532013-12-12 23:59:08 +00001280 struct mv_xor_chan *chan;
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001281 dma_cap_mask_t cap_mask;
1282 int irq;
1283
Thomas Petazzoni77757292015-07-08 16:28:19 +02001284 if (i >= max_channels)
1285 continue;
1286
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001287 dma_cap_zero(cap_mask);
Thomas Petazzoni6d8f7ab2015-07-08 16:28:16 +02001288 dma_cap_set(DMA_MEMCPY, cap_mask);
1289 dma_cap_set(DMA_XOR, cap_mask);
1290 dma_cap_set(DMA_INTERRUPT, cap_mask);
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001291
1292 irq = irq_of_parse_and_map(np, 0);
Thomas Petazzonif8eb9e72012-11-22 18:22:12 +01001293 if (!irq) {
1294 ret = -ENODEV;
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001295 goto err_channel_add;
1296 }
1297
Russell King0be82532013-12-12 23:59:08 +00001298 chan = mv_xor_channel_add(xordev, pdev, i,
Gregory CLEMENTdd130c62016-04-29 09:49:06 +02001299 cap_mask, irq);
Russell King0be82532013-12-12 23:59:08 +00001300 if (IS_ERR(chan)) {
1301 ret = PTR_ERR(chan);
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001302 irq_dispose_mapping(irq);
1303 goto err_channel_add;
1304 }
1305
Russell King0be82532013-12-12 23:59:08 +00001306 xordev->channels[i] = chan;
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001307 i++;
1308 }
1309 } else if (pdata && pdata->channels) {
Thomas Petazzoni77757292015-07-08 16:28:19 +02001310 for (i = 0; i < max_channels; i++) {
Thomas Petazzonie39f6ec2012-10-30 11:56:26 +01001311 struct mv_xor_channel_data *cd;
Russell King0be82532013-12-12 23:59:08 +00001312 struct mv_xor_chan *chan;
Thomas Petazzoni60d151f2012-10-29 16:54:49 +01001313 int irq;
1314
1315 cd = &pdata->channels[i];
1316 if (!cd) {
1317 ret = -ENODEV;
1318 goto err_channel_add;
1319 }
1320
1321 irq = platform_get_irq(pdev, i);
1322 if (irq < 0) {
1323 ret = irq;
1324 goto err_channel_add;
1325 }
1326
Russell King0be82532013-12-12 23:59:08 +00001327 chan = mv_xor_channel_add(xordev, pdev, i,
Gregory CLEMENTdd130c62016-04-29 09:49:06 +02001328 cd->cap_mask, irq);
Russell King0be82532013-12-12 23:59:08 +00001329 if (IS_ERR(chan)) {
1330 ret = PTR_ERR(chan);
Thomas Petazzoni60d151f2012-10-29 16:54:49 +01001331 goto err_channel_add;
1332 }
Russell King0be82532013-12-12 23:59:08 +00001333
1334 xordev->channels[i] = chan;
Thomas Petazzoni60d151f2012-10-29 16:54:49 +01001335 }
1336 }
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001337
1338 return 0;
Thomas Petazzoni60d151f2012-10-29 16:54:49 +01001339
1340err_channel_add:
1341 for (i = 0; i < MV_XOR_MAX_CHANNELS; i++)
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001342 if (xordev->channels[i]) {
Thomas Petazzoniab6e4392013-01-06 11:10:43 +01001343 mv_xor_channel_remove(xordev->channels[i]);
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001344 if (pdev->dev.of_node)
1345 irq_dispose_mapping(xordev->channels[i]->irq);
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001346 }
Thomas Petazzoni60d151f2012-10-29 16:54:49 +01001347
Thomas Petazzonidab92062013-01-06 11:10:44 +01001348 if (!IS_ERR(xordev->clk)) {
1349 clk_disable_unprepare(xordev->clk);
1350 clk_put(xordev->clk);
1351 }
1352
Thomas Petazzoni60d151f2012-10-29 16:54:49 +01001353 return ret;
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001354}
1355
Thomas Petazzoni61971652012-10-30 12:05:40 +01001356static struct platform_driver mv_xor_driver = {
1357 .probe = mv_xor_probe,
Thomas Petazzoni8b648432015-12-22 11:43:29 +01001358 .suspend = mv_xor_suspend,
1359 .resume = mv_xor_resume,
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001360 .driver = {
Thomas Petazzonif7d12ef2012-11-15 16:47:58 +01001361 .name = MV_XOR_NAME,
1362 .of_match_table = of_match_ptr(mv_xor_dt_ids),
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001363 },
1364};
1365
1366
1367static int __init mv_xor_init(void)
1368{
Thomas Petazzoni61971652012-10-30 12:05:40 +01001369 return platform_driver_register(&mv_xor_driver);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001370}
Paul Gortmaker25cf68d2015-08-21 16:27:49 -04001371device_initcall(mv_xor_init);
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001372
Paul Gortmaker25cf68d2015-08-21 16:27:49 -04001373/*
Saeed Bisharaff7b0472008-07-08 11:58:36 -07001374MODULE_AUTHOR("Saeed Bishara <saeed@marvell.com>");
1375MODULE_DESCRIPTION("DMA engine driver for Marvell's XOR engine");
1376MODULE_LICENSE("GPL");
Paul Gortmaker25cf68d2015-08-21 16:27:49 -04001377*/