Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1 | /* |
| 2 | * offload engine driver for the Intel Xscale series of i/o processors |
| 3 | * Copyright © 2006, Intel Corporation. |
| 4 | * |
| 5 | * This program is free software; you can redistribute it and/or modify it |
| 6 | * under the terms and conditions of the GNU General Public License, |
| 7 | * version 2, as published by the Free Software Foundation. |
| 8 | * |
| 9 | * This program is distributed in the hope it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 12 | * more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU General Public License along with |
| 15 | * this program; if not, write to the Free Software Foundation, Inc., |
| 16 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. |
| 17 | * |
| 18 | */ |
| 19 | |
| 20 | /* |
| 21 | * This driver supports the asynchrounous DMA copy and RAID engines available |
| 22 | * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) |
| 23 | */ |
| 24 | |
| 25 | #include <linux/init.h> |
| 26 | #include <linux/module.h> |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 27 | #include <linux/delay.h> |
| 28 | #include <linux/dma-mapping.h> |
| 29 | #include <linux/spinlock.h> |
| 30 | #include <linux/interrupt.h> |
| 31 | #include <linux/platform_device.h> |
| 32 | #include <linux/memory.h> |
| 33 | #include <linux/ioport.h> |
Dan Williams | f6dbf65 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 34 | #include <linux/raid/pq.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 35 | #include <linux/slab.h> |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 36 | |
Russell King | a09e64f | 2008-08-05 16:14:15 +0100 | [diff] [blame] | 37 | #include <mach/adma.h> |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 38 | |
Russell King - ARM Linux | d2ebfb3 | 2012-03-06 22:34:26 +0000 | [diff] [blame] | 39 | #include "dmaengine.h" |
| 40 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 41 | #define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common) |
| 42 | #define to_iop_adma_device(dev) \ |
| 43 | container_of(dev, struct iop_adma_device, common) |
| 44 | #define tx_to_iop_adma_slot(tx) \ |
| 45 | container_of(tx, struct iop_adma_desc_slot, async_tx) |
| 46 | |
| 47 | /** |
| 48 | * iop_adma_free_slots - flags descriptor slots for reuse |
| 49 | * @slot: Slot to free |
| 50 | * Caller must hold &iop_chan->lock while calling this function |
| 51 | */ |
| 52 | static void iop_adma_free_slots(struct iop_adma_desc_slot *slot) |
| 53 | { |
| 54 | int stride = slot->slots_per_op; |
| 55 | |
| 56 | while (stride--) { |
| 57 | slot->slots_per_op = 0; |
| 58 | slot = list_entry(slot->slot_node.next, |
| 59 | struct iop_adma_desc_slot, |
| 60 | slot_node); |
| 61 | } |
| 62 | } |
| 63 | |
Dan Williams | 7bf649a | 2009-08-28 14:32:04 -0700 | [diff] [blame] | 64 | static void |
| 65 | iop_desc_unmap(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc) |
| 66 | { |
| 67 | struct dma_async_tx_descriptor *tx = &desc->async_tx; |
| 68 | struct iop_adma_desc_slot *unmap = desc->group_head; |
| 69 | struct device *dev = &iop_chan->device->pdev->dev; |
| 70 | u32 len = unmap->unmap_len; |
| 71 | enum dma_ctrl_flags flags = tx->flags; |
| 72 | u32 src_cnt; |
| 73 | dma_addr_t addr; |
| 74 | dma_addr_t dest; |
| 75 | |
| 76 | src_cnt = unmap->unmap_src_cnt; |
| 77 | dest = iop_desc_get_dest_addr(unmap, iop_chan); |
| 78 | if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) { |
| 79 | enum dma_data_direction dir; |
| 80 | |
| 81 | if (src_cnt > 1) /* is xor? */ |
| 82 | dir = DMA_BIDIRECTIONAL; |
| 83 | else |
| 84 | dir = DMA_FROM_DEVICE; |
| 85 | |
| 86 | dma_unmap_page(dev, dest, len, dir); |
| 87 | } |
| 88 | |
| 89 | if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) { |
| 90 | while (src_cnt--) { |
| 91 | addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt); |
| 92 | if (addr == dest) |
| 93 | continue; |
| 94 | dma_unmap_page(dev, addr, len, DMA_TO_DEVICE); |
| 95 | } |
| 96 | } |
| 97 | desc->group_head = NULL; |
| 98 | } |
| 99 | |
| 100 | static void |
| 101 | iop_desc_unmap_pq(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc) |
| 102 | { |
| 103 | struct dma_async_tx_descriptor *tx = &desc->async_tx; |
| 104 | struct iop_adma_desc_slot *unmap = desc->group_head; |
| 105 | struct device *dev = &iop_chan->device->pdev->dev; |
| 106 | u32 len = unmap->unmap_len; |
| 107 | enum dma_ctrl_flags flags = tx->flags; |
| 108 | u32 src_cnt = unmap->unmap_src_cnt; |
| 109 | dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan); |
| 110 | dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan); |
| 111 | int i; |
| 112 | |
| 113 | if (tx->flags & DMA_PREP_CONTINUE) |
| 114 | src_cnt -= 3; |
| 115 | |
| 116 | if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) { |
| 117 | dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL); |
| 118 | dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL); |
| 119 | } |
| 120 | |
| 121 | if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) { |
| 122 | dma_addr_t addr; |
| 123 | |
| 124 | for (i = 0; i < src_cnt; i++) { |
| 125 | addr = iop_desc_get_src_addr(unmap, iop_chan, i); |
| 126 | dma_unmap_page(dev, addr, len, DMA_TO_DEVICE); |
| 127 | } |
| 128 | if (desc->pq_check_result) { |
| 129 | dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE); |
| 130 | dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE); |
| 131 | } |
| 132 | } |
| 133 | |
| 134 | desc->group_head = NULL; |
| 135 | } |
| 136 | |
| 137 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 138 | static dma_cookie_t |
| 139 | iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, |
| 140 | struct iop_adma_chan *iop_chan, dma_cookie_t cookie) |
| 141 | { |
Dan Williams | 507fbec | 2009-08-29 19:12:39 -0700 | [diff] [blame] | 142 | struct dma_async_tx_descriptor *tx = &desc->async_tx; |
| 143 | |
| 144 | BUG_ON(tx->cookie < 0); |
| 145 | if (tx->cookie > 0) { |
| 146 | cookie = tx->cookie; |
| 147 | tx->cookie = 0; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 148 | |
| 149 | /* call the callback (must not sleep or submit new |
| 150 | * operations to this channel) |
| 151 | */ |
Dan Williams | 507fbec | 2009-08-29 19:12:39 -0700 | [diff] [blame] | 152 | if (tx->callback) |
| 153 | tx->callback(tx->callback_param); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 154 | |
| 155 | /* unmap dma addresses |
| 156 | * (unmap_single vs unmap_page?) |
| 157 | */ |
| 158 | if (desc->group_head && desc->unmap_len) { |
Dan Williams | 7bf649a | 2009-08-28 14:32:04 -0700 | [diff] [blame] | 159 | if (iop_desc_is_pq(desc)) |
| 160 | iop_desc_unmap_pq(iop_chan, desc); |
| 161 | else |
| 162 | iop_desc_unmap(iop_chan, desc); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 163 | } |
| 164 | } |
| 165 | |
| 166 | /* run dependent operations */ |
Dan Williams | 507fbec | 2009-08-29 19:12:39 -0700 | [diff] [blame] | 167 | dma_run_dependencies(tx); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 168 | |
| 169 | return cookie; |
| 170 | } |
| 171 | |
| 172 | static int |
| 173 | iop_adma_clean_slot(struct iop_adma_desc_slot *desc, |
| 174 | struct iop_adma_chan *iop_chan) |
| 175 | { |
| 176 | /* the client is allowed to attach dependent operations |
| 177 | * until 'ack' is set |
| 178 | */ |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 179 | if (!async_tx_test_ack(&desc->async_tx)) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 180 | return 0; |
| 181 | |
| 182 | /* leave the last descriptor in the chain |
| 183 | * so we can append to it |
| 184 | */ |
| 185 | if (desc->chain_node.next == &iop_chan->chain) |
| 186 | return 1; |
| 187 | |
| 188 | dev_dbg(iop_chan->device->common.dev, |
| 189 | "\tfree slot: %d slots_per_op: %d\n", |
| 190 | desc->idx, desc->slots_per_op); |
| 191 | |
| 192 | list_del(&desc->chain_node); |
| 193 | iop_adma_free_slots(desc); |
| 194 | |
| 195 | return 0; |
| 196 | } |
| 197 | |
| 198 | static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) |
| 199 | { |
| 200 | struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL; |
| 201 | dma_cookie_t cookie = 0; |
| 202 | u32 current_desc = iop_chan_get_current_descriptor(iop_chan); |
| 203 | int busy = iop_chan_is_busy(iop_chan); |
| 204 | int seen_current = 0, slot_cnt = 0, slots_per_op = 0; |
| 205 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 206 | dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 207 | /* free completed slots from the chain starting with |
| 208 | * the oldest descriptor |
| 209 | */ |
| 210 | list_for_each_entry_safe(iter, _iter, &iop_chan->chain, |
| 211 | chain_node) { |
| 212 | pr_debug("\tcookie: %d slot: %d busy: %d " |
| 213 | "this_desc: %#x next_desc: %#x ack: %d\n", |
| 214 | iter->async_tx.cookie, iter->idx, busy, |
| 215 | iter->async_tx.phys, iop_desc_get_next_desc(iter), |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 216 | async_tx_test_ack(&iter->async_tx)); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 217 | prefetch(_iter); |
| 218 | prefetch(&_iter->async_tx); |
| 219 | |
| 220 | /* do not advance past the current descriptor loaded into the |
| 221 | * hardware channel, subsequent descriptors are either in |
| 222 | * process or have not been submitted |
| 223 | */ |
| 224 | if (seen_current) |
| 225 | break; |
| 226 | |
| 227 | /* stop the search if we reach the current descriptor and the |
| 228 | * channel is busy, or if it appears that the current descriptor |
| 229 | * needs to be re-read (i.e. has been appended to) |
| 230 | */ |
| 231 | if (iter->async_tx.phys == current_desc) { |
| 232 | BUG_ON(seen_current++); |
| 233 | if (busy || iop_desc_get_next_desc(iter)) |
| 234 | break; |
| 235 | } |
| 236 | |
| 237 | /* detect the start of a group transaction */ |
| 238 | if (!slot_cnt && !slots_per_op) { |
| 239 | slot_cnt = iter->slot_cnt; |
| 240 | slots_per_op = iter->slots_per_op; |
| 241 | if (slot_cnt <= slots_per_op) { |
| 242 | slot_cnt = 0; |
| 243 | slots_per_op = 0; |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | if (slot_cnt) { |
| 248 | pr_debug("\tgroup++\n"); |
| 249 | if (!grp_start) |
| 250 | grp_start = iter; |
| 251 | slot_cnt -= slots_per_op; |
| 252 | } |
| 253 | |
| 254 | /* all the members of a group are complete */ |
| 255 | if (slots_per_op != 0 && slot_cnt == 0) { |
| 256 | struct iop_adma_desc_slot *grp_iter, *_grp_iter; |
| 257 | int end_of_chain = 0; |
| 258 | pr_debug("\tgroup end\n"); |
| 259 | |
| 260 | /* collect the total results */ |
| 261 | if (grp_start->xor_check_result) { |
| 262 | u32 zero_sum_result = 0; |
| 263 | slot_cnt = grp_start->slot_cnt; |
| 264 | grp_iter = grp_start; |
| 265 | |
| 266 | list_for_each_entry_from(grp_iter, |
| 267 | &iop_chan->chain, chain_node) { |
| 268 | zero_sum_result |= |
| 269 | iop_desc_get_zero_result(grp_iter); |
| 270 | pr_debug("\titer%d result: %d\n", |
| 271 | grp_iter->idx, zero_sum_result); |
| 272 | slot_cnt -= slots_per_op; |
| 273 | if (slot_cnt == 0) |
| 274 | break; |
| 275 | } |
| 276 | pr_debug("\tgrp_start->xor_check_result: %p\n", |
| 277 | grp_start->xor_check_result); |
| 278 | *grp_start->xor_check_result = zero_sum_result; |
| 279 | } |
| 280 | |
| 281 | /* clean up the group */ |
| 282 | slot_cnt = grp_start->slot_cnt; |
| 283 | grp_iter = grp_start; |
| 284 | list_for_each_entry_safe_from(grp_iter, _grp_iter, |
| 285 | &iop_chan->chain, chain_node) { |
| 286 | cookie = iop_adma_run_tx_complete_actions( |
| 287 | grp_iter, iop_chan, cookie); |
| 288 | |
| 289 | slot_cnt -= slots_per_op; |
| 290 | end_of_chain = iop_adma_clean_slot(grp_iter, |
| 291 | iop_chan); |
| 292 | |
| 293 | if (slot_cnt == 0 || end_of_chain) |
| 294 | break; |
| 295 | } |
| 296 | |
| 297 | /* the group should be complete at this point */ |
| 298 | BUG_ON(slot_cnt); |
| 299 | |
| 300 | slots_per_op = 0; |
| 301 | grp_start = NULL; |
| 302 | if (end_of_chain) |
| 303 | break; |
| 304 | else |
| 305 | continue; |
| 306 | } else if (slots_per_op) /* wait for group completion */ |
| 307 | continue; |
| 308 | |
| 309 | /* write back zero sum results (single descriptor case) */ |
| 310 | if (iter->xor_check_result && iter->async_tx.cookie) |
| 311 | *iter->xor_check_result = |
| 312 | iop_desc_get_zero_result(iter); |
| 313 | |
| 314 | cookie = iop_adma_run_tx_complete_actions( |
| 315 | iter, iop_chan, cookie); |
| 316 | |
| 317 | if (iop_adma_clean_slot(iter, iop_chan)) |
| 318 | break; |
| 319 | } |
| 320 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 321 | if (cookie > 0) { |
Russell King - ARM Linux | 4d4e58d | 2012-03-06 22:34:06 +0000 | [diff] [blame] | 322 | iop_chan->common.completed_cookie = cookie; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 323 | pr_debug("\tcompleted cookie %d\n", cookie); |
| 324 | } |
| 325 | } |
| 326 | |
| 327 | static void |
| 328 | iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) |
| 329 | { |
| 330 | spin_lock_bh(&iop_chan->lock); |
| 331 | __iop_adma_slot_cleanup(iop_chan); |
| 332 | spin_unlock_bh(&iop_chan->lock); |
| 333 | } |
| 334 | |
| 335 | static void iop_adma_tasklet(unsigned long data) |
| 336 | { |
Dan Williams | 19242d7 | 2008-04-17 20:17:25 -0700 | [diff] [blame] | 337 | struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data; |
| 338 | |
Dan Williams | 72be12f | 2009-07-14 13:38:29 -0700 | [diff] [blame] | 339 | /* lockdep will flag depedency submissions as potentially |
| 340 | * recursive locking, this is not the case as a dependency |
| 341 | * submission will never recurse a channels submit routine. |
| 342 | * There are checks in async_tx.c to prevent this. |
| 343 | */ |
| 344 | spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING); |
Dan Williams | 19242d7 | 2008-04-17 20:17:25 -0700 | [diff] [blame] | 345 | __iop_adma_slot_cleanup(iop_chan); |
| 346 | spin_unlock(&iop_chan->lock); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 347 | } |
| 348 | |
| 349 | static struct iop_adma_desc_slot * |
| 350 | iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots, |
| 351 | int slots_per_op) |
| 352 | { |
| 353 | struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL; |
Denis Cheng | e73ef9a | 2008-02-02 19:30:01 -0700 | [diff] [blame] | 354 | LIST_HEAD(chain); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 355 | int slots_found, retry = 0; |
| 356 | |
| 357 | /* start search from the last allocated descrtiptor |
| 358 | * if a contiguous allocation can not be found start searching |
| 359 | * from the beginning of the list |
| 360 | */ |
| 361 | retry: |
| 362 | slots_found = 0; |
| 363 | if (retry == 0) |
| 364 | iter = iop_chan->last_used; |
| 365 | else |
| 366 | iter = list_entry(&iop_chan->all_slots, |
| 367 | struct iop_adma_desc_slot, |
| 368 | slot_node); |
| 369 | |
| 370 | list_for_each_entry_safe_continue( |
| 371 | iter, _iter, &iop_chan->all_slots, slot_node) { |
| 372 | prefetch(_iter); |
| 373 | prefetch(&_iter->async_tx); |
| 374 | if (iter->slots_per_op) { |
| 375 | /* give up after finding the first busy slot |
| 376 | * on the second pass through the list |
| 377 | */ |
| 378 | if (retry) |
| 379 | break; |
| 380 | |
| 381 | slots_found = 0; |
| 382 | continue; |
| 383 | } |
| 384 | |
| 385 | /* start the allocation if the slot is correctly aligned */ |
| 386 | if (!slots_found++) { |
| 387 | if (iop_desc_is_aligned(iter, slots_per_op)) |
| 388 | alloc_start = iter; |
| 389 | else { |
| 390 | slots_found = 0; |
| 391 | continue; |
| 392 | } |
| 393 | } |
| 394 | |
| 395 | if (slots_found == num_slots) { |
| 396 | struct iop_adma_desc_slot *alloc_tail = NULL; |
| 397 | struct iop_adma_desc_slot *last_used = NULL; |
| 398 | iter = alloc_start; |
| 399 | while (num_slots) { |
| 400 | int i; |
| 401 | dev_dbg(iop_chan->device->common.dev, |
| 402 | "allocated slot: %d " |
| 403 | "(desc %p phys: %#x) slots_per_op %d\n", |
| 404 | iter->idx, iter->hw_desc, |
| 405 | iter->async_tx.phys, slots_per_op); |
| 406 | |
| 407 | /* pre-ack all but the last descriptor */ |
| 408 | if (num_slots != slots_per_op) |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 409 | async_tx_ack(&iter->async_tx); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 410 | |
| 411 | list_add_tail(&iter->chain_node, &chain); |
| 412 | alloc_tail = iter; |
| 413 | iter->async_tx.cookie = 0; |
| 414 | iter->slot_cnt = num_slots; |
| 415 | iter->xor_check_result = NULL; |
| 416 | for (i = 0; i < slots_per_op; i++) { |
| 417 | iter->slots_per_op = slots_per_op - i; |
| 418 | last_used = iter; |
| 419 | iter = list_entry(iter->slot_node.next, |
| 420 | struct iop_adma_desc_slot, |
| 421 | slot_node); |
| 422 | } |
| 423 | num_slots -= slots_per_op; |
| 424 | } |
| 425 | alloc_tail->group_head = alloc_start; |
| 426 | alloc_tail->async_tx.cookie = -EBUSY; |
Dan Williams | 308136d | 2009-09-08 17:53:02 -0700 | [diff] [blame] | 427 | list_splice(&chain, &alloc_tail->tx_list); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 428 | iop_chan->last_used = last_used; |
| 429 | iop_desc_clear_next_desc(alloc_start); |
| 430 | iop_desc_clear_next_desc(alloc_tail); |
| 431 | return alloc_tail; |
| 432 | } |
| 433 | } |
| 434 | if (!retry++) |
| 435 | goto retry; |
| 436 | |
Dan Williams | c7141d0 | 2008-07-17 17:59:56 -0700 | [diff] [blame] | 437 | /* perform direct reclaim if the allocation fails */ |
| 438 | __iop_adma_slot_cleanup(iop_chan); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 439 | |
| 440 | return NULL; |
| 441 | } |
| 442 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 443 | static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan) |
| 444 | { |
| 445 | dev_dbg(iop_chan->device->common.dev, "pending: %d\n", |
| 446 | iop_chan->pending); |
| 447 | |
| 448 | if (iop_chan->pending >= IOP_ADMA_THRESHOLD) { |
| 449 | iop_chan->pending = 0; |
| 450 | iop_chan_append(iop_chan); |
| 451 | } |
| 452 | } |
| 453 | |
| 454 | static dma_cookie_t |
| 455 | iop_adma_tx_submit(struct dma_async_tx_descriptor *tx) |
| 456 | { |
| 457 | struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx); |
| 458 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan); |
| 459 | struct iop_adma_desc_slot *grp_start, *old_chain_tail; |
| 460 | int slot_cnt; |
| 461 | int slots_per_op; |
| 462 | dma_cookie_t cookie; |
Dan Williams | 137cb55 | 2008-11-11 13:12:33 -0700 | [diff] [blame] | 463 | dma_addr_t next_dma; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 464 | |
| 465 | grp_start = sw_desc->group_head; |
| 466 | slot_cnt = grp_start->slot_cnt; |
| 467 | slots_per_op = grp_start->slots_per_op; |
| 468 | |
| 469 | spin_lock_bh(&iop_chan->lock); |
Russell King - ARM Linux | 884485e | 2012-03-06 22:34:46 +0000 | [diff] [blame] | 470 | cookie = dma_cookie_assign(tx); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 471 | |
| 472 | old_chain_tail = list_entry(iop_chan->chain.prev, |
| 473 | struct iop_adma_desc_slot, chain_node); |
Dan Williams | 308136d | 2009-09-08 17:53:02 -0700 | [diff] [blame] | 474 | list_splice_init(&sw_desc->tx_list, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 475 | &old_chain_tail->chain_node); |
| 476 | |
| 477 | /* fix up the hardware chain */ |
Dan Williams | 137cb55 | 2008-11-11 13:12:33 -0700 | [diff] [blame] | 478 | next_dma = grp_start->async_tx.phys; |
| 479 | iop_desc_set_next_desc(old_chain_tail, next_dma); |
| 480 | BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */ |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 481 | |
Dan Williams | 137cb55 | 2008-11-11 13:12:33 -0700 | [diff] [blame] | 482 | /* check for pre-chained descriptors */ |
Dan Williams | 65e5038 | 2008-11-11 13:12:33 -0700 | [diff] [blame] | 483 | iop_paranoia(iop_desc_get_next_desc(sw_desc)); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 484 | |
| 485 | /* increment the pending count by the number of slots |
| 486 | * memcpy operations have a 1:1 (slot:operation) relation |
| 487 | * other operations are heavier and will pop the threshold |
| 488 | * more often. |
| 489 | */ |
| 490 | iop_chan->pending += slot_cnt; |
| 491 | iop_adma_check_threshold(iop_chan); |
| 492 | spin_unlock_bh(&iop_chan->lock); |
| 493 | |
| 494 | dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n", |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 495 | __func__, sw_desc->async_tx.cookie, sw_desc->idx); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 496 | |
| 497 | return cookie; |
| 498 | } |
| 499 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 500 | static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan); |
| 501 | static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan); |
| 502 | |
Dan Williams | 5eb907a | 2008-07-17 17:59:56 -0700 | [diff] [blame] | 503 | /** |
| 504 | * iop_adma_alloc_chan_resources - returns the number of allocated descriptors |
| 505 | * @chan - allocate descriptor resources for this channel |
| 506 | * @client - current client requesting the channel be ready for requests |
| 507 | * |
| 508 | * Note: We keep the slots for 1 operation on iop_chan->chain at all times. To |
| 509 | * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be |
| 510 | * greater than 2x the number slots needed to satisfy a device->max_xor |
| 511 | * request. |
| 512 | * */ |
Dan Williams | aa1e6f1 | 2009-01-06 11:38:17 -0700 | [diff] [blame] | 513 | static int iop_adma_alloc_chan_resources(struct dma_chan *chan) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 514 | { |
| 515 | char *hw_desc; |
| 516 | int idx; |
| 517 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 518 | struct iop_adma_desc_slot *slot = NULL; |
| 519 | int init = iop_chan->slots_allocated ? 0 : 1; |
| 520 | struct iop_adma_platform_data *plat_data = |
| 521 | iop_chan->device->pdev->dev.platform_data; |
| 522 | int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE; |
| 523 | |
| 524 | /* Allocate descriptor slots */ |
| 525 | do { |
| 526 | idx = iop_chan->slots_allocated; |
| 527 | if (idx == num_descs_in_pool) |
| 528 | break; |
| 529 | |
| 530 | slot = kzalloc(sizeof(*slot), GFP_KERNEL); |
| 531 | if (!slot) { |
| 532 | printk(KERN_INFO "IOP ADMA Channel only initialized" |
| 533 | " %d descriptor slots", idx); |
| 534 | break; |
| 535 | } |
| 536 | hw_desc = (char *) iop_chan->device->dma_desc_pool_virt; |
| 537 | slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE]; |
| 538 | |
| 539 | dma_async_tx_descriptor_init(&slot->async_tx, chan); |
| 540 | slot->async_tx.tx_submit = iop_adma_tx_submit; |
Dan Williams | 308136d | 2009-09-08 17:53:02 -0700 | [diff] [blame] | 541 | INIT_LIST_HEAD(&slot->tx_list); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 542 | INIT_LIST_HEAD(&slot->chain_node); |
| 543 | INIT_LIST_HEAD(&slot->slot_node); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 544 | hw_desc = (char *) iop_chan->device->dma_desc_pool; |
| 545 | slot->async_tx.phys = |
| 546 | (dma_addr_t) &hw_desc[idx * IOP_ADMA_SLOT_SIZE]; |
| 547 | slot->idx = idx; |
| 548 | |
| 549 | spin_lock_bh(&iop_chan->lock); |
| 550 | iop_chan->slots_allocated++; |
| 551 | list_add_tail(&slot->slot_node, &iop_chan->all_slots); |
| 552 | spin_unlock_bh(&iop_chan->lock); |
| 553 | } while (iop_chan->slots_allocated < num_descs_in_pool); |
| 554 | |
| 555 | if (idx && !iop_chan->last_used) |
| 556 | iop_chan->last_used = list_entry(iop_chan->all_slots.next, |
| 557 | struct iop_adma_desc_slot, |
| 558 | slot_node); |
| 559 | |
| 560 | dev_dbg(iop_chan->device->common.dev, |
| 561 | "allocated %d descriptor slots last_used: %p\n", |
| 562 | iop_chan->slots_allocated, iop_chan->last_used); |
| 563 | |
| 564 | /* initialize the channel and the chain with a null operation */ |
| 565 | if (init) { |
| 566 | if (dma_has_cap(DMA_MEMCPY, |
| 567 | iop_chan->device->common.cap_mask)) |
| 568 | iop_chan_start_null_memcpy(iop_chan); |
| 569 | else if (dma_has_cap(DMA_XOR, |
| 570 | iop_chan->device->common.cap_mask)) |
| 571 | iop_chan_start_null_xor(iop_chan); |
| 572 | else |
| 573 | BUG(); |
| 574 | } |
| 575 | |
| 576 | return (idx > 0) ? idx : -ENOMEM; |
| 577 | } |
| 578 | |
| 579 | static struct dma_async_tx_descriptor * |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 580 | iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 581 | { |
| 582 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 583 | struct iop_adma_desc_slot *sw_desc, *grp_start; |
| 584 | int slot_cnt, slots_per_op; |
| 585 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 586 | dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 587 | |
| 588 | spin_lock_bh(&iop_chan->lock); |
| 589 | slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan); |
| 590 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 591 | if (sw_desc) { |
| 592 | grp_start = sw_desc->group_head; |
| 593 | iop_desc_init_interrupt(grp_start, iop_chan); |
| 594 | grp_start->unmap_len = 0; |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 595 | sw_desc->async_tx.flags = flags; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 596 | } |
| 597 | spin_unlock_bh(&iop_chan->lock); |
| 598 | |
| 599 | return sw_desc ? &sw_desc->async_tx : NULL; |
| 600 | } |
| 601 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 602 | static struct dma_async_tx_descriptor * |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 603 | iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest, |
Dan Williams | d4c56f9 | 2008-02-02 19:49:58 -0700 | [diff] [blame] | 604 | dma_addr_t dma_src, size_t len, unsigned long flags) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 605 | { |
| 606 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 607 | struct iop_adma_desc_slot *sw_desc, *grp_start; |
| 608 | int slot_cnt, slots_per_op; |
| 609 | |
| 610 | if (unlikely(!len)) |
| 611 | return NULL; |
Coly Li | e2ec771 | 2011-03-27 01:26:52 +0800 | [diff] [blame] | 612 | BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 613 | |
| 614 | dev_dbg(iop_chan->device->common.dev, "%s len: %u\n", |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 615 | __func__, len); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 616 | |
| 617 | spin_lock_bh(&iop_chan->lock); |
| 618 | slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op); |
| 619 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 620 | if (sw_desc) { |
| 621 | grp_start = sw_desc->group_head; |
Dan Williams | d4c56f9 | 2008-02-02 19:49:58 -0700 | [diff] [blame] | 622 | iop_desc_init_memcpy(grp_start, flags); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 623 | iop_desc_set_byte_count(grp_start, iop_chan, len); |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 624 | iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); |
| 625 | iop_desc_set_memcpy_src_addr(grp_start, dma_src); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 626 | sw_desc->unmap_src_cnt = 1; |
| 627 | sw_desc->unmap_len = len; |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 628 | sw_desc->async_tx.flags = flags; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 629 | } |
| 630 | spin_unlock_bh(&iop_chan->lock); |
| 631 | |
| 632 | return sw_desc ? &sw_desc->async_tx : NULL; |
| 633 | } |
| 634 | |
| 635 | static struct dma_async_tx_descriptor * |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 636 | iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest, |
| 637 | dma_addr_t *dma_src, unsigned int src_cnt, size_t len, |
Dan Williams | d4c56f9 | 2008-02-02 19:49:58 -0700 | [diff] [blame] | 638 | unsigned long flags) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 639 | { |
| 640 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 641 | struct iop_adma_desc_slot *sw_desc, *grp_start; |
| 642 | int slot_cnt, slots_per_op; |
| 643 | |
| 644 | if (unlikely(!len)) |
| 645 | return NULL; |
Coly Li | e2ec771 | 2011-03-27 01:26:52 +0800 | [diff] [blame] | 646 | BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 647 | |
| 648 | dev_dbg(iop_chan->device->common.dev, |
Dan Williams | d4c56f9 | 2008-02-02 19:49:58 -0700 | [diff] [blame] | 649 | "%s src_cnt: %d len: %u flags: %lx\n", |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 650 | __func__, src_cnt, len, flags); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 651 | |
| 652 | spin_lock_bh(&iop_chan->lock); |
| 653 | slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op); |
| 654 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 655 | if (sw_desc) { |
| 656 | grp_start = sw_desc->group_head; |
Dan Williams | d4c56f9 | 2008-02-02 19:49:58 -0700 | [diff] [blame] | 657 | iop_desc_init_xor(grp_start, src_cnt, flags); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 658 | iop_desc_set_byte_count(grp_start, iop_chan, len); |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 659 | iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 660 | sw_desc->unmap_src_cnt = src_cnt; |
| 661 | sw_desc->unmap_len = len; |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 662 | sw_desc->async_tx.flags = flags; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 663 | while (src_cnt--) |
| 664 | iop_desc_set_xor_src_addr(grp_start, src_cnt, |
| 665 | dma_src[src_cnt]); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 666 | } |
| 667 | spin_unlock_bh(&iop_chan->lock); |
| 668 | |
| 669 | return sw_desc ? &sw_desc->async_tx : NULL; |
| 670 | } |
| 671 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 672 | static struct dma_async_tx_descriptor * |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 673 | iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src, |
| 674 | unsigned int src_cnt, size_t len, u32 *result, |
| 675 | unsigned long flags) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 676 | { |
| 677 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 678 | struct iop_adma_desc_slot *sw_desc, *grp_start; |
| 679 | int slot_cnt, slots_per_op; |
| 680 | |
| 681 | if (unlikely(!len)) |
| 682 | return NULL; |
| 683 | |
| 684 | dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n", |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 685 | __func__, src_cnt, len); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 686 | |
| 687 | spin_lock_bh(&iop_chan->lock); |
| 688 | slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op); |
| 689 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 690 | if (sw_desc) { |
| 691 | grp_start = sw_desc->group_head; |
Dan Williams | d4c56f9 | 2008-02-02 19:49:58 -0700 | [diff] [blame] | 692 | iop_desc_init_zero_sum(grp_start, src_cnt, flags); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 693 | iop_desc_set_zero_sum_byte_count(grp_start, len); |
| 694 | grp_start->xor_check_result = result; |
| 695 | pr_debug("\t%s: grp_start->xor_check_result: %p\n", |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 696 | __func__, grp_start->xor_check_result); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 697 | sw_desc->unmap_src_cnt = src_cnt; |
| 698 | sw_desc->unmap_len = len; |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 699 | sw_desc->async_tx.flags = flags; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 700 | while (src_cnt--) |
| 701 | iop_desc_set_zero_sum_src_addr(grp_start, src_cnt, |
| 702 | dma_src[src_cnt]); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 703 | } |
| 704 | spin_unlock_bh(&iop_chan->lock); |
| 705 | |
| 706 | return sw_desc ? &sw_desc->async_tx : NULL; |
| 707 | } |
| 708 | |
Dan Williams | 7bf649a | 2009-08-28 14:32:04 -0700 | [diff] [blame] | 709 | static struct dma_async_tx_descriptor * |
| 710 | iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, |
| 711 | unsigned int src_cnt, const unsigned char *scf, size_t len, |
| 712 | unsigned long flags) |
| 713 | { |
| 714 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 715 | struct iop_adma_desc_slot *sw_desc, *g; |
| 716 | int slot_cnt, slots_per_op; |
| 717 | int continue_srcs; |
| 718 | |
| 719 | if (unlikely(!len)) |
| 720 | return NULL; |
| 721 | BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); |
| 722 | |
| 723 | dev_dbg(iop_chan->device->common.dev, |
| 724 | "%s src_cnt: %d len: %u flags: %lx\n", |
| 725 | __func__, src_cnt, len, flags); |
| 726 | |
| 727 | if (dmaf_p_disabled_continue(flags)) |
| 728 | continue_srcs = 1+src_cnt; |
| 729 | else if (dmaf_continue(flags)) |
| 730 | continue_srcs = 3+src_cnt; |
| 731 | else |
| 732 | continue_srcs = 0+src_cnt; |
| 733 | |
| 734 | spin_lock_bh(&iop_chan->lock); |
| 735 | slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op); |
| 736 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 737 | if (sw_desc) { |
| 738 | int i; |
| 739 | |
| 740 | g = sw_desc->group_head; |
| 741 | iop_desc_set_byte_count(g, iop_chan, len); |
| 742 | |
| 743 | /* even if P is disabled its destination address (bits |
| 744 | * [3:0]) must match Q. It is ok if P points to an |
| 745 | * invalid address, it won't be written. |
| 746 | */ |
| 747 | if (flags & DMA_PREP_PQ_DISABLE_P) |
| 748 | dst[0] = dst[1] & 0x7; |
| 749 | |
| 750 | iop_desc_set_pq_addr(g, dst); |
| 751 | sw_desc->unmap_src_cnt = src_cnt; |
| 752 | sw_desc->unmap_len = len; |
| 753 | sw_desc->async_tx.flags = flags; |
| 754 | for (i = 0; i < src_cnt; i++) |
| 755 | iop_desc_set_pq_src_addr(g, i, src[i], scf[i]); |
| 756 | |
| 757 | /* if we are continuing a previous operation factor in |
| 758 | * the old p and q values, see the comment for dma_maxpq |
| 759 | * in include/linux/dmaengine.h |
| 760 | */ |
| 761 | if (dmaf_p_disabled_continue(flags)) |
| 762 | iop_desc_set_pq_src_addr(g, i++, dst[1], 1); |
| 763 | else if (dmaf_continue(flags)) { |
| 764 | iop_desc_set_pq_src_addr(g, i++, dst[0], 0); |
| 765 | iop_desc_set_pq_src_addr(g, i++, dst[1], 1); |
| 766 | iop_desc_set_pq_src_addr(g, i++, dst[1], 0); |
| 767 | } |
| 768 | iop_desc_init_pq(g, i, flags); |
| 769 | } |
| 770 | spin_unlock_bh(&iop_chan->lock); |
| 771 | |
| 772 | return sw_desc ? &sw_desc->async_tx : NULL; |
| 773 | } |
| 774 | |
| 775 | static struct dma_async_tx_descriptor * |
| 776 | iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, |
| 777 | unsigned int src_cnt, const unsigned char *scf, |
| 778 | size_t len, enum sum_check_flags *pqres, |
| 779 | unsigned long flags) |
| 780 | { |
| 781 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 782 | struct iop_adma_desc_slot *sw_desc, *g; |
| 783 | int slot_cnt, slots_per_op; |
| 784 | |
| 785 | if (unlikely(!len)) |
| 786 | return NULL; |
| 787 | BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); |
| 788 | |
| 789 | dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n", |
| 790 | __func__, src_cnt, len); |
| 791 | |
| 792 | spin_lock_bh(&iop_chan->lock); |
| 793 | slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op); |
| 794 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 795 | if (sw_desc) { |
| 796 | /* for validate operations p and q are tagged onto the |
| 797 | * end of the source list |
| 798 | */ |
| 799 | int pq_idx = src_cnt; |
| 800 | |
| 801 | g = sw_desc->group_head; |
| 802 | iop_desc_init_pq_zero_sum(g, src_cnt+2, flags); |
| 803 | iop_desc_set_pq_zero_sum_byte_count(g, len); |
| 804 | g->pq_check_result = pqres; |
| 805 | pr_debug("\t%s: g->pq_check_result: %p\n", |
| 806 | __func__, g->pq_check_result); |
| 807 | sw_desc->unmap_src_cnt = src_cnt+2; |
| 808 | sw_desc->unmap_len = len; |
| 809 | sw_desc->async_tx.flags = flags; |
| 810 | while (src_cnt--) |
| 811 | iop_desc_set_pq_zero_sum_src_addr(g, src_cnt, |
| 812 | src[src_cnt], |
| 813 | scf[src_cnt]); |
| 814 | iop_desc_set_pq_zero_sum_addr(g, pq_idx, src); |
| 815 | } |
| 816 | spin_unlock_bh(&iop_chan->lock); |
| 817 | |
| 818 | return sw_desc ? &sw_desc->async_tx : NULL; |
| 819 | } |
| 820 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 821 | static void iop_adma_free_chan_resources(struct dma_chan *chan) |
| 822 | { |
| 823 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 824 | struct iop_adma_desc_slot *iter, *_iter; |
| 825 | int in_use_descs = 0; |
| 826 | |
| 827 | iop_adma_slot_cleanup(iop_chan); |
| 828 | |
| 829 | spin_lock_bh(&iop_chan->lock); |
| 830 | list_for_each_entry_safe(iter, _iter, &iop_chan->chain, |
| 831 | chain_node) { |
| 832 | in_use_descs++; |
| 833 | list_del(&iter->chain_node); |
| 834 | } |
| 835 | list_for_each_entry_safe_reverse( |
| 836 | iter, _iter, &iop_chan->all_slots, slot_node) { |
| 837 | list_del(&iter->slot_node); |
| 838 | kfree(iter); |
| 839 | iop_chan->slots_allocated--; |
| 840 | } |
| 841 | iop_chan->last_used = NULL; |
| 842 | |
| 843 | dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n", |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 844 | __func__, iop_chan->slots_allocated); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 845 | spin_unlock_bh(&iop_chan->lock); |
| 846 | |
| 847 | /* one is ok since we left it on there on purpose */ |
| 848 | if (in_use_descs > 1) |
| 849 | printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n", |
| 850 | in_use_descs - 1); |
| 851 | } |
| 852 | |
| 853 | /** |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 854 | * iop_adma_status - poll the status of an ADMA transaction |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 855 | * @chan: ADMA channel handle |
| 856 | * @cookie: ADMA transaction identifier |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 857 | * @txstate: a holder for the current state of the channel or NULL |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 858 | */ |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 859 | static enum dma_status iop_adma_status(struct dma_chan *chan, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 860 | dma_cookie_t cookie, |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 861 | struct dma_tx_state *txstate) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 862 | { |
| 863 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
Vinod Koul | 949ff5b | 2012-03-13 11:58:12 +0530 | [diff] [blame] | 864 | int ret; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 865 | |
Russell King - ARM Linux | 96a2af4 | 2012-03-06 22:35:27 +0000 | [diff] [blame] | 866 | ret = dma_cookie_status(chan, cookie, txstate); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 867 | if (ret == DMA_SUCCESS) |
| 868 | return ret; |
| 869 | |
| 870 | iop_adma_slot_cleanup(iop_chan); |
| 871 | |
Russell King - ARM Linux | 96a2af4 | 2012-03-06 22:35:27 +0000 | [diff] [blame] | 872 | return dma_cookie_status(chan, cookie, txstate); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 873 | } |
| 874 | |
| 875 | static irqreturn_t iop_adma_eot_handler(int irq, void *data) |
| 876 | { |
| 877 | struct iop_adma_chan *chan = data; |
| 878 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 879 | dev_dbg(chan->device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 880 | |
| 881 | tasklet_schedule(&chan->irq_tasklet); |
| 882 | |
| 883 | iop_adma_device_clear_eot_status(chan); |
| 884 | |
| 885 | return IRQ_HANDLED; |
| 886 | } |
| 887 | |
| 888 | static irqreturn_t iop_adma_eoc_handler(int irq, void *data) |
| 889 | { |
| 890 | struct iop_adma_chan *chan = data; |
| 891 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 892 | dev_dbg(chan->device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 893 | |
| 894 | tasklet_schedule(&chan->irq_tasklet); |
| 895 | |
| 896 | iop_adma_device_clear_eoc_status(chan); |
| 897 | |
| 898 | return IRQ_HANDLED; |
| 899 | } |
| 900 | |
| 901 | static irqreturn_t iop_adma_err_handler(int irq, void *data) |
| 902 | { |
| 903 | struct iop_adma_chan *chan = data; |
| 904 | unsigned long status = iop_chan_get_status(chan); |
| 905 | |
Joe Perches | 1ba151c | 2012-10-28 01:05:44 -0700 | [diff] [blame] | 906 | dev_err(chan->device->common.dev, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 907 | "error ( %s%s%s%s%s%s%s)\n", |
| 908 | iop_is_err_int_parity(status, chan) ? "int_parity " : "", |
| 909 | iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "", |
| 910 | iop_is_err_int_tabort(status, chan) ? "int_tabort " : "", |
| 911 | iop_is_err_int_mabort(status, chan) ? "int_mabort " : "", |
| 912 | iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "", |
| 913 | iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "", |
| 914 | iop_is_err_split_tx(status, chan) ? "split_tx " : ""); |
| 915 | |
| 916 | iop_adma_device_clear_err_status(chan); |
| 917 | |
| 918 | BUG(); |
| 919 | |
| 920 | return IRQ_HANDLED; |
| 921 | } |
| 922 | |
| 923 | static void iop_adma_issue_pending(struct dma_chan *chan) |
| 924 | { |
| 925 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 926 | |
| 927 | if (iop_chan->pending) { |
| 928 | iop_chan->pending = 0; |
| 929 | iop_chan_append(iop_chan); |
| 930 | } |
| 931 | } |
| 932 | |
| 933 | /* |
| 934 | * Perform a transaction to verify the HW works. |
| 935 | */ |
| 936 | #define IOP_ADMA_TEST_SIZE 2000 |
| 937 | |
Bill Pemberton | 463a1f8 | 2012-11-19 13:22:55 -0500 | [diff] [blame] | 938 | static int iop_adma_memcpy_self_test(struct iop_adma_device *device) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 939 | { |
| 940 | int i; |
| 941 | void *src, *dest; |
| 942 | dma_addr_t src_dma, dest_dma; |
| 943 | struct dma_chan *dma_chan; |
| 944 | dma_cookie_t cookie; |
| 945 | struct dma_async_tx_descriptor *tx; |
| 946 | int err = 0; |
| 947 | struct iop_adma_chan *iop_chan; |
| 948 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 949 | dev_dbg(device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 950 | |
Christophe Jaillet | eccf214 | 2008-05-20 16:33:06 -0700 | [diff] [blame] | 951 | src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 952 | if (!src) |
| 953 | return -ENOMEM; |
Christophe Jaillet | eccf214 | 2008-05-20 16:33:06 -0700 | [diff] [blame] | 954 | dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 955 | if (!dest) { |
| 956 | kfree(src); |
| 957 | return -ENOMEM; |
| 958 | } |
| 959 | |
| 960 | /* Fill in src buffer */ |
| 961 | for (i = 0; i < IOP_ADMA_TEST_SIZE; i++) |
| 962 | ((u8 *) src)[i] = (u8)i; |
| 963 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 964 | /* Start copy, using first DMA channel */ |
| 965 | dma_chan = container_of(device->common.channels.next, |
| 966 | struct dma_chan, |
| 967 | device_node); |
Dan Williams | aa1e6f1 | 2009-01-06 11:38:17 -0700 | [diff] [blame] | 968 | if (iop_adma_alloc_chan_resources(dma_chan) < 1) { |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 969 | err = -ENODEV; |
| 970 | goto out; |
| 971 | } |
| 972 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 973 | dest_dma = dma_map_single(dma_chan->device->dev, dest, |
| 974 | IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 975 | src_dma = dma_map_single(dma_chan->device->dev, src, |
| 976 | IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE); |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 977 | tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma, |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 978 | IOP_ADMA_TEST_SIZE, |
| 979 | DMA_PREP_INTERRUPT | DMA_CTRL_ACK); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 980 | |
| 981 | cookie = iop_adma_tx_submit(tx); |
| 982 | iop_adma_issue_pending(dma_chan); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 983 | msleep(1); |
| 984 | |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 985 | if (iop_adma_status(dma_chan, cookie, NULL) != |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 986 | DMA_SUCCESS) { |
Joe Perches | 1ba151c | 2012-10-28 01:05:44 -0700 | [diff] [blame] | 987 | dev_err(dma_chan->device->dev, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 988 | "Self-test copy timed out, disabling\n"); |
| 989 | err = -ENODEV; |
| 990 | goto free_resources; |
| 991 | } |
| 992 | |
| 993 | iop_chan = to_iop_adma_chan(dma_chan); |
| 994 | dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, |
| 995 | IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); |
| 996 | if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) { |
Joe Perches | 1ba151c | 2012-10-28 01:05:44 -0700 | [diff] [blame] | 997 | dev_err(dma_chan->device->dev, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 998 | "Self-test copy failed compare, disabling\n"); |
| 999 | err = -ENODEV; |
| 1000 | goto free_resources; |
| 1001 | } |
| 1002 | |
| 1003 | free_resources: |
| 1004 | iop_adma_free_chan_resources(dma_chan); |
| 1005 | out: |
| 1006 | kfree(src); |
| 1007 | kfree(dest); |
| 1008 | return err; |
| 1009 | } |
| 1010 | |
| 1011 | #define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */ |
Bill Pemberton | 463a1f8 | 2012-11-19 13:22:55 -0500 | [diff] [blame] | 1012 | static int |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 1013 | iop_adma_xor_val_self_test(struct iop_adma_device *device) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1014 | { |
| 1015 | int i, src_idx; |
| 1016 | struct page *dest; |
| 1017 | struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST]; |
| 1018 | struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 1019 | dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; |
Olof Johansson | f9f0a7d | 2013-07-08 15:59:35 -0700 | [diff] [blame] | 1020 | dma_addr_t dest_dma; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1021 | struct dma_async_tx_descriptor *tx; |
| 1022 | struct dma_chan *dma_chan; |
| 1023 | dma_cookie_t cookie; |
| 1024 | u8 cmp_byte = 0; |
| 1025 | u32 cmp_word; |
| 1026 | u32 zero_sum_result; |
| 1027 | int err = 0; |
| 1028 | struct iop_adma_chan *iop_chan; |
| 1029 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 1030 | dev_dbg(device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1031 | |
| 1032 | for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { |
| 1033 | xor_srcs[src_idx] = alloc_page(GFP_KERNEL); |
Roel Kluin | a09b09a | 2009-02-25 13:56:21 +0100 | [diff] [blame] | 1034 | if (!xor_srcs[src_idx]) { |
| 1035 | while (src_idx--) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1036 | __free_page(xor_srcs[src_idx]); |
Roel Kluin | a09b09a | 2009-02-25 13:56:21 +0100 | [diff] [blame] | 1037 | return -ENOMEM; |
| 1038 | } |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1039 | } |
| 1040 | |
| 1041 | dest = alloc_page(GFP_KERNEL); |
Roel Kluin | a09b09a | 2009-02-25 13:56:21 +0100 | [diff] [blame] | 1042 | if (!dest) { |
| 1043 | while (src_idx--) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1044 | __free_page(xor_srcs[src_idx]); |
Roel Kluin | a09b09a | 2009-02-25 13:56:21 +0100 | [diff] [blame] | 1045 | return -ENOMEM; |
| 1046 | } |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1047 | |
| 1048 | /* Fill in src buffers */ |
| 1049 | for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { |
| 1050 | u8 *ptr = page_address(xor_srcs[src_idx]); |
| 1051 | for (i = 0; i < PAGE_SIZE; i++) |
| 1052 | ptr[i] = (1 << src_idx); |
| 1053 | } |
| 1054 | |
| 1055 | for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) |
| 1056 | cmp_byte ^= (u8) (1 << src_idx); |
| 1057 | |
| 1058 | cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | |
| 1059 | (cmp_byte << 8) | cmp_byte; |
| 1060 | |
| 1061 | memset(page_address(dest), 0, PAGE_SIZE); |
| 1062 | |
| 1063 | dma_chan = container_of(device->common.channels.next, |
| 1064 | struct dma_chan, |
| 1065 | device_node); |
Dan Williams | aa1e6f1 | 2009-01-06 11:38:17 -0700 | [diff] [blame] | 1066 | if (iop_adma_alloc_chan_resources(dma_chan) < 1) { |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1067 | err = -ENODEV; |
| 1068 | goto out; |
| 1069 | } |
| 1070 | |
| 1071 | /* test xor */ |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1072 | dest_dma = dma_map_page(dma_chan->device->dev, dest, 0, |
| 1073 | PAGE_SIZE, DMA_FROM_DEVICE); |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 1074 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) |
| 1075 | dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i], |
| 1076 | 0, PAGE_SIZE, DMA_TO_DEVICE); |
| 1077 | tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs, |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 1078 | IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE, |
| 1079 | DMA_PREP_INTERRUPT | DMA_CTRL_ACK); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1080 | |
| 1081 | cookie = iop_adma_tx_submit(tx); |
| 1082 | iop_adma_issue_pending(dma_chan); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1083 | msleep(8); |
| 1084 | |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1085 | if (iop_adma_status(dma_chan, cookie, NULL) != |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1086 | DMA_SUCCESS) { |
Joe Perches | 1ba151c | 2012-10-28 01:05:44 -0700 | [diff] [blame] | 1087 | dev_err(dma_chan->device->dev, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1088 | "Self-test xor timed out, disabling\n"); |
| 1089 | err = -ENODEV; |
| 1090 | goto free_resources; |
| 1091 | } |
| 1092 | |
| 1093 | iop_chan = to_iop_adma_chan(dma_chan); |
| 1094 | dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, |
| 1095 | PAGE_SIZE, DMA_FROM_DEVICE); |
| 1096 | for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { |
| 1097 | u32 *ptr = page_address(dest); |
| 1098 | if (ptr[i] != cmp_word) { |
Joe Perches | 1ba151c | 2012-10-28 01:05:44 -0700 | [diff] [blame] | 1099 | dev_err(dma_chan->device->dev, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1100 | "Self-test xor failed compare, disabling\n"); |
| 1101 | err = -ENODEV; |
| 1102 | goto free_resources; |
| 1103 | } |
| 1104 | } |
| 1105 | dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma, |
| 1106 | PAGE_SIZE, DMA_TO_DEVICE); |
| 1107 | |
| 1108 | /* skip zero sum if the capability is not present */ |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 1109 | if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask)) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1110 | goto free_resources; |
| 1111 | |
| 1112 | /* zero sum the sources with the destintation page */ |
| 1113 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) |
| 1114 | zero_sum_srcs[i] = xor_srcs[i]; |
| 1115 | zero_sum_srcs[i] = dest; |
| 1116 | |
| 1117 | zero_sum_result = 1; |
| 1118 | |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 1119 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) |
| 1120 | dma_srcs[i] = dma_map_page(dma_chan->device->dev, |
| 1121 | zero_sum_srcs[i], 0, PAGE_SIZE, |
| 1122 | DMA_TO_DEVICE); |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 1123 | tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, |
| 1124 | IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, |
| 1125 | &zero_sum_result, |
| 1126 | DMA_PREP_INTERRUPT | DMA_CTRL_ACK); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1127 | |
| 1128 | cookie = iop_adma_tx_submit(tx); |
| 1129 | iop_adma_issue_pending(dma_chan); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1130 | msleep(8); |
| 1131 | |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1132 | if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { |
Joe Perches | 1ba151c | 2012-10-28 01:05:44 -0700 | [diff] [blame] | 1133 | dev_err(dma_chan->device->dev, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1134 | "Self-test zero sum timed out, disabling\n"); |
| 1135 | err = -ENODEV; |
| 1136 | goto free_resources; |
| 1137 | } |
| 1138 | |
| 1139 | if (zero_sum_result != 0) { |
Joe Perches | 1ba151c | 2012-10-28 01:05:44 -0700 | [diff] [blame] | 1140 | dev_err(dma_chan->device->dev, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1141 | "Self-test zero sum failed compare, disabling\n"); |
| 1142 | err = -ENODEV; |
| 1143 | goto free_resources; |
| 1144 | } |
| 1145 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1146 | /* test for non-zero parity sum */ |
| 1147 | zero_sum_result = 0; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 1148 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) |
| 1149 | dma_srcs[i] = dma_map_page(dma_chan->device->dev, |
| 1150 | zero_sum_srcs[i], 0, PAGE_SIZE, |
| 1151 | DMA_TO_DEVICE); |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 1152 | tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, |
| 1153 | IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, |
| 1154 | &zero_sum_result, |
| 1155 | DMA_PREP_INTERRUPT | DMA_CTRL_ACK); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1156 | |
| 1157 | cookie = iop_adma_tx_submit(tx); |
| 1158 | iop_adma_issue_pending(dma_chan); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1159 | msleep(8); |
| 1160 | |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1161 | if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { |
Joe Perches | 1ba151c | 2012-10-28 01:05:44 -0700 | [diff] [blame] | 1162 | dev_err(dma_chan->device->dev, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1163 | "Self-test non-zero sum timed out, disabling\n"); |
| 1164 | err = -ENODEV; |
| 1165 | goto free_resources; |
| 1166 | } |
| 1167 | |
| 1168 | if (zero_sum_result != 1) { |
Joe Perches | 1ba151c | 2012-10-28 01:05:44 -0700 | [diff] [blame] | 1169 | dev_err(dma_chan->device->dev, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1170 | "Self-test non-zero sum failed compare, disabling\n"); |
| 1171 | err = -ENODEV; |
| 1172 | goto free_resources; |
| 1173 | } |
| 1174 | |
| 1175 | free_resources: |
| 1176 | iop_adma_free_chan_resources(dma_chan); |
| 1177 | out: |
| 1178 | src_idx = IOP_ADMA_NUM_SRC_TEST; |
| 1179 | while (src_idx--) |
| 1180 | __free_page(xor_srcs[src_idx]); |
| 1181 | __free_page(dest); |
| 1182 | return err; |
| 1183 | } |
| 1184 | |
Wei Yongquan | 0261f74 | 2010-12-29 20:30:55 +0800 | [diff] [blame] | 1185 | #ifdef CONFIG_RAID6_PQ |
Bill Pemberton | 463a1f8 | 2012-11-19 13:22:55 -0500 | [diff] [blame] | 1186 | static int |
Dan Williams | f6dbf65 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 1187 | iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device) |
| 1188 | { |
| 1189 | /* combined sources, software pq results, and extra hw pq results */ |
| 1190 | struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2]; |
| 1191 | /* ptr to the extra hw pq buffers defined above */ |
| 1192 | struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2]; |
| 1193 | /* address conversion buffers (dma_map / page_address) */ |
| 1194 | void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2]; |
Don Morris | 3d9ea9e | 2012-03-15 11:07:30 -0700 | [diff] [blame] | 1195 | dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST+2]; |
| 1196 | dma_addr_t *pq_dest = &pq_src[IOP_ADMA_NUM_SRC_TEST]; |
Dan Williams | f6dbf65 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 1197 | |
| 1198 | int i; |
| 1199 | struct dma_async_tx_descriptor *tx; |
| 1200 | struct dma_chan *dma_chan; |
| 1201 | dma_cookie_t cookie; |
| 1202 | u32 zero_sum_result; |
| 1203 | int err = 0; |
| 1204 | struct device *dev; |
| 1205 | |
| 1206 | dev_dbg(device->common.dev, "%s\n", __func__); |
| 1207 | |
| 1208 | for (i = 0; i < ARRAY_SIZE(pq); i++) { |
| 1209 | pq[i] = alloc_page(GFP_KERNEL); |
| 1210 | if (!pq[i]) { |
| 1211 | while (i--) |
| 1212 | __free_page(pq[i]); |
| 1213 | return -ENOMEM; |
| 1214 | } |
| 1215 | } |
| 1216 | |
| 1217 | /* Fill in src buffers */ |
| 1218 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) { |
| 1219 | pq_sw[i] = page_address(pq[i]); |
| 1220 | memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE); |
| 1221 | } |
| 1222 | pq_sw[i] = page_address(pq[i]); |
| 1223 | pq_sw[i+1] = page_address(pq[i+1]); |
| 1224 | |
| 1225 | dma_chan = container_of(device->common.channels.next, |
| 1226 | struct dma_chan, |
| 1227 | device_node); |
| 1228 | if (iop_adma_alloc_chan_resources(dma_chan) < 1) { |
| 1229 | err = -ENODEV; |
| 1230 | goto out; |
| 1231 | } |
| 1232 | |
| 1233 | dev = dma_chan->device->dev; |
| 1234 | |
| 1235 | /* initialize the dests */ |
| 1236 | memset(page_address(pq_hw[0]), 0 , PAGE_SIZE); |
| 1237 | memset(page_address(pq_hw[1]), 0 , PAGE_SIZE); |
| 1238 | |
| 1239 | /* test pq */ |
| 1240 | pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE); |
| 1241 | pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE); |
| 1242 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) |
| 1243 | pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, |
| 1244 | DMA_TO_DEVICE); |
| 1245 | |
| 1246 | tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src, |
| 1247 | IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp, |
| 1248 | PAGE_SIZE, |
| 1249 | DMA_PREP_INTERRUPT | |
| 1250 | DMA_CTRL_ACK); |
| 1251 | |
| 1252 | cookie = iop_adma_tx_submit(tx); |
| 1253 | iop_adma_issue_pending(dma_chan); |
| 1254 | msleep(8); |
| 1255 | |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1256 | if (iop_adma_status(dma_chan, cookie, NULL) != |
Dan Williams | f6dbf65 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 1257 | DMA_SUCCESS) { |
| 1258 | dev_err(dev, "Self-test pq timed out, disabling\n"); |
| 1259 | err = -ENODEV; |
| 1260 | goto free_resources; |
| 1261 | } |
| 1262 | |
| 1263 | raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw); |
| 1264 | |
| 1265 | if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST], |
| 1266 | page_address(pq_hw[0]), PAGE_SIZE) != 0) { |
| 1267 | dev_err(dev, "Self-test p failed compare, disabling\n"); |
| 1268 | err = -ENODEV; |
| 1269 | goto free_resources; |
| 1270 | } |
| 1271 | if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1], |
| 1272 | page_address(pq_hw[1]), PAGE_SIZE) != 0) { |
| 1273 | dev_err(dev, "Self-test q failed compare, disabling\n"); |
| 1274 | err = -ENODEV; |
| 1275 | goto free_resources; |
| 1276 | } |
| 1277 | |
| 1278 | /* test correct zero sum using the software generated pq values */ |
| 1279 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) |
| 1280 | pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, |
| 1281 | DMA_TO_DEVICE); |
| 1282 | |
| 1283 | zero_sum_result = ~0; |
| 1284 | tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], |
| 1285 | pq_src, IOP_ADMA_NUM_SRC_TEST, |
| 1286 | raid6_gfexp, PAGE_SIZE, &zero_sum_result, |
| 1287 | DMA_PREP_INTERRUPT|DMA_CTRL_ACK); |
| 1288 | |
| 1289 | cookie = iop_adma_tx_submit(tx); |
| 1290 | iop_adma_issue_pending(dma_chan); |
| 1291 | msleep(8); |
| 1292 | |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1293 | if (iop_adma_status(dma_chan, cookie, NULL) != |
Dan Williams | f6dbf65 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 1294 | DMA_SUCCESS) { |
| 1295 | dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n"); |
| 1296 | err = -ENODEV; |
| 1297 | goto free_resources; |
| 1298 | } |
| 1299 | |
| 1300 | if (zero_sum_result != 0) { |
| 1301 | dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n", |
| 1302 | zero_sum_result); |
| 1303 | err = -ENODEV; |
| 1304 | goto free_resources; |
| 1305 | } |
| 1306 | |
| 1307 | /* test incorrect zero sum */ |
| 1308 | i = IOP_ADMA_NUM_SRC_TEST; |
| 1309 | memset(pq_sw[i] + 100, 0, 100); |
| 1310 | memset(pq_sw[i+1] + 200, 0, 200); |
| 1311 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) |
| 1312 | pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, |
| 1313 | DMA_TO_DEVICE); |
| 1314 | |
| 1315 | zero_sum_result = 0; |
| 1316 | tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], |
| 1317 | pq_src, IOP_ADMA_NUM_SRC_TEST, |
| 1318 | raid6_gfexp, PAGE_SIZE, &zero_sum_result, |
| 1319 | DMA_PREP_INTERRUPT|DMA_CTRL_ACK); |
| 1320 | |
| 1321 | cookie = iop_adma_tx_submit(tx); |
| 1322 | iop_adma_issue_pending(dma_chan); |
| 1323 | msleep(8); |
| 1324 | |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1325 | if (iop_adma_status(dma_chan, cookie, NULL) != |
Dan Williams | f6dbf65 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 1326 | DMA_SUCCESS) { |
| 1327 | dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n"); |
| 1328 | err = -ENODEV; |
| 1329 | goto free_resources; |
| 1330 | } |
| 1331 | |
| 1332 | if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) { |
| 1333 | dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n", |
| 1334 | zero_sum_result); |
| 1335 | err = -ENODEV; |
| 1336 | goto free_resources; |
| 1337 | } |
| 1338 | |
| 1339 | free_resources: |
| 1340 | iop_adma_free_chan_resources(dma_chan); |
| 1341 | out: |
| 1342 | i = ARRAY_SIZE(pq); |
| 1343 | while (i--) |
| 1344 | __free_page(pq[i]); |
| 1345 | return err; |
| 1346 | } |
| 1347 | #endif |
| 1348 | |
Greg Kroah-Hartman | 4bf27b8 | 2012-12-21 15:09:59 -0800 | [diff] [blame] | 1349 | static int iop_adma_remove(struct platform_device *dev) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1350 | { |
| 1351 | struct iop_adma_device *device = platform_get_drvdata(dev); |
| 1352 | struct dma_chan *chan, *_chan; |
| 1353 | struct iop_adma_chan *iop_chan; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1354 | struct iop_adma_platform_data *plat_data = dev->dev.platform_data; |
| 1355 | |
| 1356 | dma_async_device_unregister(&device->common); |
| 1357 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1358 | dma_free_coherent(&dev->dev, plat_data->pool_size, |
| 1359 | device->dma_desc_pool_virt, device->dma_desc_pool); |
| 1360 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1361 | list_for_each_entry_safe(chan, _chan, &device->common.channels, |
| 1362 | device_node) { |
| 1363 | iop_chan = to_iop_adma_chan(chan); |
| 1364 | list_del(&chan->device_node); |
| 1365 | kfree(iop_chan); |
| 1366 | } |
| 1367 | kfree(device); |
| 1368 | |
| 1369 | return 0; |
| 1370 | } |
| 1371 | |
Bill Pemberton | 463a1f8 | 2012-11-19 13:22:55 -0500 | [diff] [blame] | 1372 | static int iop_adma_probe(struct platform_device *pdev) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1373 | { |
| 1374 | struct resource *res; |
| 1375 | int ret = 0, i; |
| 1376 | struct iop_adma_device *adev; |
| 1377 | struct iop_adma_chan *iop_chan; |
| 1378 | struct dma_device *dma_dev; |
| 1379 | struct iop_adma_platform_data *plat_data = pdev->dev.platform_data; |
| 1380 | |
| 1381 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
| 1382 | if (!res) |
| 1383 | return -ENODEV; |
| 1384 | |
| 1385 | if (!devm_request_mem_region(&pdev->dev, res->start, |
H Hartley Sweeten | 2e032b6 | 2009-12-11 21:24:33 -0700 | [diff] [blame] | 1386 | resource_size(res), pdev->name)) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1387 | return -EBUSY; |
| 1388 | |
| 1389 | adev = kzalloc(sizeof(*adev), GFP_KERNEL); |
| 1390 | if (!adev) |
| 1391 | return -ENOMEM; |
| 1392 | dma_dev = &adev->common; |
| 1393 | |
| 1394 | /* allocate coherent memory for hardware descriptors |
| 1395 | * note: writecombine gives slightly better performance, but |
| 1396 | * requires that we explicitly flush the writes |
| 1397 | */ |
| 1398 | if ((adev->dma_desc_pool_virt = dma_alloc_writecombine(&pdev->dev, |
| 1399 | plat_data->pool_size, |
| 1400 | &adev->dma_desc_pool, |
| 1401 | GFP_KERNEL)) == NULL) { |
| 1402 | ret = -ENOMEM; |
| 1403 | goto err_free_adev; |
| 1404 | } |
| 1405 | |
Masanari Iida | b6695e4 | 2012-01-27 23:00:53 +0900 | [diff] [blame] | 1406 | dev_dbg(&pdev->dev, "%s: allocated descriptor pool virt %p phys %p\n", |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 1407 | __func__, adev->dma_desc_pool_virt, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1408 | (void *) adev->dma_desc_pool); |
| 1409 | |
| 1410 | adev->id = plat_data->hw_id; |
| 1411 | |
| 1412 | /* discover transaction capabilites from the platform data */ |
| 1413 | dma_dev->cap_mask = plat_data->cap_mask; |
| 1414 | |
| 1415 | adev->pdev = pdev; |
| 1416 | platform_set_drvdata(pdev, adev); |
| 1417 | |
| 1418 | INIT_LIST_HEAD(&dma_dev->channels); |
| 1419 | |
| 1420 | /* set base routines */ |
| 1421 | dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources; |
| 1422 | dma_dev->device_free_chan_resources = iop_adma_free_chan_resources; |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1423 | dma_dev->device_tx_status = iop_adma_status; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1424 | dma_dev->device_issue_pending = iop_adma_issue_pending; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1425 | dma_dev->dev = &pdev->dev; |
| 1426 | |
| 1427 | /* set prep routines based on capability */ |
| 1428 | if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) |
| 1429 | dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1430 | if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { |
| 1431 | dma_dev->max_xor = iop_adma_get_max_xor(); |
| 1432 | dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor; |
| 1433 | } |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 1434 | if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask)) |
| 1435 | dma_dev->device_prep_dma_xor_val = |
| 1436 | iop_adma_prep_dma_xor_val; |
Dan Williams | 7bf649a | 2009-08-28 14:32:04 -0700 | [diff] [blame] | 1437 | if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { |
| 1438 | dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0); |
| 1439 | dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq; |
| 1440 | } |
| 1441 | if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) |
| 1442 | dma_dev->device_prep_dma_pq_val = |
| 1443 | iop_adma_prep_dma_pq_val; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1444 | if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) |
| 1445 | dma_dev->device_prep_dma_interrupt = |
| 1446 | iop_adma_prep_dma_interrupt; |
| 1447 | |
| 1448 | iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL); |
| 1449 | if (!iop_chan) { |
| 1450 | ret = -ENOMEM; |
| 1451 | goto err_free_dma; |
| 1452 | } |
| 1453 | iop_chan->device = adev; |
| 1454 | |
| 1455 | iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start, |
H Hartley Sweeten | 2e032b6 | 2009-12-11 21:24:33 -0700 | [diff] [blame] | 1456 | resource_size(res)); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1457 | if (!iop_chan->mmr_base) { |
| 1458 | ret = -ENOMEM; |
| 1459 | goto err_free_iop_chan; |
| 1460 | } |
| 1461 | tasklet_init(&iop_chan->irq_tasklet, iop_adma_tasklet, (unsigned long) |
| 1462 | iop_chan); |
| 1463 | |
| 1464 | /* clear errors before enabling interrupts */ |
| 1465 | iop_adma_device_clear_err_status(iop_chan); |
| 1466 | |
| 1467 | for (i = 0; i < 3; i++) { |
| 1468 | irq_handler_t handler[] = { iop_adma_eot_handler, |
| 1469 | iop_adma_eoc_handler, |
| 1470 | iop_adma_err_handler }; |
| 1471 | int irq = platform_get_irq(pdev, i); |
| 1472 | if (irq < 0) { |
| 1473 | ret = -ENXIO; |
| 1474 | goto err_free_iop_chan; |
| 1475 | } else { |
| 1476 | ret = devm_request_irq(&pdev->dev, irq, |
| 1477 | handler[i], 0, pdev->name, iop_chan); |
| 1478 | if (ret) |
| 1479 | goto err_free_iop_chan; |
| 1480 | } |
| 1481 | } |
| 1482 | |
| 1483 | spin_lock_init(&iop_chan->lock); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1484 | INIT_LIST_HEAD(&iop_chan->chain); |
| 1485 | INIT_LIST_HEAD(&iop_chan->all_slots); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1486 | iop_chan->common.device = dma_dev; |
Russell King - ARM Linux | 8ac6954 | 2012-03-06 22:36:27 +0000 | [diff] [blame] | 1487 | dma_cookie_init(&iop_chan->common); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1488 | list_add_tail(&iop_chan->common.device_node, &dma_dev->channels); |
| 1489 | |
| 1490 | if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { |
| 1491 | ret = iop_adma_memcpy_self_test(adev); |
| 1492 | dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret); |
| 1493 | if (ret) |
| 1494 | goto err_free_iop_chan; |
| 1495 | } |
| 1496 | |
Bartlomiej Zolnierkiewicz | 48a9db4 | 2013-07-03 15:05:06 -0700 | [diff] [blame] | 1497 | if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 1498 | ret = iop_adma_xor_val_self_test(adev); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1499 | dev_dbg(&pdev->dev, "xor self test returned %d\n", ret); |
| 1500 | if (ret) |
| 1501 | goto err_free_iop_chan; |
| 1502 | } |
| 1503 | |
Dan Williams | f6dbf65 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 1504 | if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) && |
| 1505 | dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) { |
Wei Yongquan | 0261f74 | 2010-12-29 20:30:55 +0800 | [diff] [blame] | 1506 | #ifdef CONFIG_RAID6_PQ |
Dan Williams | f6dbf65 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 1507 | ret = iop_adma_pq_zero_sum_self_test(adev); |
| 1508 | dev_dbg(&pdev->dev, "pq self test returned %d\n", ret); |
| 1509 | #else |
| 1510 | /* can not test raid6, so do not publish capability */ |
| 1511 | dma_cap_clear(DMA_PQ, dma_dev->cap_mask); |
| 1512 | dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask); |
| 1513 | ret = 0; |
| 1514 | #endif |
| 1515 | if (ret) |
| 1516 | goto err_free_iop_chan; |
| 1517 | } |
| 1518 | |
Olof Johansson | f9f0a7d | 2013-07-08 15:59:35 -0700 | [diff] [blame] | 1519 | dev_info(&pdev->dev, "Intel(R) IOP: ( %s%s%s%s%s%s)\n", |
Joe Perches | 1ba151c | 2012-10-28 01:05:44 -0700 | [diff] [blame] | 1520 | dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "", |
| 1521 | dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "", |
| 1522 | dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", |
| 1523 | dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "", |
Joe Perches | 1ba151c | 2012-10-28 01:05:44 -0700 | [diff] [blame] | 1524 | dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "", |
| 1525 | dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : ""); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1526 | |
| 1527 | dma_async_device_register(dma_dev); |
| 1528 | goto out; |
| 1529 | |
| 1530 | err_free_iop_chan: |
| 1531 | kfree(iop_chan); |
| 1532 | err_free_dma: |
| 1533 | dma_free_coherent(&adev->pdev->dev, plat_data->pool_size, |
| 1534 | adev->dma_desc_pool_virt, adev->dma_desc_pool); |
| 1535 | err_free_adev: |
| 1536 | kfree(adev); |
| 1537 | out: |
| 1538 | return ret; |
| 1539 | } |
| 1540 | |
| 1541 | static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan) |
| 1542 | { |
| 1543 | struct iop_adma_desc_slot *sw_desc, *grp_start; |
| 1544 | dma_cookie_t cookie; |
| 1545 | int slot_cnt, slots_per_op; |
| 1546 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 1547 | dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1548 | |
| 1549 | spin_lock_bh(&iop_chan->lock); |
| 1550 | slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op); |
| 1551 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 1552 | if (sw_desc) { |
| 1553 | grp_start = sw_desc->group_head; |
| 1554 | |
Dan Williams | 308136d | 2009-09-08 17:53:02 -0700 | [diff] [blame] | 1555 | list_splice_init(&sw_desc->tx_list, &iop_chan->chain); |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 1556 | async_tx_ack(&sw_desc->async_tx); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1557 | iop_desc_init_memcpy(grp_start, 0); |
| 1558 | iop_desc_set_byte_count(grp_start, iop_chan, 0); |
| 1559 | iop_desc_set_dest_addr(grp_start, iop_chan, 0); |
| 1560 | iop_desc_set_memcpy_src_addr(grp_start, 0); |
| 1561 | |
Russell King - ARM Linux | 2a926e4 | 2012-03-06 22:36:07 +0000 | [diff] [blame] | 1562 | cookie = dma_cookie_assign(&sw_desc->async_tx); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1563 | |
| 1564 | /* initialize the completed cookie to be less than |
| 1565 | * the most recently used cookie |
| 1566 | */ |
Russell King - ARM Linux | 4d4e58d | 2012-03-06 22:34:06 +0000 | [diff] [blame] | 1567 | iop_chan->common.completed_cookie = cookie - 1; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1568 | |
| 1569 | /* channel should not be busy */ |
| 1570 | BUG_ON(iop_chan_is_busy(iop_chan)); |
| 1571 | |
| 1572 | /* clear any prior error-status bits */ |
| 1573 | iop_adma_device_clear_err_status(iop_chan); |
| 1574 | |
| 1575 | /* disable operation */ |
| 1576 | iop_chan_disable(iop_chan); |
| 1577 | |
| 1578 | /* set the descriptor address */ |
| 1579 | iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); |
| 1580 | |
| 1581 | /* 1/ don't add pre-chained descriptors |
| 1582 | * 2/ dummy read to flush next_desc write |
| 1583 | */ |
| 1584 | BUG_ON(iop_desc_get_next_desc(sw_desc)); |
| 1585 | |
| 1586 | /* run the descriptor */ |
| 1587 | iop_chan_enable(iop_chan); |
| 1588 | } else |
Joe Perches | 1ba151c | 2012-10-28 01:05:44 -0700 | [diff] [blame] | 1589 | dev_err(iop_chan->device->common.dev, |
| 1590 | "failed to allocate null descriptor\n"); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1591 | spin_unlock_bh(&iop_chan->lock); |
| 1592 | } |
| 1593 | |
| 1594 | static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan) |
| 1595 | { |
| 1596 | struct iop_adma_desc_slot *sw_desc, *grp_start; |
| 1597 | dma_cookie_t cookie; |
| 1598 | int slot_cnt, slots_per_op; |
| 1599 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 1600 | dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1601 | |
| 1602 | spin_lock_bh(&iop_chan->lock); |
| 1603 | slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op); |
| 1604 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 1605 | if (sw_desc) { |
| 1606 | grp_start = sw_desc->group_head; |
Dan Williams | 308136d | 2009-09-08 17:53:02 -0700 | [diff] [blame] | 1607 | list_splice_init(&sw_desc->tx_list, &iop_chan->chain); |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 1608 | async_tx_ack(&sw_desc->async_tx); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1609 | iop_desc_init_null_xor(grp_start, 2, 0); |
| 1610 | iop_desc_set_byte_count(grp_start, iop_chan, 0); |
| 1611 | iop_desc_set_dest_addr(grp_start, iop_chan, 0); |
| 1612 | iop_desc_set_xor_src_addr(grp_start, 0, 0); |
| 1613 | iop_desc_set_xor_src_addr(grp_start, 1, 0); |
| 1614 | |
Russell King - ARM Linux | 2a926e4 | 2012-03-06 22:36:07 +0000 | [diff] [blame] | 1615 | cookie = dma_cookie_assign(&sw_desc->async_tx); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1616 | |
| 1617 | /* initialize the completed cookie to be less than |
| 1618 | * the most recently used cookie |
| 1619 | */ |
Russell King - ARM Linux | 4d4e58d | 2012-03-06 22:34:06 +0000 | [diff] [blame] | 1620 | iop_chan->common.completed_cookie = cookie - 1; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1621 | |
| 1622 | /* channel should not be busy */ |
| 1623 | BUG_ON(iop_chan_is_busy(iop_chan)); |
| 1624 | |
| 1625 | /* clear any prior error-status bits */ |
| 1626 | iop_adma_device_clear_err_status(iop_chan); |
| 1627 | |
| 1628 | /* disable operation */ |
| 1629 | iop_chan_disable(iop_chan); |
| 1630 | |
| 1631 | /* set the descriptor address */ |
| 1632 | iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); |
| 1633 | |
| 1634 | /* 1/ don't add pre-chained descriptors |
| 1635 | * 2/ dummy read to flush next_desc write |
| 1636 | */ |
| 1637 | BUG_ON(iop_desc_get_next_desc(sw_desc)); |
| 1638 | |
| 1639 | /* run the descriptor */ |
| 1640 | iop_chan_enable(iop_chan); |
| 1641 | } else |
Joe Perches | 1ba151c | 2012-10-28 01:05:44 -0700 | [diff] [blame] | 1642 | dev_err(iop_chan->device->common.dev, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1643 | "failed to allocate null descriptor\n"); |
| 1644 | spin_unlock_bh(&iop_chan->lock); |
| 1645 | } |
| 1646 | |
| 1647 | static struct platform_driver iop_adma_driver = { |
| 1648 | .probe = iop_adma_probe, |
Bill Pemberton | a7d6e3e | 2012-11-19 13:20:04 -0500 | [diff] [blame] | 1649 | .remove = iop_adma_remove, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1650 | .driver = { |
| 1651 | .owner = THIS_MODULE, |
| 1652 | .name = "iop-adma", |
| 1653 | }, |
| 1654 | }; |
| 1655 | |
Axel Lin | c94e910 | 2011-11-26 15:11:12 +0800 | [diff] [blame] | 1656 | module_platform_driver(iop_adma_driver); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1657 | |
| 1658 | MODULE_AUTHOR("Intel Corporation"); |
| 1659 | MODULE_DESCRIPTION("IOP ADMA Engine Driver"); |
| 1660 | MODULE_LICENSE("GPL"); |
Axel Lin | c94e910 | 2011-11-26 15:11:12 +0800 | [diff] [blame] | 1661 | MODULE_ALIAS("platform:iop-adma"); |