Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1 | /* |
| 2 | * offload engine driver for the Intel Xscale series of i/o processors |
| 3 | * Copyright © 2006, Intel Corporation. |
| 4 | * |
| 5 | * This program is free software; you can redistribute it and/or modify it |
| 6 | * under the terms and conditions of the GNU General Public License, |
| 7 | * version 2, as published by the Free Software Foundation. |
| 8 | * |
| 9 | * This program is distributed in the hope it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 12 | * more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU General Public License along with |
| 15 | * this program; if not, write to the Free Software Foundation, Inc., |
| 16 | * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. |
| 17 | * |
| 18 | */ |
| 19 | |
| 20 | /* |
| 21 | * This driver supports the asynchrounous DMA copy and RAID engines available |
| 22 | * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) |
| 23 | */ |
| 24 | |
| 25 | #include <linux/init.h> |
| 26 | #include <linux/module.h> |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 27 | #include <linux/delay.h> |
| 28 | #include <linux/dma-mapping.h> |
| 29 | #include <linux/spinlock.h> |
| 30 | #include <linux/interrupt.h> |
| 31 | #include <linux/platform_device.h> |
| 32 | #include <linux/memory.h> |
| 33 | #include <linux/ioport.h> |
Dan Williams | f6dbf651 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 34 | #include <linux/raid/pq.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 35 | #include <linux/slab.h> |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 36 | |
Russell King | a09e64f | 2008-08-05 16:14:15 +0100 | [diff] [blame] | 37 | #include <mach/adma.h> |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 38 | |
| 39 | #define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common) |
| 40 | #define to_iop_adma_device(dev) \ |
| 41 | container_of(dev, struct iop_adma_device, common) |
| 42 | #define tx_to_iop_adma_slot(tx) \ |
| 43 | container_of(tx, struct iop_adma_desc_slot, async_tx) |
| 44 | |
| 45 | /** |
| 46 | * iop_adma_free_slots - flags descriptor slots for reuse |
| 47 | * @slot: Slot to free |
| 48 | * Caller must hold &iop_chan->lock while calling this function |
| 49 | */ |
| 50 | static void iop_adma_free_slots(struct iop_adma_desc_slot *slot) |
| 51 | { |
| 52 | int stride = slot->slots_per_op; |
| 53 | |
| 54 | while (stride--) { |
| 55 | slot->slots_per_op = 0; |
| 56 | slot = list_entry(slot->slot_node.next, |
| 57 | struct iop_adma_desc_slot, |
| 58 | slot_node); |
| 59 | } |
| 60 | } |
| 61 | |
Dan Williams | 7bf649a | 2009-08-28 14:32:04 -0700 | [diff] [blame] | 62 | static void |
| 63 | iop_desc_unmap(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc) |
| 64 | { |
| 65 | struct dma_async_tx_descriptor *tx = &desc->async_tx; |
| 66 | struct iop_adma_desc_slot *unmap = desc->group_head; |
| 67 | struct device *dev = &iop_chan->device->pdev->dev; |
| 68 | u32 len = unmap->unmap_len; |
| 69 | enum dma_ctrl_flags flags = tx->flags; |
| 70 | u32 src_cnt; |
| 71 | dma_addr_t addr; |
| 72 | dma_addr_t dest; |
| 73 | |
| 74 | src_cnt = unmap->unmap_src_cnt; |
| 75 | dest = iop_desc_get_dest_addr(unmap, iop_chan); |
| 76 | if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) { |
| 77 | enum dma_data_direction dir; |
| 78 | |
| 79 | if (src_cnt > 1) /* is xor? */ |
| 80 | dir = DMA_BIDIRECTIONAL; |
| 81 | else |
| 82 | dir = DMA_FROM_DEVICE; |
| 83 | |
| 84 | dma_unmap_page(dev, dest, len, dir); |
| 85 | } |
| 86 | |
| 87 | if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) { |
| 88 | while (src_cnt--) { |
| 89 | addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt); |
| 90 | if (addr == dest) |
| 91 | continue; |
| 92 | dma_unmap_page(dev, addr, len, DMA_TO_DEVICE); |
| 93 | } |
| 94 | } |
| 95 | desc->group_head = NULL; |
| 96 | } |
| 97 | |
| 98 | static void |
| 99 | iop_desc_unmap_pq(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc) |
| 100 | { |
| 101 | struct dma_async_tx_descriptor *tx = &desc->async_tx; |
| 102 | struct iop_adma_desc_slot *unmap = desc->group_head; |
| 103 | struct device *dev = &iop_chan->device->pdev->dev; |
| 104 | u32 len = unmap->unmap_len; |
| 105 | enum dma_ctrl_flags flags = tx->flags; |
| 106 | u32 src_cnt = unmap->unmap_src_cnt; |
| 107 | dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan); |
| 108 | dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan); |
| 109 | int i; |
| 110 | |
| 111 | if (tx->flags & DMA_PREP_CONTINUE) |
| 112 | src_cnt -= 3; |
| 113 | |
| 114 | if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) { |
| 115 | dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL); |
| 116 | dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL); |
| 117 | } |
| 118 | |
| 119 | if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) { |
| 120 | dma_addr_t addr; |
| 121 | |
| 122 | for (i = 0; i < src_cnt; i++) { |
| 123 | addr = iop_desc_get_src_addr(unmap, iop_chan, i); |
| 124 | dma_unmap_page(dev, addr, len, DMA_TO_DEVICE); |
| 125 | } |
| 126 | if (desc->pq_check_result) { |
| 127 | dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE); |
| 128 | dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE); |
| 129 | } |
| 130 | } |
| 131 | |
| 132 | desc->group_head = NULL; |
| 133 | } |
| 134 | |
| 135 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 136 | static dma_cookie_t |
| 137 | iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, |
| 138 | struct iop_adma_chan *iop_chan, dma_cookie_t cookie) |
| 139 | { |
Dan Williams | 507fbec | 2009-08-29 19:12:39 -0700 | [diff] [blame] | 140 | struct dma_async_tx_descriptor *tx = &desc->async_tx; |
| 141 | |
| 142 | BUG_ON(tx->cookie < 0); |
| 143 | if (tx->cookie > 0) { |
| 144 | cookie = tx->cookie; |
| 145 | tx->cookie = 0; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 146 | |
| 147 | /* call the callback (must not sleep or submit new |
| 148 | * operations to this channel) |
| 149 | */ |
Dan Williams | 507fbec | 2009-08-29 19:12:39 -0700 | [diff] [blame] | 150 | if (tx->callback) |
| 151 | tx->callback(tx->callback_param); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 152 | |
| 153 | /* unmap dma addresses |
| 154 | * (unmap_single vs unmap_page?) |
| 155 | */ |
| 156 | if (desc->group_head && desc->unmap_len) { |
Dan Williams | 7bf649a | 2009-08-28 14:32:04 -0700 | [diff] [blame] | 157 | if (iop_desc_is_pq(desc)) |
| 158 | iop_desc_unmap_pq(iop_chan, desc); |
| 159 | else |
| 160 | iop_desc_unmap(iop_chan, desc); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 161 | } |
| 162 | } |
| 163 | |
| 164 | /* run dependent operations */ |
Dan Williams | 507fbec | 2009-08-29 19:12:39 -0700 | [diff] [blame] | 165 | dma_run_dependencies(tx); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 166 | |
| 167 | return cookie; |
| 168 | } |
| 169 | |
| 170 | static int |
| 171 | iop_adma_clean_slot(struct iop_adma_desc_slot *desc, |
| 172 | struct iop_adma_chan *iop_chan) |
| 173 | { |
| 174 | /* the client is allowed to attach dependent operations |
| 175 | * until 'ack' is set |
| 176 | */ |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 177 | if (!async_tx_test_ack(&desc->async_tx)) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 178 | return 0; |
| 179 | |
| 180 | /* leave the last descriptor in the chain |
| 181 | * so we can append to it |
| 182 | */ |
| 183 | if (desc->chain_node.next == &iop_chan->chain) |
| 184 | return 1; |
| 185 | |
| 186 | dev_dbg(iop_chan->device->common.dev, |
| 187 | "\tfree slot: %d slots_per_op: %d\n", |
| 188 | desc->idx, desc->slots_per_op); |
| 189 | |
| 190 | list_del(&desc->chain_node); |
| 191 | iop_adma_free_slots(desc); |
| 192 | |
| 193 | return 0; |
| 194 | } |
| 195 | |
| 196 | static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) |
| 197 | { |
| 198 | struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL; |
| 199 | dma_cookie_t cookie = 0; |
| 200 | u32 current_desc = iop_chan_get_current_descriptor(iop_chan); |
| 201 | int busy = iop_chan_is_busy(iop_chan); |
| 202 | int seen_current = 0, slot_cnt = 0, slots_per_op = 0; |
| 203 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 204 | dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 205 | /* free completed slots from the chain starting with |
| 206 | * the oldest descriptor |
| 207 | */ |
| 208 | list_for_each_entry_safe(iter, _iter, &iop_chan->chain, |
| 209 | chain_node) { |
| 210 | pr_debug("\tcookie: %d slot: %d busy: %d " |
| 211 | "this_desc: %#x next_desc: %#x ack: %d\n", |
| 212 | iter->async_tx.cookie, iter->idx, busy, |
| 213 | iter->async_tx.phys, iop_desc_get_next_desc(iter), |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 214 | async_tx_test_ack(&iter->async_tx)); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 215 | prefetch(_iter); |
| 216 | prefetch(&_iter->async_tx); |
| 217 | |
| 218 | /* do not advance past the current descriptor loaded into the |
| 219 | * hardware channel, subsequent descriptors are either in |
| 220 | * process or have not been submitted |
| 221 | */ |
| 222 | if (seen_current) |
| 223 | break; |
| 224 | |
| 225 | /* stop the search if we reach the current descriptor and the |
| 226 | * channel is busy, or if it appears that the current descriptor |
| 227 | * needs to be re-read (i.e. has been appended to) |
| 228 | */ |
| 229 | if (iter->async_tx.phys == current_desc) { |
| 230 | BUG_ON(seen_current++); |
| 231 | if (busy || iop_desc_get_next_desc(iter)) |
| 232 | break; |
| 233 | } |
| 234 | |
| 235 | /* detect the start of a group transaction */ |
| 236 | if (!slot_cnt && !slots_per_op) { |
| 237 | slot_cnt = iter->slot_cnt; |
| 238 | slots_per_op = iter->slots_per_op; |
| 239 | if (slot_cnt <= slots_per_op) { |
| 240 | slot_cnt = 0; |
| 241 | slots_per_op = 0; |
| 242 | } |
| 243 | } |
| 244 | |
| 245 | if (slot_cnt) { |
| 246 | pr_debug("\tgroup++\n"); |
| 247 | if (!grp_start) |
| 248 | grp_start = iter; |
| 249 | slot_cnt -= slots_per_op; |
| 250 | } |
| 251 | |
| 252 | /* all the members of a group are complete */ |
| 253 | if (slots_per_op != 0 && slot_cnt == 0) { |
| 254 | struct iop_adma_desc_slot *grp_iter, *_grp_iter; |
| 255 | int end_of_chain = 0; |
| 256 | pr_debug("\tgroup end\n"); |
| 257 | |
| 258 | /* collect the total results */ |
| 259 | if (grp_start->xor_check_result) { |
| 260 | u32 zero_sum_result = 0; |
| 261 | slot_cnt = grp_start->slot_cnt; |
| 262 | grp_iter = grp_start; |
| 263 | |
| 264 | list_for_each_entry_from(grp_iter, |
| 265 | &iop_chan->chain, chain_node) { |
| 266 | zero_sum_result |= |
| 267 | iop_desc_get_zero_result(grp_iter); |
| 268 | pr_debug("\titer%d result: %d\n", |
| 269 | grp_iter->idx, zero_sum_result); |
| 270 | slot_cnt -= slots_per_op; |
| 271 | if (slot_cnt == 0) |
| 272 | break; |
| 273 | } |
| 274 | pr_debug("\tgrp_start->xor_check_result: %p\n", |
| 275 | grp_start->xor_check_result); |
| 276 | *grp_start->xor_check_result = zero_sum_result; |
| 277 | } |
| 278 | |
| 279 | /* clean up the group */ |
| 280 | slot_cnt = grp_start->slot_cnt; |
| 281 | grp_iter = grp_start; |
| 282 | list_for_each_entry_safe_from(grp_iter, _grp_iter, |
| 283 | &iop_chan->chain, chain_node) { |
| 284 | cookie = iop_adma_run_tx_complete_actions( |
| 285 | grp_iter, iop_chan, cookie); |
| 286 | |
| 287 | slot_cnt -= slots_per_op; |
| 288 | end_of_chain = iop_adma_clean_slot(grp_iter, |
| 289 | iop_chan); |
| 290 | |
| 291 | if (slot_cnt == 0 || end_of_chain) |
| 292 | break; |
| 293 | } |
| 294 | |
| 295 | /* the group should be complete at this point */ |
| 296 | BUG_ON(slot_cnt); |
| 297 | |
| 298 | slots_per_op = 0; |
| 299 | grp_start = NULL; |
| 300 | if (end_of_chain) |
| 301 | break; |
| 302 | else |
| 303 | continue; |
| 304 | } else if (slots_per_op) /* wait for group completion */ |
| 305 | continue; |
| 306 | |
| 307 | /* write back zero sum results (single descriptor case) */ |
| 308 | if (iter->xor_check_result && iter->async_tx.cookie) |
| 309 | *iter->xor_check_result = |
| 310 | iop_desc_get_zero_result(iter); |
| 311 | |
| 312 | cookie = iop_adma_run_tx_complete_actions( |
| 313 | iter, iop_chan, cookie); |
| 314 | |
| 315 | if (iop_adma_clean_slot(iter, iop_chan)) |
| 316 | break; |
| 317 | } |
| 318 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 319 | if (cookie > 0) { |
| 320 | iop_chan->completed_cookie = cookie; |
| 321 | pr_debug("\tcompleted cookie %d\n", cookie); |
| 322 | } |
| 323 | } |
| 324 | |
| 325 | static void |
| 326 | iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan) |
| 327 | { |
| 328 | spin_lock_bh(&iop_chan->lock); |
| 329 | __iop_adma_slot_cleanup(iop_chan); |
| 330 | spin_unlock_bh(&iop_chan->lock); |
| 331 | } |
| 332 | |
| 333 | static void iop_adma_tasklet(unsigned long data) |
| 334 | { |
Dan Williams | 19242d7 | 2008-04-17 20:17:25 -0700 | [diff] [blame] | 335 | struct iop_adma_chan *iop_chan = (struct iop_adma_chan *) data; |
| 336 | |
Dan Williams | 72be12f | 2009-07-14 13:38:29 -0700 | [diff] [blame] | 337 | /* lockdep will flag depedency submissions as potentially |
| 338 | * recursive locking, this is not the case as a dependency |
| 339 | * submission will never recurse a channels submit routine. |
| 340 | * There are checks in async_tx.c to prevent this. |
| 341 | */ |
| 342 | spin_lock_nested(&iop_chan->lock, SINGLE_DEPTH_NESTING); |
Dan Williams | 19242d7 | 2008-04-17 20:17:25 -0700 | [diff] [blame] | 343 | __iop_adma_slot_cleanup(iop_chan); |
| 344 | spin_unlock(&iop_chan->lock); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 345 | } |
| 346 | |
| 347 | static struct iop_adma_desc_slot * |
| 348 | iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots, |
| 349 | int slots_per_op) |
| 350 | { |
| 351 | struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL; |
Denis Cheng | e73ef9a | 2008-02-02 19:30:01 -0700 | [diff] [blame] | 352 | LIST_HEAD(chain); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 353 | int slots_found, retry = 0; |
| 354 | |
| 355 | /* start search from the last allocated descrtiptor |
| 356 | * if a contiguous allocation can not be found start searching |
| 357 | * from the beginning of the list |
| 358 | */ |
| 359 | retry: |
| 360 | slots_found = 0; |
| 361 | if (retry == 0) |
| 362 | iter = iop_chan->last_used; |
| 363 | else |
| 364 | iter = list_entry(&iop_chan->all_slots, |
| 365 | struct iop_adma_desc_slot, |
| 366 | slot_node); |
| 367 | |
| 368 | list_for_each_entry_safe_continue( |
| 369 | iter, _iter, &iop_chan->all_slots, slot_node) { |
| 370 | prefetch(_iter); |
| 371 | prefetch(&_iter->async_tx); |
| 372 | if (iter->slots_per_op) { |
| 373 | /* give up after finding the first busy slot |
| 374 | * on the second pass through the list |
| 375 | */ |
| 376 | if (retry) |
| 377 | break; |
| 378 | |
| 379 | slots_found = 0; |
| 380 | continue; |
| 381 | } |
| 382 | |
| 383 | /* start the allocation if the slot is correctly aligned */ |
| 384 | if (!slots_found++) { |
| 385 | if (iop_desc_is_aligned(iter, slots_per_op)) |
| 386 | alloc_start = iter; |
| 387 | else { |
| 388 | slots_found = 0; |
| 389 | continue; |
| 390 | } |
| 391 | } |
| 392 | |
| 393 | if (slots_found == num_slots) { |
| 394 | struct iop_adma_desc_slot *alloc_tail = NULL; |
| 395 | struct iop_adma_desc_slot *last_used = NULL; |
| 396 | iter = alloc_start; |
| 397 | while (num_slots) { |
| 398 | int i; |
| 399 | dev_dbg(iop_chan->device->common.dev, |
| 400 | "allocated slot: %d " |
| 401 | "(desc %p phys: %#x) slots_per_op %d\n", |
| 402 | iter->idx, iter->hw_desc, |
| 403 | iter->async_tx.phys, slots_per_op); |
| 404 | |
| 405 | /* pre-ack all but the last descriptor */ |
| 406 | if (num_slots != slots_per_op) |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 407 | async_tx_ack(&iter->async_tx); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 408 | |
| 409 | list_add_tail(&iter->chain_node, &chain); |
| 410 | alloc_tail = iter; |
| 411 | iter->async_tx.cookie = 0; |
| 412 | iter->slot_cnt = num_slots; |
| 413 | iter->xor_check_result = NULL; |
| 414 | for (i = 0; i < slots_per_op; i++) { |
| 415 | iter->slots_per_op = slots_per_op - i; |
| 416 | last_used = iter; |
| 417 | iter = list_entry(iter->slot_node.next, |
| 418 | struct iop_adma_desc_slot, |
| 419 | slot_node); |
| 420 | } |
| 421 | num_slots -= slots_per_op; |
| 422 | } |
| 423 | alloc_tail->group_head = alloc_start; |
| 424 | alloc_tail->async_tx.cookie = -EBUSY; |
Dan Williams | 308136d | 2009-09-08 17:53:02 -0700 | [diff] [blame] | 425 | list_splice(&chain, &alloc_tail->tx_list); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 426 | iop_chan->last_used = last_used; |
| 427 | iop_desc_clear_next_desc(alloc_start); |
| 428 | iop_desc_clear_next_desc(alloc_tail); |
| 429 | return alloc_tail; |
| 430 | } |
| 431 | } |
| 432 | if (!retry++) |
| 433 | goto retry; |
| 434 | |
Dan Williams | c7141d0 | 2008-07-17 17:59:56 -0700 | [diff] [blame] | 435 | /* perform direct reclaim if the allocation fails */ |
| 436 | __iop_adma_slot_cleanup(iop_chan); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 437 | |
| 438 | return NULL; |
| 439 | } |
| 440 | |
| 441 | static dma_cookie_t |
| 442 | iop_desc_assign_cookie(struct iop_adma_chan *iop_chan, |
| 443 | struct iop_adma_desc_slot *desc) |
| 444 | { |
| 445 | dma_cookie_t cookie = iop_chan->common.cookie; |
| 446 | cookie++; |
| 447 | if (cookie < 0) |
| 448 | cookie = 1; |
| 449 | iop_chan->common.cookie = desc->async_tx.cookie = cookie; |
| 450 | return cookie; |
| 451 | } |
| 452 | |
| 453 | static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan) |
| 454 | { |
| 455 | dev_dbg(iop_chan->device->common.dev, "pending: %d\n", |
| 456 | iop_chan->pending); |
| 457 | |
| 458 | if (iop_chan->pending >= IOP_ADMA_THRESHOLD) { |
| 459 | iop_chan->pending = 0; |
| 460 | iop_chan_append(iop_chan); |
| 461 | } |
| 462 | } |
| 463 | |
| 464 | static dma_cookie_t |
| 465 | iop_adma_tx_submit(struct dma_async_tx_descriptor *tx) |
| 466 | { |
| 467 | struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx); |
| 468 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan); |
| 469 | struct iop_adma_desc_slot *grp_start, *old_chain_tail; |
| 470 | int slot_cnt; |
| 471 | int slots_per_op; |
| 472 | dma_cookie_t cookie; |
Dan Williams | 137cb55 | 2008-11-11 13:12:33 -0700 | [diff] [blame] | 473 | dma_addr_t next_dma; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 474 | |
| 475 | grp_start = sw_desc->group_head; |
| 476 | slot_cnt = grp_start->slot_cnt; |
| 477 | slots_per_op = grp_start->slots_per_op; |
| 478 | |
| 479 | spin_lock_bh(&iop_chan->lock); |
| 480 | cookie = iop_desc_assign_cookie(iop_chan, sw_desc); |
| 481 | |
| 482 | old_chain_tail = list_entry(iop_chan->chain.prev, |
| 483 | struct iop_adma_desc_slot, chain_node); |
Dan Williams | 308136d | 2009-09-08 17:53:02 -0700 | [diff] [blame] | 484 | list_splice_init(&sw_desc->tx_list, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 485 | &old_chain_tail->chain_node); |
| 486 | |
| 487 | /* fix up the hardware chain */ |
Dan Williams | 137cb55 | 2008-11-11 13:12:33 -0700 | [diff] [blame] | 488 | next_dma = grp_start->async_tx.phys; |
| 489 | iop_desc_set_next_desc(old_chain_tail, next_dma); |
| 490 | BUG_ON(iop_desc_get_next_desc(old_chain_tail) != next_dma); /* flush */ |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 491 | |
Dan Williams | 137cb55 | 2008-11-11 13:12:33 -0700 | [diff] [blame] | 492 | /* check for pre-chained descriptors */ |
Dan Williams | 65e5038 | 2008-11-11 13:12:33 -0700 | [diff] [blame] | 493 | iop_paranoia(iop_desc_get_next_desc(sw_desc)); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 494 | |
| 495 | /* increment the pending count by the number of slots |
| 496 | * memcpy operations have a 1:1 (slot:operation) relation |
| 497 | * other operations are heavier and will pop the threshold |
| 498 | * more often. |
| 499 | */ |
| 500 | iop_chan->pending += slot_cnt; |
| 501 | iop_adma_check_threshold(iop_chan); |
| 502 | spin_unlock_bh(&iop_chan->lock); |
| 503 | |
| 504 | dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n", |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 505 | __func__, sw_desc->async_tx.cookie, sw_desc->idx); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 506 | |
| 507 | return cookie; |
| 508 | } |
| 509 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 510 | static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan); |
| 511 | static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan); |
| 512 | |
Dan Williams | 5eb907a | 2008-07-17 17:59:56 -0700 | [diff] [blame] | 513 | /** |
| 514 | * iop_adma_alloc_chan_resources - returns the number of allocated descriptors |
| 515 | * @chan - allocate descriptor resources for this channel |
| 516 | * @client - current client requesting the channel be ready for requests |
| 517 | * |
| 518 | * Note: We keep the slots for 1 operation on iop_chan->chain at all times. To |
| 519 | * avoid deadlock, via async_xor, num_descs_in_pool must at a minimum be |
| 520 | * greater than 2x the number slots needed to satisfy a device->max_xor |
| 521 | * request. |
| 522 | * */ |
Dan Williams | aa1e6f1 | 2009-01-06 11:38:17 -0700 | [diff] [blame] | 523 | static int iop_adma_alloc_chan_resources(struct dma_chan *chan) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 524 | { |
| 525 | char *hw_desc; |
| 526 | int idx; |
| 527 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 528 | struct iop_adma_desc_slot *slot = NULL; |
| 529 | int init = iop_chan->slots_allocated ? 0 : 1; |
| 530 | struct iop_adma_platform_data *plat_data = |
| 531 | iop_chan->device->pdev->dev.platform_data; |
| 532 | int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE; |
| 533 | |
| 534 | /* Allocate descriptor slots */ |
| 535 | do { |
| 536 | idx = iop_chan->slots_allocated; |
| 537 | if (idx == num_descs_in_pool) |
| 538 | break; |
| 539 | |
| 540 | slot = kzalloc(sizeof(*slot), GFP_KERNEL); |
| 541 | if (!slot) { |
| 542 | printk(KERN_INFO "IOP ADMA Channel only initialized" |
| 543 | " %d descriptor slots", idx); |
| 544 | break; |
| 545 | } |
| 546 | hw_desc = (char *) iop_chan->device->dma_desc_pool_virt; |
| 547 | slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE]; |
| 548 | |
| 549 | dma_async_tx_descriptor_init(&slot->async_tx, chan); |
| 550 | slot->async_tx.tx_submit = iop_adma_tx_submit; |
Dan Williams | 308136d | 2009-09-08 17:53:02 -0700 | [diff] [blame] | 551 | INIT_LIST_HEAD(&slot->tx_list); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 552 | INIT_LIST_HEAD(&slot->chain_node); |
| 553 | INIT_LIST_HEAD(&slot->slot_node); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 554 | hw_desc = (char *) iop_chan->device->dma_desc_pool; |
| 555 | slot->async_tx.phys = |
| 556 | (dma_addr_t) &hw_desc[idx * IOP_ADMA_SLOT_SIZE]; |
| 557 | slot->idx = idx; |
| 558 | |
| 559 | spin_lock_bh(&iop_chan->lock); |
| 560 | iop_chan->slots_allocated++; |
| 561 | list_add_tail(&slot->slot_node, &iop_chan->all_slots); |
| 562 | spin_unlock_bh(&iop_chan->lock); |
| 563 | } while (iop_chan->slots_allocated < num_descs_in_pool); |
| 564 | |
| 565 | if (idx && !iop_chan->last_used) |
| 566 | iop_chan->last_used = list_entry(iop_chan->all_slots.next, |
| 567 | struct iop_adma_desc_slot, |
| 568 | slot_node); |
| 569 | |
| 570 | dev_dbg(iop_chan->device->common.dev, |
| 571 | "allocated %d descriptor slots last_used: %p\n", |
| 572 | iop_chan->slots_allocated, iop_chan->last_used); |
| 573 | |
| 574 | /* initialize the channel and the chain with a null operation */ |
| 575 | if (init) { |
| 576 | if (dma_has_cap(DMA_MEMCPY, |
| 577 | iop_chan->device->common.cap_mask)) |
| 578 | iop_chan_start_null_memcpy(iop_chan); |
| 579 | else if (dma_has_cap(DMA_XOR, |
| 580 | iop_chan->device->common.cap_mask)) |
| 581 | iop_chan_start_null_xor(iop_chan); |
| 582 | else |
| 583 | BUG(); |
| 584 | } |
| 585 | |
| 586 | return (idx > 0) ? idx : -ENOMEM; |
| 587 | } |
| 588 | |
| 589 | static struct dma_async_tx_descriptor * |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 590 | iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 591 | { |
| 592 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 593 | struct iop_adma_desc_slot *sw_desc, *grp_start; |
| 594 | int slot_cnt, slots_per_op; |
| 595 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 596 | dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 597 | |
| 598 | spin_lock_bh(&iop_chan->lock); |
| 599 | slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan); |
| 600 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 601 | if (sw_desc) { |
| 602 | grp_start = sw_desc->group_head; |
| 603 | iop_desc_init_interrupt(grp_start, iop_chan); |
| 604 | grp_start->unmap_len = 0; |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 605 | sw_desc->async_tx.flags = flags; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 606 | } |
| 607 | spin_unlock_bh(&iop_chan->lock); |
| 608 | |
| 609 | return sw_desc ? &sw_desc->async_tx : NULL; |
| 610 | } |
| 611 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 612 | static struct dma_async_tx_descriptor * |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 613 | iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest, |
Dan Williams | d4c56f9 | 2008-02-02 19:49:58 -0700 | [diff] [blame] | 614 | dma_addr_t dma_src, size_t len, unsigned long flags) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 615 | { |
| 616 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 617 | struct iop_adma_desc_slot *sw_desc, *grp_start; |
| 618 | int slot_cnt, slots_per_op; |
| 619 | |
| 620 | if (unlikely(!len)) |
| 621 | return NULL; |
Coly Li | e2ec771 | 2011-03-27 01:26:52 +0800 | [diff] [blame] | 622 | BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 623 | |
| 624 | dev_dbg(iop_chan->device->common.dev, "%s len: %u\n", |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 625 | __func__, len); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 626 | |
| 627 | spin_lock_bh(&iop_chan->lock); |
| 628 | slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op); |
| 629 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 630 | if (sw_desc) { |
| 631 | grp_start = sw_desc->group_head; |
Dan Williams | d4c56f9 | 2008-02-02 19:49:58 -0700 | [diff] [blame] | 632 | iop_desc_init_memcpy(grp_start, flags); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 633 | iop_desc_set_byte_count(grp_start, iop_chan, len); |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 634 | iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); |
| 635 | iop_desc_set_memcpy_src_addr(grp_start, dma_src); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 636 | sw_desc->unmap_src_cnt = 1; |
| 637 | sw_desc->unmap_len = len; |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 638 | sw_desc->async_tx.flags = flags; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 639 | } |
| 640 | spin_unlock_bh(&iop_chan->lock); |
| 641 | |
| 642 | return sw_desc ? &sw_desc->async_tx : NULL; |
| 643 | } |
| 644 | |
| 645 | static struct dma_async_tx_descriptor * |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 646 | iop_adma_prep_dma_memset(struct dma_chan *chan, dma_addr_t dma_dest, |
Dan Williams | d4c56f9 | 2008-02-02 19:49:58 -0700 | [diff] [blame] | 647 | int value, size_t len, unsigned long flags) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 648 | { |
| 649 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 650 | struct iop_adma_desc_slot *sw_desc, *grp_start; |
| 651 | int slot_cnt, slots_per_op; |
| 652 | |
| 653 | if (unlikely(!len)) |
| 654 | return NULL; |
Coly Li | e2ec771 | 2011-03-27 01:26:52 +0800 | [diff] [blame] | 655 | BUG_ON(len > IOP_ADMA_MAX_BYTE_COUNT); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 656 | |
| 657 | dev_dbg(iop_chan->device->common.dev, "%s len: %u\n", |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 658 | __func__, len); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 659 | |
| 660 | spin_lock_bh(&iop_chan->lock); |
| 661 | slot_cnt = iop_chan_memset_slot_count(len, &slots_per_op); |
| 662 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 663 | if (sw_desc) { |
| 664 | grp_start = sw_desc->group_head; |
Dan Williams | d4c56f9 | 2008-02-02 19:49:58 -0700 | [diff] [blame] | 665 | iop_desc_init_memset(grp_start, flags); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 666 | iop_desc_set_byte_count(grp_start, iop_chan, len); |
| 667 | iop_desc_set_block_fill_val(grp_start, value); |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 668 | iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 669 | sw_desc->unmap_src_cnt = 1; |
| 670 | sw_desc->unmap_len = len; |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 671 | sw_desc->async_tx.flags = flags; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 672 | } |
| 673 | spin_unlock_bh(&iop_chan->lock); |
| 674 | |
| 675 | return sw_desc ? &sw_desc->async_tx : NULL; |
| 676 | } |
| 677 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 678 | static struct dma_async_tx_descriptor * |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 679 | iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest, |
| 680 | dma_addr_t *dma_src, unsigned int src_cnt, size_t len, |
Dan Williams | d4c56f9 | 2008-02-02 19:49:58 -0700 | [diff] [blame] | 681 | unsigned long flags) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 682 | { |
| 683 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 684 | struct iop_adma_desc_slot *sw_desc, *grp_start; |
| 685 | int slot_cnt, slots_per_op; |
| 686 | |
| 687 | if (unlikely(!len)) |
| 688 | return NULL; |
Coly Li | e2ec771 | 2011-03-27 01:26:52 +0800 | [diff] [blame] | 689 | BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 690 | |
| 691 | dev_dbg(iop_chan->device->common.dev, |
Dan Williams | d4c56f9 | 2008-02-02 19:49:58 -0700 | [diff] [blame] | 692 | "%s src_cnt: %d len: %u flags: %lx\n", |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 693 | __func__, src_cnt, len, flags); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 694 | |
| 695 | spin_lock_bh(&iop_chan->lock); |
| 696 | slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op); |
| 697 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 698 | if (sw_desc) { |
| 699 | grp_start = sw_desc->group_head; |
Dan Williams | d4c56f9 | 2008-02-02 19:49:58 -0700 | [diff] [blame] | 700 | iop_desc_init_xor(grp_start, src_cnt, flags); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 701 | iop_desc_set_byte_count(grp_start, iop_chan, len); |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 702 | iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 703 | sw_desc->unmap_src_cnt = src_cnt; |
| 704 | sw_desc->unmap_len = len; |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 705 | sw_desc->async_tx.flags = flags; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 706 | while (src_cnt--) |
| 707 | iop_desc_set_xor_src_addr(grp_start, src_cnt, |
| 708 | dma_src[src_cnt]); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 709 | } |
| 710 | spin_unlock_bh(&iop_chan->lock); |
| 711 | |
| 712 | return sw_desc ? &sw_desc->async_tx : NULL; |
| 713 | } |
| 714 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 715 | static struct dma_async_tx_descriptor * |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 716 | iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src, |
| 717 | unsigned int src_cnt, size_t len, u32 *result, |
| 718 | unsigned long flags) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 719 | { |
| 720 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 721 | struct iop_adma_desc_slot *sw_desc, *grp_start; |
| 722 | int slot_cnt, slots_per_op; |
| 723 | |
| 724 | if (unlikely(!len)) |
| 725 | return NULL; |
| 726 | |
| 727 | dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n", |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 728 | __func__, src_cnt, len); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 729 | |
| 730 | spin_lock_bh(&iop_chan->lock); |
| 731 | slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op); |
| 732 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 733 | if (sw_desc) { |
| 734 | grp_start = sw_desc->group_head; |
Dan Williams | d4c56f9 | 2008-02-02 19:49:58 -0700 | [diff] [blame] | 735 | iop_desc_init_zero_sum(grp_start, src_cnt, flags); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 736 | iop_desc_set_zero_sum_byte_count(grp_start, len); |
| 737 | grp_start->xor_check_result = result; |
| 738 | pr_debug("\t%s: grp_start->xor_check_result: %p\n", |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 739 | __func__, grp_start->xor_check_result); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 740 | sw_desc->unmap_src_cnt = src_cnt; |
| 741 | sw_desc->unmap_len = len; |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 742 | sw_desc->async_tx.flags = flags; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 743 | while (src_cnt--) |
| 744 | iop_desc_set_zero_sum_src_addr(grp_start, src_cnt, |
| 745 | dma_src[src_cnt]); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 746 | } |
| 747 | spin_unlock_bh(&iop_chan->lock); |
| 748 | |
| 749 | return sw_desc ? &sw_desc->async_tx : NULL; |
| 750 | } |
| 751 | |
Dan Williams | 7bf649a | 2009-08-28 14:32:04 -0700 | [diff] [blame] | 752 | static struct dma_async_tx_descriptor * |
| 753 | iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, |
| 754 | unsigned int src_cnt, const unsigned char *scf, size_t len, |
| 755 | unsigned long flags) |
| 756 | { |
| 757 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 758 | struct iop_adma_desc_slot *sw_desc, *g; |
| 759 | int slot_cnt, slots_per_op; |
| 760 | int continue_srcs; |
| 761 | |
| 762 | if (unlikely(!len)) |
| 763 | return NULL; |
| 764 | BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); |
| 765 | |
| 766 | dev_dbg(iop_chan->device->common.dev, |
| 767 | "%s src_cnt: %d len: %u flags: %lx\n", |
| 768 | __func__, src_cnt, len, flags); |
| 769 | |
| 770 | if (dmaf_p_disabled_continue(flags)) |
| 771 | continue_srcs = 1+src_cnt; |
| 772 | else if (dmaf_continue(flags)) |
| 773 | continue_srcs = 3+src_cnt; |
| 774 | else |
| 775 | continue_srcs = 0+src_cnt; |
| 776 | |
| 777 | spin_lock_bh(&iop_chan->lock); |
| 778 | slot_cnt = iop_chan_pq_slot_count(len, continue_srcs, &slots_per_op); |
| 779 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 780 | if (sw_desc) { |
| 781 | int i; |
| 782 | |
| 783 | g = sw_desc->group_head; |
| 784 | iop_desc_set_byte_count(g, iop_chan, len); |
| 785 | |
| 786 | /* even if P is disabled its destination address (bits |
| 787 | * [3:0]) must match Q. It is ok if P points to an |
| 788 | * invalid address, it won't be written. |
| 789 | */ |
| 790 | if (flags & DMA_PREP_PQ_DISABLE_P) |
| 791 | dst[0] = dst[1] & 0x7; |
| 792 | |
| 793 | iop_desc_set_pq_addr(g, dst); |
| 794 | sw_desc->unmap_src_cnt = src_cnt; |
| 795 | sw_desc->unmap_len = len; |
| 796 | sw_desc->async_tx.flags = flags; |
| 797 | for (i = 0; i < src_cnt; i++) |
| 798 | iop_desc_set_pq_src_addr(g, i, src[i], scf[i]); |
| 799 | |
| 800 | /* if we are continuing a previous operation factor in |
| 801 | * the old p and q values, see the comment for dma_maxpq |
| 802 | * in include/linux/dmaengine.h |
| 803 | */ |
| 804 | if (dmaf_p_disabled_continue(flags)) |
| 805 | iop_desc_set_pq_src_addr(g, i++, dst[1], 1); |
| 806 | else if (dmaf_continue(flags)) { |
| 807 | iop_desc_set_pq_src_addr(g, i++, dst[0], 0); |
| 808 | iop_desc_set_pq_src_addr(g, i++, dst[1], 1); |
| 809 | iop_desc_set_pq_src_addr(g, i++, dst[1], 0); |
| 810 | } |
| 811 | iop_desc_init_pq(g, i, flags); |
| 812 | } |
| 813 | spin_unlock_bh(&iop_chan->lock); |
| 814 | |
| 815 | return sw_desc ? &sw_desc->async_tx : NULL; |
| 816 | } |
| 817 | |
| 818 | static struct dma_async_tx_descriptor * |
| 819 | iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, |
| 820 | unsigned int src_cnt, const unsigned char *scf, |
| 821 | size_t len, enum sum_check_flags *pqres, |
| 822 | unsigned long flags) |
| 823 | { |
| 824 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 825 | struct iop_adma_desc_slot *sw_desc, *g; |
| 826 | int slot_cnt, slots_per_op; |
| 827 | |
| 828 | if (unlikely(!len)) |
| 829 | return NULL; |
| 830 | BUG_ON(len > IOP_ADMA_XOR_MAX_BYTE_COUNT); |
| 831 | |
| 832 | dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n", |
| 833 | __func__, src_cnt, len); |
| 834 | |
| 835 | spin_lock_bh(&iop_chan->lock); |
| 836 | slot_cnt = iop_chan_pq_zero_sum_slot_count(len, src_cnt + 2, &slots_per_op); |
| 837 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 838 | if (sw_desc) { |
| 839 | /* for validate operations p and q are tagged onto the |
| 840 | * end of the source list |
| 841 | */ |
| 842 | int pq_idx = src_cnt; |
| 843 | |
| 844 | g = sw_desc->group_head; |
| 845 | iop_desc_init_pq_zero_sum(g, src_cnt+2, flags); |
| 846 | iop_desc_set_pq_zero_sum_byte_count(g, len); |
| 847 | g->pq_check_result = pqres; |
| 848 | pr_debug("\t%s: g->pq_check_result: %p\n", |
| 849 | __func__, g->pq_check_result); |
| 850 | sw_desc->unmap_src_cnt = src_cnt+2; |
| 851 | sw_desc->unmap_len = len; |
| 852 | sw_desc->async_tx.flags = flags; |
| 853 | while (src_cnt--) |
| 854 | iop_desc_set_pq_zero_sum_src_addr(g, src_cnt, |
| 855 | src[src_cnt], |
| 856 | scf[src_cnt]); |
| 857 | iop_desc_set_pq_zero_sum_addr(g, pq_idx, src); |
| 858 | } |
| 859 | spin_unlock_bh(&iop_chan->lock); |
| 860 | |
| 861 | return sw_desc ? &sw_desc->async_tx : NULL; |
| 862 | } |
| 863 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 864 | static void iop_adma_free_chan_resources(struct dma_chan *chan) |
| 865 | { |
| 866 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 867 | struct iop_adma_desc_slot *iter, *_iter; |
| 868 | int in_use_descs = 0; |
| 869 | |
| 870 | iop_adma_slot_cleanup(iop_chan); |
| 871 | |
| 872 | spin_lock_bh(&iop_chan->lock); |
| 873 | list_for_each_entry_safe(iter, _iter, &iop_chan->chain, |
| 874 | chain_node) { |
| 875 | in_use_descs++; |
| 876 | list_del(&iter->chain_node); |
| 877 | } |
| 878 | list_for_each_entry_safe_reverse( |
| 879 | iter, _iter, &iop_chan->all_slots, slot_node) { |
| 880 | list_del(&iter->slot_node); |
| 881 | kfree(iter); |
| 882 | iop_chan->slots_allocated--; |
| 883 | } |
| 884 | iop_chan->last_used = NULL; |
| 885 | |
| 886 | dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n", |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 887 | __func__, iop_chan->slots_allocated); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 888 | spin_unlock_bh(&iop_chan->lock); |
| 889 | |
| 890 | /* one is ok since we left it on there on purpose */ |
| 891 | if (in_use_descs > 1) |
| 892 | printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n", |
| 893 | in_use_descs - 1); |
| 894 | } |
| 895 | |
| 896 | /** |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 897 | * iop_adma_status - poll the status of an ADMA transaction |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 898 | * @chan: ADMA channel handle |
| 899 | * @cookie: ADMA transaction identifier |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 900 | * @txstate: a holder for the current state of the channel or NULL |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 901 | */ |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 902 | static enum dma_status iop_adma_status(struct dma_chan *chan, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 903 | dma_cookie_t cookie, |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 904 | struct dma_tx_state *txstate) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 905 | { |
| 906 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 907 | dma_cookie_t last_used; |
| 908 | dma_cookie_t last_complete; |
| 909 | enum dma_status ret; |
| 910 | |
| 911 | last_used = chan->cookie; |
| 912 | last_complete = iop_chan->completed_cookie; |
Dan Williams | bca3469 | 2010-03-26 16:52:10 -0700 | [diff] [blame] | 913 | dma_set_tx_state(txstate, last_complete, last_used, 0); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 914 | ret = dma_async_is_complete(cookie, last_complete, last_used); |
| 915 | if (ret == DMA_SUCCESS) |
| 916 | return ret; |
| 917 | |
| 918 | iop_adma_slot_cleanup(iop_chan); |
| 919 | |
| 920 | last_used = chan->cookie; |
| 921 | last_complete = iop_chan->completed_cookie; |
Dan Williams | bca3469 | 2010-03-26 16:52:10 -0700 | [diff] [blame] | 922 | dma_set_tx_state(txstate, last_complete, last_used, 0); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 923 | |
| 924 | return dma_async_is_complete(cookie, last_complete, last_used); |
| 925 | } |
| 926 | |
| 927 | static irqreturn_t iop_adma_eot_handler(int irq, void *data) |
| 928 | { |
| 929 | struct iop_adma_chan *chan = data; |
| 930 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 931 | dev_dbg(chan->device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 932 | |
| 933 | tasklet_schedule(&chan->irq_tasklet); |
| 934 | |
| 935 | iop_adma_device_clear_eot_status(chan); |
| 936 | |
| 937 | return IRQ_HANDLED; |
| 938 | } |
| 939 | |
| 940 | static irqreturn_t iop_adma_eoc_handler(int irq, void *data) |
| 941 | { |
| 942 | struct iop_adma_chan *chan = data; |
| 943 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 944 | dev_dbg(chan->device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 945 | |
| 946 | tasklet_schedule(&chan->irq_tasklet); |
| 947 | |
| 948 | iop_adma_device_clear_eoc_status(chan); |
| 949 | |
| 950 | return IRQ_HANDLED; |
| 951 | } |
| 952 | |
| 953 | static irqreturn_t iop_adma_err_handler(int irq, void *data) |
| 954 | { |
| 955 | struct iop_adma_chan *chan = data; |
| 956 | unsigned long status = iop_chan_get_status(chan); |
| 957 | |
| 958 | dev_printk(KERN_ERR, chan->device->common.dev, |
| 959 | "error ( %s%s%s%s%s%s%s)\n", |
| 960 | iop_is_err_int_parity(status, chan) ? "int_parity " : "", |
| 961 | iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "", |
| 962 | iop_is_err_int_tabort(status, chan) ? "int_tabort " : "", |
| 963 | iop_is_err_int_mabort(status, chan) ? "int_mabort " : "", |
| 964 | iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "", |
| 965 | iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "", |
| 966 | iop_is_err_split_tx(status, chan) ? "split_tx " : ""); |
| 967 | |
| 968 | iop_adma_device_clear_err_status(chan); |
| 969 | |
| 970 | BUG(); |
| 971 | |
| 972 | return IRQ_HANDLED; |
| 973 | } |
| 974 | |
| 975 | static void iop_adma_issue_pending(struct dma_chan *chan) |
| 976 | { |
| 977 | struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan); |
| 978 | |
| 979 | if (iop_chan->pending) { |
| 980 | iop_chan->pending = 0; |
| 981 | iop_chan_append(iop_chan); |
| 982 | } |
| 983 | } |
| 984 | |
| 985 | /* |
| 986 | * Perform a transaction to verify the HW works. |
| 987 | */ |
| 988 | #define IOP_ADMA_TEST_SIZE 2000 |
| 989 | |
| 990 | static int __devinit iop_adma_memcpy_self_test(struct iop_adma_device *device) |
| 991 | { |
| 992 | int i; |
| 993 | void *src, *dest; |
| 994 | dma_addr_t src_dma, dest_dma; |
| 995 | struct dma_chan *dma_chan; |
| 996 | dma_cookie_t cookie; |
| 997 | struct dma_async_tx_descriptor *tx; |
| 998 | int err = 0; |
| 999 | struct iop_adma_chan *iop_chan; |
| 1000 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 1001 | dev_dbg(device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1002 | |
Christophe Jaillet | eccf214 | 2008-05-20 16:33:06 -0700 | [diff] [blame] | 1003 | src = kmalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1004 | if (!src) |
| 1005 | return -ENOMEM; |
Christophe Jaillet | eccf214 | 2008-05-20 16:33:06 -0700 | [diff] [blame] | 1006 | dest = kzalloc(IOP_ADMA_TEST_SIZE, GFP_KERNEL); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1007 | if (!dest) { |
| 1008 | kfree(src); |
| 1009 | return -ENOMEM; |
| 1010 | } |
| 1011 | |
| 1012 | /* Fill in src buffer */ |
| 1013 | for (i = 0; i < IOP_ADMA_TEST_SIZE; i++) |
| 1014 | ((u8 *) src)[i] = (u8)i; |
| 1015 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1016 | /* Start copy, using first DMA channel */ |
| 1017 | dma_chan = container_of(device->common.channels.next, |
| 1018 | struct dma_chan, |
| 1019 | device_node); |
Dan Williams | aa1e6f1 | 2009-01-06 11:38:17 -0700 | [diff] [blame] | 1020 | if (iop_adma_alloc_chan_resources(dma_chan) < 1) { |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1021 | err = -ENODEV; |
| 1022 | goto out; |
| 1023 | } |
| 1024 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1025 | dest_dma = dma_map_single(dma_chan->device->dev, dest, |
| 1026 | IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1027 | src_dma = dma_map_single(dma_chan->device->dev, src, |
| 1028 | IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE); |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 1029 | tx = iop_adma_prep_dma_memcpy(dma_chan, dest_dma, src_dma, |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 1030 | IOP_ADMA_TEST_SIZE, |
| 1031 | DMA_PREP_INTERRUPT | DMA_CTRL_ACK); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1032 | |
| 1033 | cookie = iop_adma_tx_submit(tx); |
| 1034 | iop_adma_issue_pending(dma_chan); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1035 | msleep(1); |
| 1036 | |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1037 | if (iop_adma_status(dma_chan, cookie, NULL) != |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1038 | DMA_SUCCESS) { |
| 1039 | dev_printk(KERN_ERR, dma_chan->device->dev, |
| 1040 | "Self-test copy timed out, disabling\n"); |
| 1041 | err = -ENODEV; |
| 1042 | goto free_resources; |
| 1043 | } |
| 1044 | |
| 1045 | iop_chan = to_iop_adma_chan(dma_chan); |
| 1046 | dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, |
| 1047 | IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE); |
| 1048 | if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) { |
| 1049 | dev_printk(KERN_ERR, dma_chan->device->dev, |
| 1050 | "Self-test copy failed compare, disabling\n"); |
| 1051 | err = -ENODEV; |
| 1052 | goto free_resources; |
| 1053 | } |
| 1054 | |
| 1055 | free_resources: |
| 1056 | iop_adma_free_chan_resources(dma_chan); |
| 1057 | out: |
| 1058 | kfree(src); |
| 1059 | kfree(dest); |
| 1060 | return err; |
| 1061 | } |
| 1062 | |
| 1063 | #define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */ |
| 1064 | static int __devinit |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 1065 | iop_adma_xor_val_self_test(struct iop_adma_device *device) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1066 | { |
| 1067 | int i, src_idx; |
| 1068 | struct page *dest; |
| 1069 | struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST]; |
| 1070 | struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 1071 | dma_addr_t dma_srcs[IOP_ADMA_NUM_SRC_TEST + 1]; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1072 | dma_addr_t dma_addr, dest_dma; |
| 1073 | struct dma_async_tx_descriptor *tx; |
| 1074 | struct dma_chan *dma_chan; |
| 1075 | dma_cookie_t cookie; |
| 1076 | u8 cmp_byte = 0; |
| 1077 | u32 cmp_word; |
| 1078 | u32 zero_sum_result; |
| 1079 | int err = 0; |
| 1080 | struct iop_adma_chan *iop_chan; |
| 1081 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 1082 | dev_dbg(device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1083 | |
| 1084 | for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { |
| 1085 | xor_srcs[src_idx] = alloc_page(GFP_KERNEL); |
Roel Kluin | a09b09a | 2009-02-25 13:56:21 +0100 | [diff] [blame] | 1086 | if (!xor_srcs[src_idx]) { |
| 1087 | while (src_idx--) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1088 | __free_page(xor_srcs[src_idx]); |
Roel Kluin | a09b09a | 2009-02-25 13:56:21 +0100 | [diff] [blame] | 1089 | return -ENOMEM; |
| 1090 | } |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1091 | } |
| 1092 | |
| 1093 | dest = alloc_page(GFP_KERNEL); |
Roel Kluin | a09b09a | 2009-02-25 13:56:21 +0100 | [diff] [blame] | 1094 | if (!dest) { |
| 1095 | while (src_idx--) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1096 | __free_page(xor_srcs[src_idx]); |
Roel Kluin | a09b09a | 2009-02-25 13:56:21 +0100 | [diff] [blame] | 1097 | return -ENOMEM; |
| 1098 | } |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1099 | |
| 1100 | /* Fill in src buffers */ |
| 1101 | for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) { |
| 1102 | u8 *ptr = page_address(xor_srcs[src_idx]); |
| 1103 | for (i = 0; i < PAGE_SIZE; i++) |
| 1104 | ptr[i] = (1 << src_idx); |
| 1105 | } |
| 1106 | |
| 1107 | for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) |
| 1108 | cmp_byte ^= (u8) (1 << src_idx); |
| 1109 | |
| 1110 | cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | |
| 1111 | (cmp_byte << 8) | cmp_byte; |
| 1112 | |
| 1113 | memset(page_address(dest), 0, PAGE_SIZE); |
| 1114 | |
| 1115 | dma_chan = container_of(device->common.channels.next, |
| 1116 | struct dma_chan, |
| 1117 | device_node); |
Dan Williams | aa1e6f1 | 2009-01-06 11:38:17 -0700 | [diff] [blame] | 1118 | if (iop_adma_alloc_chan_resources(dma_chan) < 1) { |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1119 | err = -ENODEV; |
| 1120 | goto out; |
| 1121 | } |
| 1122 | |
| 1123 | /* test xor */ |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1124 | dest_dma = dma_map_page(dma_chan->device->dev, dest, 0, |
| 1125 | PAGE_SIZE, DMA_FROM_DEVICE); |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 1126 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) |
| 1127 | dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i], |
| 1128 | 0, PAGE_SIZE, DMA_TO_DEVICE); |
| 1129 | tx = iop_adma_prep_dma_xor(dma_chan, dest_dma, dma_srcs, |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 1130 | IOP_ADMA_NUM_SRC_TEST, PAGE_SIZE, |
| 1131 | DMA_PREP_INTERRUPT | DMA_CTRL_ACK); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1132 | |
| 1133 | cookie = iop_adma_tx_submit(tx); |
| 1134 | iop_adma_issue_pending(dma_chan); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1135 | msleep(8); |
| 1136 | |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1137 | if (iop_adma_status(dma_chan, cookie, NULL) != |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1138 | DMA_SUCCESS) { |
| 1139 | dev_printk(KERN_ERR, dma_chan->device->dev, |
| 1140 | "Self-test xor timed out, disabling\n"); |
| 1141 | err = -ENODEV; |
| 1142 | goto free_resources; |
| 1143 | } |
| 1144 | |
| 1145 | iop_chan = to_iop_adma_chan(dma_chan); |
| 1146 | dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma, |
| 1147 | PAGE_SIZE, DMA_FROM_DEVICE); |
| 1148 | for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) { |
| 1149 | u32 *ptr = page_address(dest); |
| 1150 | if (ptr[i] != cmp_word) { |
| 1151 | dev_printk(KERN_ERR, dma_chan->device->dev, |
| 1152 | "Self-test xor failed compare, disabling\n"); |
| 1153 | err = -ENODEV; |
| 1154 | goto free_resources; |
| 1155 | } |
| 1156 | } |
| 1157 | dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma, |
| 1158 | PAGE_SIZE, DMA_TO_DEVICE); |
| 1159 | |
| 1160 | /* skip zero sum if the capability is not present */ |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 1161 | if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask)) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1162 | goto free_resources; |
| 1163 | |
| 1164 | /* zero sum the sources with the destintation page */ |
| 1165 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) |
| 1166 | zero_sum_srcs[i] = xor_srcs[i]; |
| 1167 | zero_sum_srcs[i] = dest; |
| 1168 | |
| 1169 | zero_sum_result = 1; |
| 1170 | |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 1171 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) |
| 1172 | dma_srcs[i] = dma_map_page(dma_chan->device->dev, |
| 1173 | zero_sum_srcs[i], 0, PAGE_SIZE, |
| 1174 | DMA_TO_DEVICE); |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 1175 | tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, |
| 1176 | IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, |
| 1177 | &zero_sum_result, |
| 1178 | DMA_PREP_INTERRUPT | DMA_CTRL_ACK); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1179 | |
| 1180 | cookie = iop_adma_tx_submit(tx); |
| 1181 | iop_adma_issue_pending(dma_chan); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1182 | msleep(8); |
| 1183 | |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1184 | if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1185 | dev_printk(KERN_ERR, dma_chan->device->dev, |
| 1186 | "Self-test zero sum timed out, disabling\n"); |
| 1187 | err = -ENODEV; |
| 1188 | goto free_resources; |
| 1189 | } |
| 1190 | |
| 1191 | if (zero_sum_result != 0) { |
| 1192 | dev_printk(KERN_ERR, dma_chan->device->dev, |
| 1193 | "Self-test zero sum failed compare, disabling\n"); |
| 1194 | err = -ENODEV; |
| 1195 | goto free_resources; |
| 1196 | } |
| 1197 | |
| 1198 | /* test memset */ |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1199 | dma_addr = dma_map_page(dma_chan->device->dev, dest, 0, |
| 1200 | PAGE_SIZE, DMA_FROM_DEVICE); |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 1201 | tx = iop_adma_prep_dma_memset(dma_chan, dma_addr, 0, PAGE_SIZE, |
| 1202 | DMA_PREP_INTERRUPT | DMA_CTRL_ACK); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1203 | |
| 1204 | cookie = iop_adma_tx_submit(tx); |
| 1205 | iop_adma_issue_pending(dma_chan); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1206 | msleep(8); |
| 1207 | |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1208 | if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1209 | dev_printk(KERN_ERR, dma_chan->device->dev, |
| 1210 | "Self-test memset timed out, disabling\n"); |
| 1211 | err = -ENODEV; |
| 1212 | goto free_resources; |
| 1213 | } |
| 1214 | |
| 1215 | for (i = 0; i < PAGE_SIZE/sizeof(u32); i++) { |
| 1216 | u32 *ptr = page_address(dest); |
| 1217 | if (ptr[i]) { |
| 1218 | dev_printk(KERN_ERR, dma_chan->device->dev, |
| 1219 | "Self-test memset failed compare, disabling\n"); |
| 1220 | err = -ENODEV; |
| 1221 | goto free_resources; |
| 1222 | } |
| 1223 | } |
| 1224 | |
| 1225 | /* test for non-zero parity sum */ |
| 1226 | zero_sum_result = 0; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 1227 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) |
| 1228 | dma_srcs[i] = dma_map_page(dma_chan->device->dev, |
| 1229 | zero_sum_srcs[i], 0, PAGE_SIZE, |
| 1230 | DMA_TO_DEVICE); |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 1231 | tx = iop_adma_prep_dma_xor_val(dma_chan, dma_srcs, |
| 1232 | IOP_ADMA_NUM_SRC_TEST + 1, PAGE_SIZE, |
| 1233 | &zero_sum_result, |
| 1234 | DMA_PREP_INTERRUPT | DMA_CTRL_ACK); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1235 | |
| 1236 | cookie = iop_adma_tx_submit(tx); |
| 1237 | iop_adma_issue_pending(dma_chan); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1238 | msleep(8); |
| 1239 | |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1240 | if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1241 | dev_printk(KERN_ERR, dma_chan->device->dev, |
| 1242 | "Self-test non-zero sum timed out, disabling\n"); |
| 1243 | err = -ENODEV; |
| 1244 | goto free_resources; |
| 1245 | } |
| 1246 | |
| 1247 | if (zero_sum_result != 1) { |
| 1248 | dev_printk(KERN_ERR, dma_chan->device->dev, |
| 1249 | "Self-test non-zero sum failed compare, disabling\n"); |
| 1250 | err = -ENODEV; |
| 1251 | goto free_resources; |
| 1252 | } |
| 1253 | |
| 1254 | free_resources: |
| 1255 | iop_adma_free_chan_resources(dma_chan); |
| 1256 | out: |
| 1257 | src_idx = IOP_ADMA_NUM_SRC_TEST; |
| 1258 | while (src_idx--) |
| 1259 | __free_page(xor_srcs[src_idx]); |
| 1260 | __free_page(dest); |
| 1261 | return err; |
| 1262 | } |
| 1263 | |
Wei Yongquan | 0261f74 | 2010-12-29 20:30:55 +0800 | [diff] [blame] | 1264 | #ifdef CONFIG_RAID6_PQ |
Dan Williams | f6dbf651 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 1265 | static int __devinit |
| 1266 | iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device) |
| 1267 | { |
| 1268 | /* combined sources, software pq results, and extra hw pq results */ |
| 1269 | struct page *pq[IOP_ADMA_NUM_SRC_TEST+2+2]; |
| 1270 | /* ptr to the extra hw pq buffers defined above */ |
| 1271 | struct page **pq_hw = &pq[IOP_ADMA_NUM_SRC_TEST+2]; |
| 1272 | /* address conversion buffers (dma_map / page_address) */ |
| 1273 | void *pq_sw[IOP_ADMA_NUM_SRC_TEST+2]; |
| 1274 | dma_addr_t pq_src[IOP_ADMA_NUM_SRC_TEST]; |
| 1275 | dma_addr_t pq_dest[2]; |
| 1276 | |
| 1277 | int i; |
| 1278 | struct dma_async_tx_descriptor *tx; |
| 1279 | struct dma_chan *dma_chan; |
| 1280 | dma_cookie_t cookie; |
| 1281 | u32 zero_sum_result; |
| 1282 | int err = 0; |
| 1283 | struct device *dev; |
| 1284 | |
| 1285 | dev_dbg(device->common.dev, "%s\n", __func__); |
| 1286 | |
| 1287 | for (i = 0; i < ARRAY_SIZE(pq); i++) { |
| 1288 | pq[i] = alloc_page(GFP_KERNEL); |
| 1289 | if (!pq[i]) { |
| 1290 | while (i--) |
| 1291 | __free_page(pq[i]); |
| 1292 | return -ENOMEM; |
| 1293 | } |
| 1294 | } |
| 1295 | |
| 1296 | /* Fill in src buffers */ |
| 1297 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) { |
| 1298 | pq_sw[i] = page_address(pq[i]); |
| 1299 | memset(pq_sw[i], 0x11111111 * (1<<i), PAGE_SIZE); |
| 1300 | } |
| 1301 | pq_sw[i] = page_address(pq[i]); |
| 1302 | pq_sw[i+1] = page_address(pq[i+1]); |
| 1303 | |
| 1304 | dma_chan = container_of(device->common.channels.next, |
| 1305 | struct dma_chan, |
| 1306 | device_node); |
| 1307 | if (iop_adma_alloc_chan_resources(dma_chan) < 1) { |
| 1308 | err = -ENODEV; |
| 1309 | goto out; |
| 1310 | } |
| 1311 | |
| 1312 | dev = dma_chan->device->dev; |
| 1313 | |
| 1314 | /* initialize the dests */ |
| 1315 | memset(page_address(pq_hw[0]), 0 , PAGE_SIZE); |
| 1316 | memset(page_address(pq_hw[1]), 0 , PAGE_SIZE); |
| 1317 | |
| 1318 | /* test pq */ |
| 1319 | pq_dest[0] = dma_map_page(dev, pq_hw[0], 0, PAGE_SIZE, DMA_FROM_DEVICE); |
| 1320 | pq_dest[1] = dma_map_page(dev, pq_hw[1], 0, PAGE_SIZE, DMA_FROM_DEVICE); |
| 1321 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) |
| 1322 | pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, |
| 1323 | DMA_TO_DEVICE); |
| 1324 | |
| 1325 | tx = iop_adma_prep_dma_pq(dma_chan, pq_dest, pq_src, |
| 1326 | IOP_ADMA_NUM_SRC_TEST, (u8 *)raid6_gfexp, |
| 1327 | PAGE_SIZE, |
| 1328 | DMA_PREP_INTERRUPT | |
| 1329 | DMA_CTRL_ACK); |
| 1330 | |
| 1331 | cookie = iop_adma_tx_submit(tx); |
| 1332 | iop_adma_issue_pending(dma_chan); |
| 1333 | msleep(8); |
| 1334 | |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1335 | if (iop_adma_status(dma_chan, cookie, NULL) != |
Dan Williams | f6dbf651 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 1336 | DMA_SUCCESS) { |
| 1337 | dev_err(dev, "Self-test pq timed out, disabling\n"); |
| 1338 | err = -ENODEV; |
| 1339 | goto free_resources; |
| 1340 | } |
| 1341 | |
| 1342 | raid6_call.gen_syndrome(IOP_ADMA_NUM_SRC_TEST+2, PAGE_SIZE, pq_sw); |
| 1343 | |
| 1344 | if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST], |
| 1345 | page_address(pq_hw[0]), PAGE_SIZE) != 0) { |
| 1346 | dev_err(dev, "Self-test p failed compare, disabling\n"); |
| 1347 | err = -ENODEV; |
| 1348 | goto free_resources; |
| 1349 | } |
| 1350 | if (memcmp(pq_sw[IOP_ADMA_NUM_SRC_TEST+1], |
| 1351 | page_address(pq_hw[1]), PAGE_SIZE) != 0) { |
| 1352 | dev_err(dev, "Self-test q failed compare, disabling\n"); |
| 1353 | err = -ENODEV; |
| 1354 | goto free_resources; |
| 1355 | } |
| 1356 | |
| 1357 | /* test correct zero sum using the software generated pq values */ |
| 1358 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) |
| 1359 | pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, |
| 1360 | DMA_TO_DEVICE); |
| 1361 | |
| 1362 | zero_sum_result = ~0; |
| 1363 | tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], |
| 1364 | pq_src, IOP_ADMA_NUM_SRC_TEST, |
| 1365 | raid6_gfexp, PAGE_SIZE, &zero_sum_result, |
| 1366 | DMA_PREP_INTERRUPT|DMA_CTRL_ACK); |
| 1367 | |
| 1368 | cookie = iop_adma_tx_submit(tx); |
| 1369 | iop_adma_issue_pending(dma_chan); |
| 1370 | msleep(8); |
| 1371 | |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1372 | if (iop_adma_status(dma_chan, cookie, NULL) != |
Dan Williams | f6dbf651 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 1373 | DMA_SUCCESS) { |
| 1374 | dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n"); |
| 1375 | err = -ENODEV; |
| 1376 | goto free_resources; |
| 1377 | } |
| 1378 | |
| 1379 | if (zero_sum_result != 0) { |
| 1380 | dev_err(dev, "Self-test pq-zero-sum failed to validate: %x\n", |
| 1381 | zero_sum_result); |
| 1382 | err = -ENODEV; |
| 1383 | goto free_resources; |
| 1384 | } |
| 1385 | |
| 1386 | /* test incorrect zero sum */ |
| 1387 | i = IOP_ADMA_NUM_SRC_TEST; |
| 1388 | memset(pq_sw[i] + 100, 0, 100); |
| 1389 | memset(pq_sw[i+1] + 200, 0, 200); |
| 1390 | for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 2; i++) |
| 1391 | pq_src[i] = dma_map_page(dev, pq[i], 0, PAGE_SIZE, |
| 1392 | DMA_TO_DEVICE); |
| 1393 | |
| 1394 | zero_sum_result = 0; |
| 1395 | tx = iop_adma_prep_dma_pq_val(dma_chan, &pq_src[IOP_ADMA_NUM_SRC_TEST], |
| 1396 | pq_src, IOP_ADMA_NUM_SRC_TEST, |
| 1397 | raid6_gfexp, PAGE_SIZE, &zero_sum_result, |
| 1398 | DMA_PREP_INTERRUPT|DMA_CTRL_ACK); |
| 1399 | |
| 1400 | cookie = iop_adma_tx_submit(tx); |
| 1401 | iop_adma_issue_pending(dma_chan); |
| 1402 | msleep(8); |
| 1403 | |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1404 | if (iop_adma_status(dma_chan, cookie, NULL) != |
Dan Williams | f6dbf651 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 1405 | DMA_SUCCESS) { |
| 1406 | dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n"); |
| 1407 | err = -ENODEV; |
| 1408 | goto free_resources; |
| 1409 | } |
| 1410 | |
| 1411 | if (zero_sum_result != (SUM_CHECK_P_RESULT | SUM_CHECK_Q_RESULT)) { |
| 1412 | dev_err(dev, "Self-test !pq-zero-sum failed to validate: %x\n", |
| 1413 | zero_sum_result); |
| 1414 | err = -ENODEV; |
| 1415 | goto free_resources; |
| 1416 | } |
| 1417 | |
| 1418 | free_resources: |
| 1419 | iop_adma_free_chan_resources(dma_chan); |
| 1420 | out: |
| 1421 | i = ARRAY_SIZE(pq); |
| 1422 | while (i--) |
| 1423 | __free_page(pq[i]); |
| 1424 | return err; |
| 1425 | } |
| 1426 | #endif |
| 1427 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1428 | static int __devexit iop_adma_remove(struct platform_device *dev) |
| 1429 | { |
| 1430 | struct iop_adma_device *device = platform_get_drvdata(dev); |
| 1431 | struct dma_chan *chan, *_chan; |
| 1432 | struct iop_adma_chan *iop_chan; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1433 | struct iop_adma_platform_data *plat_data = dev->dev.platform_data; |
| 1434 | |
| 1435 | dma_async_device_unregister(&device->common); |
| 1436 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1437 | dma_free_coherent(&dev->dev, plat_data->pool_size, |
| 1438 | device->dma_desc_pool_virt, device->dma_desc_pool); |
| 1439 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1440 | list_for_each_entry_safe(chan, _chan, &device->common.channels, |
| 1441 | device_node) { |
| 1442 | iop_chan = to_iop_adma_chan(chan); |
| 1443 | list_del(&chan->device_node); |
| 1444 | kfree(iop_chan); |
| 1445 | } |
| 1446 | kfree(device); |
| 1447 | |
| 1448 | return 0; |
| 1449 | } |
| 1450 | |
| 1451 | static int __devinit iop_adma_probe(struct platform_device *pdev) |
| 1452 | { |
| 1453 | struct resource *res; |
| 1454 | int ret = 0, i; |
| 1455 | struct iop_adma_device *adev; |
| 1456 | struct iop_adma_chan *iop_chan; |
| 1457 | struct dma_device *dma_dev; |
| 1458 | struct iop_adma_platform_data *plat_data = pdev->dev.platform_data; |
| 1459 | |
| 1460 | res = platform_get_resource(pdev, IORESOURCE_MEM, 0); |
| 1461 | if (!res) |
| 1462 | return -ENODEV; |
| 1463 | |
| 1464 | if (!devm_request_mem_region(&pdev->dev, res->start, |
H Hartley Sweeten | 2e032b6 | 2009-12-11 21:24:33 -0700 | [diff] [blame] | 1465 | resource_size(res), pdev->name)) |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1466 | return -EBUSY; |
| 1467 | |
| 1468 | adev = kzalloc(sizeof(*adev), GFP_KERNEL); |
| 1469 | if (!adev) |
| 1470 | return -ENOMEM; |
| 1471 | dma_dev = &adev->common; |
| 1472 | |
| 1473 | /* allocate coherent memory for hardware descriptors |
| 1474 | * note: writecombine gives slightly better performance, but |
| 1475 | * requires that we explicitly flush the writes |
| 1476 | */ |
| 1477 | if ((adev->dma_desc_pool_virt = dma_alloc_writecombine(&pdev->dev, |
| 1478 | plat_data->pool_size, |
| 1479 | &adev->dma_desc_pool, |
| 1480 | GFP_KERNEL)) == NULL) { |
| 1481 | ret = -ENOMEM; |
| 1482 | goto err_free_adev; |
| 1483 | } |
| 1484 | |
| 1485 | dev_dbg(&pdev->dev, "%s: allocted descriptor pool virt %p phys %p\n", |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 1486 | __func__, adev->dma_desc_pool_virt, |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1487 | (void *) adev->dma_desc_pool); |
| 1488 | |
| 1489 | adev->id = plat_data->hw_id; |
| 1490 | |
| 1491 | /* discover transaction capabilites from the platform data */ |
| 1492 | dma_dev->cap_mask = plat_data->cap_mask; |
| 1493 | |
| 1494 | adev->pdev = pdev; |
| 1495 | platform_set_drvdata(pdev, adev); |
| 1496 | |
| 1497 | INIT_LIST_HEAD(&dma_dev->channels); |
| 1498 | |
| 1499 | /* set base routines */ |
| 1500 | dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources; |
| 1501 | dma_dev->device_free_chan_resources = iop_adma_free_chan_resources; |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 1502 | dma_dev->device_tx_status = iop_adma_status; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1503 | dma_dev->device_issue_pending = iop_adma_issue_pending; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1504 | dma_dev->dev = &pdev->dev; |
| 1505 | |
| 1506 | /* set prep routines based on capability */ |
| 1507 | if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) |
| 1508 | dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy; |
| 1509 | if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) |
| 1510 | dma_dev->device_prep_dma_memset = iop_adma_prep_dma_memset; |
| 1511 | if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { |
| 1512 | dma_dev->max_xor = iop_adma_get_max_xor(); |
| 1513 | dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor; |
| 1514 | } |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 1515 | if (dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask)) |
| 1516 | dma_dev->device_prep_dma_xor_val = |
| 1517 | iop_adma_prep_dma_xor_val; |
Dan Williams | 7bf649a | 2009-08-28 14:32:04 -0700 | [diff] [blame] | 1518 | if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { |
| 1519 | dma_set_maxpq(dma_dev, iop_adma_get_max_pq(), 0); |
| 1520 | dma_dev->device_prep_dma_pq = iop_adma_prep_dma_pq; |
| 1521 | } |
| 1522 | if (dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) |
| 1523 | dma_dev->device_prep_dma_pq_val = |
| 1524 | iop_adma_prep_dma_pq_val; |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1525 | if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) |
| 1526 | dma_dev->device_prep_dma_interrupt = |
| 1527 | iop_adma_prep_dma_interrupt; |
| 1528 | |
| 1529 | iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL); |
| 1530 | if (!iop_chan) { |
| 1531 | ret = -ENOMEM; |
| 1532 | goto err_free_dma; |
| 1533 | } |
| 1534 | iop_chan->device = adev; |
| 1535 | |
| 1536 | iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start, |
H Hartley Sweeten | 2e032b6 | 2009-12-11 21:24:33 -0700 | [diff] [blame] | 1537 | resource_size(res)); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1538 | if (!iop_chan->mmr_base) { |
| 1539 | ret = -ENOMEM; |
| 1540 | goto err_free_iop_chan; |
| 1541 | } |
| 1542 | tasklet_init(&iop_chan->irq_tasklet, iop_adma_tasklet, (unsigned long) |
| 1543 | iop_chan); |
| 1544 | |
| 1545 | /* clear errors before enabling interrupts */ |
| 1546 | iop_adma_device_clear_err_status(iop_chan); |
| 1547 | |
| 1548 | for (i = 0; i < 3; i++) { |
| 1549 | irq_handler_t handler[] = { iop_adma_eot_handler, |
| 1550 | iop_adma_eoc_handler, |
| 1551 | iop_adma_err_handler }; |
| 1552 | int irq = platform_get_irq(pdev, i); |
| 1553 | if (irq < 0) { |
| 1554 | ret = -ENXIO; |
| 1555 | goto err_free_iop_chan; |
| 1556 | } else { |
| 1557 | ret = devm_request_irq(&pdev->dev, irq, |
| 1558 | handler[i], 0, pdev->name, iop_chan); |
| 1559 | if (ret) |
| 1560 | goto err_free_iop_chan; |
| 1561 | } |
| 1562 | } |
| 1563 | |
| 1564 | spin_lock_init(&iop_chan->lock); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1565 | INIT_LIST_HEAD(&iop_chan->chain); |
| 1566 | INIT_LIST_HEAD(&iop_chan->all_slots); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1567 | iop_chan->common.device = dma_dev; |
| 1568 | list_add_tail(&iop_chan->common.device_node, &dma_dev->channels); |
| 1569 | |
| 1570 | if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { |
| 1571 | ret = iop_adma_memcpy_self_test(adev); |
| 1572 | dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret); |
| 1573 | if (ret) |
| 1574 | goto err_free_iop_chan; |
| 1575 | } |
| 1576 | |
| 1577 | if (dma_has_cap(DMA_XOR, dma_dev->cap_mask) || |
Dan Williams | f6dbf651 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 1578 | dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) { |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 1579 | ret = iop_adma_xor_val_self_test(adev); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1580 | dev_dbg(&pdev->dev, "xor self test returned %d\n", ret); |
| 1581 | if (ret) |
| 1582 | goto err_free_iop_chan; |
| 1583 | } |
| 1584 | |
Dan Williams | f6dbf651 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 1585 | if (dma_has_cap(DMA_PQ, dma_dev->cap_mask) && |
| 1586 | dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask)) { |
Wei Yongquan | 0261f74 | 2010-12-29 20:30:55 +0800 | [diff] [blame] | 1587 | #ifdef CONFIG_RAID6_PQ |
Dan Williams | f6dbf651 | 2009-08-29 19:12:40 -0700 | [diff] [blame] | 1588 | ret = iop_adma_pq_zero_sum_self_test(adev); |
| 1589 | dev_dbg(&pdev->dev, "pq self test returned %d\n", ret); |
| 1590 | #else |
| 1591 | /* can not test raid6, so do not publish capability */ |
| 1592 | dma_cap_clear(DMA_PQ, dma_dev->cap_mask); |
| 1593 | dma_cap_clear(DMA_PQ_VAL, dma_dev->cap_mask); |
| 1594 | ret = 0; |
| 1595 | #endif |
| 1596 | if (ret) |
| 1597 | goto err_free_iop_chan; |
| 1598 | } |
| 1599 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1600 | dev_printk(KERN_INFO, &pdev->dev, "Intel(R) IOP: " |
Dan Williams | 9308add | 2009-09-08 17:42:52 -0700 | [diff] [blame] | 1601 | "( %s%s%s%s%s%s%s)\n", |
Dan Williams | b2f46fd | 2009-07-14 12:20:36 -0700 | [diff] [blame] | 1602 | dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : "", |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 1603 | dma_has_cap(DMA_PQ_VAL, dma_dev->cap_mask) ? "pq_val " : "", |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1604 | dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", |
Dan Williams | 099f53c | 2009-04-08 14:28:37 -0700 | [diff] [blame] | 1605 | dma_has_cap(DMA_XOR_VAL, dma_dev->cap_mask) ? "xor_val " : "", |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1606 | dma_has_cap(DMA_MEMSET, dma_dev->cap_mask) ? "fill " : "", |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1607 | dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "", |
| 1608 | dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : ""); |
| 1609 | |
| 1610 | dma_async_device_register(dma_dev); |
| 1611 | goto out; |
| 1612 | |
| 1613 | err_free_iop_chan: |
| 1614 | kfree(iop_chan); |
| 1615 | err_free_dma: |
| 1616 | dma_free_coherent(&adev->pdev->dev, plat_data->pool_size, |
| 1617 | adev->dma_desc_pool_virt, adev->dma_desc_pool); |
| 1618 | err_free_adev: |
| 1619 | kfree(adev); |
| 1620 | out: |
| 1621 | return ret; |
| 1622 | } |
| 1623 | |
| 1624 | static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan) |
| 1625 | { |
| 1626 | struct iop_adma_desc_slot *sw_desc, *grp_start; |
| 1627 | dma_cookie_t cookie; |
| 1628 | int slot_cnt, slots_per_op; |
| 1629 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 1630 | dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1631 | |
| 1632 | spin_lock_bh(&iop_chan->lock); |
| 1633 | slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op); |
| 1634 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 1635 | if (sw_desc) { |
| 1636 | grp_start = sw_desc->group_head; |
| 1637 | |
Dan Williams | 308136d | 2009-09-08 17:53:02 -0700 | [diff] [blame] | 1638 | list_splice_init(&sw_desc->tx_list, &iop_chan->chain); |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 1639 | async_tx_ack(&sw_desc->async_tx); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1640 | iop_desc_init_memcpy(grp_start, 0); |
| 1641 | iop_desc_set_byte_count(grp_start, iop_chan, 0); |
| 1642 | iop_desc_set_dest_addr(grp_start, iop_chan, 0); |
| 1643 | iop_desc_set_memcpy_src_addr(grp_start, 0); |
| 1644 | |
| 1645 | cookie = iop_chan->common.cookie; |
| 1646 | cookie++; |
| 1647 | if (cookie <= 1) |
| 1648 | cookie = 2; |
| 1649 | |
| 1650 | /* initialize the completed cookie to be less than |
| 1651 | * the most recently used cookie |
| 1652 | */ |
| 1653 | iop_chan->completed_cookie = cookie - 1; |
| 1654 | iop_chan->common.cookie = sw_desc->async_tx.cookie = cookie; |
| 1655 | |
| 1656 | /* channel should not be busy */ |
| 1657 | BUG_ON(iop_chan_is_busy(iop_chan)); |
| 1658 | |
| 1659 | /* clear any prior error-status bits */ |
| 1660 | iop_adma_device_clear_err_status(iop_chan); |
| 1661 | |
| 1662 | /* disable operation */ |
| 1663 | iop_chan_disable(iop_chan); |
| 1664 | |
| 1665 | /* set the descriptor address */ |
| 1666 | iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); |
| 1667 | |
| 1668 | /* 1/ don't add pre-chained descriptors |
| 1669 | * 2/ dummy read to flush next_desc write |
| 1670 | */ |
| 1671 | BUG_ON(iop_desc_get_next_desc(sw_desc)); |
| 1672 | |
| 1673 | /* run the descriptor */ |
| 1674 | iop_chan_enable(iop_chan); |
| 1675 | } else |
| 1676 | dev_printk(KERN_ERR, iop_chan->device->common.dev, |
| 1677 | "failed to allocate null descriptor\n"); |
| 1678 | spin_unlock_bh(&iop_chan->lock); |
| 1679 | } |
| 1680 | |
| 1681 | static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan) |
| 1682 | { |
| 1683 | struct iop_adma_desc_slot *sw_desc, *grp_start; |
| 1684 | dma_cookie_t cookie; |
| 1685 | int slot_cnt, slots_per_op; |
| 1686 | |
Harvey Harrison | 3d9b525 | 2008-03-13 17:45:28 -0700 | [diff] [blame] | 1687 | dev_dbg(iop_chan->device->common.dev, "%s\n", __func__); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1688 | |
| 1689 | spin_lock_bh(&iop_chan->lock); |
| 1690 | slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op); |
| 1691 | sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op); |
| 1692 | if (sw_desc) { |
| 1693 | grp_start = sw_desc->group_head; |
Dan Williams | 308136d | 2009-09-08 17:53:02 -0700 | [diff] [blame] | 1694 | list_splice_init(&sw_desc->tx_list, &iop_chan->chain); |
Dan Williams | 636bdea | 2008-04-17 20:17:26 -0700 | [diff] [blame] | 1695 | async_tx_ack(&sw_desc->async_tx); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1696 | iop_desc_init_null_xor(grp_start, 2, 0); |
| 1697 | iop_desc_set_byte_count(grp_start, iop_chan, 0); |
| 1698 | iop_desc_set_dest_addr(grp_start, iop_chan, 0); |
| 1699 | iop_desc_set_xor_src_addr(grp_start, 0, 0); |
| 1700 | iop_desc_set_xor_src_addr(grp_start, 1, 0); |
| 1701 | |
| 1702 | cookie = iop_chan->common.cookie; |
| 1703 | cookie++; |
| 1704 | if (cookie <= 1) |
| 1705 | cookie = 2; |
| 1706 | |
| 1707 | /* initialize the completed cookie to be less than |
| 1708 | * the most recently used cookie |
| 1709 | */ |
| 1710 | iop_chan->completed_cookie = cookie - 1; |
| 1711 | iop_chan->common.cookie = sw_desc->async_tx.cookie = cookie; |
| 1712 | |
| 1713 | /* channel should not be busy */ |
| 1714 | BUG_ON(iop_chan_is_busy(iop_chan)); |
| 1715 | |
| 1716 | /* clear any prior error-status bits */ |
| 1717 | iop_adma_device_clear_err_status(iop_chan); |
| 1718 | |
| 1719 | /* disable operation */ |
| 1720 | iop_chan_disable(iop_chan); |
| 1721 | |
| 1722 | /* set the descriptor address */ |
| 1723 | iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys); |
| 1724 | |
| 1725 | /* 1/ don't add pre-chained descriptors |
| 1726 | * 2/ dummy read to flush next_desc write |
| 1727 | */ |
| 1728 | BUG_ON(iop_desc_get_next_desc(sw_desc)); |
| 1729 | |
| 1730 | /* run the descriptor */ |
| 1731 | iop_chan_enable(iop_chan); |
| 1732 | } else |
| 1733 | dev_printk(KERN_ERR, iop_chan->device->common.dev, |
| 1734 | "failed to allocate null descriptor\n"); |
| 1735 | spin_unlock_bh(&iop_chan->lock); |
| 1736 | } |
| 1737 | |
Kay Sievers | ebabe27 | 2008-07-08 11:58:28 -0700 | [diff] [blame] | 1738 | MODULE_ALIAS("platform:iop-adma"); |
| 1739 | |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1740 | static struct platform_driver iop_adma_driver = { |
| 1741 | .probe = iop_adma_probe, |
Russell King | bdf602b | 2009-03-03 13:43:47 +0000 | [diff] [blame] | 1742 | .remove = __devexit_p(iop_adma_remove), |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1743 | .driver = { |
| 1744 | .owner = THIS_MODULE, |
| 1745 | .name = "iop-adma", |
| 1746 | }, |
| 1747 | }; |
| 1748 | |
| 1749 | static int __init iop_adma_init (void) |
| 1750 | { |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1751 | return platform_driver_register(&iop_adma_driver); |
| 1752 | } |
| 1753 | |
| 1754 | static void __exit iop_adma_exit (void) |
| 1755 | { |
| 1756 | platform_driver_unregister(&iop_adma_driver); |
| 1757 | return; |
| 1758 | } |
Rusty Russell | af49d92 | 2007-10-16 23:26:27 -0700 | [diff] [blame] | 1759 | module_exit(iop_adma_exit); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1760 | module_init(iop_adma_init); |
Dan Williams | c211092 | 2007-01-02 13:52:26 -0700 | [diff] [blame] | 1761 | |
| 1762 | MODULE_AUTHOR("Intel Corporation"); |
| 1763 | MODULE_DESCRIPTION("IOP ADMA Engine Driver"); |
| 1764 | MODULE_LICENSE("GPL"); |