Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 1 | /* |
Shannon Nelson | 43d6e36 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 2 | * Intel I/OAT DMA Linux driver |
Dave Jiang | 85596a1 | 2015-08-11 08:48:10 -0700 | [diff] [blame] | 3 | * Copyright(c) 2004 - 2015 Intel Corporation. |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 4 | * |
| 5 | * This program is free software; you can redistribute it and/or modify it |
Shannon Nelson | 43d6e36 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 6 | * under the terms and conditions of the GNU General Public License, |
| 7 | * version 2, as published by the Free Software Foundation. |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 12 | * more details. |
| 13 | * |
Shannon Nelson | 43d6e36 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 14 | * The full GNU General Public License is included in this distribution in |
| 15 | * the file called "COPYING". |
| 16 | * |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 17 | */ |
| 18 | |
| 19 | /* |
| 20 | * This driver supports an Intel I/OAT DMA engine, which does asynchronous |
| 21 | * copy operations. |
| 22 | */ |
| 23 | |
| 24 | #include <linux/init.h> |
| 25 | #include <linux/module.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 26 | #include <linux/slab.h> |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 27 | #include <linux/pci.h> |
| 28 | #include <linux/interrupt.h> |
| 29 | #include <linux/dmaengine.h> |
| 30 | #include <linux/delay.h> |
David S. Miller | 6b00c92 | 2006-05-23 17:37:58 -0700 | [diff] [blame] | 31 | #include <linux/dma-mapping.h> |
Maciej Sosnowski | 09177e8 | 2008-07-22 10:07:33 -0700 | [diff] [blame] | 32 | #include <linux/workqueue.h> |
Paul Gortmaker | 70c7160 | 2011-05-22 16:47:17 -0400 | [diff] [blame] | 33 | #include <linux/prefetch.h> |
Dan Williams | 584ec22 | 2009-07-28 14:32:12 -0700 | [diff] [blame] | 34 | #include "dma.h" |
| 35 | #include "registers.h" |
| 36 | #include "hw.h" |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 37 | |
Russell King - ARM Linux | d2ebfb3 | 2012-03-06 22:34:26 +0000 | [diff] [blame] | 38 | #include "../dmaengine.h" |
| 39 | |
Dan Williams | 5cbafa6 | 2009-08-26 13:01:44 -0700 | [diff] [blame] | 40 | int ioat_pending_level = 4; |
Shannon Nelson | 7bb67c1 | 2007-11-14 16:59:51 -0800 | [diff] [blame] | 41 | module_param(ioat_pending_level, int, 0644); |
| 42 | MODULE_PARM_DESC(ioat_pending_level, |
| 43 | "high-water mark for pushing ioat descriptors (default: 4)"); |
| 44 | |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 45 | /** |
| 46 | * ioat_dma_do_interrupt - handler used for single vector interrupt mode |
| 47 | * @irq: interrupt id |
| 48 | * @data: interrupt data |
| 49 | */ |
| 50 | static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) |
| 51 | { |
| 52 | struct ioatdma_device *instance = data; |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 53 | struct ioatdma_chan *ioat_chan; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 54 | unsigned long attnstatus; |
| 55 | int bit; |
| 56 | u8 intrctrl; |
| 57 | |
| 58 | intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET); |
| 59 | |
| 60 | if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN)) |
| 61 | return IRQ_NONE; |
| 62 | |
| 63 | if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) { |
| 64 | writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); |
| 65 | return IRQ_NONE; |
| 66 | } |
| 67 | |
| 68 | attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); |
Akinobu Mita | 984b3f5 | 2010-03-05 13:41:37 -0800 | [diff] [blame] | 69 | for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) { |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 70 | ioat_chan = ioat_chan_by_index(instance, bit); |
| 71 | if (test_bit(IOAT_RUN, &ioat_chan->state)) |
| 72 | tasklet_schedule(&ioat_chan->cleanup_task); |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 73 | } |
| 74 | |
| 75 | writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); |
| 76 | return IRQ_HANDLED; |
| 77 | } |
| 78 | |
| 79 | /** |
| 80 | * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode |
| 81 | * @irq: interrupt id |
| 82 | * @data: interrupt data |
| 83 | */ |
| 84 | static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) |
| 85 | { |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 86 | struct ioatdma_chan *ioat_chan = data; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 87 | |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 88 | if (test_bit(IOAT_RUN, &ioat_chan->state)) |
| 89 | tasklet_schedule(&ioat_chan->cleanup_task); |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 90 | |
| 91 | return IRQ_HANDLED; |
| 92 | } |
| 93 | |
Dan Williams | 5cbafa6 | 2009-08-26 13:01:44 -0700 | [diff] [blame] | 94 | /* common channel initialization */ |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 95 | void |
| 96 | ioat_init_channel(struct ioatdma_device *device, struct ioatdma_chan *ioat_chan, |
| 97 | int idx) |
Dan Williams | 5cbafa6 | 2009-08-26 13:01:44 -0700 | [diff] [blame] | 98 | { |
| 99 | struct dma_device *dma = &device->common; |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 100 | struct dma_chan *c = &ioat_chan->dma_chan; |
Dan Williams | aa4d72a | 2010-03-03 21:21:13 -0700 | [diff] [blame] | 101 | unsigned long data = (unsigned long) c; |
Dan Williams | 5cbafa6 | 2009-08-26 13:01:44 -0700 | [diff] [blame] | 102 | |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 103 | ioat_chan->device = device; |
| 104 | ioat_chan->reg_base = device->reg_base + (0x80 * (idx + 1)); |
| 105 | spin_lock_init(&ioat_chan->cleanup_lock); |
| 106 | ioat_chan->dma_chan.device = dma; |
| 107 | dma_cookie_init(&ioat_chan->dma_chan); |
| 108 | list_add_tail(&ioat_chan->dma_chan.device_node, &dma->channels); |
| 109 | device->idx[idx] = ioat_chan; |
| 110 | init_timer(&ioat_chan->timer); |
| 111 | ioat_chan->timer.function = device->timer_fn; |
| 112 | ioat_chan->timer.data = data; |
| 113 | tasklet_init(&ioat_chan->cleanup_task, device->cleanup_fn, data); |
Dan Williams | 5cbafa6 | 2009-08-26 13:01:44 -0700 | [diff] [blame] | 114 | } |
| 115 | |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 116 | void ioat_stop(struct ioatdma_chan *ioat_chan) |
Dan Williams | da87ca4 | 2014-02-19 16:19:35 -0800 | [diff] [blame] | 117 | { |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 118 | struct ioatdma_device *device = ioat_chan->device; |
Dan Williams | da87ca4 | 2014-02-19 16:19:35 -0800 | [diff] [blame] | 119 | struct pci_dev *pdev = device->pdev; |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 120 | int chan_id = chan_num(ioat_chan); |
Dan Williams | da87ca4 | 2014-02-19 16:19:35 -0800 | [diff] [blame] | 121 | struct msix_entry *msix; |
| 122 | |
| 123 | /* 1/ stop irq from firing tasklets |
| 124 | * 2/ stop the tasklet from re-arming irqs |
| 125 | */ |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 126 | clear_bit(IOAT_RUN, &ioat_chan->state); |
Dan Williams | da87ca4 | 2014-02-19 16:19:35 -0800 | [diff] [blame] | 127 | |
| 128 | /* flush inflight interrupts */ |
| 129 | switch (device->irq_mode) { |
| 130 | case IOAT_MSIX: |
| 131 | msix = &device->msix_entries[chan_id]; |
| 132 | synchronize_irq(msix->vector); |
| 133 | break; |
| 134 | case IOAT_MSI: |
| 135 | case IOAT_INTX: |
| 136 | synchronize_irq(pdev->irq); |
| 137 | break; |
| 138 | default: |
| 139 | break; |
| 140 | } |
| 141 | |
| 142 | /* flush inflight timers */ |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 143 | del_timer_sync(&ioat_chan->timer); |
Dan Williams | da87ca4 | 2014-02-19 16:19:35 -0800 | [diff] [blame] | 144 | |
| 145 | /* flush inflight tasklet runs */ |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 146 | tasklet_kill(&ioat_chan->cleanup_task); |
Dan Williams | da87ca4 | 2014-02-19 16:19:35 -0800 | [diff] [blame] | 147 | |
| 148 | /* final cleanup now that everything is quiesced and can't re-arm */ |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 149 | device->cleanup_fn((unsigned long)&ioat_chan->dma_chan); |
Dan Williams | da87ca4 | 2014-02-19 16:19:35 -0800 | [diff] [blame] | 150 | } |
| 151 | |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 152 | dma_addr_t ioat_get_current_completion(struct ioatdma_chan *ioat_chan) |
Dan Williams | 5cbafa6 | 2009-08-26 13:01:44 -0700 | [diff] [blame] | 153 | { |
Dan Williams | 2750293 | 2012-03-23 13:36:42 -0700 | [diff] [blame] | 154 | dma_addr_t phys_complete; |
Dan Williams | 4fb9b9e | 2009-09-08 12:01:04 -0700 | [diff] [blame] | 155 | u64 completion; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 156 | |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 157 | completion = *ioat_chan->completion; |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 158 | phys_complete = ioat_chansts_to_addr(completion); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 159 | |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 160 | dev_dbg(to_dev(ioat_chan), "%s: phys_complete: %#llx\n", __func__, |
Dan Williams | 6df9183 | 2009-09-08 12:00:55 -0700 | [diff] [blame] | 161 | (unsigned long long) phys_complete); |
| 162 | |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 163 | if (is_ioat_halted(completion)) { |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 164 | u32 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET); |
| 165 | |
| 166 | dev_err(to_dev(ioat_chan), "Channel halted, chanerr = %x\n", |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 167 | chanerr); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 168 | |
| 169 | /* TODO do something to salvage the situation */ |
| 170 | } |
| 171 | |
Dan Williams | 5cbafa6 | 2009-08-26 13:01:44 -0700 | [diff] [blame] | 172 | return phys_complete; |
| 173 | } |
| 174 | |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 175 | bool ioat_cleanup_preamble(struct ioatdma_chan *ioat_chan, |
Dan Williams | 2750293 | 2012-03-23 13:36:42 -0700 | [diff] [blame] | 176 | dma_addr_t *phys_complete) |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 177 | { |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 178 | *phys_complete = ioat_get_current_completion(ioat_chan); |
| 179 | if (*phys_complete == ioat_chan->last_completion) |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 180 | return false; |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 181 | clear_bit(IOAT_COMPLETION_ACK, &ioat_chan->state); |
| 182 | mod_timer(&ioat_chan->timer, jiffies + COMPLETION_TIMEOUT); |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 183 | |
| 184 | return true; |
| 185 | } |
| 186 | |
Dan Williams | aa4d72a | 2010-03-03 21:21:13 -0700 | [diff] [blame] | 187 | enum dma_status |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 188 | ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, |
| 189 | struct dma_tx_state *txstate) |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 190 | { |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 191 | struct ioatdma_chan *ioat_chan = to_ioat_chan(c); |
| 192 | struct ioatdma_device *device = ioat_chan->device; |
Russell King - ARM Linux | 96a2af4 | 2012-03-06 22:35:27 +0000 | [diff] [blame] | 193 | enum dma_status ret; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 194 | |
Russell King - ARM Linux | 96a2af4 | 2012-03-06 22:35:27 +0000 | [diff] [blame] | 195 | ret = dma_cookie_status(c, cookie, txstate); |
Vinod Koul | 2f16f80 | 2013-10-16 20:48:52 +0530 | [diff] [blame] | 196 | if (ret == DMA_COMPLETE) |
Russell King - ARM Linux | 96a2af4 | 2012-03-06 22:35:27 +0000 | [diff] [blame] | 197 | return ret; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 198 | |
Dan Williams | aa4d72a | 2010-03-03 21:21:13 -0700 | [diff] [blame] | 199 | device->cleanup_fn((unsigned long) c); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 200 | |
Russell King - ARM Linux | 96a2af4 | 2012-03-06 22:35:27 +0000 | [diff] [blame] | 201 | return dma_cookie_status(c, cookie, txstate); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 202 | } |
| 203 | |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 204 | /* |
| 205 | * Perform a IOAT transaction to verify the HW works. |
| 206 | */ |
| 207 | #define IOAT_TEST_SIZE 2000 |
| 208 | |
Greg Kroah-Hartman | 4bf27b8 | 2012-12-21 15:09:59 -0800 | [diff] [blame] | 209 | static void ioat_dma_test_callback(void *dma_async_param) |
Shannon Nelson | 9521843 | 2007-10-18 03:07:15 -0700 | [diff] [blame] | 210 | { |
Dan Williams | b9bdcbb | 2009-01-06 11:38:22 -0700 | [diff] [blame] | 211 | struct completion *cmp = dma_async_param; |
| 212 | |
| 213 | complete(cmp); |
Shannon Nelson | 9521843 | 2007-10-18 03:07:15 -0700 | [diff] [blame] | 214 | } |
| 215 | |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 216 | /** |
| 217 | * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works. |
| 218 | * @device: device to be tested |
| 219 | */ |
Greg Kroah-Hartman | 4bf27b8 | 2012-12-21 15:09:59 -0800 | [diff] [blame] | 220 | int ioat_dma_self_test(struct ioatdma_device *device) |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 221 | { |
| 222 | int i; |
| 223 | u8 *src; |
| 224 | u8 *dest; |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 225 | struct dma_device *dma = &device->common; |
| 226 | struct device *dev = &device->pdev->dev; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 227 | struct dma_chan *dma_chan; |
Shannon Nelson | 711924b | 2007-12-17 16:20:08 -0800 | [diff] [blame] | 228 | struct dma_async_tx_descriptor *tx; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 229 | dma_addr_t dma_dest, dma_src; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 230 | dma_cookie_t cookie; |
| 231 | int err = 0; |
Dan Williams | b9bdcbb | 2009-01-06 11:38:22 -0700 | [diff] [blame] | 232 | struct completion cmp; |
Dan Williams | 0c33e1c | 2009-03-02 13:31:35 -0700 | [diff] [blame] | 233 | unsigned long tmo; |
Maciej Sosnowski | 4f005db | 2009-04-23 12:31:51 +0200 | [diff] [blame] | 234 | unsigned long flags; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 235 | |
Christoph Lameter | e94b176 | 2006-12-06 20:33:17 -0800 | [diff] [blame] | 236 | src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 237 | if (!src) |
| 238 | return -ENOMEM; |
Christoph Lameter | e94b176 | 2006-12-06 20:33:17 -0800 | [diff] [blame] | 239 | dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 240 | if (!dest) { |
| 241 | kfree(src); |
| 242 | return -ENOMEM; |
| 243 | } |
| 244 | |
| 245 | /* Fill in src buffer */ |
| 246 | for (i = 0; i < IOAT_TEST_SIZE; i++) |
| 247 | src[i] = (u8)i; |
| 248 | |
| 249 | /* Start copy, using first DMA channel */ |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 250 | dma_chan = container_of(dma->channels.next, struct dma_chan, |
Shannon Nelson | 43d6e36 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 251 | device_node); |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 252 | if (dma->device_alloc_chan_resources(dma_chan) < 1) { |
| 253 | dev_err(dev, "selftest cannot allocate chan resource\n"); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 254 | err = -ENODEV; |
| 255 | goto out; |
| 256 | } |
| 257 | |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 258 | dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE); |
Jiang Liu | 3532e56 | 2014-01-02 12:58:52 -0800 | [diff] [blame] | 259 | if (dma_mapping_error(dev, dma_src)) { |
| 260 | dev_err(dev, "mapping src buffer failed\n"); |
| 261 | goto free_resources; |
| 262 | } |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 263 | dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); |
Jiang Liu | 3532e56 | 2014-01-02 12:58:52 -0800 | [diff] [blame] | 264 | if (dma_mapping_error(dev, dma_dest)) { |
| 265 | dev_err(dev, "mapping dest buffer failed\n"); |
| 266 | goto unmap_src; |
| 267 | } |
Bartlomiej Zolnierkiewicz | 0776ae7 | 2013-10-18 19:35:33 +0200 | [diff] [blame] | 268 | flags = DMA_PREP_INTERRUPT; |
Dan Williams | 0036731 | 2008-02-02 19:49:57 -0700 | [diff] [blame] | 269 | tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src, |
Maciej Sosnowski | 4f005db | 2009-04-23 12:31:51 +0200 | [diff] [blame] | 270 | IOAT_TEST_SIZE, flags); |
Shannon Nelson | 5149fd0 | 2007-10-18 03:07:13 -0700 | [diff] [blame] | 271 | if (!tx) { |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 272 | dev_err(dev, "Self-test prep failed, disabling\n"); |
Shannon Nelson | 5149fd0 | 2007-10-18 03:07:13 -0700 | [diff] [blame] | 273 | err = -ENODEV; |
Bartlomiej Zolnierkiewicz | 522d974 | 2012-11-05 10:00:13 +0000 | [diff] [blame] | 274 | goto unmap_dma; |
Shannon Nelson | 5149fd0 | 2007-10-18 03:07:13 -0700 | [diff] [blame] | 275 | } |
| 276 | |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 277 | async_tx_ack(tx); |
Dan Williams | b9bdcbb | 2009-01-06 11:38:22 -0700 | [diff] [blame] | 278 | init_completion(&cmp); |
Shannon Nelson | 9521843 | 2007-10-18 03:07:15 -0700 | [diff] [blame] | 279 | tx->callback = ioat_dma_test_callback; |
Dan Williams | b9bdcbb | 2009-01-06 11:38:22 -0700 | [diff] [blame] | 280 | tx->callback_param = &cmp; |
Shannon Nelson | 7bb67c1 | 2007-11-14 16:59:51 -0800 | [diff] [blame] | 281 | cookie = tx->tx_submit(tx); |
Shannon Nelson | 7f2b291 | 2007-10-18 03:07:14 -0700 | [diff] [blame] | 282 | if (cookie < 0) { |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 283 | dev_err(dev, "Self-test setup failed, disabling\n"); |
Shannon Nelson | 7f2b291 | 2007-10-18 03:07:14 -0700 | [diff] [blame] | 284 | err = -ENODEV; |
Bartlomiej Zolnierkiewicz | 522d974 | 2012-11-05 10:00:13 +0000 | [diff] [blame] | 285 | goto unmap_dma; |
Shannon Nelson | 7f2b291 | 2007-10-18 03:07:14 -0700 | [diff] [blame] | 286 | } |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 287 | dma->device_issue_pending(dma_chan); |
Dan Williams | 532d3b1 | 2008-12-03 17:16:55 -0700 | [diff] [blame] | 288 | |
Dan Williams | 0c33e1c | 2009-03-02 13:31:35 -0700 | [diff] [blame] | 289 | tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 290 | |
Dan Williams | 0c33e1c | 2009-03-02 13:31:35 -0700 | [diff] [blame] | 291 | if (tmo == 0 || |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 292 | dma->device_tx_status(dma_chan, cookie, NULL) |
Vinod Koul | 2f16f80 | 2013-10-16 20:48:52 +0530 | [diff] [blame] | 293 | != DMA_COMPLETE) { |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 294 | dev_err(dev, "Self-test copy timed out, disabling\n"); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 295 | err = -ENODEV; |
Bartlomiej Zolnierkiewicz | 522d974 | 2012-11-05 10:00:13 +0000 | [diff] [blame] | 296 | goto unmap_dma; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 297 | } |
| 298 | if (memcmp(src, dest, IOAT_TEST_SIZE)) { |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 299 | dev_err(dev, "Self-test copy failed compare, disabling\n"); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 300 | err = -ENODEV; |
| 301 | goto free_resources; |
| 302 | } |
| 303 | |
Bartlomiej Zolnierkiewicz | 522d974 | 2012-11-05 10:00:13 +0000 | [diff] [blame] | 304 | unmap_dma: |
Bartlomiej Zolnierkiewicz | 522d974 | 2012-11-05 10:00:13 +0000 | [diff] [blame] | 305 | dma_unmap_single(dev, dma_dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); |
Jiang Liu | 3532e56 | 2014-01-02 12:58:52 -0800 | [diff] [blame] | 306 | unmap_src: |
| 307 | dma_unmap_single(dev, dma_src, IOAT_TEST_SIZE, DMA_TO_DEVICE); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 308 | free_resources: |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 309 | dma->device_free_chan_resources(dma_chan); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 310 | out: |
| 311 | kfree(src); |
| 312 | kfree(dest); |
| 313 | return err; |
| 314 | } |
| 315 | |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 316 | static char ioat_interrupt_style[32] = "msix"; |
| 317 | module_param_string(ioat_interrupt_style, ioat_interrupt_style, |
| 318 | sizeof(ioat_interrupt_style), 0644); |
| 319 | MODULE_PARM_DESC(ioat_interrupt_style, |
Dan Williams | 4c5d961 | 2013-11-13 16:29:52 -0800 | [diff] [blame] | 320 | "set ioat interrupt style: msix (default), msi, intx"); |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 321 | |
| 322 | /** |
| 323 | * ioat_dma_setup_interrupts - setup interrupt handler |
| 324 | * @device: ioat device |
| 325 | */ |
Dave Jiang | 8a52b9f | 2013-03-26 15:42:47 -0700 | [diff] [blame] | 326 | int ioat_dma_setup_interrupts(struct ioatdma_device *device) |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 327 | { |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 328 | struct ioatdma_chan *ioat_chan; |
Dan Williams | e6c0b69 | 2009-09-08 17:29:44 -0700 | [diff] [blame] | 329 | struct pci_dev *pdev = device->pdev; |
| 330 | struct device *dev = &pdev->dev; |
| 331 | struct msix_entry *msix; |
| 332 | int i, j, msixcnt; |
| 333 | int err = -EINVAL; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 334 | u8 intrctrl = 0; |
| 335 | |
| 336 | if (!strcmp(ioat_interrupt_style, "msix")) |
| 337 | goto msix; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 338 | if (!strcmp(ioat_interrupt_style, "msi")) |
| 339 | goto msi; |
| 340 | if (!strcmp(ioat_interrupt_style, "intx")) |
| 341 | goto intx; |
Dan Williams | e6c0b69 | 2009-09-08 17:29:44 -0700 | [diff] [blame] | 342 | dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style); |
Shannon Nelson | 5149fd0 | 2007-10-18 03:07:13 -0700 | [diff] [blame] | 343 | goto err_no_irq; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 344 | |
| 345 | msix: |
| 346 | /* The number of MSI-X vectors should equal the number of channels */ |
| 347 | msixcnt = device->common.chancnt; |
| 348 | for (i = 0; i < msixcnt; i++) |
| 349 | device->msix_entries[i].entry = i; |
| 350 | |
Alexander Gordeev | 368da99 | 2014-03-06 21:11:21 +0100 | [diff] [blame] | 351 | err = pci_enable_msix_exact(pdev, device->msix_entries, msixcnt); |
Dan Williams | 4c5d961 | 2013-11-13 16:29:52 -0800 | [diff] [blame] | 352 | if (err) |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 353 | goto msi; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 354 | |
| 355 | for (i = 0; i < msixcnt; i++) { |
Dan Williams | e6c0b69 | 2009-09-08 17:29:44 -0700 | [diff] [blame] | 356 | msix = &device->msix_entries[i]; |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 357 | ioat_chan = ioat_chan_by_index(device, i); |
Dan Williams | e6c0b69 | 2009-09-08 17:29:44 -0700 | [diff] [blame] | 358 | err = devm_request_irq(dev, msix->vector, |
| 359 | ioat_dma_do_interrupt_msix, 0, |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 360 | "ioat-msix", ioat_chan); |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 361 | if (err) { |
| 362 | for (j = 0; j < i; j++) { |
Dan Williams | e6c0b69 | 2009-09-08 17:29:44 -0700 | [diff] [blame] | 363 | msix = &device->msix_entries[j]; |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 364 | ioat_chan = ioat_chan_by_index(device, j); |
| 365 | devm_free_irq(dev, msix->vector, ioat_chan); |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 366 | } |
Dan Williams | 4c5d961 | 2013-11-13 16:29:52 -0800 | [diff] [blame] | 367 | goto msi; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 368 | } |
| 369 | } |
| 370 | intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL; |
Dave Jiang | 8a52b9f | 2013-03-26 15:42:47 -0700 | [diff] [blame] | 371 | device->irq_mode = IOAT_MSIX; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 372 | goto done; |
| 373 | |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 374 | msi: |
Dan Williams | e6c0b69 | 2009-09-08 17:29:44 -0700 | [diff] [blame] | 375 | err = pci_enable_msi(pdev); |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 376 | if (err) |
| 377 | goto intx; |
| 378 | |
Dan Williams | e6c0b69 | 2009-09-08 17:29:44 -0700 | [diff] [blame] | 379 | err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0, |
| 380 | "ioat-msi", device); |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 381 | if (err) { |
Dan Williams | e6c0b69 | 2009-09-08 17:29:44 -0700 | [diff] [blame] | 382 | pci_disable_msi(pdev); |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 383 | goto intx; |
| 384 | } |
Dan Williams | 779e561 | 2013-11-13 16:30:43 -0800 | [diff] [blame] | 385 | device->irq_mode = IOAT_MSI; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 386 | goto done; |
| 387 | |
| 388 | intx: |
Dan Williams | e6c0b69 | 2009-09-08 17:29:44 -0700 | [diff] [blame] | 389 | err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, |
| 390 | IRQF_SHARED, "ioat-intx", device); |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 391 | if (err) |
| 392 | goto err_no_irq; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 393 | |
Dave Jiang | 8a52b9f | 2013-03-26 15:42:47 -0700 | [diff] [blame] | 394 | device->irq_mode = IOAT_INTX; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 395 | done: |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 396 | if (device->intr_quirk) |
| 397 | device->intr_quirk(device); |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 398 | intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN; |
| 399 | writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET); |
| 400 | return 0; |
| 401 | |
| 402 | err_no_irq: |
| 403 | /* Disable all interrupt generation */ |
| 404 | writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET); |
Dave Jiang | 8a52b9f | 2013-03-26 15:42:47 -0700 | [diff] [blame] | 405 | device->irq_mode = IOAT_NOIRQ; |
Dan Williams | e6c0b69 | 2009-09-08 17:29:44 -0700 | [diff] [blame] | 406 | dev_err(dev, "no usable interrupts\n"); |
| 407 | return err; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 408 | } |
Dave Jiang | 8a52b9f | 2013-03-26 15:42:47 -0700 | [diff] [blame] | 409 | EXPORT_SYMBOL(ioat_dma_setup_interrupts); |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 410 | |
Dan Williams | e6c0b69 | 2009-09-08 17:29:44 -0700 | [diff] [blame] | 411 | static void ioat_disable_interrupts(struct ioatdma_device *device) |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 412 | { |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 413 | /* Disable all interrupt generation */ |
| 414 | writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET); |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 415 | } |
| 416 | |
Greg Kroah-Hartman | 4bf27b8 | 2012-12-21 15:09:59 -0800 | [diff] [blame] | 417 | int ioat_probe(struct ioatdma_device *device) |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 418 | { |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 419 | int err = -ENODEV; |
| 420 | struct dma_device *dma = &device->common; |
| 421 | struct pci_dev *pdev = device->pdev; |
Dan Williams | e6c0b69 | 2009-09-08 17:29:44 -0700 | [diff] [blame] | 422 | struct device *dev = &pdev->dev; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 423 | |
| 424 | /* DMA coherent memory pool for DMA descriptor allocations */ |
| 425 | device->dma_pool = pci_pool_create("dma_desc_pool", pdev, |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 426 | sizeof(struct ioat_dma_descriptor), |
| 427 | 64, 0); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 428 | if (!device->dma_pool) { |
| 429 | err = -ENOMEM; |
| 430 | goto err_dma_pool; |
| 431 | } |
| 432 | |
Shannon Nelson | 43d6e36 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 433 | device->completion_pool = pci_pool_create("completion_pool", pdev, |
| 434 | sizeof(u64), SMP_CACHE_BYTES, |
| 435 | SMP_CACHE_BYTES); |
Dan Williams | 5cbafa6 | 2009-08-26 13:01:44 -0700 | [diff] [blame] | 436 | |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 437 | if (!device->completion_pool) { |
| 438 | err = -ENOMEM; |
| 439 | goto err_completion_pool; |
| 440 | } |
| 441 | |
Dan Williams | 5cbafa6 | 2009-08-26 13:01:44 -0700 | [diff] [blame] | 442 | device->enumerate_channels(device); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 443 | |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 444 | dma_cap_set(DMA_MEMCPY, dma->cap_mask); |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 445 | dma->dev = &pdev->dev; |
Shannon Nelson | 7bb67c1 | 2007-11-14 16:59:51 -0800 | [diff] [blame] | 446 | |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 447 | if (!dma->chancnt) { |
Dan Williams | a6d52d7 | 2009-12-19 15:36:02 -0700 | [diff] [blame] | 448 | dev_err(dev, "channel enumeration error\n"); |
Maciej Sosnowski | 8b794b1 | 2009-02-26 11:04:54 +0100 | [diff] [blame] | 449 | goto err_setup_interrupts; |
| 450 | } |
| 451 | |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 452 | err = ioat_dma_setup_interrupts(device); |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 453 | if (err) |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 454 | goto err_setup_interrupts; |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 455 | |
Dan Williams | 9de6fc7 | 2009-09-08 17:42:58 -0700 | [diff] [blame] | 456 | err = device->self_test(device); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 457 | if (err) |
| 458 | goto err_self_test; |
| 459 | |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 460 | return 0; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 461 | |
| 462 | err_self_test: |
Dan Williams | e6c0b69 | 2009-09-08 17:29:44 -0700 | [diff] [blame] | 463 | ioat_disable_interrupts(device); |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 464 | err_setup_interrupts: |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 465 | pci_pool_destroy(device->completion_pool); |
| 466 | err_completion_pool: |
| 467 | pci_pool_destroy(device->dma_pool); |
| 468 | err_dma_pool: |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 469 | return err; |
| 470 | } |
| 471 | |
Greg Kroah-Hartman | 4bf27b8 | 2012-12-21 15:09:59 -0800 | [diff] [blame] | 472 | int ioat_register(struct ioatdma_device *device) |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 473 | { |
| 474 | int err = dma_async_device_register(&device->common); |
| 475 | |
| 476 | if (err) { |
| 477 | ioat_disable_interrupts(device); |
| 478 | pci_pool_destroy(device->completion_pool); |
| 479 | pci_pool_destroy(device->dma_pool); |
| 480 | } |
| 481 | |
| 482 | return err; |
| 483 | } |
| 484 | |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 485 | static ssize_t cap_show(struct dma_chan *c, char *page) |
| 486 | { |
| 487 | struct dma_device *dma = c->device; |
| 488 | |
Bartlomiej Zolnierkiewicz | 48a9db4 | 2013-07-03 15:05:06 -0700 | [diff] [blame] | 489 | return sprintf(page, "copy%s%s%s%s%s\n", |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 490 | dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "", |
| 491 | dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "", |
| 492 | dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "", |
| 493 | dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "", |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 494 | dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : ""); |
| 495 | |
| 496 | } |
| 497 | struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap); |
| 498 | |
| 499 | static ssize_t version_show(struct dma_chan *c, char *page) |
| 500 | { |
| 501 | struct dma_device *dma = c->device; |
| 502 | struct ioatdma_device *device = to_ioatdma_device(dma); |
| 503 | |
| 504 | return sprintf(page, "%d.%d\n", |
| 505 | device->version >> 4, device->version & 0xf); |
| 506 | } |
| 507 | struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version); |
| 508 | |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 509 | static ssize_t |
| 510 | ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page) |
| 511 | { |
| 512 | struct ioat_sysfs_entry *entry; |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 513 | struct ioatdma_chan *ioat_chan; |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 514 | |
| 515 | entry = container_of(attr, struct ioat_sysfs_entry, attr); |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 516 | ioat_chan = container_of(kobj, struct ioatdma_chan, kobj); |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 517 | |
| 518 | if (!entry->show) |
| 519 | return -EIO; |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 520 | return entry->show(&ioat_chan->dma_chan, page); |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 521 | } |
| 522 | |
Emese Revfy | 52cf25d | 2010-01-19 02:58:23 +0100 | [diff] [blame] | 523 | const struct sysfs_ops ioat_sysfs_ops = { |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 524 | .show = ioat_attr_show, |
| 525 | }; |
| 526 | |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 527 | void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type) |
| 528 | { |
| 529 | struct dma_device *dma = &device->common; |
| 530 | struct dma_chan *c; |
| 531 | |
| 532 | list_for_each_entry(c, &dma->channels, device_node) { |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 533 | struct ioatdma_chan *ioat_chan = to_ioat_chan(c); |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 534 | struct kobject *parent = &c->dev->device.kobj; |
| 535 | int err; |
| 536 | |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 537 | err = kobject_init_and_add(&ioat_chan->kobj, type, |
| 538 | parent, "quickdata"); |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 539 | if (err) { |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 540 | dev_warn(to_dev(ioat_chan), |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 541 | "sysfs init error (%d), continuing...\n", err); |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 542 | kobject_put(&ioat_chan->kobj); |
| 543 | set_bit(IOAT_KOBJ_INIT_FAIL, &ioat_chan->state); |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 544 | } |
| 545 | } |
| 546 | } |
| 547 | |
| 548 | void ioat_kobject_del(struct ioatdma_device *device) |
| 549 | { |
| 550 | struct dma_device *dma = &device->common; |
| 551 | struct dma_chan *c; |
| 552 | |
| 553 | list_for_each_entry(c, &dma->channels, device_node) { |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 554 | struct ioatdma_chan *ioat_chan = to_ioat_chan(c); |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 555 | |
Dave Jiang | 5a97688 | 2015-08-11 08:48:21 -0700 | [diff] [blame^] | 556 | if (!test_bit(IOAT_KOBJ_INIT_FAIL, &ioat_chan->state)) { |
| 557 | kobject_del(&ioat_chan->kobj); |
| 558 | kobject_put(&ioat_chan->kobj); |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 559 | } |
| 560 | } |
| 561 | } |
| 562 | |
Greg Kroah-Hartman | 4bf27b8 | 2012-12-21 15:09:59 -0800 | [diff] [blame] | 563 | void ioat_dma_remove(struct ioatdma_device *device) |
Dan Aloni | 428ed60 | 2007-03-08 09:57:36 -0800 | [diff] [blame] | 564 | { |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 565 | struct dma_device *dma = &device->common; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 566 | |
Dan Williams | e6c0b69 | 2009-09-08 17:29:44 -0700 | [diff] [blame] | 567 | ioat_disable_interrupts(device); |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 568 | |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 569 | ioat_kobject_del(device); |
| 570 | |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 571 | dma_async_device_unregister(dma); |
Shannon Nelson | dfe2299 | 2007-10-18 03:07:13 -0700 | [diff] [blame] | 572 | |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 573 | pci_pool_destroy(device->dma_pool); |
| 574 | pci_pool_destroy(device->completion_pool); |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 575 | |
Dan Williams | dcbc853 | 2009-07-28 14:44:50 -0700 | [diff] [blame] | 576 | INIT_LIST_HEAD(&dma->channels); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 577 | } |