Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 1 | /* |
Maciej Sosnowski | 211a22c | 2009-02-26 11:05:43 +0100 | [diff] [blame] | 2 | * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved. |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify it |
| 5 | * under the terms of the GNU General Public License as published by the Free |
| 6 | * Software Foundation; either version 2 of the License, or (at your option) |
| 7 | * any later version. |
| 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 12 | * more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU General Public License along with |
| 15 | * this program; if not, write to the Free Software Foundation, Inc., 59 |
| 16 | * Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
| 17 | * |
| 18 | * The full GNU General Public License is included in this distribution in the |
| 19 | * file called COPYING. |
| 20 | */ |
| 21 | #ifndef IOATDMA_H |
| 22 | #define IOATDMA_H |
| 23 | |
| 24 | #include <linux/dmaengine.h> |
Dan Williams | 584ec22 | 2009-07-28 14:32:12 -0700 | [diff] [blame] | 25 | #include "hw.h" |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 26 | #include "registers.h" |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 27 | #include <linux/init.h> |
| 28 | #include <linux/dmapool.h> |
| 29 | #include <linux/cache.h> |
David S. Miller | 57c651f | 2006-05-23 17:39:49 -0700 | [diff] [blame] | 30 | #include <linux/pci_ids.h> |
Maciej Sosnowski | 16a37ac | 2008-07-22 17:30:57 -0700 | [diff] [blame] | 31 | #include <net/tcp.h> |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 32 | |
Dan Williams | 3208ca5 | 2009-09-10 11:27:36 -0700 | [diff] [blame] | 33 | #define IOAT_DMA_VERSION "4.00" |
Shannon Nelson | 5149fd0 | 2007-10-18 03:07:13 -0700 | [diff] [blame] | 34 | |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 35 | #define IOAT_LOW_COMPLETION_MASK 0xffffffc0 |
Shannon Nelson | 7bb67c1 | 2007-11-14 16:59:51 -0800 | [diff] [blame] | 36 | #define IOAT_DMA_DCA_ANY_CPU ~0 |
| 37 | |
Dan Williams | 1f27adc2 | 2009-09-08 17:29:02 -0700 | [diff] [blame] | 38 | #define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common) |
| 39 | #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 40 | #define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, txd) |
| 41 | #define to_dev(ioat_chan) (&(ioat_chan)->device->pdev->dev) |
Dave Jiang | 3f09ede | 2013-03-26 15:43:09 -0700 | [diff] [blame^] | 42 | #define to_pdev(ioat_chan) ((ioat_chan)->device->pdev) |
Dan Williams | 1f27adc2 | 2009-09-08 17:29:02 -0700 | [diff] [blame] | 43 | |
| 44 | #define chan_num(ch) ((int)((ch)->reg_base - (ch)->device->reg_base) / 0x80) |
| 45 | |
Dan Williams | 1f27adc2 | 2009-09-08 17:29:02 -0700 | [diff] [blame] | 46 | /* |
| 47 | * workaround for IOAT ver.3.0 null descriptor issue |
| 48 | * (channel returns error when size is 0) |
| 49 | */ |
| 50 | #define NULL_DESC_BUFFER_SIZE 1 |
| 51 | |
Dave Jiang | 8a52b9f | 2013-03-26 15:42:47 -0700 | [diff] [blame] | 52 | enum ioat_irq_mode { |
| 53 | IOAT_NOIRQ = 0, |
| 54 | IOAT_MSIX, |
| 55 | IOAT_MSIX_SINGLE, |
| 56 | IOAT_MSI, |
| 57 | IOAT_INTX |
| 58 | }; |
| 59 | |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 60 | /** |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 61 | * struct ioatdma_device - internal representation of a IOAT device |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 62 | * @pdev: PCI-Express device |
| 63 | * @reg_base: MMIO register space base address |
| 64 | * @dma_pool: for allocating DMA descriptors |
| 65 | * @common: embedded struct dma_device |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 66 | * @version: version of ioatdma device |
Shannon Nelson | 7bb67c1 | 2007-11-14 16:59:51 -0800 | [diff] [blame] | 67 | * @msix_entries: irq handlers |
| 68 | * @idx: per channel data |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 69 | * @dca: direct cache access context |
| 70 | * @intr_quirk: interrupt setup quirk (for ioat_v1 devices) |
Dan Williams | 5cbafa6 | 2009-08-26 13:01:44 -0700 | [diff] [blame] | 71 | * @enumerate_channels: hw version specific channel enumeration |
Dan Williams | a6d52d7 | 2009-12-19 15:36:02 -0700 | [diff] [blame] | 72 | * @reset_hw: hw version specific channel (re)initialization |
Dan Williams | aa4d72a | 2010-03-03 21:21:13 -0700 | [diff] [blame] | 73 | * @cleanup_fn: select between the v2 and v3 cleanup routines |
Dan Williams | bf40a68 | 2009-09-08 17:42:55 -0700 | [diff] [blame] | 74 | * @timer_fn: select between the v2 and v3 timer watchdog routines |
Dan Williams | 9de6fc7 | 2009-09-08 17:42:58 -0700 | [diff] [blame] | 75 | * @self_test: hardware version specific self test for each supported op type |
Dan Williams | bf40a68 | 2009-09-08 17:42:55 -0700 | [diff] [blame] | 76 | * |
| 77 | * Note: the v3 cleanup routine supports raid operations |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 78 | */ |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 79 | struct ioatdma_device { |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 80 | struct pci_dev *pdev; |
Al Viro | 47b1653 | 2006-10-10 22:45:47 +0100 | [diff] [blame] | 81 | void __iomem *reg_base; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 82 | struct pci_pool *dma_pool; |
| 83 | struct pci_pool *completion_pool; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 84 | struct dma_device common; |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 85 | u8 version; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 86 | struct msix_entry msix_entries[4]; |
Dan Williams | dcbc853 | 2009-07-28 14:44:50 -0700 | [diff] [blame] | 87 | struct ioat_chan_common *idx[4]; |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 88 | struct dca_provider *dca; |
Dave Jiang | 8a52b9f | 2013-03-26 15:42:47 -0700 | [diff] [blame] | 89 | enum ioat_irq_mode irq_mode; |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 90 | void (*intr_quirk)(struct ioatdma_device *device); |
Dan Williams | 5cbafa6 | 2009-08-26 13:01:44 -0700 | [diff] [blame] | 91 | int (*enumerate_channels)(struct ioatdma_device *device); |
Dan Williams | a6d52d7 | 2009-12-19 15:36:02 -0700 | [diff] [blame] | 92 | int (*reset_hw)(struct ioat_chan_common *chan); |
Dan Williams | aa4d72a | 2010-03-03 21:21:13 -0700 | [diff] [blame] | 93 | void (*cleanup_fn)(unsigned long data); |
Dan Williams | bf40a68 | 2009-09-08 17:42:55 -0700 | [diff] [blame] | 94 | void (*timer_fn)(unsigned long data); |
Dan Williams | 9de6fc7 | 2009-09-08 17:42:58 -0700 | [diff] [blame] | 95 | int (*self_test)(struct ioatdma_device *device); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 96 | }; |
| 97 | |
Dan Williams | dcbc853 | 2009-07-28 14:44:50 -0700 | [diff] [blame] | 98 | struct ioat_chan_common { |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 99 | struct dma_chan common; |
Al Viro | 47b1653 | 2006-10-10 22:45:47 +0100 | [diff] [blame] | 100 | void __iomem *reg_base; |
Dan Williams | 2750293 | 2012-03-23 13:36:42 -0700 | [diff] [blame] | 101 | dma_addr_t last_completion; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 102 | spinlock_t cleanup_lock; |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 103 | unsigned long state; |
| 104 | #define IOAT_COMPLETION_PENDING 0 |
| 105 | #define IOAT_COMPLETION_ACK 1 |
| 106 | #define IOAT_RESET_PENDING 2 |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 107 | #define IOAT_KOBJ_INIT_FAIL 3 |
Dan Williams | 074cc47 | 2010-05-01 15:22:55 -0700 | [diff] [blame] | 108 | #define IOAT_RESHAPE_PENDING 4 |
Dan Williams | 556ab45 | 2010-07-23 15:47:56 -0700 | [diff] [blame] | 109 | #define IOAT_RUN 5 |
Dave Jiang | 4dec23d | 2013-02-07 14:38:32 -0700 | [diff] [blame] | 110 | #define IOAT_CHAN_ACTIVE 6 |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 111 | struct timer_list timer; |
| 112 | #define COMPLETION_TIMEOUT msecs_to_jiffies(100) |
Dan Williams | a309218 | 2009-09-08 12:02:01 -0700 | [diff] [blame] | 113 | #define IDLE_TIMEOUT msecs_to_jiffies(2000) |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 114 | #define RESET_DELAY msecs_to_jiffies(100) |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 115 | struct ioatdma_device *device; |
Dan Williams | 4fb9b9e | 2009-09-08 12:01:04 -0700 | [diff] [blame] | 116 | dma_addr_t completion_dma; |
| 117 | u64 *completion; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 118 | struct tasklet_struct cleanup_task; |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 119 | struct kobject kobj; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 120 | }; |
| 121 | |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 122 | struct ioat_sysfs_entry { |
| 123 | struct attribute attr; |
| 124 | ssize_t (*show)(struct dma_chan *, char *); |
| 125 | }; |
Dan Williams | 5cbafa6 | 2009-08-26 13:01:44 -0700 | [diff] [blame] | 126 | |
Dan Williams | dcbc853 | 2009-07-28 14:44:50 -0700 | [diff] [blame] | 127 | /** |
| 128 | * struct ioat_dma_chan - internal representation of a DMA channel |
| 129 | */ |
| 130 | struct ioat_dma_chan { |
| 131 | struct ioat_chan_common base; |
| 132 | |
| 133 | size_t xfercap; /* XFERCAP register value expanded out */ |
| 134 | |
| 135 | spinlock_t desc_lock; |
| 136 | struct list_head free_desc; |
| 137 | struct list_head used_desc; |
| 138 | |
| 139 | int pending; |
Dan Williams | dcbc853 | 2009-07-28 14:44:50 -0700 | [diff] [blame] | 140 | u16 desccount; |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 141 | u16 active; |
Dan Williams | dcbc853 | 2009-07-28 14:44:50 -0700 | [diff] [blame] | 142 | }; |
| 143 | |
| 144 | static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c) |
| 145 | { |
| 146 | return container_of(c, struct ioat_chan_common, common); |
| 147 | } |
| 148 | |
| 149 | static inline struct ioat_dma_chan *to_ioat_chan(struct dma_chan *c) |
| 150 | { |
| 151 | struct ioat_chan_common *chan = to_chan_common(c); |
| 152 | |
| 153 | return container_of(chan, struct ioat_dma_chan, base); |
| 154 | } |
| 155 | |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 156 | /* wrapper around hardware descriptor format + additional software fields */ |
| 157 | |
| 158 | /** |
| 159 | * struct ioat_desc_sw - wrapper around hardware descriptor |
Dan Williams | 2aec048 | 2009-09-08 17:42:54 -0700 | [diff] [blame] | 160 | * @hw: hardware DMA descriptor (for memcpy) |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 161 | * @node: this descriptor will either be on the free list, |
Dan Williams | ea25968a | 2009-09-08 17:53:02 -0700 | [diff] [blame] | 162 | * or attached to a transaction list (tx_list) |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 163 | * @txd: the generic software descriptor for all engines |
Dan Williams | 6df9183 | 2009-09-08 12:00:55 -0700 | [diff] [blame] | 164 | * @id: identifier for debug |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 165 | */ |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 166 | struct ioat_desc_sw { |
| 167 | struct ioat_dma_descriptor *hw; |
| 168 | struct list_head node; |
Shannon Nelson | 7f2b291 | 2007-10-18 03:07:14 -0700 | [diff] [blame] | 169 | size_t len; |
Dan Williams | ea25968a | 2009-09-08 17:53:02 -0700 | [diff] [blame] | 170 | struct list_head tx_list; |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 171 | struct dma_async_tx_descriptor txd; |
Dan Williams | 6df9183 | 2009-09-08 12:00:55 -0700 | [diff] [blame] | 172 | #ifdef DEBUG |
| 173 | int id; |
| 174 | #endif |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 175 | }; |
| 176 | |
Dan Williams | 6df9183 | 2009-09-08 12:00:55 -0700 | [diff] [blame] | 177 | #ifdef DEBUG |
| 178 | #define set_desc_id(desc, i) ((desc)->id = (i)) |
| 179 | #define desc_id(desc) ((desc)->id) |
| 180 | #else |
| 181 | #define set_desc_id(desc, i) |
| 182 | #define desc_id(desc) (0) |
| 183 | #endif |
| 184 | |
| 185 | static inline void |
| 186 | __dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw, |
| 187 | struct dma_async_tx_descriptor *tx, int id) |
| 188 | { |
| 189 | struct device *dev = to_dev(chan); |
| 190 | |
| 191 | dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x" |
Dave Jiang | 50f9f97 | 2013-03-04 10:59:54 -0700 | [diff] [blame] | 192 | " ctl: %#10.8x (op: %#x int_en: %d compl: %d)\n", id, |
Dan Williams | 6df9183 | 2009-09-08 12:00:55 -0700 | [diff] [blame] | 193 | (unsigned long long) tx->phys, |
| 194 | (unsigned long long) hw->next, tx->cookie, tx->flags, |
| 195 | hw->ctl, hw->ctl_f.op, hw->ctl_f.int_en, hw->ctl_f.compl_write); |
| 196 | } |
| 197 | |
| 198 | #define dump_desc_dbg(c, d) \ |
| 199 | ({ if (d) __dump_desc_dbg(&c->base, d->hw, &d->txd, desc_id(d)); 0; }) |
| 200 | |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 201 | static inline void ioat_set_tcp_copy_break(unsigned long copybreak) |
Maciej Sosnowski | 16a37ac | 2008-07-22 17:30:57 -0700 | [diff] [blame] | 202 | { |
| 203 | #ifdef CONFIG_NET_DMA |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 204 | sysctl_tcp_dma_copybreak = copybreak; |
Maciej Sosnowski | 16a37ac | 2008-07-22 17:30:57 -0700 | [diff] [blame] | 205 | #endif |
| 206 | } |
| 207 | |
Dan Williams | 5cbafa6 | 2009-08-26 13:01:44 -0700 | [diff] [blame] | 208 | static inline struct ioat_chan_common * |
| 209 | ioat_chan_by_index(struct ioatdma_device *device, int index) |
| 210 | { |
| 211 | return device->idx[index]; |
| 212 | } |
| 213 | |
Dave Jiang | d92a8d7 | 2013-03-26 15:42:41 -0700 | [diff] [blame] | 214 | static inline u64 ioat_chansts_32(struct ioat_chan_common *chan) |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 215 | { |
| 216 | u8 ver = chan->device->version; |
| 217 | u64 status; |
| 218 | u32 status_lo; |
| 219 | |
| 220 | /* We need to read the low address first as this causes the |
| 221 | * chipset to latch the upper bits for the subsequent read |
| 222 | */ |
| 223 | status_lo = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver)); |
| 224 | status = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver)); |
| 225 | status <<= 32; |
| 226 | status |= status_lo; |
| 227 | |
| 228 | return status; |
| 229 | } |
| 230 | |
Dave Jiang | d92a8d7 | 2013-03-26 15:42:41 -0700 | [diff] [blame] | 231 | #if BITS_PER_LONG == 64 |
| 232 | |
| 233 | static inline u64 ioat_chansts(struct ioat_chan_common *chan) |
| 234 | { |
| 235 | u8 ver = chan->device->version; |
| 236 | u64 status; |
| 237 | |
| 238 | /* With IOAT v3.3 the status register is 64bit. */ |
| 239 | if (ver >= IOAT_VER_3_3) |
| 240 | status = readq(chan->reg_base + IOAT_CHANSTS_OFFSET(ver)); |
| 241 | else |
| 242 | status = ioat_chansts_32(chan); |
| 243 | |
| 244 | return status; |
| 245 | } |
| 246 | |
| 247 | #else |
| 248 | #define ioat_chansts ioat_chansts_32 |
| 249 | #endif |
| 250 | |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 251 | static inline void ioat_start(struct ioat_chan_common *chan) |
| 252 | { |
| 253 | u8 ver = chan->device->version; |
| 254 | |
| 255 | writeb(IOAT_CHANCMD_START, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); |
| 256 | } |
| 257 | |
| 258 | static inline u64 ioat_chansts_to_addr(u64 status) |
| 259 | { |
| 260 | return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; |
| 261 | } |
| 262 | |
| 263 | static inline u32 ioat_chanerr(struct ioat_chan_common *chan) |
| 264 | { |
| 265 | return readl(chan->reg_base + IOAT_CHANERR_OFFSET); |
| 266 | } |
| 267 | |
| 268 | static inline void ioat_suspend(struct ioat_chan_common *chan) |
| 269 | { |
| 270 | u8 ver = chan->device->version; |
| 271 | |
| 272 | writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); |
| 273 | } |
| 274 | |
Dan Williams | a6d52d7 | 2009-12-19 15:36:02 -0700 | [diff] [blame] | 275 | static inline void ioat_reset(struct ioat_chan_common *chan) |
| 276 | { |
| 277 | u8 ver = chan->device->version; |
| 278 | |
| 279 | writeb(IOAT_CHANCMD_RESET, chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); |
| 280 | } |
| 281 | |
| 282 | static inline bool ioat_reset_pending(struct ioat_chan_common *chan) |
| 283 | { |
| 284 | u8 ver = chan->device->version; |
| 285 | u8 cmd; |
| 286 | |
| 287 | cmd = readb(chan->reg_base + IOAT_CHANCMD_OFFSET(ver)); |
| 288 | return (cmd & IOAT_CHANCMD_RESET) == IOAT_CHANCMD_RESET; |
| 289 | } |
| 290 | |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 291 | static inline void ioat_set_chainaddr(struct ioat_dma_chan *ioat, u64 addr) |
| 292 | { |
| 293 | struct ioat_chan_common *chan = &ioat->base; |
| 294 | |
| 295 | writel(addr & 0x00000000FFFFFFFF, |
| 296 | chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW); |
| 297 | writel(addr >> 32, |
| 298 | chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH); |
| 299 | } |
| 300 | |
| 301 | static inline bool is_ioat_active(unsigned long status) |
| 302 | { |
| 303 | return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE); |
| 304 | } |
| 305 | |
| 306 | static inline bool is_ioat_idle(unsigned long status) |
| 307 | { |
| 308 | return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_DONE); |
| 309 | } |
| 310 | |
| 311 | static inline bool is_ioat_halted(unsigned long status) |
| 312 | { |
| 313 | return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED); |
| 314 | } |
| 315 | |
| 316 | static inline bool is_ioat_suspended(unsigned long status) |
| 317 | { |
| 318 | return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED); |
| 319 | } |
| 320 | |
| 321 | /* channel was fatally programmed */ |
| 322 | static inline bool is_ioat_bug(unsigned long err) |
| 323 | { |
Dan Williams | b57014d | 2009-11-19 17:10:07 -0700 | [diff] [blame] | 324 | return !!err; |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 325 | } |
| 326 | |
Dan Williams | bf40a68 | 2009-09-08 17:42:55 -0700 | [diff] [blame] | 327 | static inline void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len, |
| 328 | int direction, enum dma_ctrl_flags flags, bool dst) |
| 329 | { |
| 330 | if ((dst && (flags & DMA_COMPL_DEST_UNMAP_SINGLE)) || |
| 331 | (!dst && (flags & DMA_COMPL_SRC_UNMAP_SINGLE))) |
| 332 | pci_unmap_single(pdev, addr, len, direction); |
| 333 | else |
| 334 | pci_unmap_page(pdev, addr, len, direction); |
| 335 | } |
| 336 | |
Greg Kroah-Hartman | 4bf27b8 | 2012-12-21 15:09:59 -0800 | [diff] [blame] | 337 | int ioat_probe(struct ioatdma_device *device); |
| 338 | int ioat_register(struct ioatdma_device *device); |
| 339 | int ioat1_dma_probe(struct ioatdma_device *dev, int dca); |
| 340 | int ioat_dma_self_test(struct ioatdma_device *device); |
| 341 | void ioat_dma_remove(struct ioatdma_device *device); |
| 342 | struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase); |
Dan Williams | 2750293 | 2012-03-23 13:36:42 -0700 | [diff] [blame] | 343 | dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan); |
Dan Williams | 5cbafa6 | 2009-08-26 13:01:44 -0700 | [diff] [blame] | 344 | void ioat_init_channel(struct ioatdma_device *device, |
Dan Williams | aa4d72a | 2010-03-03 21:21:13 -0700 | [diff] [blame] | 345 | struct ioat_chan_common *chan, int idx); |
Linus Walleij | 0793448 | 2010-03-26 16:50:49 -0700 | [diff] [blame] | 346 | enum dma_status ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, |
| 347 | struct dma_tx_state *txstate); |
Dan Williams | 5cbafa6 | 2009-08-26 13:01:44 -0700 | [diff] [blame] | 348 | void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags, |
| 349 | size_t len, struct ioat_dma_descriptor *hw); |
Dan Williams | 09c8a5b | 2009-09-08 12:01:49 -0700 | [diff] [blame] | 350 | bool ioat_cleanup_preamble(struct ioat_chan_common *chan, |
Dan Williams | 2750293 | 2012-03-23 13:36:42 -0700 | [diff] [blame] | 351 | dma_addr_t *phys_complete); |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 352 | void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type); |
| 353 | void ioat_kobject_del(struct ioatdma_device *device); |
Dave Jiang | 8a52b9f | 2013-03-26 15:42:47 -0700 | [diff] [blame] | 354 | int ioat_dma_setup_interrupts(struct ioatdma_device *device); |
Emese Revfy | 52cf25d | 2010-01-19 02:58:23 +0100 | [diff] [blame] | 355 | extern const struct sysfs_ops ioat_sysfs_ops; |
Dan Williams | 5669e31 | 2009-09-08 17:42:56 -0700 | [diff] [blame] | 356 | extern struct ioat_sysfs_entry ioat_version_attr; |
| 357 | extern struct ioat_sysfs_entry ioat_cap_attr; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 358 | #endif /* IOATDMA_H */ |