Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 1 | /* |
Maciej Sosnowski | 211a22c | 2009-02-26 11:05:43 +0100 | [diff] [blame] | 2 | * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved. |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify it |
| 5 | * under the terms of the GNU General Public License as published by the Free |
| 6 | * Software Foundation; either version 2 of the License, or (at your option) |
| 7 | * any later version. |
| 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 12 | * more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU General Public License along with |
| 15 | * this program; if not, write to the Free Software Foundation, Inc., 59 |
| 16 | * Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
| 17 | * |
| 18 | * The full GNU General Public License is included in this distribution in the |
| 19 | * file called COPYING. |
| 20 | */ |
| 21 | #ifndef IOATDMA_H |
| 22 | #define IOATDMA_H |
| 23 | |
| 24 | #include <linux/dmaengine.h> |
Dan Williams | 584ec22 | 2009-07-28 14:32:12 -0700 | [diff] [blame] | 25 | #include "hw.h" |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 26 | #include <linux/init.h> |
| 27 | #include <linux/dmapool.h> |
| 28 | #include <linux/cache.h> |
David S. Miller | 57c651f | 2006-05-23 17:39:49 -0700 | [diff] [blame] | 29 | #include <linux/pci_ids.h> |
Maciej Sosnowski | 16a37ac | 2008-07-22 17:30:57 -0700 | [diff] [blame] | 30 | #include <net/tcp.h> |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 31 | |
Maciej Sosnowski | 211a22c | 2009-02-26 11:05:43 +0100 | [diff] [blame] | 32 | #define IOAT_DMA_VERSION "3.64" |
Shannon Nelson | 5149fd0 | 2007-10-18 03:07:13 -0700 | [diff] [blame] | 33 | |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 34 | #define IOAT_LOW_COMPLETION_MASK 0xffffffc0 |
Shannon Nelson | 7bb67c1 | 2007-11-14 16:59:51 -0800 | [diff] [blame] | 35 | #define IOAT_DMA_DCA_ANY_CPU ~0 |
Maciej Sosnowski | 09177e8 | 2008-07-22 10:07:33 -0700 | [diff] [blame] | 36 | #define IOAT_WATCHDOG_PERIOD (2 * HZ) |
Shannon Nelson | 7bb67c1 | 2007-11-14 16:59:51 -0800 | [diff] [blame] | 37 | |
Dan Williams | 1f27adc2 | 2009-09-08 17:29:02 -0700 | [diff] [blame] | 38 | #define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common) |
| 39 | #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 40 | #define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, txd) |
| 41 | #define to_dev(ioat_chan) (&(ioat_chan)->device->pdev->dev) |
Dan Williams | 1f27adc2 | 2009-09-08 17:29:02 -0700 | [diff] [blame] | 42 | |
| 43 | #define chan_num(ch) ((int)((ch)->reg_base - (ch)->device->reg_base) / 0x80) |
| 44 | |
| 45 | #define RESET_DELAY msecs_to_jiffies(100) |
| 46 | #define WATCHDOG_DELAY round_jiffies(msecs_to_jiffies(2000)) |
| 47 | |
| 48 | /* |
| 49 | * workaround for IOAT ver.3.0 null descriptor issue |
| 50 | * (channel returns error when size is 0) |
| 51 | */ |
| 52 | #define NULL_DESC_BUFFER_SIZE 1 |
| 53 | |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 54 | /** |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 55 | * struct ioatdma_device - internal representation of a IOAT device |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 56 | * @pdev: PCI-Express device |
| 57 | * @reg_base: MMIO register space base address |
| 58 | * @dma_pool: for allocating DMA descriptors |
| 59 | * @common: embedded struct dma_device |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 60 | * @version: version of ioatdma device |
Shannon Nelson | 7bb67c1 | 2007-11-14 16:59:51 -0800 | [diff] [blame] | 61 | * @msix_entries: irq handlers |
| 62 | * @idx: per channel data |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 63 | * @dca: direct cache access context |
| 64 | * @intr_quirk: interrupt setup quirk (for ioat_v1 devices) |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 65 | */ |
| 66 | |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 67 | struct ioatdma_device { |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 68 | struct pci_dev *pdev; |
Al Viro | 47b1653 | 2006-10-10 22:45:47 +0100 | [diff] [blame] | 69 | void __iomem *reg_base; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 70 | struct pci_pool *dma_pool; |
| 71 | struct pci_pool *completion_pool; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 72 | struct dma_device common; |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 73 | u8 version; |
Maciej Sosnowski | 09177e8 | 2008-07-22 10:07:33 -0700 | [diff] [blame] | 74 | struct delayed_work work; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 75 | struct msix_entry msix_entries[4]; |
Dan Williams | dcbc853 | 2009-07-28 14:44:50 -0700 | [diff] [blame^] | 76 | struct ioat_chan_common *idx[4]; |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 77 | struct dca_provider *dca; |
| 78 | void (*intr_quirk)(struct ioatdma_device *device); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 79 | }; |
| 80 | |
Dan Williams | dcbc853 | 2009-07-28 14:44:50 -0700 | [diff] [blame^] | 81 | struct ioat_chan_common { |
Al Viro | 47b1653 | 2006-10-10 22:45:47 +0100 | [diff] [blame] | 82 | void __iomem *reg_base; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 83 | |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 84 | unsigned long last_completion; |
Maciej Sosnowski | 09177e8 | 2008-07-22 10:07:33 -0700 | [diff] [blame] | 85 | unsigned long last_completion_time; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 86 | |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 87 | spinlock_t cleanup_lock; |
Dan Williams | dcbc853 | 2009-07-28 14:44:50 -0700 | [diff] [blame^] | 88 | dma_cookie_t completed_cookie; |
Maciej Sosnowski | 09177e8 | 2008-07-22 10:07:33 -0700 | [diff] [blame] | 89 | unsigned long watchdog_completion; |
| 90 | int watchdog_tcp_cookie; |
| 91 | u32 watchdog_last_tcp_cookie; |
| 92 | struct delayed_work work; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 93 | |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 94 | struct ioatdma_device *device; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 95 | struct dma_chan common; |
| 96 | |
| 97 | dma_addr_t completion_addr; |
| 98 | union { |
| 99 | u64 full; /* HW completion writeback */ |
| 100 | struct { |
| 101 | u32 low; |
| 102 | u32 high; |
| 103 | }; |
| 104 | } *completion_virt; |
Maciej Sosnowski | 09177e8 | 2008-07-22 10:07:33 -0700 | [diff] [blame] | 105 | unsigned long last_compl_desc_addr_hw; |
Shannon Nelson | 3e03745 | 2007-10-16 01:27:40 -0700 | [diff] [blame] | 106 | struct tasklet_struct cleanup_task; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 107 | }; |
| 108 | |
Dan Williams | dcbc853 | 2009-07-28 14:44:50 -0700 | [diff] [blame^] | 109 | /** |
| 110 | * struct ioat_dma_chan - internal representation of a DMA channel |
| 111 | */ |
| 112 | struct ioat_dma_chan { |
| 113 | struct ioat_chan_common base; |
| 114 | |
| 115 | size_t xfercap; /* XFERCAP register value expanded out */ |
| 116 | |
| 117 | spinlock_t desc_lock; |
| 118 | struct list_head free_desc; |
| 119 | struct list_head used_desc; |
| 120 | |
| 121 | int pending; |
| 122 | u16 dmacount; |
| 123 | u16 desccount; |
| 124 | }; |
| 125 | |
| 126 | static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c) |
| 127 | { |
| 128 | return container_of(c, struct ioat_chan_common, common); |
| 129 | } |
| 130 | |
| 131 | static inline struct ioat_dma_chan *to_ioat_chan(struct dma_chan *c) |
| 132 | { |
| 133 | struct ioat_chan_common *chan = to_chan_common(c); |
| 134 | |
| 135 | return container_of(chan, struct ioat_dma_chan, base); |
| 136 | } |
| 137 | |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 138 | /* wrapper around hardware descriptor format + additional software fields */ |
| 139 | |
| 140 | /** |
| 141 | * struct ioat_desc_sw - wrapper around hardware descriptor |
| 142 | * @hw: hardware DMA descriptor |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 143 | * @node: this descriptor will either be on the free list, |
| 144 | * or attached to a transaction list (async_tx.tx_list) |
| 145 | * @tx_cnt: number of descriptors required to complete the transaction |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 146 | * @txd: the generic software descriptor for all engines |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 147 | */ |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 148 | struct ioat_desc_sw { |
| 149 | struct ioat_dma_descriptor *hw; |
| 150 | struct list_head node; |
Dan Williams | 7405f74 | 2007-01-02 11:10:43 -0700 | [diff] [blame] | 151 | int tx_cnt; |
Shannon Nelson | 7f2b291 | 2007-10-18 03:07:14 -0700 | [diff] [blame] | 152 | size_t len; |
| 153 | dma_addr_t src; |
| 154 | dma_addr_t dst; |
Dan Williams | bc3c702 | 2009-07-28 14:33:42 -0700 | [diff] [blame] | 155 | struct dma_async_tx_descriptor txd; |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 156 | }; |
| 157 | |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 158 | static inline void ioat_set_tcp_copy_break(unsigned long copybreak) |
Maciej Sosnowski | 16a37ac | 2008-07-22 17:30:57 -0700 | [diff] [blame] | 159 | { |
| 160 | #ifdef CONFIG_NET_DMA |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 161 | sysctl_tcp_dma_copybreak = copybreak; |
Maciej Sosnowski | 16a37ac | 2008-07-22 17:30:57 -0700 | [diff] [blame] | 162 | #endif |
| 163 | } |
| 164 | |
Dan Williams | f2427e2 | 2009-07-28 14:42:38 -0700 | [diff] [blame] | 165 | int ioat1_dma_probe(struct ioatdma_device *dev, int dca); |
| 166 | int ioat2_dma_probe(struct ioatdma_device *dev, int dca); |
| 167 | int ioat3_dma_probe(struct ioatdma_device *dev, int dca); |
Shannon Nelson | 8ab8956 | 2007-10-16 01:27:39 -0700 | [diff] [blame] | 168 | void ioat_dma_remove(struct ioatdma_device *device); |
Shannon Nelson | 7bb67c1 | 2007-11-14 16:59:51 -0800 | [diff] [blame] | 169 | struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase); |
| 170 | struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase); |
Maciej Sosnowski | 7f1b358 | 2008-07-22 17:30:57 -0700 | [diff] [blame] | 171 | struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase); |
Chris Leech | 0bbd5f4 | 2006-05-23 17:35:34 -0700 | [diff] [blame] | 172 | #endif /* IOATDMA_H */ |