blob: 1226e35f2709e73516547751a2502360b9c50566 [file] [log] [blame]
Chris Leech0bbd5f42006-05-23 17:35:34 -07001/*
Maciej Sosnowski211a22c2009-02-26 11:05:43 +01002 * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
Chris Leech0bbd5f42006-05-23 17:35:34 -07003 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
20 */
21#ifndef IOATDMA_H
22#define IOATDMA_H
23
24#include <linux/dmaengine.h>
Dan Williams584ec222009-07-28 14:32:12 -070025#include "hw.h"
Chris Leech0bbd5f42006-05-23 17:35:34 -070026#include <linux/init.h>
27#include <linux/dmapool.h>
28#include <linux/cache.h>
David S. Miller57c651f2006-05-23 17:39:49 -070029#include <linux/pci_ids.h>
Maciej Sosnowski16a37ac2008-07-22 17:30:57 -070030#include <net/tcp.h>
Chris Leech0bbd5f42006-05-23 17:35:34 -070031
Maciej Sosnowski211a22c2009-02-26 11:05:43 +010032#define IOAT_DMA_VERSION "3.64"
Shannon Nelson5149fd02007-10-18 03:07:13 -070033
Chris Leech0bbd5f42006-05-23 17:35:34 -070034#define IOAT_LOW_COMPLETION_MASK 0xffffffc0
Shannon Nelson7bb67c12007-11-14 16:59:51 -080035#define IOAT_DMA_DCA_ANY_CPU ~0
Maciej Sosnowski09177e82008-07-22 10:07:33 -070036#define IOAT_WATCHDOG_PERIOD (2 * HZ)
Shannon Nelson7bb67c12007-11-14 16:59:51 -080037
Dan Williams1f27adc22009-09-08 17:29:02 -070038#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
39#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common)
40#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
Dan Williamsbc3c7022009-07-28 14:33:42 -070041#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, txd)
42#define to_dev(ioat_chan) (&(ioat_chan)->device->pdev->dev)
Dan Williams1f27adc22009-09-08 17:29:02 -070043
44#define chan_num(ch) ((int)((ch)->reg_base - (ch)->device->reg_base) / 0x80)
45
46#define RESET_DELAY msecs_to_jiffies(100)
47#define WATCHDOG_DELAY round_jiffies(msecs_to_jiffies(2000))
48
49/*
50 * workaround for IOAT ver.3.0 null descriptor issue
51 * (channel returns error when size is 0)
52 */
53#define NULL_DESC_BUFFER_SIZE 1
54
Chris Leech0bbd5f42006-05-23 17:35:34 -070055/**
Shannon Nelson8ab89562007-10-16 01:27:39 -070056 * struct ioatdma_device - internal representation of a IOAT device
Chris Leech0bbd5f42006-05-23 17:35:34 -070057 * @pdev: PCI-Express device
58 * @reg_base: MMIO register space base address
59 * @dma_pool: for allocating DMA descriptors
60 * @common: embedded struct dma_device
Shannon Nelson8ab89562007-10-16 01:27:39 -070061 * @version: version of ioatdma device
Shannon Nelson7bb67c12007-11-14 16:59:51 -080062 * @msix_entries: irq handlers
63 * @idx: per channel data
Dan Williamsf2427e22009-07-28 14:42:38 -070064 * @dca: direct cache access context
65 * @intr_quirk: interrupt setup quirk (for ioat_v1 devices)
Chris Leech0bbd5f42006-05-23 17:35:34 -070066 */
67
Shannon Nelson8ab89562007-10-16 01:27:39 -070068struct ioatdma_device {
Chris Leech0bbd5f42006-05-23 17:35:34 -070069 struct pci_dev *pdev;
Al Viro47b16532006-10-10 22:45:47 +010070 void __iomem *reg_base;
Chris Leech0bbd5f42006-05-23 17:35:34 -070071 struct pci_pool *dma_pool;
72 struct pci_pool *completion_pool;
Chris Leech0bbd5f42006-05-23 17:35:34 -070073 struct dma_device common;
Shannon Nelson8ab89562007-10-16 01:27:39 -070074 u8 version;
Maciej Sosnowski09177e82008-07-22 10:07:33 -070075 struct delayed_work work;
Shannon Nelson3e037452007-10-16 01:27:40 -070076 struct msix_entry msix_entries[4];
77 struct ioat_dma_chan *idx[4];
Dan Williamsf2427e22009-07-28 14:42:38 -070078 struct dca_provider *dca;
79 void (*intr_quirk)(struct ioatdma_device *device);
Chris Leech0bbd5f42006-05-23 17:35:34 -070080};
81
82/**
83 * struct ioat_dma_chan - internal representation of a DMA channel
Chris Leech0bbd5f42006-05-23 17:35:34 -070084 */
Chris Leech0bbd5f42006-05-23 17:35:34 -070085struct ioat_dma_chan {
86
Al Viro47b16532006-10-10 22:45:47 +010087 void __iomem *reg_base;
Chris Leech0bbd5f42006-05-23 17:35:34 -070088
89 dma_cookie_t completed_cookie;
90 unsigned long last_completion;
Maciej Sosnowski09177e82008-07-22 10:07:33 -070091 unsigned long last_completion_time;
Chris Leech0bbd5f42006-05-23 17:35:34 -070092
Shannon Nelson711924b2007-12-17 16:20:08 -080093 size_t xfercap; /* XFERCAP register value expanded out */
Chris Leech0bbd5f42006-05-23 17:35:34 -070094
95 spinlock_t cleanup_lock;
96 spinlock_t desc_lock;
97 struct list_head free_desc;
98 struct list_head used_desc;
Maciej Sosnowski09177e82008-07-22 10:07:33 -070099 unsigned long watchdog_completion;
100 int watchdog_tcp_cookie;
101 u32 watchdog_last_tcp_cookie;
102 struct delayed_work work;
Chris Leech0bbd5f42006-05-23 17:35:34 -0700103
104 int pending;
Shannon Nelson7bb67c12007-11-14 16:59:51 -0800105 int dmacount;
106 int desccount;
Chris Leech0bbd5f42006-05-23 17:35:34 -0700107
Shannon Nelson8ab89562007-10-16 01:27:39 -0700108 struct ioatdma_device *device;
Chris Leech0bbd5f42006-05-23 17:35:34 -0700109 struct dma_chan common;
110
111 dma_addr_t completion_addr;
112 union {
113 u64 full; /* HW completion writeback */
114 struct {
115 u32 low;
116 u32 high;
117 };
118 } *completion_virt;
Maciej Sosnowski09177e82008-07-22 10:07:33 -0700119 unsigned long last_compl_desc_addr_hw;
Shannon Nelson3e037452007-10-16 01:27:40 -0700120 struct tasklet_struct cleanup_task;
Chris Leech0bbd5f42006-05-23 17:35:34 -0700121};
122
123/* wrapper around hardware descriptor format + additional software fields */
124
125/**
126 * struct ioat_desc_sw - wrapper around hardware descriptor
127 * @hw: hardware DMA descriptor
Dan Williams7405f742007-01-02 11:10:43 -0700128 * @node: this descriptor will either be on the free list,
129 * or attached to a transaction list (async_tx.tx_list)
130 * @tx_cnt: number of descriptors required to complete the transaction
Dan Williamsbc3c7022009-07-28 14:33:42 -0700131 * @txd: the generic software descriptor for all engines
Chris Leech0bbd5f42006-05-23 17:35:34 -0700132 */
Chris Leech0bbd5f42006-05-23 17:35:34 -0700133struct ioat_desc_sw {
134 struct ioat_dma_descriptor *hw;
135 struct list_head node;
Dan Williams7405f742007-01-02 11:10:43 -0700136 int tx_cnt;
Shannon Nelson7f2b2912007-10-18 03:07:14 -0700137 size_t len;
138 dma_addr_t src;
139 dma_addr_t dst;
Dan Williamsbc3c7022009-07-28 14:33:42 -0700140 struct dma_async_tx_descriptor txd;
Chris Leech0bbd5f42006-05-23 17:35:34 -0700141};
142
Dan Williamsf2427e22009-07-28 14:42:38 -0700143static inline void ioat_set_tcp_copy_break(unsigned long copybreak)
Maciej Sosnowski16a37ac2008-07-22 17:30:57 -0700144{
145 #ifdef CONFIG_NET_DMA
Dan Williamsf2427e22009-07-28 14:42:38 -0700146 sysctl_tcp_dma_copybreak = copybreak;
Maciej Sosnowski16a37ac2008-07-22 17:30:57 -0700147 #endif
148}
149
Dan Williamsf2427e22009-07-28 14:42:38 -0700150int ioat1_dma_probe(struct ioatdma_device *dev, int dca);
151int ioat2_dma_probe(struct ioatdma_device *dev, int dca);
152int ioat3_dma_probe(struct ioatdma_device *dev, int dca);
Shannon Nelson8ab89562007-10-16 01:27:39 -0700153void ioat_dma_remove(struct ioatdma_device *device);
Shannon Nelson7bb67c12007-11-14 16:59:51 -0800154struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase);
155struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
Maciej Sosnowski7f1b3582008-07-22 17:30:57 -0700156struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
Chris Leech0bbd5f42006-05-23 17:35:34 -0700157#endif /* IOATDMA_H */