blob: 470292767e68e81e390e1b9065954fd2546e04e6 [file] [log] [blame]
Dan Williams5cbafa62009-08-26 13:01:44 -07001/*
2 * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
20 */
21#ifndef IOATDMA_V2_H
22#define IOATDMA_V2_H
23
24#include <linux/dmaengine.h>
Dan Williamsabb12df2010-05-01 15:22:54 -070025#include <linux/circ_buf.h>
Dan Williams5cbafa62009-08-26 13:01:44 -070026#include "dma.h"
27#include "hw.h"
28
29
30extern int ioat_pending_level;
Dan Williamsbf40a682009-09-08 17:42:55 -070031extern int ioat_ring_alloc_order;
Dan Williams5cbafa62009-08-26 13:01:44 -070032
33/*
34 * workaround for IOAT ver.3.0 null descriptor issue
35 * (channel returns error when size is 0)
36 */
37#define NULL_DESC_BUFFER_SIZE 1
38
39#define IOAT_MAX_ORDER 16
40#define ioat_get_alloc_order() \
41 (min(ioat_ring_alloc_order, IOAT_MAX_ORDER))
Dan Williamsa3092182009-09-08 12:02:01 -070042#define ioat_get_max_alloc_order() \
43 (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER))
Dan Williams5cbafa62009-08-26 13:01:44 -070044
45/* struct ioat2_dma_chan - ioat v2 / v3 channel attributes
46 * @base: common ioat channel parameters
47 * @xfercap_log; log2 of channel max transfer length (for fast division)
48 * @head: allocated index
49 * @issued: hardware notification point
50 * @tail: cleanup index
Dan Williams5cbafa62009-08-26 13:01:44 -070051 * @dmacount: identical to 'head' except for occasionally resetting to zero
52 * @alloc_order: log2 of the number of allocated descriptors
Dan Williams074cc472010-05-01 15:22:55 -070053 * @produce: number of descriptors to produce at submit time
Dan Williams5cbafa62009-08-26 13:01:44 -070054 * @ring: software ring buffer implementation of hardware ring
Dan Williams074cc472010-05-01 15:22:55 -070055 * @prep_lock: serializes descriptor preparation (producers)
Dan Williams5cbafa62009-08-26 13:01:44 -070056 */
57struct ioat2_dma_chan {
58 struct ioat_chan_common base;
59 size_t xfercap_log;
60 u16 head;
61 u16 issued;
62 u16 tail;
63 u16 dmacount;
64 u16 alloc_order;
Dan Williams074cc472010-05-01 15:22:55 -070065 u16 produce;
Dan Williams5cbafa62009-08-26 13:01:44 -070066 struct ioat_ring_ent **ring;
Dan Williams074cc472010-05-01 15:22:55 -070067 spinlock_t prep_lock;
Dan Williams5cbafa62009-08-26 13:01:44 -070068};
69
70static inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c)
71{
72 struct ioat_chan_common *chan = to_chan_common(c);
73
74 return container_of(chan, struct ioat2_dma_chan, base);
75}
76
Dave Jiang21b764e2012-04-04 16:10:35 -070077static inline u32 ioat2_ring_size(struct ioat2_dma_chan *ioat)
Dan Williams5cbafa62009-08-26 13:01:44 -070078{
Dan Williamsabb12df2010-05-01 15:22:54 -070079 return 1 << ioat->alloc_order;
Dan Williams5cbafa62009-08-26 13:01:44 -070080}
81
82/* count of descriptors in flight with the engine */
83static inline u16 ioat2_ring_active(struct ioat2_dma_chan *ioat)
84{
Dan Williamsabb12df2010-05-01 15:22:54 -070085 return CIRC_CNT(ioat->head, ioat->tail, ioat2_ring_size(ioat));
Dan Williams5cbafa62009-08-26 13:01:44 -070086}
87
88/* count of descriptors pending submission to hardware */
89static inline u16 ioat2_ring_pending(struct ioat2_dma_chan *ioat)
90{
Dan Williamsabb12df2010-05-01 15:22:54 -070091 return CIRC_CNT(ioat->head, ioat->issued, ioat2_ring_size(ioat));
Dan Williams5cbafa62009-08-26 13:01:44 -070092}
93
Dave Jiang21b764e2012-04-04 16:10:35 -070094static inline u32 ioat2_ring_space(struct ioat2_dma_chan *ioat)
Dan Williams5cbafa62009-08-26 13:01:44 -070095{
Dan Williamsabb12df2010-05-01 15:22:54 -070096 return ioat2_ring_size(ioat) - ioat2_ring_active(ioat);
Dan Williams5cbafa62009-08-26 13:01:44 -070097}
98
Dan Williams5cbafa62009-08-26 13:01:44 -070099static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len)
100{
101 u16 num_descs = len >> ioat->xfercap_log;
102
103 num_descs += !!(len & ((1 << ioat->xfercap_log) - 1));
104 return num_descs;
105}
106
Dan Williams2aec0482009-09-08 17:42:54 -0700107/**
108 * struct ioat_ring_ent - wrapper around hardware descriptor
109 * @hw: hardware DMA descriptor (for memcpy)
110 * @fill: hardware fill descriptor
111 * @xor: hardware xor descriptor
112 * @xor_ex: hardware xor extension descriptor
113 * @pq: hardware pq descriptor
114 * @pq_ex: hardware pq extension descriptor
115 * @pqu: hardware pq update descriptor
116 * @raw: hardware raw (un-typed) descriptor
117 * @txd: the generic software descriptor for all engines
118 * @len: total transaction length for unmap
Dan Williamsb094ad32009-09-08 17:42:57 -0700119 * @result: asynchronous result of validate operations
Dan Williams2aec0482009-09-08 17:42:54 -0700120 * @id: identifier for debug
121 */
122
Dan Williams5cbafa62009-08-26 13:01:44 -0700123struct ioat_ring_ent {
Dan Williams2aec0482009-09-08 17:42:54 -0700124 union {
125 struct ioat_dma_descriptor *hw;
Dan Williams2aec0482009-09-08 17:42:54 -0700126 struct ioat_xor_descriptor *xor;
127 struct ioat_xor_ext_descriptor *xor_ex;
128 struct ioat_pq_descriptor *pq;
129 struct ioat_pq_ext_descriptor *pq_ex;
130 struct ioat_pq_update_descriptor *pqu;
131 struct ioat_raw_descriptor *raw;
132 };
Dan Williams5cbafa62009-08-26 13:01:44 -0700133 size_t len;
Dan Williams162b96e2009-09-08 17:53:04 -0700134 struct dma_async_tx_descriptor txd;
Dan Williamsb094ad32009-09-08 17:42:57 -0700135 enum sum_check_flags *result;
Dan Williams6df91832009-09-08 12:00:55 -0700136 #ifdef DEBUG
137 int id;
138 #endif
Dave Jiang7727eaa2013-04-15 10:25:56 -0700139 struct ioat_sed_ent *sed;
Dan Williams5cbafa62009-08-26 13:01:44 -0700140};
141
142static inline struct ioat_ring_ent *
143ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx)
144{
Dan Williamsabb12df2010-05-01 15:22:54 -0700145 return ioat->ring[idx & (ioat2_ring_size(ioat) - 1)];
Dan Williams5cbafa62009-08-26 13:01:44 -0700146}
147
Dan Williams09c8a5b2009-09-08 12:01:49 -0700148static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr)
149{
150 struct ioat_chan_common *chan = &ioat->base;
151
152 writel(addr & 0x00000000FFFFFFFF,
153 chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
154 writel(addr >> 32,
155 chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
156}
157
Greg Kroah-Hartman4bf27b82012-12-21 15:09:59 -0800158int ioat2_dma_probe(struct ioatdma_device *dev, int dca);
159int ioat3_dma_probe(struct ioatdma_device *dev, int dca);
160struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
161struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
Dan Williams074cc472010-05-01 15:22:55 -0700162int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs);
Dan Williamsbf40a682009-09-08 17:42:55 -0700163int ioat2_enumerate_channels(struct ioatdma_device *device);
164struct dma_async_tx_descriptor *
165ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
166 dma_addr_t dma_src, size_t len, unsigned long flags);
167void ioat2_issue_pending(struct dma_chan *chan);
168int ioat2_alloc_chan_resources(struct dma_chan *c);
169void ioat2_free_chan_resources(struct dma_chan *c);
Dan Williamsbf40a682009-09-08 17:42:55 -0700170void __ioat2_restart_chan(struct ioat2_dma_chan *ioat);
171bool reshape_ring(struct ioat2_dma_chan *ioat, int order);
Dan Williamsb094ad32009-09-08 17:42:57 -0700172void __ioat2_issue_pending(struct ioat2_dma_chan *ioat);
Dan Williamsaa4d72a2010-03-03 21:21:13 -0700173void ioat2_cleanup_event(unsigned long data);
Dan Williamse3232712009-09-08 17:43:02 -0700174void ioat2_timer_event(unsigned long data);
Dan Williamsa6d52d72009-12-19 15:36:02 -0700175int ioat2_quiesce(struct ioat_chan_common *chan, unsigned long tmo);
176int ioat2_reset_sync(struct ioat_chan_common *chan, unsigned long tmo);
Dan Williams5669e312009-09-08 17:42:56 -0700177extern struct kobj_type ioat2_ktype;
Dan Williams162b96e2009-09-08 17:53:04 -0700178extern struct kmem_cache *ioat2_cache;
Dan Williams5cbafa62009-08-26 13:01:44 -0700179#endif /* IOATDMA_V2_H */