blob: 969d4d928455b237ac97deda8f7a1e1cf38b63df [file] [log] [blame]
Steve Wiseb038ced2007-02-12 16:16:18 -08001/*
2 * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
Steve Wiseb038ced2007-02-12 16:16:18 -08003 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#ifndef __CXIO_WR_H__
33#define __CXIO_WR_H__
34
35#include <asm/io.h>
36#include <linux/pci.h>
37#include <linux/timer.h>
38#include "firmware_exports.h"
39
40#define T3_MAX_SGE 4
Steve Wise1860cdf2007-04-26 15:21:09 -050041#define T3_MAX_INLINE 64
Steve Wiseb038ced2007-02-12 16:16:18 -080042
43#define Q_EMPTY(rptr,wptr) ((rptr)==(wptr))
44#define Q_FULL(rptr,wptr,size_log2) ( (((wptr)-(rptr))>>(size_log2)) && \
45 ((rptr)!=(wptr)) )
46#define Q_GENBIT(ptr,size_log2) (!(((ptr)>>size_log2)&0x1))
47#define Q_FREECNT(rptr,wptr,size_log2) ((1UL<<size_log2)-((wptr)-(rptr)))
48#define Q_COUNT(rptr,wptr) ((wptr)-(rptr))
49#define Q_PTR2IDX(ptr,size_log2) (ptr & ((1UL<<size_log2)-1))
50
51static inline void ring_doorbell(void __iomem *doorbell, u32 qpid)
52{
53 writel(((1<<31) | qpid), doorbell);
54}
55
56#define SEQ32_GE(x,y) (!( (((u32) (x)) - ((u32) (y))) & 0x80000000 ))
57
58enum t3_wr_flags {
59 T3_COMPLETION_FLAG = 0x01,
60 T3_NOTIFY_FLAG = 0x02,
61 T3_SOLICITED_EVENT_FLAG = 0x04,
62 T3_READ_FENCE_FLAG = 0x08,
63 T3_LOCAL_FENCE_FLAG = 0x10
64} __attribute__ ((packed));
65
66enum t3_wr_opcode {
67 T3_WR_BP = FW_WROPCODE_RI_BYPASS,
68 T3_WR_SEND = FW_WROPCODE_RI_SEND,
69 T3_WR_WRITE = FW_WROPCODE_RI_RDMA_WRITE,
70 T3_WR_READ = FW_WROPCODE_RI_RDMA_READ,
71 T3_WR_INV_STAG = FW_WROPCODE_RI_LOCAL_INV,
72 T3_WR_BIND = FW_WROPCODE_RI_BIND_MW,
73 T3_WR_RCV = FW_WROPCODE_RI_RECEIVE,
74 T3_WR_INIT = FW_WROPCODE_RI_RDMA_INIT,
75 T3_WR_QP_MOD = FW_WROPCODE_RI_MODIFY_QP
76} __attribute__ ((packed));
77
78enum t3_rdma_opcode {
79 T3_RDMA_WRITE, /* IETF RDMAP v1.0 ... */
80 T3_READ_REQ,
81 T3_READ_RESP,
82 T3_SEND,
83 T3_SEND_WITH_INV,
84 T3_SEND_WITH_SE,
85 T3_SEND_WITH_SE_INV,
86 T3_TERMINATE,
87 T3_RDMA_INIT, /* CHELSIO RI specific ... */
88 T3_BIND_MW,
89 T3_FAST_REGISTER,
90 T3_LOCAL_INV,
91 T3_QP_MOD,
92 T3_BYPASS
93} __attribute__ ((packed));
94
95static inline enum t3_rdma_opcode wr2opcode(enum t3_wr_opcode wrop)
96{
97 switch (wrop) {
98 case T3_WR_BP: return T3_BYPASS;
99 case T3_WR_SEND: return T3_SEND;
100 case T3_WR_WRITE: return T3_RDMA_WRITE;
101 case T3_WR_READ: return T3_READ_REQ;
102 case T3_WR_INV_STAG: return T3_LOCAL_INV;
103 case T3_WR_BIND: return T3_BIND_MW;
104 case T3_WR_INIT: return T3_RDMA_INIT;
105 case T3_WR_QP_MOD: return T3_QP_MOD;
106 default: break;
107 }
108 return -1;
109}
110
111
112/* Work request id */
113union t3_wrid {
114 struct {
115 u32 hi;
116 u32 low;
117 } id0;
118 u64 id1;
119};
120
121#define WRID(wrid) (wrid.id1)
122#define WRID_GEN(wrid) (wrid.id0.wr_gen)
123#define WRID_IDX(wrid) (wrid.id0.wr_idx)
124#define WRID_LO(wrid) (wrid.id0.wr_lo)
125
126struct fw_riwrh {
127 __be32 op_seop_flags;
128 __be32 gen_tid_len;
129};
130
131#define S_FW_RIWR_OP 24
132#define M_FW_RIWR_OP 0xff
133#define V_FW_RIWR_OP(x) ((x) << S_FW_RIWR_OP)
134#define G_FW_RIWR_OP(x) ((((x) >> S_FW_RIWR_OP)) & M_FW_RIWR_OP)
135
136#define S_FW_RIWR_SOPEOP 22
137#define M_FW_RIWR_SOPEOP 0x3
138#define V_FW_RIWR_SOPEOP(x) ((x) << S_FW_RIWR_SOPEOP)
139
140#define S_FW_RIWR_FLAGS 8
141#define M_FW_RIWR_FLAGS 0x3fffff
142#define V_FW_RIWR_FLAGS(x) ((x) << S_FW_RIWR_FLAGS)
143#define G_FW_RIWR_FLAGS(x) ((((x) >> S_FW_RIWR_FLAGS)) & M_FW_RIWR_FLAGS)
144
145#define S_FW_RIWR_TID 8
146#define V_FW_RIWR_TID(x) ((x) << S_FW_RIWR_TID)
147
148#define S_FW_RIWR_LEN 0
149#define V_FW_RIWR_LEN(x) ((x) << S_FW_RIWR_LEN)
150
151#define S_FW_RIWR_GEN 31
152#define V_FW_RIWR_GEN(x) ((x) << S_FW_RIWR_GEN)
153
154struct t3_sge {
155 __be32 stag;
156 __be32 len;
157 __be64 to;
158};
159
160/* If num_sgle is zero, flit 5+ contains immediate data.*/
161struct t3_send_wr {
162 struct fw_riwrh wrh; /* 0 */
163 union t3_wrid wrid; /* 1 */
164
165 u8 rdmaop; /* 2 */
166 u8 reserved[3];
167 __be32 rem_stag;
168 __be32 plen; /* 3 */
169 __be32 num_sgle;
170 struct t3_sge sgl[T3_MAX_SGE]; /* 4+ */
171};
172
173struct t3_local_inv_wr {
174 struct fw_riwrh wrh; /* 0 */
175 union t3_wrid wrid; /* 1 */
176 __be32 stag; /* 2 */
177 __be32 reserved3;
178};
179
180struct t3_rdma_write_wr {
181 struct fw_riwrh wrh; /* 0 */
182 union t3_wrid wrid; /* 1 */
183 u8 rdmaop; /* 2 */
184 u8 reserved[3];
185 __be32 stag_sink;
186 __be64 to_sink; /* 3 */
187 __be32 plen; /* 4 */
188 __be32 num_sgle;
189 struct t3_sge sgl[T3_MAX_SGE]; /* 5+ */
190};
191
192struct t3_rdma_read_wr {
193 struct fw_riwrh wrh; /* 0 */
194 union t3_wrid wrid; /* 1 */
195 u8 rdmaop; /* 2 */
196 u8 reserved[3];
197 __be32 rem_stag;
198 __be64 rem_to; /* 3 */
199 __be32 local_stag; /* 4 */
200 __be32 local_len;
201 __be64 local_to; /* 5 */
202};
203
204enum t3_addr_type {
205 T3_VA_BASED_TO = 0x0,
206 T3_ZERO_BASED_TO = 0x1
207} __attribute__ ((packed));
208
209enum t3_mem_perms {
210 T3_MEM_ACCESS_LOCAL_READ = 0x1,
211 T3_MEM_ACCESS_LOCAL_WRITE = 0x2,
212 T3_MEM_ACCESS_REM_READ = 0x4,
213 T3_MEM_ACCESS_REM_WRITE = 0x8
214} __attribute__ ((packed));
215
216struct t3_bind_mw_wr {
217 struct fw_riwrh wrh; /* 0 */
218 union t3_wrid wrid; /* 1 */
219 u16 reserved; /* 2 */
220 u8 type;
221 u8 perms;
222 __be32 mr_stag;
223 __be32 mw_stag; /* 3 */
224 __be32 mw_len;
225 __be64 mw_va; /* 4 */
226 __be32 mr_pbl_addr; /* 5 */
227 u8 reserved2[3];
228 u8 mr_pagesz;
229};
230
231struct t3_receive_wr {
232 struct fw_riwrh wrh; /* 0 */
233 union t3_wrid wrid; /* 1 */
234 u8 pagesz[T3_MAX_SGE];
235 __be32 num_sgle; /* 2 */
236 struct t3_sge sgl[T3_MAX_SGE]; /* 3+ */
237 __be32 pbl_addr[T3_MAX_SGE];
238};
239
240struct t3_bypass_wr {
241 struct fw_riwrh wrh;
242 union t3_wrid wrid; /* 1 */
243};
244
245struct t3_modify_qp_wr {
246 struct fw_riwrh wrh; /* 0 */
247 union t3_wrid wrid; /* 1 */
248 __be32 flags; /* 2 */
249 __be32 quiesce; /* 2 */
250 __be32 max_ird; /* 3 */
251 __be32 max_ord; /* 3 */
252 __be64 sge_cmd; /* 4 */
253 __be64 ctx1; /* 5 */
254 __be64 ctx0; /* 6 */
255};
256
257enum t3_modify_qp_flags {
258 MODQP_QUIESCE = 0x01,
259 MODQP_MAX_IRD = 0x02,
260 MODQP_MAX_ORD = 0x04,
261 MODQP_WRITE_EC = 0x08,
262 MODQP_READ_EC = 0x10,
263};
264
265
266enum t3_mpa_attrs {
267 uP_RI_MPA_RX_MARKER_ENABLE = 0x1,
268 uP_RI_MPA_TX_MARKER_ENABLE = 0x2,
269 uP_RI_MPA_CRC_ENABLE = 0x4,
270 uP_RI_MPA_IETF_ENABLE = 0x8
271} __attribute__ ((packed));
272
273enum t3_qp_caps {
274 uP_RI_QP_RDMA_READ_ENABLE = 0x01,
275 uP_RI_QP_RDMA_WRITE_ENABLE = 0x02,
276 uP_RI_QP_BIND_ENABLE = 0x04,
277 uP_RI_QP_FAST_REGISTER_ENABLE = 0x08,
278 uP_RI_QP_STAG0_ENABLE = 0x10
279} __attribute__ ((packed));
280
281struct t3_rdma_init_attr {
282 u32 tid;
283 u32 qpid;
284 u32 pdid;
285 u32 scqid;
286 u32 rcqid;
287 u32 rq_addr;
288 u32 rq_size;
289 enum t3_mpa_attrs mpaattrs;
290 enum t3_qp_caps qpcaps;
291 u16 tcp_emss;
292 u32 ord;
293 u32 ird;
294 u64 qp_dma_addr;
295 u32 qp_dma_size;
296 u32 flags;
Steve Wisede3d3532007-05-14 13:27:27 -0500297 u32 irs;
Steve Wiseb038ced2007-02-12 16:16:18 -0800298};
299
300struct t3_rdma_init_wr {
301 struct fw_riwrh wrh; /* 0 */
302 union t3_wrid wrid; /* 1 */
303 __be32 qpid; /* 2 */
304 __be32 pdid;
305 __be32 scqid; /* 3 */
306 __be32 rcqid;
307 __be32 rq_addr; /* 4 */
308 __be32 rq_size;
309 u8 mpaattrs; /* 5 */
310 u8 qpcaps;
311 __be16 ulpdu_size;
312 __be32 flags; /* bits 31-1 - reservered */
313 /* bit 0 - set if RECV posted */
314 __be32 ord; /* 6 */
315 __be32 ird;
316 __be64 qp_dma_addr; /* 7 */
317 __be32 qp_dma_size; /* 8 */
Roland Dreier1d6e6582008-01-25 14:15:42 -0800318 __be32 irs;
Steve Wiseb038ced2007-02-12 16:16:18 -0800319};
320
321struct t3_genbit {
322 u64 flit[15];
323 __be64 genbit;
324};
325
326enum rdma_init_wr_flags {
Steve Wisec6b5b502008-01-21 14:42:13 -0600327 RECVS_POSTED = (1<<0),
328 PRIV_QP = (1<<1),
Steve Wiseb038ced2007-02-12 16:16:18 -0800329};
330
331union t3_wr {
332 struct t3_send_wr send;
333 struct t3_rdma_write_wr write;
334 struct t3_rdma_read_wr read;
335 struct t3_receive_wr recv;
336 struct t3_local_inv_wr local_inv;
337 struct t3_bind_mw_wr bind;
338 struct t3_bypass_wr bypass;
339 struct t3_rdma_init_wr init;
340 struct t3_modify_qp_wr qp_mod;
341 struct t3_genbit genbit;
342 u64 flit[16];
343};
344
345#define T3_SQ_CQE_FLIT 13
346#define T3_SQ_COOKIE_FLIT 14
347
348#define T3_RQ_COOKIE_FLIT 13
349#define T3_RQ_CQE_FLIT 14
350
351static inline enum t3_wr_opcode fw_riwrh_opcode(struct fw_riwrh *wqe)
352{
353 return G_FW_RIWR_OP(be32_to_cpu(wqe->op_seop_flags));
354}
355
356static inline void build_fw_riwrh(struct fw_riwrh *wqe, enum t3_wr_opcode op,
357 enum t3_wr_flags flags, u8 genbit, u32 tid,
358 u8 len)
359{
360 wqe->op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(op) |
361 V_FW_RIWR_SOPEOP(M_FW_RIWR_SOPEOP) |
362 V_FW_RIWR_FLAGS(flags));
363 wmb();
364 wqe->gen_tid_len = cpu_to_be32(V_FW_RIWR_GEN(genbit) |
365 V_FW_RIWR_TID(tid) |
366 V_FW_RIWR_LEN(len));
367 /* 2nd gen bit... */
368 ((union t3_wr *)wqe)->genbit.genbit = cpu_to_be64(genbit);
369}
370
371/*
372 * T3 ULP2_TX commands
373 */
374enum t3_utx_mem_op {
375 T3_UTX_MEM_READ = 2,
376 T3_UTX_MEM_WRITE = 3
377};
378
379/* T3 MC7 RDMA TPT entry format */
380
381enum tpt_mem_type {
382 TPT_NON_SHARED_MR = 0x0,
383 TPT_SHARED_MR = 0x1,
384 TPT_MW = 0x2,
385 TPT_MW_RELAXED_PROTECTION = 0x3
386};
387
388enum tpt_addr_type {
389 TPT_ZBTO = 0,
390 TPT_VATO = 1
391};
392
393enum tpt_mem_perm {
394 TPT_LOCAL_READ = 0x8,
395 TPT_LOCAL_WRITE = 0x4,
396 TPT_REMOTE_READ = 0x2,
397 TPT_REMOTE_WRITE = 0x1
398};
399
400struct tpt_entry {
401 __be32 valid_stag_pdid;
402 __be32 flags_pagesize_qpid;
403
404 __be32 rsvd_pbl_addr;
405 __be32 len;
406 __be32 va_hi;
407 __be32 va_low_or_fbo;
408
409 __be32 rsvd_bind_cnt_or_pstag;
410 __be32 rsvd_pbl_size;
411};
412
413#define S_TPT_VALID 31
414#define V_TPT_VALID(x) ((x) << S_TPT_VALID)
415#define F_TPT_VALID V_TPT_VALID(1U)
416
417#define S_TPT_STAG_KEY 23
418#define M_TPT_STAG_KEY 0xFF
419#define V_TPT_STAG_KEY(x) ((x) << S_TPT_STAG_KEY)
420#define G_TPT_STAG_KEY(x) (((x) >> S_TPT_STAG_KEY) & M_TPT_STAG_KEY)
421
422#define S_TPT_STAG_STATE 22
423#define V_TPT_STAG_STATE(x) ((x) << S_TPT_STAG_STATE)
424#define F_TPT_STAG_STATE V_TPT_STAG_STATE(1U)
425
426#define S_TPT_STAG_TYPE 20
427#define M_TPT_STAG_TYPE 0x3
428#define V_TPT_STAG_TYPE(x) ((x) << S_TPT_STAG_TYPE)
429#define G_TPT_STAG_TYPE(x) (((x) >> S_TPT_STAG_TYPE) & M_TPT_STAG_TYPE)
430
431#define S_TPT_PDID 0
432#define M_TPT_PDID 0xFFFFF
433#define V_TPT_PDID(x) ((x) << S_TPT_PDID)
434#define G_TPT_PDID(x) (((x) >> S_TPT_PDID) & M_TPT_PDID)
435
436#define S_TPT_PERM 28
437#define M_TPT_PERM 0xF
438#define V_TPT_PERM(x) ((x) << S_TPT_PERM)
439#define G_TPT_PERM(x) (((x) >> S_TPT_PERM) & M_TPT_PERM)
440
441#define S_TPT_REM_INV_DIS 27
442#define V_TPT_REM_INV_DIS(x) ((x) << S_TPT_REM_INV_DIS)
443#define F_TPT_REM_INV_DIS V_TPT_REM_INV_DIS(1U)
444
445#define S_TPT_ADDR_TYPE 26
446#define V_TPT_ADDR_TYPE(x) ((x) << S_TPT_ADDR_TYPE)
447#define F_TPT_ADDR_TYPE V_TPT_ADDR_TYPE(1U)
448
449#define S_TPT_MW_BIND_ENABLE 25
450#define V_TPT_MW_BIND_ENABLE(x) ((x) << S_TPT_MW_BIND_ENABLE)
451#define F_TPT_MW_BIND_ENABLE V_TPT_MW_BIND_ENABLE(1U)
452
453#define S_TPT_PAGE_SIZE 20
454#define M_TPT_PAGE_SIZE 0x1F
455#define V_TPT_PAGE_SIZE(x) ((x) << S_TPT_PAGE_SIZE)
456#define G_TPT_PAGE_SIZE(x) (((x) >> S_TPT_PAGE_SIZE) & M_TPT_PAGE_SIZE)
457
458#define S_TPT_PBL_ADDR 0
459#define M_TPT_PBL_ADDR 0x1FFFFFFF
460#define V_TPT_PBL_ADDR(x) ((x) << S_TPT_PBL_ADDR)
461#define G_TPT_PBL_ADDR(x) (((x) >> S_TPT_PBL_ADDR) & M_TPT_PBL_ADDR)
462
463#define S_TPT_QPID 0
464#define M_TPT_QPID 0xFFFFF
465#define V_TPT_QPID(x) ((x) << S_TPT_QPID)
466#define G_TPT_QPID(x) (((x) >> S_TPT_QPID) & M_TPT_QPID)
467
468#define S_TPT_PSTAG 0
469#define M_TPT_PSTAG 0xFFFFFF
470#define V_TPT_PSTAG(x) ((x) << S_TPT_PSTAG)
471#define G_TPT_PSTAG(x) (((x) >> S_TPT_PSTAG) & M_TPT_PSTAG)
472
473#define S_TPT_PBL_SIZE 0
474#define M_TPT_PBL_SIZE 0xFFFFF
475#define V_TPT_PBL_SIZE(x) ((x) << S_TPT_PBL_SIZE)
476#define G_TPT_PBL_SIZE(x) (((x) >> S_TPT_PBL_SIZE) & M_TPT_PBL_SIZE)
477
478/*
479 * CQE defs
480 */
481struct t3_cqe {
482 __be32 header;
483 __be32 len;
484 union {
485 struct {
486 __be32 stag;
487 __be32 msn;
488 } rcqe;
489 struct {
490 u32 wrid_hi;
491 u32 wrid_low;
492 } scqe;
493 } u;
494};
495
496#define S_CQE_OOO 31
497#define M_CQE_OOO 0x1
498#define G_CQE_OOO(x) ((((x) >> S_CQE_OOO)) & M_CQE_OOO)
499#define V_CEQ_OOO(x) ((x)<<S_CQE_OOO)
500
501#define S_CQE_QPID 12
502#define M_CQE_QPID 0x7FFFF
503#define G_CQE_QPID(x) ((((x) >> S_CQE_QPID)) & M_CQE_QPID)
504#define V_CQE_QPID(x) ((x)<<S_CQE_QPID)
505
506#define S_CQE_SWCQE 11
507#define M_CQE_SWCQE 0x1
508#define G_CQE_SWCQE(x) ((((x) >> S_CQE_SWCQE)) & M_CQE_SWCQE)
509#define V_CQE_SWCQE(x) ((x)<<S_CQE_SWCQE)
510
511#define S_CQE_GENBIT 10
512#define M_CQE_GENBIT 0x1
513#define G_CQE_GENBIT(x) (((x) >> S_CQE_GENBIT) & M_CQE_GENBIT)
514#define V_CQE_GENBIT(x) ((x)<<S_CQE_GENBIT)
515
516#define S_CQE_STATUS 5
517#define M_CQE_STATUS 0x1F
518#define G_CQE_STATUS(x) ((((x) >> S_CQE_STATUS)) & M_CQE_STATUS)
519#define V_CQE_STATUS(x) ((x)<<S_CQE_STATUS)
520
521#define S_CQE_TYPE 4
522#define M_CQE_TYPE 0x1
523#define G_CQE_TYPE(x) ((((x) >> S_CQE_TYPE)) & M_CQE_TYPE)
524#define V_CQE_TYPE(x) ((x)<<S_CQE_TYPE)
525
526#define S_CQE_OPCODE 0
527#define M_CQE_OPCODE 0xF
528#define G_CQE_OPCODE(x) ((((x) >> S_CQE_OPCODE)) & M_CQE_OPCODE)
529#define V_CQE_OPCODE(x) ((x)<<S_CQE_OPCODE)
530
531#define SW_CQE(x) (G_CQE_SWCQE(be32_to_cpu((x).header)))
532#define CQE_OOO(x) (G_CQE_OOO(be32_to_cpu((x).header)))
533#define CQE_QPID(x) (G_CQE_QPID(be32_to_cpu((x).header)))
534#define CQE_GENBIT(x) (G_CQE_GENBIT(be32_to_cpu((x).header)))
535#define CQE_TYPE(x) (G_CQE_TYPE(be32_to_cpu((x).header)))
536#define SQ_TYPE(x) (CQE_TYPE((x)))
537#define RQ_TYPE(x) (!CQE_TYPE((x)))
538#define CQE_STATUS(x) (G_CQE_STATUS(be32_to_cpu((x).header)))
539#define CQE_OPCODE(x) (G_CQE_OPCODE(be32_to_cpu((x).header)))
540
541#define CQE_LEN(x) (be32_to_cpu((x).len))
542
543/* used for RQ completion processing */
544#define CQE_WRID_STAG(x) (be32_to_cpu((x).u.rcqe.stag))
545#define CQE_WRID_MSN(x) (be32_to_cpu((x).u.rcqe.msn))
546
547/* used for SQ completion processing */
548#define CQE_WRID_SQ_WPTR(x) ((x).u.scqe.wrid_hi)
549#define CQE_WRID_WPTR(x) ((x).u.scqe.wrid_low)
550
551/* generic accessor macros */
552#define CQE_WRID_HI(x) ((x).u.scqe.wrid_hi)
553#define CQE_WRID_LOW(x) ((x).u.scqe.wrid_low)
554
555#define TPT_ERR_SUCCESS 0x0
556#define TPT_ERR_STAG 0x1 /* STAG invalid: either the */
557 /* STAG is offlimt, being 0, */
558 /* or STAG_key mismatch */
559#define TPT_ERR_PDID 0x2 /* PDID mismatch */
560#define TPT_ERR_QPID 0x3 /* QPID mismatch */
561#define TPT_ERR_ACCESS 0x4 /* Invalid access right */
562#define TPT_ERR_WRAP 0x5 /* Wrap error */
563#define TPT_ERR_BOUND 0x6 /* base and bounds voilation */
564#define TPT_ERR_INVALIDATE_SHARED_MR 0x7 /* attempt to invalidate a */
565 /* shared memory region */
566#define TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND 0x8 /* attempt to invalidate a */
567 /* shared memory region */
568#define TPT_ERR_ECC 0x9 /* ECC error detected */
569#define TPT_ERR_ECC_PSTAG 0xA /* ECC error detected when */
570 /* reading PSTAG for a MW */
571 /* Invalidate */
572#define TPT_ERR_PBL_ADDR_BOUND 0xB /* pbl addr out of bounds: */
573 /* software error */
574#define TPT_ERR_SWFLUSH 0xC /* SW FLUSHED */
575#define TPT_ERR_CRC 0x10 /* CRC error */
576#define TPT_ERR_MARKER 0x11 /* Marker error */
577#define TPT_ERR_PDU_LEN_ERR 0x12 /* invalid PDU length */
578#define TPT_ERR_OUT_OF_RQE 0x13 /* out of RQE */
579#define TPT_ERR_DDP_VERSION 0x14 /* wrong DDP version */
580#define TPT_ERR_RDMA_VERSION 0x15 /* wrong RDMA version */
581#define TPT_ERR_OPCODE 0x16 /* invalid rdma opcode */
582#define TPT_ERR_DDP_QUEUE_NUM 0x17 /* invalid ddp queue number */
583#define TPT_ERR_MSN 0x18 /* MSN error */
584#define TPT_ERR_TBIT 0x19 /* tag bit not set correctly */
585#define TPT_ERR_MO 0x1A /* MO not 0 for TERMINATE */
586 /* or READ_REQ */
587#define TPT_ERR_MSN_GAP 0x1B
588#define TPT_ERR_MSN_RANGE 0x1C
589#define TPT_ERR_IRD_OVERFLOW 0x1D
590#define TPT_ERR_RQE_ADDR_BOUND 0x1E /* RQE addr out of bounds: */
591 /* software error */
592#define TPT_ERR_INTERNAL_ERR 0x1F /* internal error (opcode */
593 /* mismatch) */
594
595struct t3_swsq {
596 __u64 wr_id;
597 struct t3_cqe cqe;
598 __u32 sq_wptr;
599 __be32 read_len;
600 int opcode;
601 int complete;
602 int signaled;
603};
604
605/*
606 * A T3 WQ implements both the SQ and RQ.
607 */
608struct t3_wq {
609 union t3_wr *queue; /* DMA accessable memory */
610 dma_addr_t dma_addr; /* DMA address for HW */
611 DECLARE_PCI_UNMAP_ADDR(mapping) /* unmap kruft */
612 u32 error; /* 1 once we go to ERROR */
613 u32 qpid;
614 u32 wptr; /* idx to next available WR slot */
615 u32 size_log2; /* total wq size */
616 struct t3_swsq *sq; /* SW SQ */
617 struct t3_swsq *oldest_read; /* tracks oldest pending read */
618 u32 sq_wptr; /* sq_wptr - sq_rptr == count of */
619 u32 sq_rptr; /* pending wrs */
620 u32 sq_size_log2; /* sq size */
621 u64 *rq; /* SW RQ (holds consumer wr_ids */
622 u32 rq_wptr; /* rq_wptr - rq_rptr == count of */
623 u32 rq_rptr; /* pending wrs */
624 u64 *rq_oldest_wr; /* oldest wr on the SW RQ */
625 u32 rq_size_log2; /* rq size */
626 u32 rq_addr; /* rq adapter address */
627 void __iomem *doorbell; /* kernel db */
628 u64 udb; /* user db if any */
629};
630
631struct t3_cq {
632 u32 cqid;
633 u32 rptr;
634 u32 wptr;
635 u32 size_log2;
636 dma_addr_t dma_addr;
637 DECLARE_PCI_UNMAP_ADDR(mapping)
638 struct t3_cqe *queue;
639 struct t3_cqe *sw_queue;
640 u32 sw_rptr;
641 u32 sw_wptr;
642};
643
644#define CQ_VLD_ENTRY(ptr,size_log2,cqe) (Q_GENBIT(ptr,size_log2) == \
645 CQE_GENBIT(*cqe))
646
647static inline void cxio_set_wq_in_error(struct t3_wq *wq)
648{
649 wq->queue->flit[13] = 1;
650}
651
652static inline struct t3_cqe *cxio_next_hw_cqe(struct t3_cq *cq)
653{
654 struct t3_cqe *cqe;
655
656 cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
657 if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))
658 return cqe;
659 return NULL;
660}
661
662static inline struct t3_cqe *cxio_next_sw_cqe(struct t3_cq *cq)
663{
664 struct t3_cqe *cqe;
665
666 if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
667 cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
668 return cqe;
669 }
670 return NULL;
671}
672
673static inline struct t3_cqe *cxio_next_cqe(struct t3_cq *cq)
674{
675 struct t3_cqe *cqe;
676
677 if (!Q_EMPTY(cq->sw_rptr, cq->sw_wptr)) {
678 cqe = cq->sw_queue + (Q_PTR2IDX(cq->sw_rptr, cq->size_log2));
679 return cqe;
680 }
681 cqe = cq->queue + (Q_PTR2IDX(cq->rptr, cq->size_log2));
682 if (CQ_VLD_ENTRY(cq->rptr, cq->size_log2, cqe))
683 return cqe;
684 return NULL;
685}
686
687#endif