Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 1 | /* |
| 2 | * |
| 3 | * This file is provided under a dual BSD/GPLv2 license. When using or |
| 4 | * redistributing this file, you may do so under either license. |
| 5 | * |
| 6 | * GPL LICENSE SUMMARY |
| 7 | * |
| 8 | * Copyright(c) 2015 Intel Corporation. |
| 9 | * |
| 10 | * This program is free software; you can redistribute it and/or modify |
| 11 | * it under the terms of version 2 of the GNU General Public License as |
| 12 | * published by the Free Software Foundation. |
| 13 | * |
| 14 | * This program is distributed in the hope that it will be useful, but |
| 15 | * WITHOUT ANY WARRANTY; without even the implied warranty of |
| 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 17 | * General Public License for more details. |
| 18 | * |
| 19 | * BSD LICENSE |
| 20 | * |
| 21 | * Copyright(c) 2015 Intel Corporation. |
| 22 | * |
| 23 | * Redistribution and use in source and binary forms, with or without |
| 24 | * modification, are permitted provided that the following conditions |
| 25 | * are met: |
| 26 | * |
| 27 | * - Redistributions of source code must retain the above copyright |
| 28 | * notice, this list of conditions and the following disclaimer. |
| 29 | * - Redistributions in binary form must reproduce the above copyright |
| 30 | * notice, this list of conditions and the following disclaimer in |
| 31 | * the documentation and/or other materials provided with the |
| 32 | * distribution. |
| 33 | * - Neither the name of Intel Corporation nor the names of its |
| 34 | * contributors may be used to endorse or promote products derived |
| 35 | * from this software without specific prior written permission. |
| 36 | * |
| 37 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 38 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 39 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 40 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 41 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 42 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 43 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 44 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 45 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 46 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 47 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 48 | * |
| 49 | */ |
| 50 | |
| 51 | #ifndef HFI1_VERBS_H |
| 52 | #define HFI1_VERBS_H |
| 53 | |
| 54 | #include <linux/types.h> |
| 55 | #include <linux/seqlock.h> |
| 56 | #include <linux/kernel.h> |
| 57 | #include <linux/interrupt.h> |
| 58 | #include <linux/kref.h> |
| 59 | #include <linux/workqueue.h> |
| 60 | #include <linux/kthread.h> |
| 61 | #include <linux/completion.h> |
| 62 | #include <rdma/ib_pack.h> |
| 63 | #include <rdma/ib_user_verbs.h> |
| 64 | #include <rdma/ib_mad.h> |
Dennis Dalessandro | ec3f2c1 | 2016-01-19 14:41:33 -0800 | [diff] [blame] | 65 | #include <rdma/rdma_vt.h> |
Dennis Dalessandro | 54d10c1 | 2016-01-19 14:43:01 -0800 | [diff] [blame] | 66 | #include <rdma/rdmavt_qp.h> |
Dennis Dalessandro | abd712d | 2016-01-19 14:43:22 -0800 | [diff] [blame] | 67 | #include <rdma/rdmavt_cq.h> |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 68 | |
| 69 | struct hfi1_ctxtdata; |
| 70 | struct hfi1_pportdata; |
| 71 | struct hfi1_devdata; |
| 72 | struct hfi1_packet; |
| 73 | |
| 74 | #include "iowait.h" |
| 75 | |
| 76 | #define HFI1_MAX_RDMA_ATOMIC 16 |
| 77 | #define HFI1_GUIDS_PER_PORT 5 |
| 78 | |
| 79 | /* |
| 80 | * Increment this value if any changes that break userspace ABI |
| 81 | * compatibility are made. |
| 82 | */ |
| 83 | #define HFI1_UVERBS_ABI_VERSION 2 |
| 84 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 85 | #define IB_SEQ_NAK (3 << 29) |
| 86 | |
| 87 | /* AETH NAK opcode values */ |
| 88 | #define IB_RNR_NAK 0x20 |
| 89 | #define IB_NAK_PSN_ERROR 0x60 |
| 90 | #define IB_NAK_INVALID_REQUEST 0x61 |
| 91 | #define IB_NAK_REMOTE_ACCESS_ERROR 0x62 |
| 92 | #define IB_NAK_REMOTE_OPERATIONAL_ERROR 0x63 |
| 93 | #define IB_NAK_INVALID_RD_REQUEST 0x64 |
| 94 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 95 | /* IB Performance Manager status values */ |
| 96 | #define IB_PMA_SAMPLE_STATUS_DONE 0x00 |
| 97 | #define IB_PMA_SAMPLE_STATUS_STARTED 0x01 |
| 98 | #define IB_PMA_SAMPLE_STATUS_RUNNING 0x02 |
| 99 | |
| 100 | /* Mandatory IB performance counter select values. */ |
| 101 | #define IB_PMA_PORT_XMIT_DATA cpu_to_be16(0x0001) |
| 102 | #define IB_PMA_PORT_RCV_DATA cpu_to_be16(0x0002) |
| 103 | #define IB_PMA_PORT_XMIT_PKTS cpu_to_be16(0x0003) |
| 104 | #define IB_PMA_PORT_RCV_PKTS cpu_to_be16(0x0004) |
| 105 | #define IB_PMA_PORT_XMIT_WAIT cpu_to_be16(0x0005) |
| 106 | |
| 107 | #define HFI1_VENDOR_IPG cpu_to_be16(0xFFA0) |
| 108 | |
Mike Marciniszyn | 7c091e5 | 2015-11-10 09:14:01 -0500 | [diff] [blame] | 109 | #define IB_BTH_REQ_ACK BIT(31) |
| 110 | #define IB_BTH_SOLICITED BIT(23) |
| 111 | #define IB_BTH_MIG_REQ BIT(22) |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 112 | |
| 113 | #define IB_GRH_VERSION 6 |
| 114 | #define IB_GRH_VERSION_MASK 0xF |
| 115 | #define IB_GRH_VERSION_SHIFT 28 |
| 116 | #define IB_GRH_TCLASS_MASK 0xFF |
| 117 | #define IB_GRH_TCLASS_SHIFT 20 |
| 118 | #define IB_GRH_FLOW_MASK 0xFFFFF |
| 119 | #define IB_GRH_FLOW_SHIFT 0 |
| 120 | #define IB_GRH_NEXT_HDR 0x1B |
| 121 | |
| 122 | #define IB_DEFAULT_GID_PREFIX cpu_to_be64(0xfe80000000000000ULL) |
| 123 | |
| 124 | /* flags passed by hfi1_ib_rcv() */ |
| 125 | enum { |
| 126 | HFI1_HAS_GRH = (1 << 0), |
| 127 | }; |
| 128 | |
| 129 | struct ib_reth { |
| 130 | __be64 vaddr; |
| 131 | __be32 rkey; |
| 132 | __be32 length; |
| 133 | } __packed; |
| 134 | |
| 135 | struct ib_atomic_eth { |
| 136 | __be32 vaddr[2]; /* unaligned so access as 2 32-bit words */ |
| 137 | __be32 rkey; |
| 138 | __be64 swap_data; |
| 139 | __be64 compare_data; |
| 140 | } __packed; |
| 141 | |
| 142 | union ib_ehdrs { |
| 143 | struct { |
| 144 | __be32 deth[2]; |
| 145 | __be32 imm_data; |
| 146 | } ud; |
| 147 | struct { |
| 148 | struct ib_reth reth; |
| 149 | __be32 imm_data; |
| 150 | } rc; |
| 151 | struct { |
| 152 | __be32 aeth; |
| 153 | __be32 atomic_ack_eth[2]; |
| 154 | } at; |
| 155 | __be32 imm_data; |
| 156 | __be32 aeth; |
| 157 | struct ib_atomic_eth atomic_eth; |
| 158 | } __packed; |
| 159 | |
| 160 | struct hfi1_other_headers { |
| 161 | __be32 bth[3]; |
| 162 | union ib_ehdrs u; |
| 163 | } __packed; |
| 164 | |
| 165 | /* |
| 166 | * Note that UD packets with a GRH header are 8+40+12+8 = 68 bytes |
| 167 | * long (72 w/ imm_data). Only the first 56 bytes of the IB header |
| 168 | * will be in the eager header buffer. The remaining 12 or 16 bytes |
| 169 | * are in the data buffer. |
| 170 | */ |
| 171 | struct hfi1_ib_header { |
| 172 | __be16 lrh[4]; |
| 173 | union { |
| 174 | struct { |
| 175 | struct ib_grh grh; |
| 176 | struct hfi1_other_headers oth; |
| 177 | } l; |
| 178 | struct hfi1_other_headers oth; |
| 179 | } u; |
| 180 | } __packed; |
| 181 | |
| 182 | struct ahg_ib_header { |
| 183 | struct sdma_engine *sde; |
| 184 | u32 ahgdesc[2]; |
| 185 | u16 tx_flags; |
| 186 | u8 ahgcount; |
| 187 | u8 ahgidx; |
| 188 | struct hfi1_ib_header ibh; |
| 189 | }; |
| 190 | |
| 191 | struct hfi1_pio_header { |
| 192 | __le64 pbc; |
| 193 | struct hfi1_ib_header hdr; |
| 194 | } __packed; |
| 195 | |
| 196 | /* |
| 197 | * used for force cacheline alignment for AHG |
| 198 | */ |
| 199 | struct tx_pio_header { |
| 200 | struct hfi1_pio_header phdr; |
| 201 | } ____cacheline_aligned; |
| 202 | |
| 203 | /* |
Dennis Dalessandro | 4c6829c | 2016-01-19 14:42:00 -0800 | [diff] [blame] | 204 | * hfi1 specific data structures that will be hidden from rvt after the queue |
| 205 | * pair is made common |
| 206 | */ |
Dennis Dalessandro | 4c6829c | 2016-01-19 14:42:00 -0800 | [diff] [blame] | 207 | struct hfi1_qp_priv { |
| 208 | struct ahg_ib_header *s_hdr; /* next packet header to send */ |
| 209 | struct sdma_engine *s_sde; /* current sde */ |
| 210 | u8 s_sc; /* SC[0..4] for next packet */ |
| 211 | u8 r_adefered; /* number of acks defered */ |
| 212 | struct iowait s_iowait; |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 213 | struct rvt_qp *owner; |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 214 | }; |
| 215 | |
| 216 | /* |
Dennis Dalessandro | d46e514 | 2015-11-11 00:34:37 -0500 | [diff] [blame] | 217 | * This structure is used to hold commonly lookedup and computed values during |
| 218 | * the send engine progress. |
| 219 | */ |
| 220 | struct hfi1_pkt_state { |
| 221 | struct hfi1_ibdev *dev; |
| 222 | struct hfi1_ibport *ibp; |
| 223 | struct hfi1_pportdata *ppd; |
| 224 | }; |
| 225 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 226 | #define HFI1_PSN_CREDIT 16 |
| 227 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 228 | struct hfi1_opcode_stats { |
| 229 | u64 n_packets; /* number of packets */ |
| 230 | u64 n_bytes; /* total number of bytes */ |
| 231 | }; |
| 232 | |
| 233 | struct hfi1_opcode_stats_perctx { |
| 234 | struct hfi1_opcode_stats stats[256]; |
| 235 | }; |
| 236 | |
| 237 | static inline void inc_opstats( |
| 238 | u32 tlen, |
| 239 | struct hfi1_opcode_stats *stats) |
| 240 | { |
| 241 | #ifdef CONFIG_DEBUG_FS |
| 242 | stats->n_bytes += tlen; |
| 243 | stats->n_packets++; |
| 244 | #endif |
| 245 | } |
| 246 | |
| 247 | struct hfi1_ibport { |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 248 | struct rvt_qp __rcu *qp[2]; |
Dennis Dalessandro | 4eb0688 | 2016-01-19 14:42:39 -0800 | [diff] [blame] | 249 | struct rvt_ibport rvp; |
Dennis Dalessandro | 15723f0 | 2016-01-19 14:42:17 -0800 | [diff] [blame] | 250 | struct rvt_ah *sm_ah; |
| 251 | struct rvt_ah *smi_ah; |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 252 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 253 | __be64 guids[HFI1_GUIDS_PER_PORT - 1]; /* writable GUIDs */ |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 254 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 255 | /* the first 16 entries are sl_to_vl for !OPA */ |
| 256 | u8 sl_to_sc[32]; |
| 257 | u8 sc_to_sl[32]; |
| 258 | }; |
| 259 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 260 | struct hfi1_ibdev { |
Dennis Dalessandro | ec3f2c1 | 2016-01-19 14:41:33 -0800 | [diff] [blame] | 261 | struct rvt_dev_info rdi; /* Must be first */ |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 262 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 263 | /* QP numbers are shared by all IB ports */ |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 264 | /* protect wait lists */ |
| 265 | seqlock_t iowait_lock; |
| 266 | struct list_head txwait; /* list for wait verbs_txreq */ |
| 267 | struct list_head memwait; /* list for wait kernel memory */ |
| 268 | struct list_head txreq_free; |
| 269 | struct kmem_cache *verbs_txreq_cache; |
| 270 | struct timer_list mem_timer; |
| 271 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 272 | u64 n_piowait; |
| 273 | u64 n_txwait; |
| 274 | u64 n_kmem_wait; |
Dean Luick | b421922 | 2015-10-26 10:28:35 -0400 | [diff] [blame] | 275 | u64 n_send_schedule; |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 276 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 277 | u32 n_srqs_allocated; /* number of SRQs allocated for device */ |
| 278 | spinlock_t n_srqs_lock; |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 279 | #ifdef CONFIG_DEBUG_FS |
| 280 | /* per HFI debugfs */ |
| 281 | struct dentry *hfi1_ibdev_dbg; |
| 282 | /* per HFI symlinks to above */ |
| 283 | struct dentry *hfi1_ibdev_link; |
| 284 | #endif |
| 285 | }; |
| 286 | |
| 287 | struct hfi1_verbs_counters { |
| 288 | u64 symbol_error_counter; |
| 289 | u64 link_error_recovery_counter; |
| 290 | u64 link_downed_counter; |
| 291 | u64 port_rcv_errors; |
| 292 | u64 port_rcv_remphys_errors; |
| 293 | u64 port_xmit_discards; |
| 294 | u64 port_xmit_data; |
| 295 | u64 port_rcv_data; |
| 296 | u64 port_xmit_packets; |
| 297 | u64 port_rcv_packets; |
| 298 | u32 local_link_integrity_errors; |
| 299 | u32 excessive_buffer_overrun_errors; |
| 300 | u32 vl15_dropped; |
| 301 | }; |
| 302 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 303 | static inline struct hfi1_ibdev *to_idev(struct ib_device *ibdev) |
| 304 | { |
Dennis Dalessandro | ec3f2c1 | 2016-01-19 14:41:33 -0800 | [diff] [blame] | 305 | struct rvt_dev_info *rdi; |
| 306 | |
| 307 | rdi = container_of(ibdev, struct rvt_dev_info, ibdev); |
| 308 | return container_of(rdi, struct hfi1_ibdev, rdi); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 309 | } |
| 310 | |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 311 | static inline struct rvt_qp *iowait_to_qp(struct iowait *s_iowait) |
Dennis Dalessandro | 4c6829c | 2016-01-19 14:42:00 -0800 | [diff] [blame] | 312 | { |
| 313 | struct hfi1_qp_priv *priv; |
| 314 | |
| 315 | priv = container_of(s_iowait, struct hfi1_qp_priv, s_iowait); |
| 316 | return priv->owner; |
| 317 | } |
| 318 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 319 | /* |
| 320 | * Send if not busy or waiting for I/O and either |
| 321 | * a RC response is pending or we can process send work requests. |
| 322 | */ |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 323 | static inline int hfi1_send_ok(struct rvt_qp *qp) |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 324 | { |
Dennis Dalessandro | 54d10c1 | 2016-01-19 14:43:01 -0800 | [diff] [blame] | 325 | return !(qp->s_flags & (RVT_S_BUSY | RVT_S_ANY_WAIT_IO)) && |
| 326 | (qp->s_hdrwords || (qp->s_flags & RVT_S_RESP_PENDING) || |
| 327 | !(qp->s_flags & RVT_S_ANY_WAIT_SEND)); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 328 | } |
| 329 | |
| 330 | /* |
| 331 | * This must be called with s_lock held. |
| 332 | */ |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 333 | void hfi1_bad_pqkey(struct hfi1_ibport *ibp, __be16 trap_num, u32 key, u32 sl, |
Erik E. Kahn | 5cd2411 | 2015-12-10 09:59:40 -0500 | [diff] [blame] | 334 | u32 qp1, u32 qp2, u16 lid1, u16 lid2); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 335 | void hfi1_cap_mask_chg(struct hfi1_ibport *ibp); |
| 336 | void hfi1_sys_guid_chg(struct hfi1_ibport *ibp); |
| 337 | void hfi1_node_desc_chg(struct hfi1_ibport *ibp); |
| 338 | int hfi1_process_mad(struct ib_device *ibdev, int mad_flags, u8 port, |
| 339 | const struct ib_wc *in_wc, const struct ib_grh *in_grh, |
| 340 | const struct ib_mad_hdr *in_mad, size_t in_mad_size, |
| 341 | struct ib_mad_hdr *out_mad, size_t *out_mad_size, |
| 342 | u16 *out_mad_pkey_index); |
| 343 | int hfi1_create_agents(struct hfi1_ibdev *dev); |
| 344 | void hfi1_free_agents(struct hfi1_ibdev *dev); |
| 345 | |
| 346 | /* |
| 347 | * The PSN_MASK and PSN_SHIFT allow for |
| 348 | * 1) comparing two PSNs |
| 349 | * 2) returning the PSN with any upper bits masked |
| 350 | * 3) returning the difference between to PSNs |
| 351 | * |
| 352 | * The number of significant bits in the PSN must |
| 353 | * necessarily be at least one bit less than |
| 354 | * the container holding the PSN. |
| 355 | */ |
| 356 | #ifndef CONFIG_HFI1_VERBS_31BIT_PSN |
| 357 | #define PSN_MASK 0xFFFFFF |
| 358 | #define PSN_SHIFT 8 |
| 359 | #else |
| 360 | #define PSN_MASK 0x7FFFFFFF |
| 361 | #define PSN_SHIFT 1 |
| 362 | #endif |
| 363 | #define PSN_MODIFY_MASK 0xFFFFFF |
| 364 | |
| 365 | /* Number of bits to pay attention to in the opcode for checking qp type */ |
| 366 | #define OPCODE_QP_MASK 0xE0 |
| 367 | |
| 368 | /* |
| 369 | * Compare the lower 24 bits of the msn values. |
| 370 | * Returns an integer <, ==, or > than zero. |
| 371 | */ |
| 372 | static inline int cmp_msn(u32 a, u32 b) |
| 373 | { |
| 374 | return (((int) a) - ((int) b)) << 8; |
| 375 | } |
| 376 | |
| 377 | /* |
| 378 | * Compare two PSNs |
| 379 | * Returns an integer <, ==, or > than zero. |
| 380 | */ |
| 381 | static inline int cmp_psn(u32 a, u32 b) |
| 382 | { |
| 383 | return (((int) a) - ((int) b)) << PSN_SHIFT; |
| 384 | } |
| 385 | |
| 386 | /* |
| 387 | * Return masked PSN |
| 388 | */ |
| 389 | static inline u32 mask_psn(u32 a) |
| 390 | { |
| 391 | return a & PSN_MASK; |
| 392 | } |
| 393 | |
| 394 | /* |
| 395 | * Return delta between two PSNs |
| 396 | */ |
| 397 | static inline u32 delta_psn(u32 a, u32 b) |
| 398 | { |
| 399 | return (((int)a - (int)b) << PSN_SHIFT) >> PSN_SHIFT; |
| 400 | } |
| 401 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 402 | struct verbs_txreq; |
| 403 | void hfi1_put_txreq(struct verbs_txreq *tx); |
| 404 | |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 405 | int hfi1_verbs_send(struct rvt_qp *qp, struct hfi1_pkt_state *ps); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 406 | |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 407 | void hfi1_copy_sge(struct rvt_sge_state *ss, void *data, u32 length, |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 408 | int release); |
| 409 | |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 410 | void hfi1_skip_sge(struct rvt_sge_state *ss, u32 length, int release); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 411 | |
| 412 | void hfi1_cnp_rcv(struct hfi1_packet *packet); |
| 413 | |
| 414 | void hfi1_uc_rcv(struct hfi1_packet *packet); |
| 415 | |
| 416 | void hfi1_rc_rcv(struct hfi1_packet *packet); |
| 417 | |
| 418 | void hfi1_rc_hdrerr( |
| 419 | struct hfi1_ctxtdata *rcd, |
| 420 | struct hfi1_ib_header *hdr, |
| 421 | u32 rcv_flags, |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 422 | struct rvt_qp *qp); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 423 | |
| 424 | u8 ah_to_sc(struct ib_device *ibdev, struct ib_ah_attr *ah_attr); |
| 425 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 426 | struct ib_ah *hfi1_create_qp0_ah(struct hfi1_ibport *ibp, u16 dlid); |
| 427 | |
| 428 | void hfi1_rc_rnr_retry(unsigned long arg); |
| 429 | |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 430 | void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_ib_header *hdr); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 431 | |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 432 | void hfi1_rc_error(struct rvt_qp *qp, enum ib_wc_status err); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 433 | |
| 434 | void hfi1_ud_rcv(struct hfi1_packet *packet); |
| 435 | |
| 436 | int hfi1_lookup_pkey_idx(struct hfi1_ibport *ibp, u16 pkey); |
| 437 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 438 | int hfi1_post_srq_receive(struct ib_srq *ibsrq, struct ib_recv_wr *wr, |
| 439 | struct ib_recv_wr **bad_wr); |
| 440 | |
| 441 | struct ib_srq *hfi1_create_srq(struct ib_pd *ibpd, |
| 442 | struct ib_srq_init_attr *srq_init_attr, |
| 443 | struct ib_udata *udata); |
| 444 | |
| 445 | int hfi1_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, |
| 446 | enum ib_srq_attr_mask attr_mask, |
| 447 | struct ib_udata *udata); |
| 448 | |
| 449 | int hfi1_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr); |
| 450 | |
| 451 | int hfi1_destroy_srq(struct ib_srq *ibsrq); |
| 452 | |
Dennis Dalessandro | ec4274f | 2016-01-19 14:43:44 -0800 | [diff] [blame^] | 453 | int hfi1_rvt_get_rwqe(struct rvt_qp *qp, int wr_id_only); |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 454 | |
| 455 | void hfi1_migrate_qp(struct rvt_qp *qp); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 456 | |
Dennis Dalessandro | ec4274f | 2016-01-19 14:43:44 -0800 | [diff] [blame^] | 457 | int hfi1_check_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, |
| 458 | int attr_mask, struct ib_udata *udata); |
| 459 | |
| 460 | void hfi1_modify_qp(struct rvt_qp *qp, struct ib_qp_attr *attr, |
| 461 | int attr_mask, struct ib_udata *udata); |
| 462 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 463 | int hfi1_ruc_check_hdr(struct hfi1_ibport *ibp, struct hfi1_ib_header *hdr, |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 464 | int has_grh, struct rvt_qp *qp, u32 bth0); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 465 | |
| 466 | u32 hfi1_make_grh(struct hfi1_ibport *ibp, struct ib_grh *hdr, |
| 467 | struct ib_global_route *grh, u32 hwords, u32 nwords); |
| 468 | |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 469 | void hfi1_make_ruc_header(struct rvt_qp *qp, struct hfi1_other_headers *ohdr, |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 470 | u32 bth0, u32 bth2, int middle); |
| 471 | |
Dennis Dalessandro | 83693bd | 2016-01-19 14:43:33 -0800 | [diff] [blame] | 472 | void _hfi1_do_send(struct work_struct *work); |
| 473 | |
| 474 | void hfi1_do_send(struct rvt_qp *qp); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 475 | |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 476 | void hfi1_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe, |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 477 | enum ib_wc_status status); |
| 478 | |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 479 | void hfi1_send_rc_ack(struct hfi1_ctxtdata *, struct rvt_qp *qp, int is_fecn); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 480 | |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 481 | int hfi1_make_rc_req(struct rvt_qp *qp); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 482 | |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 483 | int hfi1_make_uc_req(struct rvt_qp *qp); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 484 | |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 485 | int hfi1_make_ud_req(struct rvt_qp *qp); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 486 | |
| 487 | int hfi1_register_ib_device(struct hfi1_devdata *); |
| 488 | |
| 489 | void hfi1_unregister_ib_device(struct hfi1_devdata *); |
| 490 | |
| 491 | void hfi1_ib_rcv(struct hfi1_packet *packet); |
| 492 | |
| 493 | unsigned hfi1_get_npkeys(struct hfi1_devdata *); |
| 494 | |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 495 | int hfi1_verbs_send_dma(struct rvt_qp *qp, struct hfi1_pkt_state *ps, |
Dennis Dalessandro | d46e514 | 2015-11-11 00:34:37 -0500 | [diff] [blame] | 496 | u64 pbc); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 497 | |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 498 | int hfi1_verbs_send_pio(struct rvt_qp *qp, struct hfi1_pkt_state *ps, |
Dennis Dalessandro | d46e514 | 2015-11-11 00:34:37 -0500 | [diff] [blame] | 499 | u64 pbc); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 500 | |
Dennis Dalessandro | 895420d | 2016-01-19 14:42:28 -0800 | [diff] [blame] | 501 | struct send_context *qp_to_send_context(struct rvt_qp *qp, u8 sc5); |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 502 | |
| 503 | extern const enum ib_wc_opcode ib_hfi1_wc_opcode[]; |
| 504 | |
| 505 | extern const u8 hdr_len_by_opcode[]; |
| 506 | |
Dennis Dalessandro | 83693bd | 2016-01-19 14:43:33 -0800 | [diff] [blame] | 507 | extern const int ib_rvt_state_ops[]; |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 508 | |
| 509 | extern __be64 ib_hfi1_sys_image_guid; /* in network order */ |
| 510 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 511 | extern unsigned int hfi1_max_cqes; |
| 512 | |
| 513 | extern unsigned int hfi1_max_cqs; |
| 514 | |
| 515 | extern unsigned int hfi1_max_qp_wrs; |
| 516 | |
| 517 | extern unsigned int hfi1_max_qps; |
| 518 | |
| 519 | extern unsigned int hfi1_max_sges; |
| 520 | |
| 521 | extern unsigned int hfi1_max_mcast_grps; |
| 522 | |
| 523 | extern unsigned int hfi1_max_mcast_qp_attached; |
| 524 | |
| 525 | extern unsigned int hfi1_max_srqs; |
| 526 | |
| 527 | extern unsigned int hfi1_max_srq_sges; |
| 528 | |
| 529 | extern unsigned int hfi1_max_srq_wrs; |
| 530 | |
| 531 | extern const u32 ib_hfi1_rnr_table[]; |
| 532 | |
Mike Marciniszyn | 7724105 | 2015-07-30 15:17:43 -0400 | [diff] [blame] | 533 | #endif /* HFI1_VERBS_H */ |