Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1 | /* |
Saeed Mahameed | 6cf0a15 | 2015-04-02 17:07:30 +0300 | [diff] [blame] | 2 | * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 3 | * |
| 4 | * This software is available to you under a choice of one of two |
| 5 | * licenses. You may choose to be licensed under the terms of the GNU |
| 6 | * General Public License (GPL) Version 2, available from the file |
| 7 | * COPYING in the main directory of this source tree, or the |
| 8 | * OpenIB.org BSD license below: |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or |
| 11 | * without modification, are permitted provided that the following |
| 12 | * conditions are met: |
| 13 | * |
| 14 | * - Redistributions of source code must retain the above |
| 15 | * copyright notice, this list of conditions and the following |
| 16 | * disclaimer. |
| 17 | * |
| 18 | * - Redistributions in binary form must reproduce the above |
| 19 | * copyright notice, this list of conditions and the following |
| 20 | * disclaimer in the documentation and/or other materials |
| 21 | * provided with the distribution. |
| 22 | * |
| 23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 30 | * SOFTWARE. |
| 31 | */ |
| 32 | |
| 33 | #include <linux/module.h> |
| 34 | #include <rdma/ib_umem.h> |
| 35 | #include "mlx5_ib.h" |
| 36 | #include "user.h" |
| 37 | |
| 38 | /* not supported currently */ |
| 39 | static int wq_signature; |
| 40 | |
| 41 | enum { |
| 42 | MLX5_IB_ACK_REQ_FREQ = 8, |
| 43 | }; |
| 44 | |
| 45 | enum { |
| 46 | MLX5_IB_DEFAULT_SCHED_QUEUE = 0x83, |
| 47 | MLX5_IB_DEFAULT_QP0_SCHED_QUEUE = 0x3f, |
| 48 | MLX5_IB_LINK_TYPE_IB = 0, |
| 49 | MLX5_IB_LINK_TYPE_ETH = 1 |
| 50 | }; |
| 51 | |
| 52 | enum { |
| 53 | MLX5_IB_SQ_STRIDE = 6, |
| 54 | MLX5_IB_CACHE_LINE_SIZE = 64, |
| 55 | }; |
| 56 | |
| 57 | static const u32 mlx5_ib_opcode[] = { |
| 58 | [IB_WR_SEND] = MLX5_OPCODE_SEND, |
| 59 | [IB_WR_SEND_WITH_IMM] = MLX5_OPCODE_SEND_IMM, |
| 60 | [IB_WR_RDMA_WRITE] = MLX5_OPCODE_RDMA_WRITE, |
| 61 | [IB_WR_RDMA_WRITE_WITH_IMM] = MLX5_OPCODE_RDMA_WRITE_IMM, |
| 62 | [IB_WR_RDMA_READ] = MLX5_OPCODE_RDMA_READ, |
| 63 | [IB_WR_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_CS, |
| 64 | [IB_WR_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_FA, |
| 65 | [IB_WR_SEND_WITH_INV] = MLX5_OPCODE_SEND_INVAL, |
| 66 | [IB_WR_LOCAL_INV] = MLX5_OPCODE_UMR, |
| 67 | [IB_WR_FAST_REG_MR] = MLX5_OPCODE_UMR, |
| 68 | [IB_WR_MASKED_ATOMIC_CMP_AND_SWP] = MLX5_OPCODE_ATOMIC_MASKED_CS, |
| 69 | [IB_WR_MASKED_ATOMIC_FETCH_AND_ADD] = MLX5_OPCODE_ATOMIC_MASKED_FA, |
| 70 | [MLX5_IB_WR_UMR] = MLX5_OPCODE_UMR, |
| 71 | }; |
| 72 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 73 | |
| 74 | static int is_qp0(enum ib_qp_type qp_type) |
| 75 | { |
| 76 | return qp_type == IB_QPT_SMI; |
| 77 | } |
| 78 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 79 | static int is_sqp(enum ib_qp_type qp_type) |
| 80 | { |
| 81 | return is_qp0(qp_type) || is_qp1(qp_type); |
| 82 | } |
| 83 | |
| 84 | static void *get_wqe(struct mlx5_ib_qp *qp, int offset) |
| 85 | { |
| 86 | return mlx5_buf_offset(&qp->buf, offset); |
| 87 | } |
| 88 | |
| 89 | static void *get_recv_wqe(struct mlx5_ib_qp *qp, int n) |
| 90 | { |
| 91 | return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift)); |
| 92 | } |
| 93 | |
| 94 | void *mlx5_get_send_wqe(struct mlx5_ib_qp *qp, int n) |
| 95 | { |
| 96 | return get_wqe(qp, qp->sq.offset + (n << MLX5_IB_SQ_STRIDE)); |
| 97 | } |
| 98 | |
Haggai Eran | c1395a2 | 2014-12-11 17:04:14 +0200 | [diff] [blame] | 99 | /** |
| 100 | * mlx5_ib_read_user_wqe() - Copy a user-space WQE to kernel space. |
| 101 | * |
| 102 | * @qp: QP to copy from. |
| 103 | * @send: copy from the send queue when non-zero, use the receive queue |
| 104 | * otherwise. |
| 105 | * @wqe_index: index to start copying from. For send work queues, the |
| 106 | * wqe_index is in units of MLX5_SEND_WQE_BB. |
| 107 | * For receive work queue, it is the number of work queue |
| 108 | * element in the queue. |
| 109 | * @buffer: destination buffer. |
| 110 | * @length: maximum number of bytes to copy. |
| 111 | * |
| 112 | * Copies at least a single WQE, but may copy more data. |
| 113 | * |
| 114 | * Return: the number of bytes copied, or an error code. |
| 115 | */ |
| 116 | int mlx5_ib_read_user_wqe(struct mlx5_ib_qp *qp, int send, int wqe_index, |
| 117 | void *buffer, u32 length) |
| 118 | { |
| 119 | struct ib_device *ibdev = qp->ibqp.device; |
| 120 | struct mlx5_ib_dev *dev = to_mdev(ibdev); |
| 121 | struct mlx5_ib_wq *wq = send ? &qp->sq : &qp->rq; |
| 122 | size_t offset; |
| 123 | size_t wq_end; |
| 124 | struct ib_umem *umem = qp->umem; |
| 125 | u32 first_copy_length; |
| 126 | int wqe_length; |
| 127 | int ret; |
| 128 | |
| 129 | if (wq->wqe_cnt == 0) { |
| 130 | mlx5_ib_dbg(dev, "mlx5_ib_read_user_wqe for a QP with wqe_cnt == 0. qp_type: 0x%x\n", |
| 131 | qp->ibqp.qp_type); |
| 132 | return -EINVAL; |
| 133 | } |
| 134 | |
| 135 | offset = wq->offset + ((wqe_index % wq->wqe_cnt) << wq->wqe_shift); |
| 136 | wq_end = wq->offset + (wq->wqe_cnt << wq->wqe_shift); |
| 137 | |
| 138 | if (send && length < sizeof(struct mlx5_wqe_ctrl_seg)) |
| 139 | return -EINVAL; |
| 140 | |
| 141 | if (offset > umem->length || |
| 142 | (send && offset + sizeof(struct mlx5_wqe_ctrl_seg) > umem->length)) |
| 143 | return -EINVAL; |
| 144 | |
| 145 | first_copy_length = min_t(u32, offset + length, wq_end) - offset; |
| 146 | ret = ib_umem_copy_from(buffer, umem, offset, first_copy_length); |
| 147 | if (ret) |
| 148 | return ret; |
| 149 | |
| 150 | if (send) { |
| 151 | struct mlx5_wqe_ctrl_seg *ctrl = buffer; |
| 152 | int ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK; |
| 153 | |
| 154 | wqe_length = ds * MLX5_WQE_DS_UNITS; |
| 155 | } else { |
| 156 | wqe_length = 1 << wq->wqe_shift; |
| 157 | } |
| 158 | |
| 159 | if (wqe_length <= first_copy_length) |
| 160 | return first_copy_length; |
| 161 | |
| 162 | ret = ib_umem_copy_from(buffer + first_copy_length, umem, wq->offset, |
| 163 | wqe_length - first_copy_length); |
| 164 | if (ret) |
| 165 | return ret; |
| 166 | |
| 167 | return wqe_length; |
| 168 | } |
| 169 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 170 | static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type) |
| 171 | { |
| 172 | struct ib_qp *ibqp = &to_mibqp(qp)->ibqp; |
| 173 | struct ib_event event; |
| 174 | |
| 175 | if (type == MLX5_EVENT_TYPE_PATH_MIG) |
| 176 | to_mibqp(qp)->port = to_mibqp(qp)->alt_port; |
| 177 | |
| 178 | if (ibqp->event_handler) { |
| 179 | event.device = ibqp->device; |
| 180 | event.element.qp = ibqp; |
| 181 | switch (type) { |
| 182 | case MLX5_EVENT_TYPE_PATH_MIG: |
| 183 | event.event = IB_EVENT_PATH_MIG; |
| 184 | break; |
| 185 | case MLX5_EVENT_TYPE_COMM_EST: |
| 186 | event.event = IB_EVENT_COMM_EST; |
| 187 | break; |
| 188 | case MLX5_EVENT_TYPE_SQ_DRAINED: |
| 189 | event.event = IB_EVENT_SQ_DRAINED; |
| 190 | break; |
| 191 | case MLX5_EVENT_TYPE_SRQ_LAST_WQE: |
| 192 | event.event = IB_EVENT_QP_LAST_WQE_REACHED; |
| 193 | break; |
| 194 | case MLX5_EVENT_TYPE_WQ_CATAS_ERROR: |
| 195 | event.event = IB_EVENT_QP_FATAL; |
| 196 | break; |
| 197 | case MLX5_EVENT_TYPE_PATH_MIG_FAILED: |
| 198 | event.event = IB_EVENT_PATH_MIG_ERR; |
| 199 | break; |
| 200 | case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR: |
| 201 | event.event = IB_EVENT_QP_REQ_ERR; |
| 202 | break; |
| 203 | case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR: |
| 204 | event.event = IB_EVENT_QP_ACCESS_ERR; |
| 205 | break; |
| 206 | default: |
| 207 | pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n", type, qp->qpn); |
| 208 | return; |
| 209 | } |
| 210 | |
| 211 | ibqp->event_handler(&event, ibqp->qp_context); |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap, |
| 216 | int has_rq, struct mlx5_ib_qp *qp, struct mlx5_ib_create_qp *ucmd) |
| 217 | { |
| 218 | int wqe_size; |
| 219 | int wq_size; |
| 220 | |
| 221 | /* Sanity check RQ size before proceeding */ |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 222 | if (cap->max_recv_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 223 | return -EINVAL; |
| 224 | |
| 225 | if (!has_rq) { |
| 226 | qp->rq.max_gs = 0; |
| 227 | qp->rq.wqe_cnt = 0; |
| 228 | qp->rq.wqe_shift = 0; |
| 229 | } else { |
| 230 | if (ucmd) { |
| 231 | qp->rq.wqe_cnt = ucmd->rq_wqe_count; |
| 232 | qp->rq.wqe_shift = ucmd->rq_wqe_shift; |
| 233 | qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig; |
| 234 | qp->rq.max_post = qp->rq.wqe_cnt; |
| 235 | } else { |
| 236 | wqe_size = qp->wq_sig ? sizeof(struct mlx5_wqe_signature_seg) : 0; |
| 237 | wqe_size += cap->max_recv_sge * sizeof(struct mlx5_wqe_data_seg); |
| 238 | wqe_size = roundup_pow_of_two(wqe_size); |
| 239 | wq_size = roundup_pow_of_two(cap->max_recv_wr) * wqe_size; |
| 240 | wq_size = max_t(int, wq_size, MLX5_SEND_WQE_BB); |
| 241 | qp->rq.wqe_cnt = wq_size / wqe_size; |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 242 | if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq)) { |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 243 | mlx5_ib_dbg(dev, "wqe_size %d, max %d\n", |
| 244 | wqe_size, |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 245 | MLX5_CAP_GEN(dev->mdev, |
| 246 | max_wqe_sz_rq)); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 247 | return -EINVAL; |
| 248 | } |
| 249 | qp->rq.wqe_shift = ilog2(wqe_size); |
| 250 | qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof(struct mlx5_wqe_data_seg) - qp->wq_sig; |
| 251 | qp->rq.max_post = qp->rq.wqe_cnt; |
| 252 | } |
| 253 | } |
| 254 | |
| 255 | return 0; |
| 256 | } |
| 257 | |
| 258 | static int sq_overhead(enum ib_qp_type qp_type) |
| 259 | { |
Andi Shyti | 618af38 | 2013-07-16 15:35:01 +0200 | [diff] [blame] | 260 | int size = 0; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 261 | |
| 262 | switch (qp_type) { |
| 263 | case IB_QPT_XRC_INI: |
Eli Cohen | b125a54 | 2013-09-11 16:35:22 +0300 | [diff] [blame] | 264 | size += sizeof(struct mlx5_wqe_xrc_seg); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 265 | /* fall through */ |
| 266 | case IB_QPT_RC: |
| 267 | size += sizeof(struct mlx5_wqe_ctrl_seg) + |
| 268 | sizeof(struct mlx5_wqe_atomic_seg) + |
| 269 | sizeof(struct mlx5_wqe_raddr_seg); |
| 270 | break; |
| 271 | |
Eli Cohen | b125a54 | 2013-09-11 16:35:22 +0300 | [diff] [blame] | 272 | case IB_QPT_XRC_TGT: |
| 273 | return 0; |
| 274 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 275 | case IB_QPT_UC: |
Eli Cohen | b125a54 | 2013-09-11 16:35:22 +0300 | [diff] [blame] | 276 | size += sizeof(struct mlx5_wqe_ctrl_seg) + |
Eli Cohen | 9e65dc3 | 2014-01-28 14:52:47 +0200 | [diff] [blame] | 277 | sizeof(struct mlx5_wqe_raddr_seg) + |
| 278 | sizeof(struct mlx5_wqe_umr_ctrl_seg) + |
| 279 | sizeof(struct mlx5_mkey_seg); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 280 | break; |
| 281 | |
| 282 | case IB_QPT_UD: |
| 283 | case IB_QPT_SMI: |
| 284 | case IB_QPT_GSI: |
Eli Cohen | b125a54 | 2013-09-11 16:35:22 +0300 | [diff] [blame] | 285 | size += sizeof(struct mlx5_wqe_ctrl_seg) + |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 286 | sizeof(struct mlx5_wqe_datagram_seg); |
| 287 | break; |
| 288 | |
| 289 | case MLX5_IB_QPT_REG_UMR: |
Eli Cohen | b125a54 | 2013-09-11 16:35:22 +0300 | [diff] [blame] | 290 | size += sizeof(struct mlx5_wqe_ctrl_seg) + |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 291 | sizeof(struct mlx5_wqe_umr_ctrl_seg) + |
| 292 | sizeof(struct mlx5_mkey_seg); |
| 293 | break; |
| 294 | |
| 295 | default: |
| 296 | return -EINVAL; |
| 297 | } |
| 298 | |
| 299 | return size; |
| 300 | } |
| 301 | |
| 302 | static int calc_send_wqe(struct ib_qp_init_attr *attr) |
| 303 | { |
| 304 | int inl_size = 0; |
| 305 | int size; |
| 306 | |
| 307 | size = sq_overhead(attr->qp_type); |
| 308 | if (size < 0) |
| 309 | return size; |
| 310 | |
| 311 | if (attr->cap.max_inline_data) { |
| 312 | inl_size = size + sizeof(struct mlx5_wqe_inline_seg) + |
| 313 | attr->cap.max_inline_data; |
| 314 | } |
| 315 | |
| 316 | size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg); |
Sagi Grimberg | e1e66cc | 2014-02-23 14:19:07 +0200 | [diff] [blame] | 317 | if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN && |
| 318 | ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE) |
| 319 | return MLX5_SIG_WQE_SIZE; |
| 320 | else |
| 321 | return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 322 | } |
| 323 | |
| 324 | static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr, |
| 325 | struct mlx5_ib_qp *qp) |
| 326 | { |
| 327 | int wqe_size; |
| 328 | int wq_size; |
| 329 | |
| 330 | if (!attr->cap.max_send_wr) |
| 331 | return 0; |
| 332 | |
| 333 | wqe_size = calc_send_wqe(attr); |
| 334 | mlx5_ib_dbg(dev, "wqe_size %d\n", wqe_size); |
| 335 | if (wqe_size < 0) |
| 336 | return wqe_size; |
| 337 | |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 338 | if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) { |
Eli Cohen | b125a54 | 2013-09-11 16:35:22 +0300 | [diff] [blame] | 339 | mlx5_ib_dbg(dev, "wqe_size(%d) > max_sq_desc_sz(%d)\n", |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 340 | wqe_size, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 341 | return -EINVAL; |
| 342 | } |
| 343 | |
| 344 | qp->max_inline_data = wqe_size - sq_overhead(attr->qp_type) - |
| 345 | sizeof(struct mlx5_wqe_inline_seg); |
| 346 | attr->cap.max_inline_data = qp->max_inline_data; |
| 347 | |
Sagi Grimberg | e1e66cc | 2014-02-23 14:19:07 +0200 | [diff] [blame] | 348 | if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN) |
| 349 | qp->signature_en = true; |
| 350 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 351 | wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size); |
| 352 | qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB; |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 353 | if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) { |
Eli Cohen | b125a54 | 2013-09-11 16:35:22 +0300 | [diff] [blame] | 354 | mlx5_ib_dbg(dev, "wqe count(%d) exceeds limits(%d)\n", |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 355 | qp->sq.wqe_cnt, |
| 356 | 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz)); |
Eli Cohen | b125a54 | 2013-09-11 16:35:22 +0300 | [diff] [blame] | 357 | return -ENOMEM; |
| 358 | } |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 359 | qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB); |
| 360 | qp->sq.max_gs = attr->cap.max_send_sge; |
Eli Cohen | b125a54 | 2013-09-11 16:35:22 +0300 | [diff] [blame] | 361 | qp->sq.max_post = wq_size / wqe_size; |
| 362 | attr->cap.max_send_wr = qp->sq.max_post; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 363 | |
| 364 | return wq_size; |
| 365 | } |
| 366 | |
| 367 | static int set_user_buf_size(struct mlx5_ib_dev *dev, |
| 368 | struct mlx5_ib_qp *qp, |
| 369 | struct mlx5_ib_create_qp *ucmd) |
| 370 | { |
| 371 | int desc_sz = 1 << qp->sq.wqe_shift; |
| 372 | |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 373 | if (desc_sz > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) { |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 374 | mlx5_ib_warn(dev, "desc_sz %d, max_sq_desc_sz %d\n", |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 375 | desc_sz, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 376 | return -EINVAL; |
| 377 | } |
| 378 | |
| 379 | if (ucmd->sq_wqe_count && ((1 << ilog2(ucmd->sq_wqe_count)) != ucmd->sq_wqe_count)) { |
| 380 | mlx5_ib_warn(dev, "sq_wqe_count %d, sq_wqe_count %d\n", |
| 381 | ucmd->sq_wqe_count, ucmd->sq_wqe_count); |
| 382 | return -EINVAL; |
| 383 | } |
| 384 | |
| 385 | qp->sq.wqe_cnt = ucmd->sq_wqe_count; |
| 386 | |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 387 | if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) { |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 388 | mlx5_ib_warn(dev, "wqe_cnt %d, max_wqes %d\n", |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 389 | qp->sq.wqe_cnt, |
| 390 | 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz)); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 391 | return -EINVAL; |
| 392 | } |
| 393 | |
| 394 | qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) + |
| 395 | (qp->sq.wqe_cnt << 6); |
| 396 | |
| 397 | return 0; |
| 398 | } |
| 399 | |
| 400 | static int qp_has_rq(struct ib_qp_init_attr *attr) |
| 401 | { |
| 402 | if (attr->qp_type == IB_QPT_XRC_INI || |
| 403 | attr->qp_type == IB_QPT_XRC_TGT || attr->srq || |
| 404 | attr->qp_type == MLX5_IB_QPT_REG_UMR || |
| 405 | !attr->cap.max_recv_wr) |
| 406 | return 0; |
| 407 | |
| 408 | return 1; |
| 409 | } |
| 410 | |
Eli Cohen | c1be523 | 2014-01-14 17:45:12 +0200 | [diff] [blame] | 411 | static int first_med_uuar(void) |
| 412 | { |
| 413 | return 1; |
| 414 | } |
| 415 | |
| 416 | static int next_uuar(int n) |
| 417 | { |
| 418 | n++; |
| 419 | |
| 420 | while (((n % 4) & 2)) |
| 421 | n++; |
| 422 | |
| 423 | return n; |
| 424 | } |
| 425 | |
| 426 | static int num_med_uuar(struct mlx5_uuar_info *uuari) |
| 427 | { |
| 428 | int n; |
| 429 | |
| 430 | n = uuari->num_uars * MLX5_NON_FP_BF_REGS_PER_PAGE - |
| 431 | uuari->num_low_latency_uuars - 1; |
| 432 | |
| 433 | return n >= 0 ? n : 0; |
| 434 | } |
| 435 | |
| 436 | static int max_uuari(struct mlx5_uuar_info *uuari) |
| 437 | { |
| 438 | return uuari->num_uars * 4; |
| 439 | } |
| 440 | |
| 441 | static int first_hi_uuar(struct mlx5_uuar_info *uuari) |
| 442 | { |
| 443 | int med; |
| 444 | int i; |
| 445 | int t; |
| 446 | |
| 447 | med = num_med_uuar(uuari); |
| 448 | for (t = 0, i = first_med_uuar();; i = next_uuar(i)) { |
| 449 | t++; |
| 450 | if (t == med) |
| 451 | return next_uuar(i); |
| 452 | } |
| 453 | |
| 454 | return 0; |
| 455 | } |
| 456 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 457 | static int alloc_high_class_uuar(struct mlx5_uuar_info *uuari) |
| 458 | { |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 459 | int i; |
| 460 | |
Eli Cohen | c1be523 | 2014-01-14 17:45:12 +0200 | [diff] [blame] | 461 | for (i = first_hi_uuar(uuari); i < max_uuari(uuari); i = next_uuar(i)) { |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 462 | if (!test_bit(i, uuari->bitmap)) { |
| 463 | set_bit(i, uuari->bitmap); |
| 464 | uuari->count[i]++; |
| 465 | return i; |
| 466 | } |
| 467 | } |
| 468 | |
| 469 | return -ENOMEM; |
| 470 | } |
| 471 | |
| 472 | static int alloc_med_class_uuar(struct mlx5_uuar_info *uuari) |
| 473 | { |
Eli Cohen | c1be523 | 2014-01-14 17:45:12 +0200 | [diff] [blame] | 474 | int minidx = first_med_uuar(); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 475 | int i; |
| 476 | |
Eli Cohen | c1be523 | 2014-01-14 17:45:12 +0200 | [diff] [blame] | 477 | for (i = first_med_uuar(); i < first_hi_uuar(uuari); i = next_uuar(i)) { |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 478 | if (uuari->count[i] < uuari->count[minidx]) |
| 479 | minidx = i; |
| 480 | } |
| 481 | |
| 482 | uuari->count[minidx]++; |
| 483 | return minidx; |
| 484 | } |
| 485 | |
| 486 | static int alloc_uuar(struct mlx5_uuar_info *uuari, |
| 487 | enum mlx5_ib_latency_class lat) |
| 488 | { |
| 489 | int uuarn = -EINVAL; |
| 490 | |
| 491 | mutex_lock(&uuari->lock); |
| 492 | switch (lat) { |
| 493 | case MLX5_IB_LATENCY_CLASS_LOW: |
| 494 | uuarn = 0; |
| 495 | uuari->count[uuarn]++; |
| 496 | break; |
| 497 | |
| 498 | case MLX5_IB_LATENCY_CLASS_MEDIUM: |
Eli Cohen | 78c0f98 | 2014-01-30 13:49:48 +0200 | [diff] [blame] | 499 | if (uuari->ver < 2) |
| 500 | uuarn = -ENOMEM; |
| 501 | else |
| 502 | uuarn = alloc_med_class_uuar(uuari); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 503 | break; |
| 504 | |
| 505 | case MLX5_IB_LATENCY_CLASS_HIGH: |
Eli Cohen | 78c0f98 | 2014-01-30 13:49:48 +0200 | [diff] [blame] | 506 | if (uuari->ver < 2) |
| 507 | uuarn = -ENOMEM; |
| 508 | else |
| 509 | uuarn = alloc_high_class_uuar(uuari); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 510 | break; |
| 511 | |
| 512 | case MLX5_IB_LATENCY_CLASS_FAST_PATH: |
| 513 | uuarn = 2; |
| 514 | break; |
| 515 | } |
| 516 | mutex_unlock(&uuari->lock); |
| 517 | |
| 518 | return uuarn; |
| 519 | } |
| 520 | |
| 521 | static void free_med_class_uuar(struct mlx5_uuar_info *uuari, int uuarn) |
| 522 | { |
| 523 | clear_bit(uuarn, uuari->bitmap); |
| 524 | --uuari->count[uuarn]; |
| 525 | } |
| 526 | |
| 527 | static void free_high_class_uuar(struct mlx5_uuar_info *uuari, int uuarn) |
| 528 | { |
| 529 | clear_bit(uuarn, uuari->bitmap); |
| 530 | --uuari->count[uuarn]; |
| 531 | } |
| 532 | |
| 533 | static void free_uuar(struct mlx5_uuar_info *uuari, int uuarn) |
| 534 | { |
| 535 | int nuuars = uuari->num_uars * MLX5_BF_REGS_PER_PAGE; |
| 536 | int high_uuar = nuuars - uuari->num_low_latency_uuars; |
| 537 | |
| 538 | mutex_lock(&uuari->lock); |
| 539 | if (uuarn == 0) { |
| 540 | --uuari->count[uuarn]; |
| 541 | goto out; |
| 542 | } |
| 543 | |
| 544 | if (uuarn < high_uuar) { |
| 545 | free_med_class_uuar(uuari, uuarn); |
| 546 | goto out; |
| 547 | } |
| 548 | |
| 549 | free_high_class_uuar(uuari, uuarn); |
| 550 | |
| 551 | out: |
| 552 | mutex_unlock(&uuari->lock); |
| 553 | } |
| 554 | |
| 555 | static enum mlx5_qp_state to_mlx5_state(enum ib_qp_state state) |
| 556 | { |
| 557 | switch (state) { |
| 558 | case IB_QPS_RESET: return MLX5_QP_STATE_RST; |
| 559 | case IB_QPS_INIT: return MLX5_QP_STATE_INIT; |
| 560 | case IB_QPS_RTR: return MLX5_QP_STATE_RTR; |
| 561 | case IB_QPS_RTS: return MLX5_QP_STATE_RTS; |
| 562 | case IB_QPS_SQD: return MLX5_QP_STATE_SQD; |
| 563 | case IB_QPS_SQE: return MLX5_QP_STATE_SQER; |
| 564 | case IB_QPS_ERR: return MLX5_QP_STATE_ERR; |
| 565 | default: return -1; |
| 566 | } |
| 567 | } |
| 568 | |
| 569 | static int to_mlx5_st(enum ib_qp_type type) |
| 570 | { |
| 571 | switch (type) { |
| 572 | case IB_QPT_RC: return MLX5_QP_ST_RC; |
| 573 | case IB_QPT_UC: return MLX5_QP_ST_UC; |
| 574 | case IB_QPT_UD: return MLX5_QP_ST_UD; |
| 575 | case MLX5_IB_QPT_REG_UMR: return MLX5_QP_ST_REG_UMR; |
| 576 | case IB_QPT_XRC_INI: |
| 577 | case IB_QPT_XRC_TGT: return MLX5_QP_ST_XRC; |
| 578 | case IB_QPT_SMI: return MLX5_QP_ST_QP0; |
| 579 | case IB_QPT_GSI: return MLX5_QP_ST_QP1; |
| 580 | case IB_QPT_RAW_IPV6: return MLX5_QP_ST_RAW_IPV6; |
| 581 | case IB_QPT_RAW_ETHERTYPE: return MLX5_QP_ST_RAW_ETHERTYPE; |
| 582 | case IB_QPT_RAW_PACKET: |
| 583 | case IB_QPT_MAX: |
| 584 | default: return -EINVAL; |
| 585 | } |
| 586 | } |
| 587 | |
| 588 | static int uuarn_to_uar_index(struct mlx5_uuar_info *uuari, int uuarn) |
| 589 | { |
| 590 | return uuari->uars[uuarn / MLX5_BF_REGS_PER_PAGE].index; |
| 591 | } |
| 592 | |
| 593 | static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd, |
| 594 | struct mlx5_ib_qp *qp, struct ib_udata *udata, |
| 595 | struct mlx5_create_qp_mbox_in **in, |
| 596 | struct mlx5_ib_create_qp_resp *resp, int *inlen) |
| 597 | { |
| 598 | struct mlx5_ib_ucontext *context; |
| 599 | struct mlx5_ib_create_qp ucmd; |
Eli Cohen | 9e9c47d | 2014-01-14 17:45:21 +0200 | [diff] [blame] | 600 | int page_shift = 0; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 601 | int uar_index; |
| 602 | int npages; |
Eli Cohen | 9e9c47d | 2014-01-14 17:45:21 +0200 | [diff] [blame] | 603 | u32 offset = 0; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 604 | int uuarn; |
Eli Cohen | 9e9c47d | 2014-01-14 17:45:21 +0200 | [diff] [blame] | 605 | int ncont = 0; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 606 | int err; |
| 607 | |
| 608 | err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)); |
| 609 | if (err) { |
| 610 | mlx5_ib_dbg(dev, "copy failed\n"); |
| 611 | return err; |
| 612 | } |
| 613 | |
| 614 | context = to_mucontext(pd->uobject->context); |
| 615 | /* |
| 616 | * TBD: should come from the verbs when we have the API |
| 617 | */ |
| 618 | uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_HIGH); |
| 619 | if (uuarn < 0) { |
| 620 | mlx5_ib_dbg(dev, "failed to allocate low latency UUAR\n"); |
Eli Cohen | c1be523 | 2014-01-14 17:45:12 +0200 | [diff] [blame] | 621 | mlx5_ib_dbg(dev, "reverting to medium latency\n"); |
| 622 | uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_MEDIUM); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 623 | if (uuarn < 0) { |
Eli Cohen | c1be523 | 2014-01-14 17:45:12 +0200 | [diff] [blame] | 624 | mlx5_ib_dbg(dev, "failed to allocate medium latency UUAR\n"); |
| 625 | mlx5_ib_dbg(dev, "reverting to high latency\n"); |
| 626 | uuarn = alloc_uuar(&context->uuari, MLX5_IB_LATENCY_CLASS_LOW); |
| 627 | if (uuarn < 0) { |
| 628 | mlx5_ib_warn(dev, "uuar allocation failed\n"); |
| 629 | return uuarn; |
| 630 | } |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 631 | } |
| 632 | } |
| 633 | |
| 634 | uar_index = uuarn_to_uar_index(&context->uuari, uuarn); |
| 635 | mlx5_ib_dbg(dev, "uuarn 0x%x, uar_index 0x%x\n", uuarn, uar_index); |
| 636 | |
Haggai Eran | 48fea83 | 2014-05-22 14:50:11 +0300 | [diff] [blame] | 637 | qp->rq.offset = 0; |
| 638 | qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB); |
| 639 | qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; |
| 640 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 641 | err = set_user_buf_size(dev, qp, &ucmd); |
| 642 | if (err) |
| 643 | goto err_uuar; |
| 644 | |
Eli Cohen | 9e9c47d | 2014-01-14 17:45:21 +0200 | [diff] [blame] | 645 | if (ucmd.buf_addr && qp->buf_size) { |
| 646 | qp->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, |
| 647 | qp->buf_size, 0, 0); |
| 648 | if (IS_ERR(qp->umem)) { |
| 649 | mlx5_ib_dbg(dev, "umem_get failed\n"); |
| 650 | err = PTR_ERR(qp->umem); |
| 651 | goto err_uuar; |
| 652 | } |
| 653 | } else { |
| 654 | qp->umem = NULL; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 655 | } |
| 656 | |
Eli Cohen | 9e9c47d | 2014-01-14 17:45:21 +0200 | [diff] [blame] | 657 | if (qp->umem) { |
| 658 | mlx5_ib_cont_pages(qp->umem, ucmd.buf_addr, &npages, &page_shift, |
| 659 | &ncont, NULL); |
| 660 | err = mlx5_ib_get_buf_offset(ucmd.buf_addr, page_shift, &offset); |
| 661 | if (err) { |
| 662 | mlx5_ib_warn(dev, "bad offset\n"); |
| 663 | goto err_umem; |
| 664 | } |
| 665 | mlx5_ib_dbg(dev, "addr 0x%llx, size %d, npages %d, page_shift %d, ncont %d, offset %d\n", |
| 666 | ucmd.buf_addr, qp->buf_size, npages, page_shift, ncont, offset); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 667 | } |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 668 | |
| 669 | *inlen = sizeof(**in) + sizeof(*(*in)->pas) * ncont; |
| 670 | *in = mlx5_vzalloc(*inlen); |
| 671 | if (!*in) { |
| 672 | err = -ENOMEM; |
| 673 | goto err_umem; |
| 674 | } |
Eli Cohen | 9e9c47d | 2014-01-14 17:45:21 +0200 | [diff] [blame] | 675 | if (qp->umem) |
| 676 | mlx5_ib_populate_pas(dev, qp->umem, page_shift, (*in)->pas, 0); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 677 | (*in)->ctx.log_pg_sz_remote_qpn = |
Eli Cohen | 1b77d2b | 2013-10-24 12:01:03 +0300 | [diff] [blame] | 678 | cpu_to_be32((page_shift - MLX5_ADAPTER_PAGE_SHIFT) << 24); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 679 | (*in)->ctx.params2 = cpu_to_be32(offset << 6); |
| 680 | |
| 681 | (*in)->ctx.qp_counter_set_usr_page = cpu_to_be32(uar_index); |
| 682 | resp->uuar_index = uuarn; |
| 683 | qp->uuarn = uuarn; |
| 684 | |
| 685 | err = mlx5_ib_db_map_user(context, ucmd.db_addr, &qp->db); |
| 686 | if (err) { |
| 687 | mlx5_ib_dbg(dev, "map failed\n"); |
| 688 | goto err_free; |
| 689 | } |
| 690 | |
| 691 | err = ib_copy_to_udata(udata, resp, sizeof(*resp)); |
| 692 | if (err) { |
| 693 | mlx5_ib_dbg(dev, "copy failed\n"); |
| 694 | goto err_unmap; |
| 695 | } |
| 696 | qp->create_type = MLX5_QP_USER; |
| 697 | |
| 698 | return 0; |
| 699 | |
| 700 | err_unmap: |
| 701 | mlx5_ib_db_unmap_user(context, &qp->db); |
| 702 | |
| 703 | err_free: |
Al Viro | 479163f | 2014-11-20 08:13:57 +0000 | [diff] [blame] | 704 | kvfree(*in); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 705 | |
| 706 | err_umem: |
Eli Cohen | 9e9c47d | 2014-01-14 17:45:21 +0200 | [diff] [blame] | 707 | if (qp->umem) |
| 708 | ib_umem_release(qp->umem); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 709 | |
| 710 | err_uuar: |
| 711 | free_uuar(&context->uuari, uuarn); |
| 712 | return err; |
| 713 | } |
| 714 | |
| 715 | static void destroy_qp_user(struct ib_pd *pd, struct mlx5_ib_qp *qp) |
| 716 | { |
| 717 | struct mlx5_ib_ucontext *context; |
| 718 | |
| 719 | context = to_mucontext(pd->uobject->context); |
| 720 | mlx5_ib_db_unmap_user(context, &qp->db); |
Eli Cohen | 9e9c47d | 2014-01-14 17:45:21 +0200 | [diff] [blame] | 721 | if (qp->umem) |
| 722 | ib_umem_release(qp->umem); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 723 | free_uuar(&context->uuari, qp->uuarn); |
| 724 | } |
| 725 | |
| 726 | static int create_kernel_qp(struct mlx5_ib_dev *dev, |
| 727 | struct ib_qp_init_attr *init_attr, |
| 728 | struct mlx5_ib_qp *qp, |
| 729 | struct mlx5_create_qp_mbox_in **in, int *inlen) |
| 730 | { |
| 731 | enum mlx5_ib_latency_class lc = MLX5_IB_LATENCY_CLASS_LOW; |
| 732 | struct mlx5_uuar_info *uuari; |
| 733 | int uar_index; |
| 734 | int uuarn; |
| 735 | int err; |
| 736 | |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 737 | uuari = &dev->mdev->priv.uuari; |
Or Gerlitz | 652c1a0 | 2014-06-25 16:44:14 +0300 | [diff] [blame] | 738 | if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN | IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)) |
Eli Cohen | 1a4c3a3 | 2014-02-06 17:41:25 +0200 | [diff] [blame] | 739 | return -EINVAL; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 740 | |
| 741 | if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR) |
| 742 | lc = MLX5_IB_LATENCY_CLASS_FAST_PATH; |
| 743 | |
| 744 | uuarn = alloc_uuar(uuari, lc); |
| 745 | if (uuarn < 0) { |
| 746 | mlx5_ib_dbg(dev, "\n"); |
| 747 | return -ENOMEM; |
| 748 | } |
| 749 | |
| 750 | qp->bf = &uuari->bfs[uuarn]; |
| 751 | uar_index = qp->bf->uar->index; |
| 752 | |
| 753 | err = calc_sq_size(dev, init_attr, qp); |
| 754 | if (err < 0) { |
| 755 | mlx5_ib_dbg(dev, "err %d\n", err); |
| 756 | goto err_uuar; |
| 757 | } |
| 758 | |
| 759 | qp->rq.offset = 0; |
| 760 | qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift; |
| 761 | qp->buf_size = err + (qp->rq.wqe_cnt << qp->rq.wqe_shift); |
| 762 | |
Amir Vadai | 64ffaa2 | 2015-05-28 22:28:38 +0300 | [diff] [blame] | 763 | err = mlx5_buf_alloc(dev->mdev, qp->buf_size, &qp->buf); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 764 | if (err) { |
| 765 | mlx5_ib_dbg(dev, "err %d\n", err); |
| 766 | goto err_uuar; |
| 767 | } |
| 768 | |
| 769 | qp->sq.qend = mlx5_get_send_wqe(qp, qp->sq.wqe_cnt); |
| 770 | *inlen = sizeof(**in) + sizeof(*(*in)->pas) * qp->buf.npages; |
| 771 | *in = mlx5_vzalloc(*inlen); |
| 772 | if (!*in) { |
| 773 | err = -ENOMEM; |
| 774 | goto err_buf; |
| 775 | } |
| 776 | (*in)->ctx.qp_counter_set_usr_page = cpu_to_be32(uar_index); |
Eli Cohen | 1b77d2b | 2013-10-24 12:01:03 +0300 | [diff] [blame] | 777 | (*in)->ctx.log_pg_sz_remote_qpn = |
| 778 | cpu_to_be32((qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT) << 24); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 779 | /* Set "fast registration enabled" for all kernel QPs */ |
| 780 | (*in)->ctx.params1 |= cpu_to_be32(1 << 11); |
| 781 | (*in)->ctx.sq_crq_size |= cpu_to_be16(1 << 4); |
| 782 | |
| 783 | mlx5_fill_page_array(&qp->buf, (*in)->pas); |
| 784 | |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 785 | err = mlx5_db_alloc(dev->mdev, &qp->db); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 786 | if (err) { |
| 787 | mlx5_ib_dbg(dev, "err %d\n", err); |
| 788 | goto err_free; |
| 789 | } |
| 790 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 791 | qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wrid), GFP_KERNEL); |
| 792 | qp->sq.wr_data = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wr_data), GFP_KERNEL); |
| 793 | qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof(*qp->rq.wrid), GFP_KERNEL); |
| 794 | qp->sq.w_list = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.w_list), GFP_KERNEL); |
| 795 | qp->sq.wqe_head = kmalloc(qp->sq.wqe_cnt * sizeof(*qp->sq.wqe_head), GFP_KERNEL); |
| 796 | |
| 797 | if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid || |
| 798 | !qp->sq.w_list || !qp->sq.wqe_head) { |
| 799 | err = -ENOMEM; |
| 800 | goto err_wrid; |
| 801 | } |
| 802 | qp->create_type = MLX5_QP_KERNEL; |
| 803 | |
| 804 | return 0; |
| 805 | |
| 806 | err_wrid: |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 807 | mlx5_db_free(dev->mdev, &qp->db); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 808 | kfree(qp->sq.wqe_head); |
| 809 | kfree(qp->sq.w_list); |
| 810 | kfree(qp->sq.wrid); |
| 811 | kfree(qp->sq.wr_data); |
| 812 | kfree(qp->rq.wrid); |
| 813 | |
| 814 | err_free: |
Al Viro | 479163f | 2014-11-20 08:13:57 +0000 | [diff] [blame] | 815 | kvfree(*in); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 816 | |
| 817 | err_buf: |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 818 | mlx5_buf_free(dev->mdev, &qp->buf); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 819 | |
| 820 | err_uuar: |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 821 | free_uuar(&dev->mdev->priv.uuari, uuarn); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 822 | return err; |
| 823 | } |
| 824 | |
| 825 | static void destroy_qp_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp) |
| 826 | { |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 827 | mlx5_db_free(dev->mdev, &qp->db); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 828 | kfree(qp->sq.wqe_head); |
| 829 | kfree(qp->sq.w_list); |
| 830 | kfree(qp->sq.wrid); |
| 831 | kfree(qp->sq.wr_data); |
| 832 | kfree(qp->rq.wrid); |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 833 | mlx5_buf_free(dev->mdev, &qp->buf); |
| 834 | free_uuar(&dev->mdev->priv.uuari, qp->bf->uuarn); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 835 | } |
| 836 | |
| 837 | static __be32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr) |
| 838 | { |
| 839 | if (attr->srq || (attr->qp_type == IB_QPT_XRC_TGT) || |
| 840 | (attr->qp_type == IB_QPT_XRC_INI)) |
| 841 | return cpu_to_be32(MLX5_SRQ_RQ); |
| 842 | else if (!qp->has_rq) |
| 843 | return cpu_to_be32(MLX5_ZERO_LEN_RQ); |
| 844 | else |
| 845 | return cpu_to_be32(MLX5_NON_ZERO_RQ); |
| 846 | } |
| 847 | |
| 848 | static int is_connected(enum ib_qp_type qp_type) |
| 849 | { |
| 850 | if (qp_type == IB_QPT_RC || qp_type == IB_QPT_UC) |
| 851 | return 1; |
| 852 | |
| 853 | return 0; |
| 854 | } |
| 855 | |
| 856 | static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd, |
| 857 | struct ib_qp_init_attr *init_attr, |
| 858 | struct ib_udata *udata, struct mlx5_ib_qp *qp) |
| 859 | { |
| 860 | struct mlx5_ib_resources *devr = &dev->devr; |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 861 | struct mlx5_core_dev *mdev = dev->mdev; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 862 | struct mlx5_ib_create_qp_resp resp; |
| 863 | struct mlx5_create_qp_mbox_in *in; |
| 864 | struct mlx5_ib_create_qp ucmd; |
| 865 | int inlen = sizeof(*in); |
| 866 | int err; |
| 867 | |
Haggai Eran | 6aec21f | 2014-12-11 17:04:23 +0200 | [diff] [blame] | 868 | mlx5_ib_odp_create_qp(qp); |
| 869 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 870 | mutex_init(&qp->mutex); |
| 871 | spin_lock_init(&qp->sq.lock); |
| 872 | spin_lock_init(&qp->rq.lock); |
| 873 | |
Eli Cohen | f360d88 | 2014-04-02 00:10:16 +0300 | [diff] [blame] | 874 | if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK) { |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 875 | if (!MLX5_CAP_GEN(mdev, block_lb_mc)) { |
Eli Cohen | f360d88 | 2014-04-02 00:10:16 +0300 | [diff] [blame] | 876 | mlx5_ib_dbg(dev, "block multicast loopback isn't supported\n"); |
| 877 | return -EINVAL; |
| 878 | } else { |
| 879 | qp->flags |= MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK; |
| 880 | } |
| 881 | } |
| 882 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 883 | if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) |
| 884 | qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE; |
| 885 | |
| 886 | if (pd && pd->uobject) { |
| 887 | if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) { |
| 888 | mlx5_ib_dbg(dev, "copy failed\n"); |
| 889 | return -EFAULT; |
| 890 | } |
| 891 | |
| 892 | qp->wq_sig = !!(ucmd.flags & MLX5_QP_FLAG_SIGNATURE); |
| 893 | qp->scat_cqe = !!(ucmd.flags & MLX5_QP_FLAG_SCATTER_CQE); |
| 894 | } else { |
| 895 | qp->wq_sig = !!wq_signature; |
| 896 | } |
| 897 | |
| 898 | qp->has_rq = qp_has_rq(init_attr); |
| 899 | err = set_rq_size(dev, &init_attr->cap, qp->has_rq, |
| 900 | qp, (pd && pd->uobject) ? &ucmd : NULL); |
| 901 | if (err) { |
| 902 | mlx5_ib_dbg(dev, "err %d\n", err); |
| 903 | return err; |
| 904 | } |
| 905 | |
| 906 | if (pd) { |
| 907 | if (pd->uobject) { |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 908 | __u32 max_wqes = |
| 909 | 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 910 | mlx5_ib_dbg(dev, "requested sq_wqe_count (%d)\n", ucmd.sq_wqe_count); |
| 911 | if (ucmd.rq_wqe_shift != qp->rq.wqe_shift || |
| 912 | ucmd.rq_wqe_count != qp->rq.wqe_cnt) { |
| 913 | mlx5_ib_dbg(dev, "invalid rq params\n"); |
| 914 | return -EINVAL; |
| 915 | } |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 916 | if (ucmd.sq_wqe_count > max_wqes) { |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 917 | mlx5_ib_dbg(dev, "requested sq_wqe_count (%d) > max allowed (%d)\n", |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 918 | ucmd.sq_wqe_count, max_wqes); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 919 | return -EINVAL; |
| 920 | } |
| 921 | err = create_user_qp(dev, pd, qp, udata, &in, &resp, &inlen); |
| 922 | if (err) |
| 923 | mlx5_ib_dbg(dev, "err %d\n", err); |
| 924 | } else { |
| 925 | err = create_kernel_qp(dev, init_attr, qp, &in, &inlen); |
| 926 | if (err) |
| 927 | mlx5_ib_dbg(dev, "err %d\n", err); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 928 | } |
| 929 | |
| 930 | if (err) |
| 931 | return err; |
| 932 | } else { |
| 933 | in = mlx5_vzalloc(sizeof(*in)); |
| 934 | if (!in) |
| 935 | return -ENOMEM; |
| 936 | |
| 937 | qp->create_type = MLX5_QP_EMPTY; |
| 938 | } |
| 939 | |
| 940 | if (is_sqp(init_attr->qp_type)) |
| 941 | qp->port = init_attr->port_num; |
| 942 | |
| 943 | in->ctx.flags = cpu_to_be32(to_mlx5_st(init_attr->qp_type) << 16 | |
| 944 | MLX5_QP_PM_MIGRATED << 11); |
| 945 | |
| 946 | if (init_attr->qp_type != MLX5_IB_QPT_REG_UMR) |
| 947 | in->ctx.flags_pd = cpu_to_be32(to_mpd(pd ? pd : devr->p0)->pdn); |
| 948 | else |
| 949 | in->ctx.flags_pd = cpu_to_be32(MLX5_QP_LAT_SENSITIVE); |
| 950 | |
| 951 | if (qp->wq_sig) |
| 952 | in->ctx.flags_pd |= cpu_to_be32(MLX5_QP_ENABLE_SIG); |
| 953 | |
Eli Cohen | f360d88 | 2014-04-02 00:10:16 +0300 | [diff] [blame] | 954 | if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK) |
| 955 | in->ctx.flags_pd |= cpu_to_be32(MLX5_QP_BLOCK_MCAST); |
| 956 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 957 | if (qp->scat_cqe && is_connected(init_attr->qp_type)) { |
| 958 | int rcqe_sz; |
| 959 | int scqe_sz; |
| 960 | |
| 961 | rcqe_sz = mlx5_ib_get_cqe_size(dev, init_attr->recv_cq); |
| 962 | scqe_sz = mlx5_ib_get_cqe_size(dev, init_attr->send_cq); |
| 963 | |
| 964 | if (rcqe_sz == 128) |
| 965 | in->ctx.cs_res = MLX5_RES_SCAT_DATA64_CQE; |
| 966 | else |
| 967 | in->ctx.cs_res = MLX5_RES_SCAT_DATA32_CQE; |
| 968 | |
| 969 | if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) { |
| 970 | if (scqe_sz == 128) |
| 971 | in->ctx.cs_req = MLX5_REQ_SCAT_DATA64_CQE; |
| 972 | else |
| 973 | in->ctx.cs_req = MLX5_REQ_SCAT_DATA32_CQE; |
| 974 | } |
| 975 | } |
| 976 | |
| 977 | if (qp->rq.wqe_cnt) { |
| 978 | in->ctx.rq_size_stride = (qp->rq.wqe_shift - 4); |
| 979 | in->ctx.rq_size_stride |= ilog2(qp->rq.wqe_cnt) << 3; |
| 980 | } |
| 981 | |
| 982 | in->ctx.rq_type_srqn = get_rx_type(qp, init_attr); |
| 983 | |
| 984 | if (qp->sq.wqe_cnt) |
| 985 | in->ctx.sq_crq_size |= cpu_to_be16(ilog2(qp->sq.wqe_cnt) << 11); |
| 986 | else |
| 987 | in->ctx.sq_crq_size |= cpu_to_be16(0x8000); |
| 988 | |
| 989 | /* Set default resources */ |
| 990 | switch (init_attr->qp_type) { |
| 991 | case IB_QPT_XRC_TGT: |
| 992 | in->ctx.cqn_recv = cpu_to_be32(to_mcq(devr->c0)->mcq.cqn); |
| 993 | in->ctx.cqn_send = cpu_to_be32(to_mcq(devr->c0)->mcq.cqn); |
| 994 | in->ctx.rq_type_srqn |= cpu_to_be32(to_msrq(devr->s0)->msrq.srqn); |
| 995 | in->ctx.xrcd = cpu_to_be32(to_mxrcd(init_attr->xrcd)->xrcdn); |
| 996 | break; |
| 997 | case IB_QPT_XRC_INI: |
| 998 | in->ctx.cqn_recv = cpu_to_be32(to_mcq(devr->c0)->mcq.cqn); |
| 999 | in->ctx.xrcd = cpu_to_be32(to_mxrcd(devr->x1)->xrcdn); |
| 1000 | in->ctx.rq_type_srqn |= cpu_to_be32(to_msrq(devr->s0)->msrq.srqn); |
| 1001 | break; |
| 1002 | default: |
| 1003 | if (init_attr->srq) { |
| 1004 | in->ctx.xrcd = cpu_to_be32(to_mxrcd(devr->x0)->xrcdn); |
| 1005 | in->ctx.rq_type_srqn |= cpu_to_be32(to_msrq(init_attr->srq)->msrq.srqn); |
| 1006 | } else { |
| 1007 | in->ctx.xrcd = cpu_to_be32(to_mxrcd(devr->x1)->xrcdn); |
Haggai Abramonvsky | 4aa17b2 | 2015-06-04 19:30:48 +0300 | [diff] [blame] | 1008 | in->ctx.rq_type_srqn |= |
| 1009 | cpu_to_be32(to_msrq(devr->s1)->msrq.srqn); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1010 | } |
| 1011 | } |
| 1012 | |
| 1013 | if (init_attr->send_cq) |
| 1014 | in->ctx.cqn_send = cpu_to_be32(to_mcq(init_attr->send_cq)->mcq.cqn); |
| 1015 | |
| 1016 | if (init_attr->recv_cq) |
| 1017 | in->ctx.cqn_recv = cpu_to_be32(to_mcq(init_attr->recv_cq)->mcq.cqn); |
| 1018 | |
| 1019 | in->ctx.db_rec_addr = cpu_to_be64(qp->db.dma); |
| 1020 | |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 1021 | err = mlx5_core_create_qp(dev->mdev, &qp->mqp, in, inlen); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1022 | if (err) { |
| 1023 | mlx5_ib_dbg(dev, "create qp failed\n"); |
| 1024 | goto err_create; |
| 1025 | } |
| 1026 | |
Al Viro | 479163f | 2014-11-20 08:13:57 +0000 | [diff] [blame] | 1027 | kvfree(in); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1028 | /* Hardware wants QPN written in big-endian order (after |
| 1029 | * shifting) for send doorbell. Precompute this value to save |
| 1030 | * a little bit when posting sends. |
| 1031 | */ |
| 1032 | qp->doorbell_qpn = swab32(qp->mqp.qpn << 8); |
| 1033 | |
| 1034 | qp->mqp.event = mlx5_ib_qp_event; |
| 1035 | |
| 1036 | return 0; |
| 1037 | |
| 1038 | err_create: |
| 1039 | if (qp->create_type == MLX5_QP_USER) |
| 1040 | destroy_qp_user(pd, qp); |
| 1041 | else if (qp->create_type == MLX5_QP_KERNEL) |
| 1042 | destroy_qp_kernel(dev, qp); |
| 1043 | |
Al Viro | 479163f | 2014-11-20 08:13:57 +0000 | [diff] [blame] | 1044 | kvfree(in); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1045 | return err; |
| 1046 | } |
| 1047 | |
| 1048 | static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq) |
| 1049 | __acquires(&send_cq->lock) __acquires(&recv_cq->lock) |
| 1050 | { |
| 1051 | if (send_cq) { |
| 1052 | if (recv_cq) { |
| 1053 | if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { |
| 1054 | spin_lock_irq(&send_cq->lock); |
| 1055 | spin_lock_nested(&recv_cq->lock, |
| 1056 | SINGLE_DEPTH_NESTING); |
| 1057 | } else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) { |
| 1058 | spin_lock_irq(&send_cq->lock); |
| 1059 | __acquire(&recv_cq->lock); |
| 1060 | } else { |
| 1061 | spin_lock_irq(&recv_cq->lock); |
| 1062 | spin_lock_nested(&send_cq->lock, |
| 1063 | SINGLE_DEPTH_NESTING); |
| 1064 | } |
| 1065 | } else { |
| 1066 | spin_lock_irq(&send_cq->lock); |
Eli Cohen | 6a4f139 | 2014-12-02 12:26:18 +0200 | [diff] [blame] | 1067 | __acquire(&recv_cq->lock); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1068 | } |
| 1069 | } else if (recv_cq) { |
| 1070 | spin_lock_irq(&recv_cq->lock); |
Eli Cohen | 6a4f139 | 2014-12-02 12:26:18 +0200 | [diff] [blame] | 1071 | __acquire(&send_cq->lock); |
| 1072 | } else { |
| 1073 | __acquire(&send_cq->lock); |
| 1074 | __acquire(&recv_cq->lock); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1075 | } |
| 1076 | } |
| 1077 | |
| 1078 | static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq) |
| 1079 | __releases(&send_cq->lock) __releases(&recv_cq->lock) |
| 1080 | { |
| 1081 | if (send_cq) { |
| 1082 | if (recv_cq) { |
| 1083 | if (send_cq->mcq.cqn < recv_cq->mcq.cqn) { |
| 1084 | spin_unlock(&recv_cq->lock); |
| 1085 | spin_unlock_irq(&send_cq->lock); |
| 1086 | } else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) { |
| 1087 | __release(&recv_cq->lock); |
| 1088 | spin_unlock_irq(&send_cq->lock); |
| 1089 | } else { |
| 1090 | spin_unlock(&send_cq->lock); |
| 1091 | spin_unlock_irq(&recv_cq->lock); |
| 1092 | } |
| 1093 | } else { |
Eli Cohen | 6a4f139 | 2014-12-02 12:26:18 +0200 | [diff] [blame] | 1094 | __release(&recv_cq->lock); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1095 | spin_unlock_irq(&send_cq->lock); |
| 1096 | } |
| 1097 | } else if (recv_cq) { |
Eli Cohen | 6a4f139 | 2014-12-02 12:26:18 +0200 | [diff] [blame] | 1098 | __release(&send_cq->lock); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1099 | spin_unlock_irq(&recv_cq->lock); |
Eli Cohen | 6a4f139 | 2014-12-02 12:26:18 +0200 | [diff] [blame] | 1100 | } else { |
| 1101 | __release(&recv_cq->lock); |
| 1102 | __release(&send_cq->lock); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1103 | } |
| 1104 | } |
| 1105 | |
| 1106 | static struct mlx5_ib_pd *get_pd(struct mlx5_ib_qp *qp) |
| 1107 | { |
| 1108 | return to_mpd(qp->ibqp.pd); |
| 1109 | } |
| 1110 | |
| 1111 | static void get_cqs(struct mlx5_ib_qp *qp, |
| 1112 | struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq) |
| 1113 | { |
| 1114 | switch (qp->ibqp.qp_type) { |
| 1115 | case IB_QPT_XRC_TGT: |
| 1116 | *send_cq = NULL; |
| 1117 | *recv_cq = NULL; |
| 1118 | break; |
| 1119 | case MLX5_IB_QPT_REG_UMR: |
| 1120 | case IB_QPT_XRC_INI: |
| 1121 | *send_cq = to_mcq(qp->ibqp.send_cq); |
| 1122 | *recv_cq = NULL; |
| 1123 | break; |
| 1124 | |
| 1125 | case IB_QPT_SMI: |
| 1126 | case IB_QPT_GSI: |
| 1127 | case IB_QPT_RC: |
| 1128 | case IB_QPT_UC: |
| 1129 | case IB_QPT_UD: |
| 1130 | case IB_QPT_RAW_IPV6: |
| 1131 | case IB_QPT_RAW_ETHERTYPE: |
| 1132 | *send_cq = to_mcq(qp->ibqp.send_cq); |
| 1133 | *recv_cq = to_mcq(qp->ibqp.recv_cq); |
| 1134 | break; |
| 1135 | |
| 1136 | case IB_QPT_RAW_PACKET: |
| 1137 | case IB_QPT_MAX: |
| 1138 | default: |
| 1139 | *send_cq = NULL; |
| 1140 | *recv_cq = NULL; |
| 1141 | break; |
| 1142 | } |
| 1143 | } |
| 1144 | |
| 1145 | static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp) |
| 1146 | { |
| 1147 | struct mlx5_ib_cq *send_cq, *recv_cq; |
| 1148 | struct mlx5_modify_qp_mbox_in *in; |
| 1149 | int err; |
| 1150 | |
| 1151 | in = kzalloc(sizeof(*in), GFP_KERNEL); |
| 1152 | if (!in) |
| 1153 | return; |
Eli Cohen | 7bef7ad | 2015-04-02 17:07:21 +0300 | [diff] [blame] | 1154 | |
Haggai Eran | 6aec21f | 2014-12-11 17:04:23 +0200 | [diff] [blame] | 1155 | if (qp->state != IB_QPS_RESET) { |
| 1156 | mlx5_ib_qp_disable_pagefaults(qp); |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 1157 | if (mlx5_core_qp_modify(dev->mdev, to_mlx5_state(qp->state), |
Haggai Abramonvsky | c3c6c9c | 2015-04-02 17:07:20 +0300 | [diff] [blame] | 1158 | MLX5_QP_STATE_RST, in, 0, &qp->mqp)) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1159 | mlx5_ib_warn(dev, "mlx5_ib: modify QP %06x to RESET failed\n", |
| 1160 | qp->mqp.qpn); |
Haggai Eran | 6aec21f | 2014-12-11 17:04:23 +0200 | [diff] [blame] | 1161 | } |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1162 | |
| 1163 | get_cqs(qp, &send_cq, &recv_cq); |
| 1164 | |
| 1165 | if (qp->create_type == MLX5_QP_KERNEL) { |
| 1166 | mlx5_ib_lock_cqs(send_cq, recv_cq); |
| 1167 | __mlx5_ib_cq_clean(recv_cq, qp->mqp.qpn, |
| 1168 | qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL); |
| 1169 | if (send_cq != recv_cq) |
| 1170 | __mlx5_ib_cq_clean(send_cq, qp->mqp.qpn, NULL); |
| 1171 | mlx5_ib_unlock_cqs(send_cq, recv_cq); |
| 1172 | } |
| 1173 | |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 1174 | err = mlx5_core_destroy_qp(dev->mdev, &qp->mqp); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1175 | if (err) |
| 1176 | mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n", qp->mqp.qpn); |
| 1177 | kfree(in); |
| 1178 | |
| 1179 | |
| 1180 | if (qp->create_type == MLX5_QP_KERNEL) |
| 1181 | destroy_qp_kernel(dev, qp); |
| 1182 | else if (qp->create_type == MLX5_QP_USER) |
| 1183 | destroy_qp_user(&get_pd(qp)->ibpd, qp); |
| 1184 | } |
| 1185 | |
| 1186 | static const char *ib_qp_type_str(enum ib_qp_type type) |
| 1187 | { |
| 1188 | switch (type) { |
| 1189 | case IB_QPT_SMI: |
| 1190 | return "IB_QPT_SMI"; |
| 1191 | case IB_QPT_GSI: |
| 1192 | return "IB_QPT_GSI"; |
| 1193 | case IB_QPT_RC: |
| 1194 | return "IB_QPT_RC"; |
| 1195 | case IB_QPT_UC: |
| 1196 | return "IB_QPT_UC"; |
| 1197 | case IB_QPT_UD: |
| 1198 | return "IB_QPT_UD"; |
| 1199 | case IB_QPT_RAW_IPV6: |
| 1200 | return "IB_QPT_RAW_IPV6"; |
| 1201 | case IB_QPT_RAW_ETHERTYPE: |
| 1202 | return "IB_QPT_RAW_ETHERTYPE"; |
| 1203 | case IB_QPT_XRC_INI: |
| 1204 | return "IB_QPT_XRC_INI"; |
| 1205 | case IB_QPT_XRC_TGT: |
| 1206 | return "IB_QPT_XRC_TGT"; |
| 1207 | case IB_QPT_RAW_PACKET: |
| 1208 | return "IB_QPT_RAW_PACKET"; |
| 1209 | case MLX5_IB_QPT_REG_UMR: |
| 1210 | return "MLX5_IB_QPT_REG_UMR"; |
| 1211 | case IB_QPT_MAX: |
| 1212 | default: |
| 1213 | return "Invalid QP type"; |
| 1214 | } |
| 1215 | } |
| 1216 | |
| 1217 | struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd, |
| 1218 | struct ib_qp_init_attr *init_attr, |
| 1219 | struct ib_udata *udata) |
| 1220 | { |
| 1221 | struct mlx5_ib_dev *dev; |
| 1222 | struct mlx5_ib_qp *qp; |
| 1223 | u16 xrcdn = 0; |
| 1224 | int err; |
| 1225 | |
| 1226 | if (pd) { |
| 1227 | dev = to_mdev(pd->device); |
| 1228 | } else { |
| 1229 | /* being cautious here */ |
| 1230 | if (init_attr->qp_type != IB_QPT_XRC_TGT && |
| 1231 | init_attr->qp_type != MLX5_IB_QPT_REG_UMR) { |
| 1232 | pr_warn("%s: no PD for transport %s\n", __func__, |
| 1233 | ib_qp_type_str(init_attr->qp_type)); |
| 1234 | return ERR_PTR(-EINVAL); |
| 1235 | } |
| 1236 | dev = to_mdev(to_mxrcd(init_attr->xrcd)->ibxrcd.device); |
| 1237 | } |
| 1238 | |
| 1239 | switch (init_attr->qp_type) { |
| 1240 | case IB_QPT_XRC_TGT: |
| 1241 | case IB_QPT_XRC_INI: |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 1242 | if (!MLX5_CAP_GEN(dev->mdev, xrc)) { |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1243 | mlx5_ib_dbg(dev, "XRC not supported\n"); |
| 1244 | return ERR_PTR(-ENOSYS); |
| 1245 | } |
| 1246 | init_attr->recv_cq = NULL; |
| 1247 | if (init_attr->qp_type == IB_QPT_XRC_TGT) { |
| 1248 | xrcdn = to_mxrcd(init_attr->xrcd)->xrcdn; |
| 1249 | init_attr->send_cq = NULL; |
| 1250 | } |
| 1251 | |
| 1252 | /* fall through */ |
| 1253 | case IB_QPT_RC: |
| 1254 | case IB_QPT_UC: |
| 1255 | case IB_QPT_UD: |
| 1256 | case IB_QPT_SMI: |
| 1257 | case IB_QPT_GSI: |
| 1258 | case MLX5_IB_QPT_REG_UMR: |
| 1259 | qp = kzalloc(sizeof(*qp), GFP_KERNEL); |
| 1260 | if (!qp) |
| 1261 | return ERR_PTR(-ENOMEM); |
| 1262 | |
| 1263 | err = create_qp_common(dev, pd, init_attr, udata, qp); |
| 1264 | if (err) { |
| 1265 | mlx5_ib_dbg(dev, "create_qp_common failed\n"); |
| 1266 | kfree(qp); |
| 1267 | return ERR_PTR(err); |
| 1268 | } |
| 1269 | |
| 1270 | if (is_qp0(init_attr->qp_type)) |
| 1271 | qp->ibqp.qp_num = 0; |
| 1272 | else if (is_qp1(init_attr->qp_type)) |
| 1273 | qp->ibqp.qp_num = 1; |
| 1274 | else |
| 1275 | qp->ibqp.qp_num = qp->mqp.qpn; |
| 1276 | |
| 1277 | mlx5_ib_dbg(dev, "ib qpnum 0x%x, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x\n", |
| 1278 | qp->ibqp.qp_num, qp->mqp.qpn, to_mcq(init_attr->recv_cq)->mcq.cqn, |
| 1279 | to_mcq(init_attr->send_cq)->mcq.cqn); |
| 1280 | |
| 1281 | qp->xrcdn = xrcdn; |
| 1282 | |
| 1283 | break; |
| 1284 | |
| 1285 | case IB_QPT_RAW_IPV6: |
| 1286 | case IB_QPT_RAW_ETHERTYPE: |
| 1287 | case IB_QPT_RAW_PACKET: |
| 1288 | case IB_QPT_MAX: |
| 1289 | default: |
| 1290 | mlx5_ib_dbg(dev, "unsupported qp type %d\n", |
| 1291 | init_attr->qp_type); |
| 1292 | /* Don't support raw QPs */ |
| 1293 | return ERR_PTR(-EINVAL); |
| 1294 | } |
| 1295 | |
| 1296 | return &qp->ibqp; |
| 1297 | } |
| 1298 | |
| 1299 | int mlx5_ib_destroy_qp(struct ib_qp *qp) |
| 1300 | { |
| 1301 | struct mlx5_ib_dev *dev = to_mdev(qp->device); |
| 1302 | struct mlx5_ib_qp *mqp = to_mqp(qp); |
| 1303 | |
| 1304 | destroy_qp_common(dev, mqp); |
| 1305 | |
| 1306 | kfree(mqp); |
| 1307 | |
| 1308 | return 0; |
| 1309 | } |
| 1310 | |
| 1311 | static __be32 to_mlx5_access_flags(struct mlx5_ib_qp *qp, const struct ib_qp_attr *attr, |
| 1312 | int attr_mask) |
| 1313 | { |
| 1314 | u32 hw_access_flags = 0; |
| 1315 | u8 dest_rd_atomic; |
| 1316 | u32 access_flags; |
| 1317 | |
| 1318 | if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) |
| 1319 | dest_rd_atomic = attr->max_dest_rd_atomic; |
| 1320 | else |
| 1321 | dest_rd_atomic = qp->resp_depth; |
| 1322 | |
| 1323 | if (attr_mask & IB_QP_ACCESS_FLAGS) |
| 1324 | access_flags = attr->qp_access_flags; |
| 1325 | else |
| 1326 | access_flags = qp->atomic_rd_en; |
| 1327 | |
| 1328 | if (!dest_rd_atomic) |
| 1329 | access_flags &= IB_ACCESS_REMOTE_WRITE; |
| 1330 | |
| 1331 | if (access_flags & IB_ACCESS_REMOTE_READ) |
| 1332 | hw_access_flags |= MLX5_QP_BIT_RRE; |
| 1333 | if (access_flags & IB_ACCESS_REMOTE_ATOMIC) |
| 1334 | hw_access_flags |= (MLX5_QP_BIT_RAE | MLX5_ATOMIC_MODE_CX); |
| 1335 | if (access_flags & IB_ACCESS_REMOTE_WRITE) |
| 1336 | hw_access_flags |= MLX5_QP_BIT_RWE; |
| 1337 | |
| 1338 | return cpu_to_be32(hw_access_flags); |
| 1339 | } |
| 1340 | |
| 1341 | enum { |
| 1342 | MLX5_PATH_FLAG_FL = 1 << 0, |
| 1343 | MLX5_PATH_FLAG_FREE_AR = 1 << 1, |
| 1344 | MLX5_PATH_FLAG_COUNTER = 1 << 2, |
| 1345 | }; |
| 1346 | |
| 1347 | static int ib_rate_to_mlx5(struct mlx5_ib_dev *dev, u8 rate) |
| 1348 | { |
| 1349 | if (rate == IB_RATE_PORT_CURRENT) { |
| 1350 | return 0; |
| 1351 | } else if (rate < IB_RATE_2_5_GBPS || rate > IB_RATE_300_GBPS) { |
| 1352 | return -EINVAL; |
| 1353 | } else { |
| 1354 | while (rate != IB_RATE_2_5_GBPS && |
| 1355 | !(1 << (rate + MLX5_STAT_RATE_OFFSET) & |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 1356 | MLX5_CAP_GEN(dev->mdev, stat_rate_support))) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1357 | --rate; |
| 1358 | } |
| 1359 | |
| 1360 | return rate + MLX5_STAT_RATE_OFFSET; |
| 1361 | } |
| 1362 | |
| 1363 | static int mlx5_set_path(struct mlx5_ib_dev *dev, const struct ib_ah_attr *ah, |
| 1364 | struct mlx5_qp_path *path, u8 port, int attr_mask, |
| 1365 | u32 path_flags, const struct ib_qp_attr *attr) |
| 1366 | { |
| 1367 | int err; |
| 1368 | |
| 1369 | path->fl = (path_flags & MLX5_PATH_FLAG_FL) ? 0x80 : 0; |
| 1370 | path->free_ar = (path_flags & MLX5_PATH_FLAG_FREE_AR) ? 0x80 : 0; |
| 1371 | |
| 1372 | if (attr_mask & IB_QP_PKEY_INDEX) |
| 1373 | path->pkey_index = attr->pkey_index; |
| 1374 | |
| 1375 | path->grh_mlid = ah->src_path_bits & 0x7f; |
| 1376 | path->rlid = cpu_to_be16(ah->dlid); |
| 1377 | |
| 1378 | if (ah->ah_flags & IB_AH_GRH) { |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 1379 | if (ah->grh.sgid_index >= |
| 1380 | dev->mdev->port_caps[port - 1].gid_table_len) { |
Joe Perches | f4f01b5 | 2015-05-08 15:58:07 -0700 | [diff] [blame] | 1381 | pr_err("sgid_index (%u) too large. max is %d\n", |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 1382 | ah->grh.sgid_index, |
| 1383 | dev->mdev->port_caps[port - 1].gid_table_len); |
Eli Cohen | f83b426 | 2014-09-14 16:47:54 +0300 | [diff] [blame] | 1384 | return -EINVAL; |
| 1385 | } |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1386 | path->grh_mlid |= 1 << 7; |
| 1387 | path->mgid_index = ah->grh.sgid_index; |
| 1388 | path->hop_limit = ah->grh.hop_limit; |
| 1389 | path->tclass_flowlabel = |
| 1390 | cpu_to_be32((ah->grh.traffic_class << 20) | |
| 1391 | (ah->grh.flow_label)); |
| 1392 | memcpy(path->rgid, ah->grh.dgid.raw, 16); |
| 1393 | } |
| 1394 | |
| 1395 | err = ib_rate_to_mlx5(dev, ah->static_rate); |
| 1396 | if (err < 0) |
| 1397 | return err; |
| 1398 | path->static_rate = err; |
| 1399 | path->port = port; |
| 1400 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1401 | if (attr_mask & IB_QP_TIMEOUT) |
| 1402 | path->ackto_lt = attr->timeout << 3; |
| 1403 | |
| 1404 | path->sl = ah->sl & 0xf; |
| 1405 | |
| 1406 | return 0; |
| 1407 | } |
| 1408 | |
| 1409 | static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_QP_ST_MAX] = { |
| 1410 | [MLX5_QP_STATE_INIT] = { |
| 1411 | [MLX5_QP_STATE_INIT] = { |
| 1412 | [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE | |
| 1413 | MLX5_QP_OPTPAR_RAE | |
| 1414 | MLX5_QP_OPTPAR_RWE | |
| 1415 | MLX5_QP_OPTPAR_PKEY_INDEX | |
| 1416 | MLX5_QP_OPTPAR_PRI_PORT, |
| 1417 | [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE | |
| 1418 | MLX5_QP_OPTPAR_PKEY_INDEX | |
| 1419 | MLX5_QP_OPTPAR_PRI_PORT, |
| 1420 | [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX | |
| 1421 | MLX5_QP_OPTPAR_Q_KEY | |
| 1422 | MLX5_QP_OPTPAR_PRI_PORT, |
| 1423 | }, |
| 1424 | [MLX5_QP_STATE_RTR] = { |
| 1425 | [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH | |
| 1426 | MLX5_QP_OPTPAR_RRE | |
| 1427 | MLX5_QP_OPTPAR_RAE | |
| 1428 | MLX5_QP_OPTPAR_RWE | |
| 1429 | MLX5_QP_OPTPAR_PKEY_INDEX, |
| 1430 | [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH | |
| 1431 | MLX5_QP_OPTPAR_RWE | |
| 1432 | MLX5_QP_OPTPAR_PKEY_INDEX, |
| 1433 | [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX | |
| 1434 | MLX5_QP_OPTPAR_Q_KEY, |
| 1435 | [MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_PKEY_INDEX | |
| 1436 | MLX5_QP_OPTPAR_Q_KEY, |
Eli Cohen | a4774e9 | 2013-09-11 16:35:32 +0300 | [diff] [blame] | 1437 | [MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH | |
| 1438 | MLX5_QP_OPTPAR_RRE | |
| 1439 | MLX5_QP_OPTPAR_RAE | |
| 1440 | MLX5_QP_OPTPAR_RWE | |
| 1441 | MLX5_QP_OPTPAR_PKEY_INDEX, |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1442 | }, |
| 1443 | }, |
| 1444 | [MLX5_QP_STATE_RTR] = { |
| 1445 | [MLX5_QP_STATE_RTS] = { |
| 1446 | [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH | |
| 1447 | MLX5_QP_OPTPAR_RRE | |
| 1448 | MLX5_QP_OPTPAR_RAE | |
| 1449 | MLX5_QP_OPTPAR_RWE | |
| 1450 | MLX5_QP_OPTPAR_PM_STATE | |
| 1451 | MLX5_QP_OPTPAR_RNR_TIMEOUT, |
| 1452 | [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH | |
| 1453 | MLX5_QP_OPTPAR_RWE | |
| 1454 | MLX5_QP_OPTPAR_PM_STATE, |
| 1455 | [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY, |
| 1456 | }, |
| 1457 | }, |
| 1458 | [MLX5_QP_STATE_RTS] = { |
| 1459 | [MLX5_QP_STATE_RTS] = { |
| 1460 | [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE | |
| 1461 | MLX5_QP_OPTPAR_RAE | |
| 1462 | MLX5_QP_OPTPAR_RWE | |
| 1463 | MLX5_QP_OPTPAR_RNR_TIMEOUT | |
Eli Cohen | c2a3431 | 2013-10-24 12:01:02 +0300 | [diff] [blame] | 1464 | MLX5_QP_OPTPAR_PM_STATE | |
| 1465 | MLX5_QP_OPTPAR_ALT_ADDR_PATH, |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1466 | [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE | |
Eli Cohen | c2a3431 | 2013-10-24 12:01:02 +0300 | [diff] [blame] | 1467 | MLX5_QP_OPTPAR_PM_STATE | |
| 1468 | MLX5_QP_OPTPAR_ALT_ADDR_PATH, |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1469 | [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY | |
| 1470 | MLX5_QP_OPTPAR_SRQN | |
| 1471 | MLX5_QP_OPTPAR_CQN_RCV, |
| 1472 | }, |
| 1473 | }, |
| 1474 | [MLX5_QP_STATE_SQER] = { |
| 1475 | [MLX5_QP_STATE_RTS] = { |
| 1476 | [MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY, |
| 1477 | [MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_Q_KEY, |
Eli Cohen | 75959f5 | 2013-09-11 16:35:31 +0300 | [diff] [blame] | 1478 | [MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE, |
Eli Cohen | a4774e9 | 2013-09-11 16:35:32 +0300 | [diff] [blame] | 1479 | [MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RNR_TIMEOUT | |
| 1480 | MLX5_QP_OPTPAR_RWE | |
| 1481 | MLX5_QP_OPTPAR_RAE | |
| 1482 | MLX5_QP_OPTPAR_RRE, |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1483 | }, |
| 1484 | }, |
| 1485 | }; |
| 1486 | |
| 1487 | static int ib_nr_to_mlx5_nr(int ib_mask) |
| 1488 | { |
| 1489 | switch (ib_mask) { |
| 1490 | case IB_QP_STATE: |
| 1491 | return 0; |
| 1492 | case IB_QP_CUR_STATE: |
| 1493 | return 0; |
| 1494 | case IB_QP_EN_SQD_ASYNC_NOTIFY: |
| 1495 | return 0; |
| 1496 | case IB_QP_ACCESS_FLAGS: |
| 1497 | return MLX5_QP_OPTPAR_RWE | MLX5_QP_OPTPAR_RRE | |
| 1498 | MLX5_QP_OPTPAR_RAE; |
| 1499 | case IB_QP_PKEY_INDEX: |
| 1500 | return MLX5_QP_OPTPAR_PKEY_INDEX; |
| 1501 | case IB_QP_PORT: |
| 1502 | return MLX5_QP_OPTPAR_PRI_PORT; |
| 1503 | case IB_QP_QKEY: |
| 1504 | return MLX5_QP_OPTPAR_Q_KEY; |
| 1505 | case IB_QP_AV: |
| 1506 | return MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH | |
| 1507 | MLX5_QP_OPTPAR_PRI_PORT; |
| 1508 | case IB_QP_PATH_MTU: |
| 1509 | return 0; |
| 1510 | case IB_QP_TIMEOUT: |
| 1511 | return MLX5_QP_OPTPAR_ACK_TIMEOUT; |
| 1512 | case IB_QP_RETRY_CNT: |
| 1513 | return MLX5_QP_OPTPAR_RETRY_COUNT; |
| 1514 | case IB_QP_RNR_RETRY: |
| 1515 | return MLX5_QP_OPTPAR_RNR_RETRY; |
| 1516 | case IB_QP_RQ_PSN: |
| 1517 | return 0; |
| 1518 | case IB_QP_MAX_QP_RD_ATOMIC: |
| 1519 | return MLX5_QP_OPTPAR_SRA_MAX; |
| 1520 | case IB_QP_ALT_PATH: |
| 1521 | return MLX5_QP_OPTPAR_ALT_ADDR_PATH; |
| 1522 | case IB_QP_MIN_RNR_TIMER: |
| 1523 | return MLX5_QP_OPTPAR_RNR_TIMEOUT; |
| 1524 | case IB_QP_SQ_PSN: |
| 1525 | return 0; |
| 1526 | case IB_QP_MAX_DEST_RD_ATOMIC: |
| 1527 | return MLX5_QP_OPTPAR_RRA_MAX | MLX5_QP_OPTPAR_RWE | |
| 1528 | MLX5_QP_OPTPAR_RRE | MLX5_QP_OPTPAR_RAE; |
| 1529 | case IB_QP_PATH_MIG_STATE: |
| 1530 | return MLX5_QP_OPTPAR_PM_STATE; |
| 1531 | case IB_QP_CAP: |
| 1532 | return 0; |
| 1533 | case IB_QP_DEST_QPN: |
| 1534 | return 0; |
| 1535 | } |
| 1536 | return 0; |
| 1537 | } |
| 1538 | |
| 1539 | static int ib_mask_to_mlx5_opt(int ib_mask) |
| 1540 | { |
| 1541 | int result = 0; |
| 1542 | int i; |
| 1543 | |
| 1544 | for (i = 0; i < 8 * sizeof(int); i++) { |
| 1545 | if ((1 << i) & ib_mask) |
| 1546 | result |= ib_nr_to_mlx5_nr(1 << i); |
| 1547 | } |
| 1548 | |
| 1549 | return result; |
| 1550 | } |
| 1551 | |
| 1552 | static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, |
| 1553 | const struct ib_qp_attr *attr, int attr_mask, |
| 1554 | enum ib_qp_state cur_state, enum ib_qp_state new_state) |
| 1555 | { |
| 1556 | struct mlx5_ib_dev *dev = to_mdev(ibqp->device); |
| 1557 | struct mlx5_ib_qp *qp = to_mqp(ibqp); |
| 1558 | struct mlx5_ib_cq *send_cq, *recv_cq; |
| 1559 | struct mlx5_qp_context *context; |
| 1560 | struct mlx5_modify_qp_mbox_in *in; |
| 1561 | struct mlx5_ib_pd *pd; |
| 1562 | enum mlx5_qp_state mlx5_cur, mlx5_new; |
| 1563 | enum mlx5_qp_optpar optpar; |
| 1564 | int sqd_event; |
| 1565 | int mlx5_st; |
| 1566 | int err; |
| 1567 | |
| 1568 | in = kzalloc(sizeof(*in), GFP_KERNEL); |
| 1569 | if (!in) |
| 1570 | return -ENOMEM; |
| 1571 | |
| 1572 | context = &in->ctx; |
| 1573 | err = to_mlx5_st(ibqp->qp_type); |
| 1574 | if (err < 0) |
| 1575 | goto out; |
| 1576 | |
| 1577 | context->flags = cpu_to_be32(err << 16); |
| 1578 | |
| 1579 | if (!(attr_mask & IB_QP_PATH_MIG_STATE)) { |
| 1580 | context->flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11); |
| 1581 | } else { |
| 1582 | switch (attr->path_mig_state) { |
| 1583 | case IB_MIG_MIGRATED: |
| 1584 | context->flags |= cpu_to_be32(MLX5_QP_PM_MIGRATED << 11); |
| 1585 | break; |
| 1586 | case IB_MIG_REARM: |
| 1587 | context->flags |= cpu_to_be32(MLX5_QP_PM_REARM << 11); |
| 1588 | break; |
| 1589 | case IB_MIG_ARMED: |
| 1590 | context->flags |= cpu_to_be32(MLX5_QP_PM_ARMED << 11); |
| 1591 | break; |
| 1592 | } |
| 1593 | } |
| 1594 | |
| 1595 | if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI) { |
| 1596 | context->mtu_msgmax = (IB_MTU_256 << 5) | 8; |
| 1597 | } else if (ibqp->qp_type == IB_QPT_UD || |
| 1598 | ibqp->qp_type == MLX5_IB_QPT_REG_UMR) { |
| 1599 | context->mtu_msgmax = (IB_MTU_4096 << 5) | 12; |
| 1600 | } else if (attr_mask & IB_QP_PATH_MTU) { |
| 1601 | if (attr->path_mtu < IB_MTU_256 || |
| 1602 | attr->path_mtu > IB_MTU_4096) { |
| 1603 | mlx5_ib_warn(dev, "invalid mtu %d\n", attr->path_mtu); |
| 1604 | err = -EINVAL; |
| 1605 | goto out; |
| 1606 | } |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 1607 | context->mtu_msgmax = (attr->path_mtu << 5) | |
| 1608 | (u8)MLX5_CAP_GEN(dev->mdev, log_max_msg); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1609 | } |
| 1610 | |
| 1611 | if (attr_mask & IB_QP_DEST_QPN) |
| 1612 | context->log_pg_sz_remote_qpn = cpu_to_be32(attr->dest_qp_num); |
| 1613 | |
| 1614 | if (attr_mask & IB_QP_PKEY_INDEX) |
| 1615 | context->pri_path.pkey_index = attr->pkey_index; |
| 1616 | |
| 1617 | /* todo implement counter_index functionality */ |
| 1618 | |
| 1619 | if (is_sqp(ibqp->qp_type)) |
| 1620 | context->pri_path.port = qp->port; |
| 1621 | |
| 1622 | if (attr_mask & IB_QP_PORT) |
| 1623 | context->pri_path.port = attr->port_num; |
| 1624 | |
| 1625 | if (attr_mask & IB_QP_AV) { |
| 1626 | err = mlx5_set_path(dev, &attr->ah_attr, &context->pri_path, |
| 1627 | attr_mask & IB_QP_PORT ? attr->port_num : qp->port, |
| 1628 | attr_mask, 0, attr); |
| 1629 | if (err) |
| 1630 | goto out; |
| 1631 | } |
| 1632 | |
| 1633 | if (attr_mask & IB_QP_TIMEOUT) |
| 1634 | context->pri_path.ackto_lt |= attr->timeout << 3; |
| 1635 | |
| 1636 | if (attr_mask & IB_QP_ALT_PATH) { |
| 1637 | err = mlx5_set_path(dev, &attr->alt_ah_attr, &context->alt_path, |
| 1638 | attr->alt_port_num, attr_mask, 0, attr); |
| 1639 | if (err) |
| 1640 | goto out; |
| 1641 | } |
| 1642 | |
| 1643 | pd = get_pd(qp); |
| 1644 | get_cqs(qp, &send_cq, &recv_cq); |
| 1645 | |
| 1646 | context->flags_pd = cpu_to_be32(pd ? pd->pdn : to_mpd(dev->devr.p0)->pdn); |
| 1647 | context->cqn_send = send_cq ? cpu_to_be32(send_cq->mcq.cqn) : 0; |
| 1648 | context->cqn_recv = recv_cq ? cpu_to_be32(recv_cq->mcq.cqn) : 0; |
| 1649 | context->params1 = cpu_to_be32(MLX5_IB_ACK_REQ_FREQ << 28); |
| 1650 | |
| 1651 | if (attr_mask & IB_QP_RNR_RETRY) |
| 1652 | context->params1 |= cpu_to_be32(attr->rnr_retry << 13); |
| 1653 | |
| 1654 | if (attr_mask & IB_QP_RETRY_CNT) |
| 1655 | context->params1 |= cpu_to_be32(attr->retry_cnt << 16); |
| 1656 | |
| 1657 | if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { |
| 1658 | if (attr->max_rd_atomic) |
| 1659 | context->params1 |= |
| 1660 | cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21); |
| 1661 | } |
| 1662 | |
| 1663 | if (attr_mask & IB_QP_SQ_PSN) |
| 1664 | context->next_send_psn = cpu_to_be32(attr->sq_psn); |
| 1665 | |
| 1666 | if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { |
| 1667 | if (attr->max_dest_rd_atomic) |
| 1668 | context->params2 |= |
| 1669 | cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21); |
| 1670 | } |
| 1671 | |
| 1672 | if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) |
| 1673 | context->params2 |= to_mlx5_access_flags(qp, attr, attr_mask); |
| 1674 | |
| 1675 | if (attr_mask & IB_QP_MIN_RNR_TIMER) |
| 1676 | context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24); |
| 1677 | |
| 1678 | if (attr_mask & IB_QP_RQ_PSN) |
| 1679 | context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn); |
| 1680 | |
| 1681 | if (attr_mask & IB_QP_QKEY) |
| 1682 | context->qkey = cpu_to_be32(attr->qkey); |
| 1683 | |
| 1684 | if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) |
| 1685 | context->db_rec_addr = cpu_to_be64(qp->db.dma); |
| 1686 | |
| 1687 | if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD && |
| 1688 | attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify) |
| 1689 | sqd_event = 1; |
| 1690 | else |
| 1691 | sqd_event = 0; |
| 1692 | |
| 1693 | if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) |
| 1694 | context->sq_crq_size |= cpu_to_be16(1 << 4); |
| 1695 | |
| 1696 | |
| 1697 | mlx5_cur = to_mlx5_state(cur_state); |
| 1698 | mlx5_new = to_mlx5_state(new_state); |
| 1699 | mlx5_st = to_mlx5_st(ibqp->qp_type); |
Eli Cohen | 07c9113 | 2013-10-24 12:01:01 +0300 | [diff] [blame] | 1700 | if (mlx5_st < 0) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1701 | goto out; |
| 1702 | |
Haggai Eran | 6aec21f | 2014-12-11 17:04:23 +0200 | [diff] [blame] | 1703 | /* If moving to a reset or error state, we must disable page faults on |
| 1704 | * this QP and flush all current page faults. Otherwise a stale page |
| 1705 | * fault may attempt to work on this QP after it is reset and moved |
| 1706 | * again to RTS, and may cause the driver and the device to get out of |
| 1707 | * sync. */ |
| 1708 | if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR && |
| 1709 | (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR)) |
| 1710 | mlx5_ib_qp_disable_pagefaults(qp); |
| 1711 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1712 | optpar = ib_mask_to_mlx5_opt(attr_mask); |
| 1713 | optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st]; |
| 1714 | in->optparam = cpu_to_be32(optpar); |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 1715 | err = mlx5_core_qp_modify(dev->mdev, to_mlx5_state(cur_state), |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1716 | to_mlx5_state(new_state), in, sqd_event, |
| 1717 | &qp->mqp); |
| 1718 | if (err) |
| 1719 | goto out; |
| 1720 | |
Haggai Eran | 6aec21f | 2014-12-11 17:04:23 +0200 | [diff] [blame] | 1721 | if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) |
| 1722 | mlx5_ib_qp_enable_pagefaults(qp); |
| 1723 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1724 | qp->state = new_state; |
| 1725 | |
| 1726 | if (attr_mask & IB_QP_ACCESS_FLAGS) |
| 1727 | qp->atomic_rd_en = attr->qp_access_flags; |
| 1728 | if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) |
| 1729 | qp->resp_depth = attr->max_dest_rd_atomic; |
| 1730 | if (attr_mask & IB_QP_PORT) |
| 1731 | qp->port = attr->port_num; |
| 1732 | if (attr_mask & IB_QP_ALT_PATH) |
| 1733 | qp->alt_port = attr->alt_port_num; |
| 1734 | |
| 1735 | /* |
| 1736 | * If we moved a kernel QP to RESET, clean up all old CQ |
| 1737 | * entries and reinitialize the QP. |
| 1738 | */ |
| 1739 | if (new_state == IB_QPS_RESET && !ibqp->uobject) { |
| 1740 | mlx5_ib_cq_clean(recv_cq, qp->mqp.qpn, |
| 1741 | ibqp->srq ? to_msrq(ibqp->srq) : NULL); |
| 1742 | if (send_cq != recv_cq) |
| 1743 | mlx5_ib_cq_clean(send_cq, qp->mqp.qpn, NULL); |
| 1744 | |
| 1745 | qp->rq.head = 0; |
| 1746 | qp->rq.tail = 0; |
| 1747 | qp->sq.head = 0; |
| 1748 | qp->sq.tail = 0; |
| 1749 | qp->sq.cur_post = 0; |
| 1750 | qp->sq.last_poll = 0; |
| 1751 | qp->db.db[MLX5_RCV_DBR] = 0; |
| 1752 | qp->db.db[MLX5_SND_DBR] = 0; |
| 1753 | } |
| 1754 | |
| 1755 | out: |
| 1756 | kfree(in); |
| 1757 | return err; |
| 1758 | } |
| 1759 | |
| 1760 | int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, |
| 1761 | int attr_mask, struct ib_udata *udata) |
| 1762 | { |
| 1763 | struct mlx5_ib_dev *dev = to_mdev(ibqp->device); |
| 1764 | struct mlx5_ib_qp *qp = to_mqp(ibqp); |
| 1765 | enum ib_qp_state cur_state, new_state; |
| 1766 | int err = -EINVAL; |
| 1767 | int port; |
| 1768 | |
| 1769 | mutex_lock(&qp->mutex); |
| 1770 | |
| 1771 | cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state; |
| 1772 | new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; |
| 1773 | |
| 1774 | if (ibqp->qp_type != MLX5_IB_QPT_REG_UMR && |
Matan Barak | dd5f03b | 2013-12-12 18:03:11 +0200 | [diff] [blame] | 1775 | !ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask, |
| 1776 | IB_LINK_LAYER_UNSPECIFIED)) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1777 | goto out; |
| 1778 | |
| 1779 | if ((attr_mask & IB_QP_PORT) && |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 1780 | (attr->port_num == 0 || |
| 1781 | attr->port_num > MLX5_CAP_GEN(dev->mdev, num_ports))) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1782 | goto out; |
| 1783 | |
| 1784 | if (attr_mask & IB_QP_PKEY_INDEX) { |
| 1785 | port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port; |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 1786 | if (attr->pkey_index >= |
| 1787 | dev->mdev->port_caps[port - 1].pkey_table_len) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1788 | goto out; |
| 1789 | } |
| 1790 | |
| 1791 | if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 1792 | attr->max_rd_atomic > |
| 1793 | (1 << MLX5_CAP_GEN(dev->mdev, log_max_ra_res_qp))) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1794 | goto out; |
| 1795 | |
| 1796 | if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 1797 | attr->max_dest_rd_atomic > |
| 1798 | (1 << MLX5_CAP_GEN(dev->mdev, log_max_ra_req_qp))) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1799 | goto out; |
| 1800 | |
| 1801 | if (cur_state == new_state && cur_state == IB_QPS_RESET) { |
| 1802 | err = 0; |
| 1803 | goto out; |
| 1804 | } |
| 1805 | |
| 1806 | err = __mlx5_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state); |
| 1807 | |
| 1808 | out: |
| 1809 | mutex_unlock(&qp->mutex); |
| 1810 | return err; |
| 1811 | } |
| 1812 | |
| 1813 | static int mlx5_wq_overflow(struct mlx5_ib_wq *wq, int nreq, struct ib_cq *ib_cq) |
| 1814 | { |
| 1815 | struct mlx5_ib_cq *cq; |
| 1816 | unsigned cur; |
| 1817 | |
| 1818 | cur = wq->head - wq->tail; |
| 1819 | if (likely(cur + nreq < wq->max_post)) |
| 1820 | return 0; |
| 1821 | |
| 1822 | cq = to_mcq(ib_cq); |
| 1823 | spin_lock(&cq->lock); |
| 1824 | cur = wq->head - wq->tail; |
| 1825 | spin_unlock(&cq->lock); |
| 1826 | |
| 1827 | return cur + nreq >= wq->max_post; |
| 1828 | } |
| 1829 | |
| 1830 | static __always_inline void set_raddr_seg(struct mlx5_wqe_raddr_seg *rseg, |
| 1831 | u64 remote_addr, u32 rkey) |
| 1832 | { |
| 1833 | rseg->raddr = cpu_to_be64(remote_addr); |
| 1834 | rseg->rkey = cpu_to_be32(rkey); |
| 1835 | rseg->reserved = 0; |
| 1836 | } |
| 1837 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1838 | static void set_datagram_seg(struct mlx5_wqe_datagram_seg *dseg, |
| 1839 | struct ib_send_wr *wr) |
| 1840 | { |
| 1841 | memcpy(&dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof(struct mlx5_av)); |
| 1842 | dseg->av.dqp_dct = cpu_to_be32(wr->wr.ud.remote_qpn | MLX5_EXTENDED_UD_AV); |
| 1843 | dseg->av.key.qkey.qkey = cpu_to_be32(wr->wr.ud.remote_qkey); |
| 1844 | } |
| 1845 | |
| 1846 | static void set_data_ptr_seg(struct mlx5_wqe_data_seg *dseg, struct ib_sge *sg) |
| 1847 | { |
| 1848 | dseg->byte_count = cpu_to_be32(sg->length); |
| 1849 | dseg->lkey = cpu_to_be32(sg->lkey); |
| 1850 | dseg->addr = cpu_to_be64(sg->addr); |
| 1851 | } |
| 1852 | |
| 1853 | static __be16 get_klm_octo(int npages) |
| 1854 | { |
| 1855 | return cpu_to_be16(ALIGN(npages, 8) / 2); |
| 1856 | } |
| 1857 | |
| 1858 | static __be64 frwr_mkey_mask(void) |
| 1859 | { |
| 1860 | u64 result; |
| 1861 | |
| 1862 | result = MLX5_MKEY_MASK_LEN | |
| 1863 | MLX5_MKEY_MASK_PAGE_SIZE | |
| 1864 | MLX5_MKEY_MASK_START_ADDR | |
| 1865 | MLX5_MKEY_MASK_EN_RINVAL | |
| 1866 | MLX5_MKEY_MASK_KEY | |
| 1867 | MLX5_MKEY_MASK_LR | |
| 1868 | MLX5_MKEY_MASK_LW | |
| 1869 | MLX5_MKEY_MASK_RR | |
| 1870 | MLX5_MKEY_MASK_RW | |
| 1871 | MLX5_MKEY_MASK_A | |
| 1872 | MLX5_MKEY_MASK_SMALL_FENCE | |
| 1873 | MLX5_MKEY_MASK_FREE; |
| 1874 | |
| 1875 | return cpu_to_be64(result); |
| 1876 | } |
| 1877 | |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 1878 | static __be64 sig_mkey_mask(void) |
| 1879 | { |
| 1880 | u64 result; |
| 1881 | |
| 1882 | result = MLX5_MKEY_MASK_LEN | |
| 1883 | MLX5_MKEY_MASK_PAGE_SIZE | |
| 1884 | MLX5_MKEY_MASK_START_ADDR | |
Sagi Grimberg | d5436ba | 2014-02-23 14:19:12 +0200 | [diff] [blame] | 1885 | MLX5_MKEY_MASK_EN_SIGERR | |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 1886 | MLX5_MKEY_MASK_EN_RINVAL | |
| 1887 | MLX5_MKEY_MASK_KEY | |
| 1888 | MLX5_MKEY_MASK_LR | |
| 1889 | MLX5_MKEY_MASK_LW | |
| 1890 | MLX5_MKEY_MASK_RR | |
| 1891 | MLX5_MKEY_MASK_RW | |
| 1892 | MLX5_MKEY_MASK_SMALL_FENCE | |
| 1893 | MLX5_MKEY_MASK_FREE | |
| 1894 | MLX5_MKEY_MASK_BSF_EN; |
| 1895 | |
| 1896 | return cpu_to_be64(result); |
| 1897 | } |
| 1898 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1899 | static void set_frwr_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr, |
| 1900 | struct ib_send_wr *wr, int li) |
| 1901 | { |
| 1902 | memset(umr, 0, sizeof(*umr)); |
| 1903 | |
| 1904 | if (li) { |
| 1905 | umr->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); |
| 1906 | umr->flags = 1 << 7; |
| 1907 | return; |
| 1908 | } |
| 1909 | |
| 1910 | umr->flags = (1 << 5); /* fail if not free */ |
| 1911 | umr->klm_octowords = get_klm_octo(wr->wr.fast_reg.page_list_len); |
| 1912 | umr->mkey_mask = frwr_mkey_mask(); |
| 1913 | } |
| 1914 | |
Haggai Eran | 968e78d | 2014-12-11 17:04:11 +0200 | [diff] [blame] | 1915 | static __be64 get_umr_reg_mr_mask(void) |
| 1916 | { |
| 1917 | u64 result; |
| 1918 | |
| 1919 | result = MLX5_MKEY_MASK_LEN | |
| 1920 | MLX5_MKEY_MASK_PAGE_SIZE | |
| 1921 | MLX5_MKEY_MASK_START_ADDR | |
| 1922 | MLX5_MKEY_MASK_PD | |
| 1923 | MLX5_MKEY_MASK_LR | |
| 1924 | MLX5_MKEY_MASK_LW | |
| 1925 | MLX5_MKEY_MASK_KEY | |
| 1926 | MLX5_MKEY_MASK_RR | |
| 1927 | MLX5_MKEY_MASK_RW | |
| 1928 | MLX5_MKEY_MASK_A | |
| 1929 | MLX5_MKEY_MASK_FREE; |
| 1930 | |
| 1931 | return cpu_to_be64(result); |
| 1932 | } |
| 1933 | |
| 1934 | static __be64 get_umr_unreg_mr_mask(void) |
| 1935 | { |
| 1936 | u64 result; |
| 1937 | |
| 1938 | result = MLX5_MKEY_MASK_FREE; |
| 1939 | |
| 1940 | return cpu_to_be64(result); |
| 1941 | } |
| 1942 | |
| 1943 | static __be64 get_umr_update_mtt_mask(void) |
| 1944 | { |
| 1945 | u64 result; |
| 1946 | |
| 1947 | result = MLX5_MKEY_MASK_FREE; |
| 1948 | |
| 1949 | return cpu_to_be64(result); |
| 1950 | } |
| 1951 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1952 | static void set_reg_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr, |
| 1953 | struct ib_send_wr *wr) |
| 1954 | { |
Haggai Eran | 968e78d | 2014-12-11 17:04:11 +0200 | [diff] [blame] | 1955 | struct mlx5_umr_wr *umrwr = (struct mlx5_umr_wr *)&wr->wr.fast_reg; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1956 | |
| 1957 | memset(umr, 0, sizeof(*umr)); |
| 1958 | |
Haggai Eran | 968e78d | 2014-12-11 17:04:11 +0200 | [diff] [blame] | 1959 | if (wr->send_flags & MLX5_IB_SEND_UMR_FAIL_IF_FREE) |
| 1960 | umr->flags = MLX5_UMR_CHECK_FREE; /* fail if free */ |
| 1961 | else |
| 1962 | umr->flags = MLX5_UMR_CHECK_NOT_FREE; /* fail if not free */ |
| 1963 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1964 | if (!(wr->send_flags & MLX5_IB_SEND_UMR_UNREG)) { |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1965 | umr->klm_octowords = get_klm_octo(umrwr->npages); |
Haggai Eran | 968e78d | 2014-12-11 17:04:11 +0200 | [diff] [blame] | 1966 | if (wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_MTT) { |
| 1967 | umr->mkey_mask = get_umr_update_mtt_mask(); |
| 1968 | umr->bsf_octowords = get_klm_octo(umrwr->target.offset); |
| 1969 | umr->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN; |
| 1970 | } else { |
| 1971 | umr->mkey_mask = get_umr_reg_mr_mask(); |
| 1972 | } |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1973 | } else { |
Haggai Eran | 968e78d | 2014-12-11 17:04:11 +0200 | [diff] [blame] | 1974 | umr->mkey_mask = get_umr_unreg_mr_mask(); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1975 | } |
| 1976 | |
| 1977 | if (!wr->num_sge) |
Haggai Eran | 968e78d | 2014-12-11 17:04:11 +0200 | [diff] [blame] | 1978 | umr->flags |= MLX5_UMR_INLINE; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1979 | } |
| 1980 | |
| 1981 | static u8 get_umr_flags(int acc) |
| 1982 | { |
| 1983 | return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) | |
| 1984 | (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) | |
| 1985 | (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) | |
| 1986 | (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) | |
Sagi Grimberg | 2ac4593 | 2014-02-23 14:19:09 +0200 | [diff] [blame] | 1987 | MLX5_PERM_LOCAL_READ | MLX5_PERM_UMR_EN; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1988 | } |
| 1989 | |
| 1990 | static void set_mkey_segment(struct mlx5_mkey_seg *seg, struct ib_send_wr *wr, |
| 1991 | int li, int *writ) |
| 1992 | { |
| 1993 | memset(seg, 0, sizeof(*seg)); |
| 1994 | if (li) { |
Haggai Eran | 968e78d | 2014-12-11 17:04:11 +0200 | [diff] [blame] | 1995 | seg->status = MLX5_MKEY_STATUS_FREE; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 1996 | return; |
| 1997 | } |
| 1998 | |
Sagi Grimberg | 2ac4593 | 2014-02-23 14:19:09 +0200 | [diff] [blame] | 1999 | seg->flags = get_umr_flags(wr->wr.fast_reg.access_flags) | |
| 2000 | MLX5_ACCESS_MODE_MTT; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2001 | *writ = seg->flags & (MLX5_PERM_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE); |
| 2002 | seg->qpn_mkey7_0 = cpu_to_be32((wr->wr.fast_reg.rkey & 0xff) | 0xffffff00); |
| 2003 | seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL); |
| 2004 | seg->start_addr = cpu_to_be64(wr->wr.fast_reg.iova_start); |
| 2005 | seg->len = cpu_to_be64(wr->wr.fast_reg.length); |
| 2006 | seg->xlt_oct_size = cpu_to_be32((wr->wr.fast_reg.page_list_len + 1) / 2); |
| 2007 | seg->log2_page_size = wr->wr.fast_reg.page_shift; |
| 2008 | } |
| 2009 | |
| 2010 | static void set_reg_mkey_segment(struct mlx5_mkey_seg *seg, struct ib_send_wr *wr) |
| 2011 | { |
Haggai Eran | 968e78d | 2014-12-11 17:04:11 +0200 | [diff] [blame] | 2012 | struct mlx5_umr_wr *umrwr = (struct mlx5_umr_wr *)&wr->wr.fast_reg; |
| 2013 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2014 | memset(seg, 0, sizeof(*seg)); |
| 2015 | if (wr->send_flags & MLX5_IB_SEND_UMR_UNREG) { |
Haggai Eran | 968e78d | 2014-12-11 17:04:11 +0200 | [diff] [blame] | 2016 | seg->status = MLX5_MKEY_STATUS_FREE; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2017 | return; |
| 2018 | } |
| 2019 | |
Haggai Eran | 968e78d | 2014-12-11 17:04:11 +0200 | [diff] [blame] | 2020 | seg->flags = convert_access(umrwr->access_flags); |
| 2021 | if (!(wr->send_flags & MLX5_IB_SEND_UMR_UPDATE_MTT)) { |
| 2022 | seg->flags_pd = cpu_to_be32(to_mpd(umrwr->pd)->pdn); |
| 2023 | seg->start_addr = cpu_to_be64(umrwr->target.virt_addr); |
| 2024 | } |
| 2025 | seg->len = cpu_to_be64(umrwr->length); |
| 2026 | seg->log2_page_size = umrwr->page_shift; |
Eli Cohen | 746b558 | 2013-10-23 09:53:14 +0300 | [diff] [blame] | 2027 | seg->qpn_mkey7_0 = cpu_to_be32(0xffffff00 | |
Haggai Eran | 968e78d | 2014-12-11 17:04:11 +0200 | [diff] [blame] | 2028 | mlx5_mkey_variant(umrwr->mkey)); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2029 | } |
| 2030 | |
| 2031 | static void set_frwr_pages(struct mlx5_wqe_data_seg *dseg, |
| 2032 | struct ib_send_wr *wr, |
| 2033 | struct mlx5_core_dev *mdev, |
| 2034 | struct mlx5_ib_pd *pd, |
| 2035 | int writ) |
| 2036 | { |
| 2037 | struct mlx5_ib_fast_reg_page_list *mfrpl = to_mfrpl(wr->wr.fast_reg.page_list); |
| 2038 | u64 *page_list = wr->wr.fast_reg.page_list->page_list; |
| 2039 | u64 perm = MLX5_EN_RD | (writ ? MLX5_EN_WR : 0); |
| 2040 | int i; |
| 2041 | |
| 2042 | for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) |
| 2043 | mfrpl->mapped_page_list[i] = cpu_to_be64(page_list[i] | perm); |
| 2044 | dseg->addr = cpu_to_be64(mfrpl->map); |
| 2045 | dseg->byte_count = cpu_to_be32(ALIGN(sizeof(u64) * wr->wr.fast_reg.page_list_len, 64)); |
Sagi Grimberg | 81fb5e2 | 2015-09-24 10:34:24 +0300 | [diff] [blame] | 2046 | dseg->lkey = cpu_to_be32(pd->ibpd.local_dma_lkey); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2047 | } |
| 2048 | |
| 2049 | static __be32 send_ieth(struct ib_send_wr *wr) |
| 2050 | { |
| 2051 | switch (wr->opcode) { |
| 2052 | case IB_WR_SEND_WITH_IMM: |
| 2053 | case IB_WR_RDMA_WRITE_WITH_IMM: |
| 2054 | return wr->ex.imm_data; |
| 2055 | |
| 2056 | case IB_WR_SEND_WITH_INV: |
| 2057 | return cpu_to_be32(wr->ex.invalidate_rkey); |
| 2058 | |
| 2059 | default: |
| 2060 | return 0; |
| 2061 | } |
| 2062 | } |
| 2063 | |
| 2064 | static u8 calc_sig(void *wqe, int size) |
| 2065 | { |
| 2066 | u8 *p = wqe; |
| 2067 | u8 res = 0; |
| 2068 | int i; |
| 2069 | |
| 2070 | for (i = 0; i < size; i++) |
| 2071 | res ^= p[i]; |
| 2072 | |
| 2073 | return ~res; |
| 2074 | } |
| 2075 | |
| 2076 | static u8 wq_sig(void *wqe) |
| 2077 | { |
| 2078 | return calc_sig(wqe, (*((u8 *)wqe + 8) & 0x3f) << 4); |
| 2079 | } |
| 2080 | |
| 2081 | static int set_data_inl_seg(struct mlx5_ib_qp *qp, struct ib_send_wr *wr, |
| 2082 | void *wqe, int *sz) |
| 2083 | { |
| 2084 | struct mlx5_wqe_inline_seg *seg; |
| 2085 | void *qend = qp->sq.qend; |
| 2086 | void *addr; |
| 2087 | int inl = 0; |
| 2088 | int copy; |
| 2089 | int len; |
| 2090 | int i; |
| 2091 | |
| 2092 | seg = wqe; |
| 2093 | wqe += sizeof(*seg); |
| 2094 | for (i = 0; i < wr->num_sge; i++) { |
| 2095 | addr = (void *)(unsigned long)(wr->sg_list[i].addr); |
| 2096 | len = wr->sg_list[i].length; |
| 2097 | inl += len; |
| 2098 | |
| 2099 | if (unlikely(inl > qp->max_inline_data)) |
| 2100 | return -ENOMEM; |
| 2101 | |
| 2102 | if (unlikely(wqe + len > qend)) { |
| 2103 | copy = qend - wqe; |
| 2104 | memcpy(wqe, addr, copy); |
| 2105 | addr += copy; |
| 2106 | len -= copy; |
| 2107 | wqe = mlx5_get_send_wqe(qp, 0); |
| 2108 | } |
| 2109 | memcpy(wqe, addr, len); |
| 2110 | wqe += len; |
| 2111 | } |
| 2112 | |
| 2113 | seg->byte_count = cpu_to_be32(inl | MLX5_INLINE_SEG); |
| 2114 | |
| 2115 | *sz = ALIGN(inl + sizeof(seg->byte_count), 16) / 16; |
| 2116 | |
| 2117 | return 0; |
| 2118 | } |
| 2119 | |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2120 | static u16 prot_field_size(enum ib_signature_type type) |
| 2121 | { |
| 2122 | switch (type) { |
| 2123 | case IB_SIG_TYPE_T10_DIF: |
| 2124 | return MLX5_DIF_SIZE; |
| 2125 | default: |
| 2126 | return 0; |
| 2127 | } |
| 2128 | } |
| 2129 | |
| 2130 | static u8 bs_selector(int block_size) |
| 2131 | { |
| 2132 | switch (block_size) { |
| 2133 | case 512: return 0x1; |
| 2134 | case 520: return 0x2; |
| 2135 | case 4096: return 0x3; |
| 2136 | case 4160: return 0x4; |
| 2137 | case 1073741824: return 0x5; |
| 2138 | default: return 0; |
| 2139 | } |
| 2140 | } |
| 2141 | |
Sagi Grimberg | 78eda2b | 2014-08-13 19:54:35 +0300 | [diff] [blame] | 2142 | static void mlx5_fill_inl_bsf(struct ib_sig_domain *domain, |
| 2143 | struct mlx5_bsf_inl *inl) |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2144 | { |
Sagi Grimberg | 142537f | 2014-08-13 19:54:32 +0300 | [diff] [blame] | 2145 | /* Valid inline section and allow BSF refresh */ |
| 2146 | inl->vld_refresh = cpu_to_be16(MLX5_BSF_INL_VALID | |
| 2147 | MLX5_BSF_REFRESH_DIF); |
| 2148 | inl->dif_apptag = cpu_to_be16(domain->sig.dif.app_tag); |
| 2149 | inl->dif_reftag = cpu_to_be32(domain->sig.dif.ref_tag); |
Sagi Grimberg | 78eda2b | 2014-08-13 19:54:35 +0300 | [diff] [blame] | 2150 | /* repeating block */ |
| 2151 | inl->rp_inv_seed = MLX5_BSF_REPEAT_BLOCK; |
| 2152 | inl->sig_type = domain->sig.dif.bg_type == IB_T10DIF_CRC ? |
| 2153 | MLX5_DIF_CRC : MLX5_DIF_IPCS; |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2154 | |
Sagi Grimberg | 78eda2b | 2014-08-13 19:54:35 +0300 | [diff] [blame] | 2155 | if (domain->sig.dif.ref_remap) |
| 2156 | inl->dif_inc_ref_guard_check |= MLX5_BSF_INC_REFTAG; |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2157 | |
Sagi Grimberg | 78eda2b | 2014-08-13 19:54:35 +0300 | [diff] [blame] | 2158 | if (domain->sig.dif.app_escape) { |
| 2159 | if (domain->sig.dif.ref_escape) |
| 2160 | inl->dif_inc_ref_guard_check |= MLX5_BSF_APPREF_ESCAPE; |
| 2161 | else |
| 2162 | inl->dif_inc_ref_guard_check |= MLX5_BSF_APPTAG_ESCAPE; |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2163 | } |
| 2164 | |
Sagi Grimberg | 78eda2b | 2014-08-13 19:54:35 +0300 | [diff] [blame] | 2165 | inl->dif_app_bitmask_check = |
| 2166 | cpu_to_be16(domain->sig.dif.apptag_check_mask); |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2167 | } |
| 2168 | |
| 2169 | static int mlx5_set_bsf(struct ib_mr *sig_mr, |
| 2170 | struct ib_sig_attrs *sig_attrs, |
| 2171 | struct mlx5_bsf *bsf, u32 data_size) |
| 2172 | { |
| 2173 | struct mlx5_core_sig_ctx *msig = to_mmr(sig_mr)->sig; |
| 2174 | struct mlx5_bsf_basic *basic = &bsf->basic; |
| 2175 | struct ib_sig_domain *mem = &sig_attrs->mem; |
| 2176 | struct ib_sig_domain *wire = &sig_attrs->wire; |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2177 | |
Sagi Grimberg | c7f44fb | 2014-05-18 18:32:40 +0300 | [diff] [blame] | 2178 | memset(bsf, 0, sizeof(*bsf)); |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2179 | |
Sagi Grimberg | 78eda2b | 2014-08-13 19:54:35 +0300 | [diff] [blame] | 2180 | /* Basic + Extended + Inline */ |
| 2181 | basic->bsf_size_sbs = 1 << 7; |
| 2182 | /* Input domain check byte mask */ |
| 2183 | basic->check_byte_mask = sig_attrs->check_mask; |
| 2184 | basic->raw_data_size = cpu_to_be32(data_size); |
| 2185 | |
| 2186 | /* Memory domain */ |
| 2187 | switch (sig_attrs->mem.sig_type) { |
| 2188 | case IB_SIG_TYPE_NONE: |
| 2189 | break; |
| 2190 | case IB_SIG_TYPE_T10_DIF: |
| 2191 | basic->mem.bs_selector = bs_selector(mem->sig.dif.pi_interval); |
| 2192 | basic->m_bfs_psv = cpu_to_be32(msig->psv_memory.psv_idx); |
| 2193 | mlx5_fill_inl_bsf(mem, &bsf->m_inl); |
| 2194 | break; |
| 2195 | default: |
| 2196 | return -EINVAL; |
| 2197 | } |
| 2198 | |
| 2199 | /* Wire domain */ |
| 2200 | switch (sig_attrs->wire.sig_type) { |
| 2201 | case IB_SIG_TYPE_NONE: |
| 2202 | break; |
| 2203 | case IB_SIG_TYPE_T10_DIF: |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2204 | if (mem->sig.dif.pi_interval == wire->sig.dif.pi_interval && |
Sagi Grimberg | 78eda2b | 2014-08-13 19:54:35 +0300 | [diff] [blame] | 2205 | mem->sig_type == wire->sig_type) { |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2206 | /* Same block structure */ |
Sagi Grimberg | 142537f | 2014-08-13 19:54:32 +0300 | [diff] [blame] | 2207 | basic->bsf_size_sbs |= 1 << 4; |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2208 | if (mem->sig.dif.bg_type == wire->sig.dif.bg_type) |
Sagi Grimberg | fd22f78 | 2014-08-13 19:54:29 +0300 | [diff] [blame] | 2209 | basic->wire.copy_byte_mask |= MLX5_CPY_GRD_MASK; |
Sagi Grimberg | c7f44fb | 2014-05-18 18:32:40 +0300 | [diff] [blame] | 2210 | if (mem->sig.dif.app_tag == wire->sig.dif.app_tag) |
Sagi Grimberg | fd22f78 | 2014-08-13 19:54:29 +0300 | [diff] [blame] | 2211 | basic->wire.copy_byte_mask |= MLX5_CPY_APP_MASK; |
Sagi Grimberg | c7f44fb | 2014-05-18 18:32:40 +0300 | [diff] [blame] | 2212 | if (mem->sig.dif.ref_tag == wire->sig.dif.ref_tag) |
Sagi Grimberg | fd22f78 | 2014-08-13 19:54:29 +0300 | [diff] [blame] | 2213 | basic->wire.copy_byte_mask |= MLX5_CPY_REF_MASK; |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2214 | } else |
| 2215 | basic->wire.bs_selector = bs_selector(wire->sig.dif.pi_interval); |
| 2216 | |
Sagi Grimberg | 142537f | 2014-08-13 19:54:32 +0300 | [diff] [blame] | 2217 | basic->w_bfs_psv = cpu_to_be32(msig->psv_wire.psv_idx); |
Sagi Grimberg | 78eda2b | 2014-08-13 19:54:35 +0300 | [diff] [blame] | 2218 | mlx5_fill_inl_bsf(wire, &bsf->w_inl); |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2219 | break; |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2220 | default: |
| 2221 | return -EINVAL; |
| 2222 | } |
| 2223 | |
| 2224 | return 0; |
| 2225 | } |
| 2226 | |
| 2227 | static int set_sig_data_segment(struct ib_send_wr *wr, struct mlx5_ib_qp *qp, |
| 2228 | void **seg, int *size) |
| 2229 | { |
| 2230 | struct ib_sig_attrs *sig_attrs = wr->wr.sig_handover.sig_attrs; |
| 2231 | struct ib_mr *sig_mr = wr->wr.sig_handover.sig_mr; |
| 2232 | struct mlx5_bsf *bsf; |
| 2233 | u32 data_len = wr->sg_list->length; |
| 2234 | u32 data_key = wr->sg_list->lkey; |
| 2235 | u64 data_va = wr->sg_list->addr; |
| 2236 | int ret; |
| 2237 | int wqe_size; |
| 2238 | |
Sagi Grimberg | 5c273b1 | 2014-05-18 18:32:39 +0300 | [diff] [blame] | 2239 | if (!wr->wr.sig_handover.prot || |
| 2240 | (data_key == wr->wr.sig_handover.prot->lkey && |
| 2241 | data_va == wr->wr.sig_handover.prot->addr && |
| 2242 | data_len == wr->wr.sig_handover.prot->length)) { |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2243 | /** |
| 2244 | * Source domain doesn't contain signature information |
Sagi Grimberg | 5c273b1 | 2014-05-18 18:32:39 +0300 | [diff] [blame] | 2245 | * or data and protection are interleaved in memory. |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2246 | * So need construct: |
| 2247 | * ------------------ |
| 2248 | * | data_klm | |
| 2249 | * ------------------ |
| 2250 | * | BSF | |
| 2251 | * ------------------ |
| 2252 | **/ |
| 2253 | struct mlx5_klm *data_klm = *seg; |
| 2254 | |
| 2255 | data_klm->bcount = cpu_to_be32(data_len); |
| 2256 | data_klm->key = cpu_to_be32(data_key); |
| 2257 | data_klm->va = cpu_to_be64(data_va); |
| 2258 | wqe_size = ALIGN(sizeof(*data_klm), 64); |
| 2259 | } else { |
| 2260 | /** |
| 2261 | * Source domain contains signature information |
| 2262 | * So need construct a strided block format: |
| 2263 | * --------------------------- |
| 2264 | * | stride_block_ctrl | |
| 2265 | * --------------------------- |
| 2266 | * | data_klm | |
| 2267 | * --------------------------- |
| 2268 | * | prot_klm | |
| 2269 | * --------------------------- |
| 2270 | * | BSF | |
| 2271 | * --------------------------- |
| 2272 | **/ |
| 2273 | struct mlx5_stride_block_ctrl_seg *sblock_ctrl; |
| 2274 | struct mlx5_stride_block_entry *data_sentry; |
| 2275 | struct mlx5_stride_block_entry *prot_sentry; |
| 2276 | u32 prot_key = wr->wr.sig_handover.prot->lkey; |
| 2277 | u64 prot_va = wr->wr.sig_handover.prot->addr; |
| 2278 | u16 block_size = sig_attrs->mem.sig.dif.pi_interval; |
| 2279 | int prot_size; |
| 2280 | |
| 2281 | sblock_ctrl = *seg; |
| 2282 | data_sentry = (void *)sblock_ctrl + sizeof(*sblock_ctrl); |
| 2283 | prot_sentry = (void *)data_sentry + sizeof(*data_sentry); |
| 2284 | |
| 2285 | prot_size = prot_field_size(sig_attrs->mem.sig_type); |
| 2286 | if (!prot_size) { |
| 2287 | pr_err("Bad block size given: %u\n", block_size); |
| 2288 | return -EINVAL; |
| 2289 | } |
| 2290 | sblock_ctrl->bcount_per_cycle = cpu_to_be32(block_size + |
| 2291 | prot_size); |
| 2292 | sblock_ctrl->op = cpu_to_be32(MLX5_STRIDE_BLOCK_OP); |
| 2293 | sblock_ctrl->repeat_count = cpu_to_be32(data_len / block_size); |
| 2294 | sblock_ctrl->num_entries = cpu_to_be16(2); |
| 2295 | |
| 2296 | data_sentry->bcount = cpu_to_be16(block_size); |
| 2297 | data_sentry->key = cpu_to_be32(data_key); |
| 2298 | data_sentry->va = cpu_to_be64(data_va); |
Sagi Grimberg | 5c273b1 | 2014-05-18 18:32:39 +0300 | [diff] [blame] | 2299 | data_sentry->stride = cpu_to_be16(block_size); |
| 2300 | |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2301 | prot_sentry->bcount = cpu_to_be16(prot_size); |
| 2302 | prot_sentry->key = cpu_to_be32(prot_key); |
Sagi Grimberg | 5c273b1 | 2014-05-18 18:32:39 +0300 | [diff] [blame] | 2303 | prot_sentry->va = cpu_to_be64(prot_va); |
| 2304 | prot_sentry->stride = cpu_to_be16(prot_size); |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2305 | |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2306 | wqe_size = ALIGN(sizeof(*sblock_ctrl) + sizeof(*data_sentry) + |
| 2307 | sizeof(*prot_sentry), 64); |
| 2308 | } |
| 2309 | |
| 2310 | *seg += wqe_size; |
| 2311 | *size += wqe_size / 16; |
| 2312 | if (unlikely((*seg == qp->sq.qend))) |
| 2313 | *seg = mlx5_get_send_wqe(qp, 0); |
| 2314 | |
| 2315 | bsf = *seg; |
| 2316 | ret = mlx5_set_bsf(sig_mr, sig_attrs, bsf, data_len); |
| 2317 | if (ret) |
| 2318 | return -EINVAL; |
| 2319 | |
| 2320 | *seg += sizeof(*bsf); |
| 2321 | *size += sizeof(*bsf) / 16; |
| 2322 | if (unlikely((*seg == qp->sq.qend))) |
| 2323 | *seg = mlx5_get_send_wqe(qp, 0); |
| 2324 | |
| 2325 | return 0; |
| 2326 | } |
| 2327 | |
| 2328 | static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg, |
| 2329 | struct ib_send_wr *wr, u32 nelements, |
| 2330 | u32 length, u32 pdn) |
| 2331 | { |
| 2332 | struct ib_mr *sig_mr = wr->wr.sig_handover.sig_mr; |
| 2333 | u32 sig_key = sig_mr->rkey; |
Sagi Grimberg | d5436ba | 2014-02-23 14:19:12 +0200 | [diff] [blame] | 2334 | u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1; |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2335 | |
| 2336 | memset(seg, 0, sizeof(*seg)); |
| 2337 | |
| 2338 | seg->flags = get_umr_flags(wr->wr.sig_handover.access_flags) | |
| 2339 | MLX5_ACCESS_MODE_KLM; |
| 2340 | seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00); |
Sagi Grimberg | d5436ba | 2014-02-23 14:19:12 +0200 | [diff] [blame] | 2341 | seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 | |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2342 | MLX5_MKEY_BSF_EN | pdn); |
| 2343 | seg->len = cpu_to_be64(length); |
| 2344 | seg->xlt_oct_size = cpu_to_be32(be16_to_cpu(get_klm_octo(nelements))); |
| 2345 | seg->bsfs_octo_size = cpu_to_be32(MLX5_MKEY_BSF_OCTO_SIZE); |
| 2346 | } |
| 2347 | |
| 2348 | static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr, |
| 2349 | struct ib_send_wr *wr, u32 nelements) |
| 2350 | { |
| 2351 | memset(umr, 0, sizeof(*umr)); |
| 2352 | |
| 2353 | umr->flags = MLX5_FLAGS_INLINE | MLX5_FLAGS_CHECK_FREE; |
| 2354 | umr->klm_octowords = get_klm_octo(nelements); |
| 2355 | umr->bsf_octowords = cpu_to_be16(MLX5_MKEY_BSF_OCTO_SIZE); |
| 2356 | umr->mkey_mask = sig_mkey_mask(); |
| 2357 | } |
| 2358 | |
| 2359 | |
| 2360 | static int set_sig_umr_wr(struct ib_send_wr *wr, struct mlx5_ib_qp *qp, |
| 2361 | void **seg, int *size) |
| 2362 | { |
| 2363 | struct mlx5_ib_mr *sig_mr = to_mmr(wr->wr.sig_handover.sig_mr); |
| 2364 | u32 pdn = get_pd(qp)->pdn; |
| 2365 | u32 klm_oct_size; |
| 2366 | int region_len, ret; |
| 2367 | |
| 2368 | if (unlikely(wr->num_sge != 1) || |
| 2369 | unlikely(wr->wr.sig_handover.access_flags & |
| 2370 | IB_ACCESS_REMOTE_ATOMIC) || |
Sagi Grimberg | d5436ba | 2014-02-23 14:19:12 +0200 | [diff] [blame] | 2371 | unlikely(!sig_mr->sig) || unlikely(!qp->signature_en) || |
| 2372 | unlikely(!sig_mr->sig->sig_status_checked)) |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2373 | return -EINVAL; |
| 2374 | |
| 2375 | /* length of the protected region, data + protection */ |
| 2376 | region_len = wr->sg_list->length; |
Sagi Grimberg | 8524867 | 2014-05-18 18:32:38 +0300 | [diff] [blame] | 2377 | if (wr->wr.sig_handover.prot && |
| 2378 | (wr->wr.sig_handover.prot->lkey != wr->sg_list->lkey || |
| 2379 | wr->wr.sig_handover.prot->addr != wr->sg_list->addr || |
| 2380 | wr->wr.sig_handover.prot->length != wr->sg_list->length)) |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2381 | region_len += wr->wr.sig_handover.prot->length; |
| 2382 | |
| 2383 | /** |
| 2384 | * KLM octoword size - if protection was provided |
| 2385 | * then we use strided block format (3 octowords), |
| 2386 | * else we use single KLM (1 octoword) |
| 2387 | **/ |
| 2388 | klm_oct_size = wr->wr.sig_handover.prot ? 3 : 1; |
| 2389 | |
| 2390 | set_sig_umr_segment(*seg, wr, klm_oct_size); |
| 2391 | *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); |
| 2392 | *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; |
| 2393 | if (unlikely((*seg == qp->sq.qend))) |
| 2394 | *seg = mlx5_get_send_wqe(qp, 0); |
| 2395 | |
| 2396 | set_sig_mkey_segment(*seg, wr, klm_oct_size, region_len, pdn); |
| 2397 | *seg += sizeof(struct mlx5_mkey_seg); |
| 2398 | *size += sizeof(struct mlx5_mkey_seg) / 16; |
| 2399 | if (unlikely((*seg == qp->sq.qend))) |
| 2400 | *seg = mlx5_get_send_wqe(qp, 0); |
| 2401 | |
| 2402 | ret = set_sig_data_segment(wr, qp, seg, size); |
| 2403 | if (ret) |
| 2404 | return ret; |
| 2405 | |
Sagi Grimberg | d5436ba | 2014-02-23 14:19:12 +0200 | [diff] [blame] | 2406 | sig_mr->sig->sig_status_checked = false; |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2407 | return 0; |
| 2408 | } |
| 2409 | |
| 2410 | static int set_psv_wr(struct ib_sig_domain *domain, |
| 2411 | u32 psv_idx, void **seg, int *size) |
| 2412 | { |
| 2413 | struct mlx5_seg_set_psv *psv_seg = *seg; |
| 2414 | |
| 2415 | memset(psv_seg, 0, sizeof(*psv_seg)); |
| 2416 | psv_seg->psv_num = cpu_to_be32(psv_idx); |
| 2417 | switch (domain->sig_type) { |
Sagi Grimberg | 78eda2b | 2014-08-13 19:54:35 +0300 | [diff] [blame] | 2418 | case IB_SIG_TYPE_NONE: |
| 2419 | break; |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2420 | case IB_SIG_TYPE_T10_DIF: |
| 2421 | psv_seg->transient_sig = cpu_to_be32(domain->sig.dif.bg << 16 | |
| 2422 | domain->sig.dif.app_tag); |
| 2423 | psv_seg->ref_tag = cpu_to_be32(domain->sig.dif.ref_tag); |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2424 | break; |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2425 | default: |
| 2426 | pr_err("Bad signature type given.\n"); |
| 2427 | return 1; |
| 2428 | } |
| 2429 | |
Sagi Grimberg | 78eda2b | 2014-08-13 19:54:35 +0300 | [diff] [blame] | 2430 | *seg += sizeof(*psv_seg); |
| 2431 | *size += sizeof(*psv_seg) / 16; |
| 2432 | |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2433 | return 0; |
| 2434 | } |
| 2435 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2436 | static int set_frwr_li_wr(void **seg, struct ib_send_wr *wr, int *size, |
| 2437 | struct mlx5_core_dev *mdev, struct mlx5_ib_pd *pd, struct mlx5_ib_qp *qp) |
| 2438 | { |
| 2439 | int writ = 0; |
| 2440 | int li; |
| 2441 | |
| 2442 | li = wr->opcode == IB_WR_LOCAL_INV ? 1 : 0; |
| 2443 | if (unlikely(wr->send_flags & IB_SEND_INLINE)) |
| 2444 | return -EINVAL; |
| 2445 | |
| 2446 | set_frwr_umr_segment(*seg, wr, li); |
| 2447 | *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); |
| 2448 | *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; |
| 2449 | if (unlikely((*seg == qp->sq.qend))) |
| 2450 | *seg = mlx5_get_send_wqe(qp, 0); |
| 2451 | set_mkey_segment(*seg, wr, li, &writ); |
| 2452 | *seg += sizeof(struct mlx5_mkey_seg); |
| 2453 | *size += sizeof(struct mlx5_mkey_seg) / 16; |
| 2454 | if (unlikely((*seg == qp->sq.qend))) |
| 2455 | *seg = mlx5_get_send_wqe(qp, 0); |
| 2456 | if (!li) { |
Eli Cohen | 9641b74 | 2013-10-23 09:53:15 +0300 | [diff] [blame] | 2457 | if (unlikely(wr->wr.fast_reg.page_list_len > |
| 2458 | wr->wr.fast_reg.page_list->max_page_list_len)) |
| 2459 | return -ENOMEM; |
| 2460 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2461 | set_frwr_pages(*seg, wr, mdev, pd, writ); |
| 2462 | *seg += sizeof(struct mlx5_wqe_data_seg); |
| 2463 | *size += (sizeof(struct mlx5_wqe_data_seg) / 16); |
| 2464 | } |
| 2465 | return 0; |
| 2466 | } |
| 2467 | |
| 2468 | static void dump_wqe(struct mlx5_ib_qp *qp, int idx, int size_16) |
| 2469 | { |
| 2470 | __be32 *p = NULL; |
| 2471 | int tidx = idx; |
| 2472 | int i, j; |
| 2473 | |
| 2474 | pr_debug("dump wqe at %p\n", mlx5_get_send_wqe(qp, tidx)); |
| 2475 | for (i = 0, j = 0; i < size_16 * 4; i += 4, j += 4) { |
| 2476 | if ((i & 0xf) == 0) { |
| 2477 | void *buf = mlx5_get_send_wqe(qp, tidx); |
| 2478 | tidx = (tidx + 1) & (qp->sq.wqe_cnt - 1); |
| 2479 | p = buf; |
| 2480 | j = 0; |
| 2481 | } |
| 2482 | pr_debug("%08x %08x %08x %08x\n", be32_to_cpu(p[j]), |
| 2483 | be32_to_cpu(p[j + 1]), be32_to_cpu(p[j + 2]), |
| 2484 | be32_to_cpu(p[j + 3])); |
| 2485 | } |
| 2486 | } |
| 2487 | |
| 2488 | static void mlx5_bf_copy(u64 __iomem *dst, u64 *src, |
| 2489 | unsigned bytecnt, struct mlx5_ib_qp *qp) |
| 2490 | { |
| 2491 | while (bytecnt > 0) { |
| 2492 | __iowrite64_copy(dst++, src++, 8); |
| 2493 | __iowrite64_copy(dst++, src++, 8); |
| 2494 | __iowrite64_copy(dst++, src++, 8); |
| 2495 | __iowrite64_copy(dst++, src++, 8); |
| 2496 | __iowrite64_copy(dst++, src++, 8); |
| 2497 | __iowrite64_copy(dst++, src++, 8); |
| 2498 | __iowrite64_copy(dst++, src++, 8); |
| 2499 | __iowrite64_copy(dst++, src++, 8); |
| 2500 | bytecnt -= 64; |
| 2501 | if (unlikely(src == qp->sq.qend)) |
| 2502 | src = mlx5_get_send_wqe(qp, 0); |
| 2503 | } |
| 2504 | } |
| 2505 | |
| 2506 | static u8 get_fence(u8 fence, struct ib_send_wr *wr) |
| 2507 | { |
| 2508 | if (unlikely(wr->opcode == IB_WR_LOCAL_INV && |
| 2509 | wr->send_flags & IB_SEND_FENCE)) |
| 2510 | return MLX5_FENCE_MODE_STRONG_ORDERING; |
| 2511 | |
| 2512 | if (unlikely(fence)) { |
| 2513 | if (wr->send_flags & IB_SEND_FENCE) |
| 2514 | return MLX5_FENCE_MODE_SMALL_AND_FENCE; |
| 2515 | else |
| 2516 | return fence; |
| 2517 | |
| 2518 | } else { |
| 2519 | return 0; |
| 2520 | } |
| 2521 | } |
| 2522 | |
Sagi Grimberg | 6e5eada | 2014-02-23 14:19:08 +0200 | [diff] [blame] | 2523 | static int begin_wqe(struct mlx5_ib_qp *qp, void **seg, |
| 2524 | struct mlx5_wqe_ctrl_seg **ctrl, |
Eli Cohen | 6a4f139 | 2014-12-02 12:26:18 +0200 | [diff] [blame] | 2525 | struct ib_send_wr *wr, unsigned *idx, |
Sagi Grimberg | 6e5eada | 2014-02-23 14:19:08 +0200 | [diff] [blame] | 2526 | int *size, int nreq) |
| 2527 | { |
| 2528 | int err = 0; |
| 2529 | |
| 2530 | if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq))) { |
| 2531 | err = -ENOMEM; |
| 2532 | return err; |
| 2533 | } |
| 2534 | |
| 2535 | *idx = qp->sq.cur_post & (qp->sq.wqe_cnt - 1); |
| 2536 | *seg = mlx5_get_send_wqe(qp, *idx); |
| 2537 | *ctrl = *seg; |
| 2538 | *(uint32_t *)(*seg + 8) = 0; |
| 2539 | (*ctrl)->imm = send_ieth(wr); |
| 2540 | (*ctrl)->fm_ce_se = qp->sq_signal_bits | |
| 2541 | (wr->send_flags & IB_SEND_SIGNALED ? |
| 2542 | MLX5_WQE_CTRL_CQ_UPDATE : 0) | |
| 2543 | (wr->send_flags & IB_SEND_SOLICITED ? |
| 2544 | MLX5_WQE_CTRL_SOLICITED : 0); |
| 2545 | |
| 2546 | *seg += sizeof(**ctrl); |
| 2547 | *size = sizeof(**ctrl) / 16; |
| 2548 | |
| 2549 | return err; |
| 2550 | } |
| 2551 | |
| 2552 | static void finish_wqe(struct mlx5_ib_qp *qp, |
| 2553 | struct mlx5_wqe_ctrl_seg *ctrl, |
| 2554 | u8 size, unsigned idx, u64 wr_id, |
| 2555 | int nreq, u8 fence, u8 next_fence, |
| 2556 | u32 mlx5_opcode) |
| 2557 | { |
| 2558 | u8 opmod = 0; |
| 2559 | |
| 2560 | ctrl->opmod_idx_opcode = cpu_to_be32(((u32)(qp->sq.cur_post) << 8) | |
| 2561 | mlx5_opcode | ((u32)opmod << 24)); |
| 2562 | ctrl->qpn_ds = cpu_to_be32(size | (qp->mqp.qpn << 8)); |
| 2563 | ctrl->fm_ce_se |= fence; |
| 2564 | qp->fm_cache = next_fence; |
| 2565 | if (unlikely(qp->wq_sig)) |
| 2566 | ctrl->signature = wq_sig(ctrl); |
| 2567 | |
| 2568 | qp->sq.wrid[idx] = wr_id; |
| 2569 | qp->sq.w_list[idx].opcode = mlx5_opcode; |
| 2570 | qp->sq.wqe_head[idx] = qp->sq.head + nreq; |
| 2571 | qp->sq.cur_post += DIV_ROUND_UP(size * 16, MLX5_SEND_WQE_BB); |
| 2572 | qp->sq.w_list[idx].next = qp->sq.cur_post; |
| 2573 | } |
| 2574 | |
| 2575 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2576 | int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, |
| 2577 | struct ib_send_wr **bad_wr) |
| 2578 | { |
| 2579 | struct mlx5_wqe_ctrl_seg *ctrl = NULL; /* compiler warning */ |
| 2580 | struct mlx5_ib_dev *dev = to_mdev(ibqp->device); |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 2581 | struct mlx5_core_dev *mdev = dev->mdev; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2582 | struct mlx5_ib_qp *qp = to_mqp(ibqp); |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2583 | struct mlx5_ib_mr *mr; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2584 | struct mlx5_wqe_data_seg *dpseg; |
| 2585 | struct mlx5_wqe_xrc_seg *xrc; |
| 2586 | struct mlx5_bf *bf = qp->bf; |
| 2587 | int uninitialized_var(size); |
| 2588 | void *qend = qp->sq.qend; |
| 2589 | unsigned long flags; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2590 | unsigned idx; |
| 2591 | int err = 0; |
| 2592 | int inl = 0; |
| 2593 | int num_sge; |
| 2594 | void *seg; |
| 2595 | int nreq; |
| 2596 | int i; |
| 2597 | u8 next_fence = 0; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2598 | u8 fence; |
| 2599 | |
| 2600 | spin_lock_irqsave(&qp->sq.lock, flags); |
| 2601 | |
| 2602 | for (nreq = 0; wr; nreq++, wr = wr->next) { |
Fabian Frederick | a8f731e | 2014-08-12 19:20:08 -0400 | [diff] [blame] | 2603 | if (unlikely(wr->opcode >= ARRAY_SIZE(mlx5_ib_opcode))) { |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2604 | mlx5_ib_warn(dev, "\n"); |
| 2605 | err = -EINVAL; |
| 2606 | *bad_wr = wr; |
| 2607 | goto out; |
| 2608 | } |
| 2609 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2610 | fence = qp->fm_cache; |
| 2611 | num_sge = wr->num_sge; |
| 2612 | if (unlikely(num_sge > qp->sq.max_gs)) { |
| 2613 | mlx5_ib_warn(dev, "\n"); |
| 2614 | err = -ENOMEM; |
| 2615 | *bad_wr = wr; |
| 2616 | goto out; |
| 2617 | } |
| 2618 | |
Sagi Grimberg | 6e5eada | 2014-02-23 14:19:08 +0200 | [diff] [blame] | 2619 | err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, nreq); |
| 2620 | if (err) { |
| 2621 | mlx5_ib_warn(dev, "\n"); |
| 2622 | err = -ENOMEM; |
| 2623 | *bad_wr = wr; |
| 2624 | goto out; |
| 2625 | } |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2626 | |
| 2627 | switch (ibqp->qp_type) { |
| 2628 | case IB_QPT_XRC_INI: |
| 2629 | xrc = seg; |
| 2630 | xrc->xrc_srqn = htonl(wr->xrc_remote_srq_num); |
| 2631 | seg += sizeof(*xrc); |
| 2632 | size += sizeof(*xrc) / 16; |
| 2633 | /* fall through */ |
| 2634 | case IB_QPT_RC: |
| 2635 | switch (wr->opcode) { |
| 2636 | case IB_WR_RDMA_READ: |
| 2637 | case IB_WR_RDMA_WRITE: |
| 2638 | case IB_WR_RDMA_WRITE_WITH_IMM: |
| 2639 | set_raddr_seg(seg, wr->wr.rdma.remote_addr, |
| 2640 | wr->wr.rdma.rkey); |
Jack Morgenstein | f241e74 | 2014-07-28 23:30:23 +0300 | [diff] [blame] | 2641 | seg += sizeof(struct mlx5_wqe_raddr_seg); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2642 | size += sizeof(struct mlx5_wqe_raddr_seg) / 16; |
| 2643 | break; |
| 2644 | |
| 2645 | case IB_WR_ATOMIC_CMP_AND_SWP: |
| 2646 | case IB_WR_ATOMIC_FETCH_AND_ADD: |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2647 | case IB_WR_MASKED_ATOMIC_CMP_AND_SWP: |
Eli Cohen | 81bea28 | 2013-09-11 16:35:30 +0300 | [diff] [blame] | 2648 | mlx5_ib_warn(dev, "Atomic operations are not supported yet\n"); |
| 2649 | err = -ENOSYS; |
| 2650 | *bad_wr = wr; |
| 2651 | goto out; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2652 | |
| 2653 | case IB_WR_LOCAL_INV: |
| 2654 | next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL; |
| 2655 | qp->sq.wr_data[idx] = IB_WR_LOCAL_INV; |
| 2656 | ctrl->imm = cpu_to_be32(wr->ex.invalidate_rkey); |
| 2657 | err = set_frwr_li_wr(&seg, wr, &size, mdev, to_mpd(ibqp->pd), qp); |
| 2658 | if (err) { |
| 2659 | mlx5_ib_warn(dev, "\n"); |
| 2660 | *bad_wr = wr; |
| 2661 | goto out; |
| 2662 | } |
| 2663 | num_sge = 0; |
| 2664 | break; |
| 2665 | |
| 2666 | case IB_WR_FAST_REG_MR: |
| 2667 | next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL; |
| 2668 | qp->sq.wr_data[idx] = IB_WR_FAST_REG_MR; |
| 2669 | ctrl->imm = cpu_to_be32(wr->wr.fast_reg.rkey); |
| 2670 | err = set_frwr_li_wr(&seg, wr, &size, mdev, to_mpd(ibqp->pd), qp); |
| 2671 | if (err) { |
| 2672 | mlx5_ib_warn(dev, "\n"); |
| 2673 | *bad_wr = wr; |
| 2674 | goto out; |
| 2675 | } |
| 2676 | num_sge = 0; |
| 2677 | break; |
| 2678 | |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2679 | case IB_WR_REG_SIG_MR: |
| 2680 | qp->sq.wr_data[idx] = IB_WR_REG_SIG_MR; |
| 2681 | mr = to_mmr(wr->wr.sig_handover.sig_mr); |
| 2682 | |
| 2683 | ctrl->imm = cpu_to_be32(mr->ibmr.rkey); |
| 2684 | err = set_sig_umr_wr(wr, qp, &seg, &size); |
| 2685 | if (err) { |
| 2686 | mlx5_ib_warn(dev, "\n"); |
| 2687 | *bad_wr = wr; |
| 2688 | goto out; |
| 2689 | } |
| 2690 | |
| 2691 | finish_wqe(qp, ctrl, size, idx, wr->wr_id, |
| 2692 | nreq, get_fence(fence, wr), |
| 2693 | next_fence, MLX5_OPCODE_UMR); |
| 2694 | /* |
| 2695 | * SET_PSV WQEs are not signaled and solicited |
| 2696 | * on error |
| 2697 | */ |
| 2698 | wr->send_flags &= ~IB_SEND_SIGNALED; |
| 2699 | wr->send_flags |= IB_SEND_SOLICITED; |
| 2700 | err = begin_wqe(qp, &seg, &ctrl, wr, |
| 2701 | &idx, &size, nreq); |
| 2702 | if (err) { |
| 2703 | mlx5_ib_warn(dev, "\n"); |
| 2704 | err = -ENOMEM; |
| 2705 | *bad_wr = wr; |
| 2706 | goto out; |
| 2707 | } |
| 2708 | |
| 2709 | err = set_psv_wr(&wr->wr.sig_handover.sig_attrs->mem, |
| 2710 | mr->sig->psv_memory.psv_idx, &seg, |
| 2711 | &size); |
| 2712 | if (err) { |
| 2713 | mlx5_ib_warn(dev, "\n"); |
| 2714 | *bad_wr = wr; |
| 2715 | goto out; |
| 2716 | } |
| 2717 | |
| 2718 | finish_wqe(qp, ctrl, size, idx, wr->wr_id, |
| 2719 | nreq, get_fence(fence, wr), |
| 2720 | next_fence, MLX5_OPCODE_SET_PSV); |
| 2721 | err = begin_wqe(qp, &seg, &ctrl, wr, |
| 2722 | &idx, &size, nreq); |
| 2723 | if (err) { |
| 2724 | mlx5_ib_warn(dev, "\n"); |
| 2725 | err = -ENOMEM; |
| 2726 | *bad_wr = wr; |
| 2727 | goto out; |
| 2728 | } |
| 2729 | |
| 2730 | next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL; |
| 2731 | err = set_psv_wr(&wr->wr.sig_handover.sig_attrs->wire, |
| 2732 | mr->sig->psv_wire.psv_idx, &seg, |
| 2733 | &size); |
| 2734 | if (err) { |
| 2735 | mlx5_ib_warn(dev, "\n"); |
| 2736 | *bad_wr = wr; |
| 2737 | goto out; |
| 2738 | } |
| 2739 | |
| 2740 | finish_wqe(qp, ctrl, size, idx, wr->wr_id, |
| 2741 | nreq, get_fence(fence, wr), |
| 2742 | next_fence, MLX5_OPCODE_SET_PSV); |
| 2743 | num_sge = 0; |
| 2744 | goto skip_psv; |
| 2745 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2746 | default: |
| 2747 | break; |
| 2748 | } |
| 2749 | break; |
| 2750 | |
| 2751 | case IB_QPT_UC: |
| 2752 | switch (wr->opcode) { |
| 2753 | case IB_WR_RDMA_WRITE: |
| 2754 | case IB_WR_RDMA_WRITE_WITH_IMM: |
| 2755 | set_raddr_seg(seg, wr->wr.rdma.remote_addr, |
| 2756 | wr->wr.rdma.rkey); |
| 2757 | seg += sizeof(struct mlx5_wqe_raddr_seg); |
| 2758 | size += sizeof(struct mlx5_wqe_raddr_seg) / 16; |
| 2759 | break; |
| 2760 | |
| 2761 | default: |
| 2762 | break; |
| 2763 | } |
| 2764 | break; |
| 2765 | |
| 2766 | case IB_QPT_UD: |
| 2767 | case IB_QPT_SMI: |
| 2768 | case IB_QPT_GSI: |
| 2769 | set_datagram_seg(seg, wr); |
Jack Morgenstein | f241e74 | 2014-07-28 23:30:23 +0300 | [diff] [blame] | 2770 | seg += sizeof(struct mlx5_wqe_datagram_seg); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2771 | size += sizeof(struct mlx5_wqe_datagram_seg) / 16; |
| 2772 | if (unlikely((seg == qend))) |
| 2773 | seg = mlx5_get_send_wqe(qp, 0); |
| 2774 | break; |
| 2775 | |
| 2776 | case MLX5_IB_QPT_REG_UMR: |
| 2777 | if (wr->opcode != MLX5_IB_WR_UMR) { |
| 2778 | err = -EINVAL; |
| 2779 | mlx5_ib_warn(dev, "bad opcode\n"); |
| 2780 | goto out; |
| 2781 | } |
| 2782 | qp->sq.wr_data[idx] = MLX5_IB_WR_UMR; |
| 2783 | ctrl->imm = cpu_to_be32(wr->wr.fast_reg.rkey); |
| 2784 | set_reg_umr_segment(seg, wr); |
| 2785 | seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); |
| 2786 | size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; |
| 2787 | if (unlikely((seg == qend))) |
| 2788 | seg = mlx5_get_send_wqe(qp, 0); |
| 2789 | set_reg_mkey_segment(seg, wr); |
| 2790 | seg += sizeof(struct mlx5_mkey_seg); |
| 2791 | size += sizeof(struct mlx5_mkey_seg) / 16; |
| 2792 | if (unlikely((seg == qend))) |
| 2793 | seg = mlx5_get_send_wqe(qp, 0); |
| 2794 | break; |
| 2795 | |
| 2796 | default: |
| 2797 | break; |
| 2798 | } |
| 2799 | |
| 2800 | if (wr->send_flags & IB_SEND_INLINE && num_sge) { |
| 2801 | int uninitialized_var(sz); |
| 2802 | |
| 2803 | err = set_data_inl_seg(qp, wr, seg, &sz); |
| 2804 | if (unlikely(err)) { |
| 2805 | mlx5_ib_warn(dev, "\n"); |
| 2806 | *bad_wr = wr; |
| 2807 | goto out; |
| 2808 | } |
| 2809 | inl = 1; |
| 2810 | size += sz; |
| 2811 | } else { |
| 2812 | dpseg = seg; |
| 2813 | for (i = 0; i < num_sge; i++) { |
| 2814 | if (unlikely(dpseg == qend)) { |
| 2815 | seg = mlx5_get_send_wqe(qp, 0); |
| 2816 | dpseg = seg; |
| 2817 | } |
| 2818 | if (likely(wr->sg_list[i].length)) { |
| 2819 | set_data_ptr_seg(dpseg, wr->sg_list + i); |
| 2820 | size += sizeof(struct mlx5_wqe_data_seg) / 16; |
| 2821 | dpseg++; |
| 2822 | } |
| 2823 | } |
| 2824 | } |
| 2825 | |
Sagi Grimberg | 6e5eada | 2014-02-23 14:19:08 +0200 | [diff] [blame] | 2826 | finish_wqe(qp, ctrl, size, idx, wr->wr_id, nreq, |
| 2827 | get_fence(fence, wr), next_fence, |
| 2828 | mlx5_ib_opcode[wr->opcode]); |
Sagi Grimberg | e663181 | 2014-02-23 14:19:11 +0200 | [diff] [blame] | 2829 | skip_psv: |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2830 | if (0) |
| 2831 | dump_wqe(qp, idx, size); |
| 2832 | } |
| 2833 | |
| 2834 | out: |
| 2835 | if (likely(nreq)) { |
| 2836 | qp->sq.head += nreq; |
| 2837 | |
| 2838 | /* Make sure that descriptors are written before |
| 2839 | * updating doorbell record and ringing the doorbell |
| 2840 | */ |
| 2841 | wmb(); |
| 2842 | |
| 2843 | qp->db.db[MLX5_SND_DBR] = cpu_to_be32(qp->sq.cur_post); |
| 2844 | |
Eli Cohen | ada388f | 2014-01-14 17:45:16 +0200 | [diff] [blame] | 2845 | /* Make sure doorbell record is visible to the HCA before |
| 2846 | * we hit doorbell */ |
| 2847 | wmb(); |
| 2848 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2849 | if (bf->need_lock) |
| 2850 | spin_lock(&bf->lock); |
Eli Cohen | 6a4f139 | 2014-12-02 12:26:18 +0200 | [diff] [blame] | 2851 | else |
| 2852 | __acquire(&bf->lock); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2853 | |
| 2854 | /* TBD enable WC */ |
| 2855 | if (0 && nreq == 1 && bf->uuarn && inl && size > 1 && size <= bf->buf_size / 16) { |
| 2856 | mlx5_bf_copy(bf->reg + bf->offset, (u64 *)ctrl, ALIGN(size * 16, 64), qp); |
| 2857 | /* wc_wmb(); */ |
| 2858 | } else { |
| 2859 | mlx5_write64((__be32 *)ctrl, bf->regreg + bf->offset, |
| 2860 | MLX5_GET_DOORBELL_LOCK(&bf->lock32)); |
| 2861 | /* Make sure doorbells don't leak out of SQ spinlock |
| 2862 | * and reach the HCA out of order. |
| 2863 | */ |
| 2864 | mmiowb(); |
| 2865 | } |
| 2866 | bf->offset ^= bf->buf_size; |
| 2867 | if (bf->need_lock) |
| 2868 | spin_unlock(&bf->lock); |
Eli Cohen | 6a4f139 | 2014-12-02 12:26:18 +0200 | [diff] [blame] | 2869 | else |
| 2870 | __release(&bf->lock); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2871 | } |
| 2872 | |
| 2873 | spin_unlock_irqrestore(&qp->sq.lock, flags); |
| 2874 | |
| 2875 | return err; |
| 2876 | } |
| 2877 | |
| 2878 | static void set_sig_seg(struct mlx5_rwqe_sig *sig, int size) |
| 2879 | { |
| 2880 | sig->signature = calc_sig(sig, size); |
| 2881 | } |
| 2882 | |
| 2883 | int mlx5_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr, |
| 2884 | struct ib_recv_wr **bad_wr) |
| 2885 | { |
| 2886 | struct mlx5_ib_qp *qp = to_mqp(ibqp); |
| 2887 | struct mlx5_wqe_data_seg *scat; |
| 2888 | struct mlx5_rwqe_sig *sig; |
| 2889 | unsigned long flags; |
| 2890 | int err = 0; |
| 2891 | int nreq; |
| 2892 | int ind; |
| 2893 | int i; |
| 2894 | |
| 2895 | spin_lock_irqsave(&qp->rq.lock, flags); |
| 2896 | |
| 2897 | ind = qp->rq.head & (qp->rq.wqe_cnt - 1); |
| 2898 | |
| 2899 | for (nreq = 0; wr; nreq++, wr = wr->next) { |
| 2900 | if (mlx5_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) { |
| 2901 | err = -ENOMEM; |
| 2902 | *bad_wr = wr; |
| 2903 | goto out; |
| 2904 | } |
| 2905 | |
| 2906 | if (unlikely(wr->num_sge > qp->rq.max_gs)) { |
| 2907 | err = -EINVAL; |
| 2908 | *bad_wr = wr; |
| 2909 | goto out; |
| 2910 | } |
| 2911 | |
| 2912 | scat = get_recv_wqe(qp, ind); |
| 2913 | if (qp->wq_sig) |
| 2914 | scat++; |
| 2915 | |
| 2916 | for (i = 0; i < wr->num_sge; i++) |
| 2917 | set_data_ptr_seg(scat + i, wr->sg_list + i); |
| 2918 | |
| 2919 | if (i < qp->rq.max_gs) { |
| 2920 | scat[i].byte_count = 0; |
| 2921 | scat[i].lkey = cpu_to_be32(MLX5_INVALID_LKEY); |
| 2922 | scat[i].addr = 0; |
| 2923 | } |
| 2924 | |
| 2925 | if (qp->wq_sig) { |
| 2926 | sig = (struct mlx5_rwqe_sig *)scat; |
| 2927 | set_sig_seg(sig, (qp->rq.max_gs + 1) << 2); |
| 2928 | } |
| 2929 | |
| 2930 | qp->rq.wrid[ind] = wr->wr_id; |
| 2931 | |
| 2932 | ind = (ind + 1) & (qp->rq.wqe_cnt - 1); |
| 2933 | } |
| 2934 | |
| 2935 | out: |
| 2936 | if (likely(nreq)) { |
| 2937 | qp->rq.head += nreq; |
| 2938 | |
| 2939 | /* Make sure that descriptors are written before |
| 2940 | * doorbell record. |
| 2941 | */ |
| 2942 | wmb(); |
| 2943 | |
| 2944 | *qp->db.db = cpu_to_be32(qp->rq.head & 0xffff); |
| 2945 | } |
| 2946 | |
| 2947 | spin_unlock_irqrestore(&qp->rq.lock, flags); |
| 2948 | |
| 2949 | return err; |
| 2950 | } |
| 2951 | |
| 2952 | static inline enum ib_qp_state to_ib_qp_state(enum mlx5_qp_state mlx5_state) |
| 2953 | { |
| 2954 | switch (mlx5_state) { |
| 2955 | case MLX5_QP_STATE_RST: return IB_QPS_RESET; |
| 2956 | case MLX5_QP_STATE_INIT: return IB_QPS_INIT; |
| 2957 | case MLX5_QP_STATE_RTR: return IB_QPS_RTR; |
| 2958 | case MLX5_QP_STATE_RTS: return IB_QPS_RTS; |
| 2959 | case MLX5_QP_STATE_SQ_DRAINING: |
| 2960 | case MLX5_QP_STATE_SQD: return IB_QPS_SQD; |
| 2961 | case MLX5_QP_STATE_SQER: return IB_QPS_SQE; |
| 2962 | case MLX5_QP_STATE_ERR: return IB_QPS_ERR; |
| 2963 | default: return -1; |
| 2964 | } |
| 2965 | } |
| 2966 | |
| 2967 | static inline enum ib_mig_state to_ib_mig_state(int mlx5_mig_state) |
| 2968 | { |
| 2969 | switch (mlx5_mig_state) { |
| 2970 | case MLX5_QP_PM_ARMED: return IB_MIG_ARMED; |
| 2971 | case MLX5_QP_PM_REARM: return IB_MIG_REARM; |
| 2972 | case MLX5_QP_PM_MIGRATED: return IB_MIG_MIGRATED; |
| 2973 | default: return -1; |
| 2974 | } |
| 2975 | } |
| 2976 | |
| 2977 | static int to_ib_qp_access_flags(int mlx5_flags) |
| 2978 | { |
| 2979 | int ib_flags = 0; |
| 2980 | |
| 2981 | if (mlx5_flags & MLX5_QP_BIT_RRE) |
| 2982 | ib_flags |= IB_ACCESS_REMOTE_READ; |
| 2983 | if (mlx5_flags & MLX5_QP_BIT_RWE) |
| 2984 | ib_flags |= IB_ACCESS_REMOTE_WRITE; |
| 2985 | if (mlx5_flags & MLX5_QP_BIT_RAE) |
| 2986 | ib_flags |= IB_ACCESS_REMOTE_ATOMIC; |
| 2987 | |
| 2988 | return ib_flags; |
| 2989 | } |
| 2990 | |
| 2991 | static void to_ib_ah_attr(struct mlx5_ib_dev *ibdev, struct ib_ah_attr *ib_ah_attr, |
| 2992 | struct mlx5_qp_path *path) |
| 2993 | { |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 2994 | struct mlx5_core_dev *dev = ibdev->mdev; |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 2995 | |
| 2996 | memset(ib_ah_attr, 0, sizeof(*ib_ah_attr)); |
| 2997 | ib_ah_attr->port_num = path->port; |
| 2998 | |
Eli Cohen | c7a08ac | 2014-10-02 12:19:42 +0300 | [diff] [blame] | 2999 | if (ib_ah_attr->port_num == 0 || |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 3000 | ib_ah_attr->port_num > MLX5_CAP_GEN(dev, num_ports)) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 3001 | return; |
| 3002 | |
| 3003 | ib_ah_attr->sl = path->sl & 0xf; |
| 3004 | |
| 3005 | ib_ah_attr->dlid = be16_to_cpu(path->rlid); |
| 3006 | ib_ah_attr->src_path_bits = path->grh_mlid & 0x7f; |
| 3007 | ib_ah_attr->static_rate = path->static_rate ? path->static_rate - 5 : 0; |
| 3008 | ib_ah_attr->ah_flags = (path->grh_mlid & (1 << 7)) ? IB_AH_GRH : 0; |
| 3009 | if (ib_ah_attr->ah_flags) { |
| 3010 | ib_ah_attr->grh.sgid_index = path->mgid_index; |
| 3011 | ib_ah_attr->grh.hop_limit = path->hop_limit; |
| 3012 | ib_ah_attr->grh.traffic_class = |
| 3013 | (be32_to_cpu(path->tclass_flowlabel) >> 20) & 0xff; |
| 3014 | ib_ah_attr->grh.flow_label = |
| 3015 | be32_to_cpu(path->tclass_flowlabel) & 0xfffff; |
| 3016 | memcpy(ib_ah_attr->grh.dgid.raw, |
| 3017 | path->rgid, sizeof(ib_ah_attr->grh.dgid.raw)); |
| 3018 | } |
| 3019 | } |
| 3020 | |
| 3021 | int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, |
| 3022 | struct ib_qp_init_attr *qp_init_attr) |
| 3023 | { |
| 3024 | struct mlx5_ib_dev *dev = to_mdev(ibqp->device); |
| 3025 | struct mlx5_ib_qp *qp = to_mqp(ibqp); |
| 3026 | struct mlx5_query_qp_mbox_out *outb; |
| 3027 | struct mlx5_qp_context *context; |
| 3028 | int mlx5_state; |
| 3029 | int err = 0; |
| 3030 | |
Haggai Eran | 6aec21f | 2014-12-11 17:04:23 +0200 | [diff] [blame] | 3031 | #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING |
| 3032 | /* |
| 3033 | * Wait for any outstanding page faults, in case the user frees memory |
| 3034 | * based upon this query's result. |
| 3035 | */ |
| 3036 | flush_workqueue(mlx5_ib_page_fault_wq); |
| 3037 | #endif |
| 3038 | |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 3039 | mutex_lock(&qp->mutex); |
| 3040 | outb = kzalloc(sizeof(*outb), GFP_KERNEL); |
| 3041 | if (!outb) { |
| 3042 | err = -ENOMEM; |
| 3043 | goto out; |
| 3044 | } |
| 3045 | context = &outb->ctx; |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 3046 | err = mlx5_core_qp_query(dev->mdev, &qp->mqp, outb, sizeof(*outb)); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 3047 | if (err) |
| 3048 | goto out_free; |
| 3049 | |
| 3050 | mlx5_state = be32_to_cpu(context->flags) >> 28; |
| 3051 | |
| 3052 | qp->state = to_ib_qp_state(mlx5_state); |
| 3053 | qp_attr->qp_state = qp->state; |
| 3054 | qp_attr->path_mtu = context->mtu_msgmax >> 5; |
| 3055 | qp_attr->path_mig_state = |
| 3056 | to_ib_mig_state((be32_to_cpu(context->flags) >> 11) & 0x3); |
| 3057 | qp_attr->qkey = be32_to_cpu(context->qkey); |
| 3058 | qp_attr->rq_psn = be32_to_cpu(context->rnr_nextrecvpsn) & 0xffffff; |
| 3059 | qp_attr->sq_psn = be32_to_cpu(context->next_send_psn) & 0xffffff; |
| 3060 | qp_attr->dest_qp_num = be32_to_cpu(context->log_pg_sz_remote_qpn) & 0xffffff; |
| 3061 | qp_attr->qp_access_flags = |
| 3062 | to_ib_qp_access_flags(be32_to_cpu(context->params2)); |
| 3063 | |
| 3064 | if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) { |
| 3065 | to_ib_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path); |
| 3066 | to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context->alt_path); |
| 3067 | qp_attr->alt_pkey_index = context->alt_path.pkey_index & 0x7f; |
| 3068 | qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num; |
| 3069 | } |
| 3070 | |
| 3071 | qp_attr->pkey_index = context->pri_path.pkey_index & 0x7f; |
| 3072 | qp_attr->port_num = context->pri_path.port; |
| 3073 | |
| 3074 | /* qp_attr->en_sqd_async_notify is only applicable in modify qp */ |
| 3075 | qp_attr->sq_draining = mlx5_state == MLX5_QP_STATE_SQ_DRAINING; |
| 3076 | |
| 3077 | qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context->params1) >> 21) & 0x7); |
| 3078 | |
| 3079 | qp_attr->max_dest_rd_atomic = |
| 3080 | 1 << ((be32_to_cpu(context->params2) >> 21) & 0x7); |
| 3081 | qp_attr->min_rnr_timer = |
| 3082 | (be32_to_cpu(context->rnr_nextrecvpsn) >> 24) & 0x1f; |
| 3083 | qp_attr->timeout = context->pri_path.ackto_lt >> 3; |
| 3084 | qp_attr->retry_cnt = (be32_to_cpu(context->params1) >> 16) & 0x7; |
| 3085 | qp_attr->rnr_retry = (be32_to_cpu(context->params1) >> 13) & 0x7; |
| 3086 | qp_attr->alt_timeout = context->alt_path.ackto_lt >> 3; |
| 3087 | qp_attr->cur_qp_state = qp_attr->qp_state; |
| 3088 | qp_attr->cap.max_recv_wr = qp->rq.wqe_cnt; |
| 3089 | qp_attr->cap.max_recv_sge = qp->rq.max_gs; |
| 3090 | |
| 3091 | if (!ibqp->uobject) { |
| 3092 | qp_attr->cap.max_send_wr = qp->sq.wqe_cnt; |
| 3093 | qp_attr->cap.max_send_sge = qp->sq.max_gs; |
| 3094 | } else { |
| 3095 | qp_attr->cap.max_send_wr = 0; |
| 3096 | qp_attr->cap.max_send_sge = 0; |
| 3097 | } |
| 3098 | |
| 3099 | /* We don't support inline sends for kernel QPs (yet), and we |
| 3100 | * don't know what userspace's value should be. |
| 3101 | */ |
| 3102 | qp_attr->cap.max_inline_data = 0; |
| 3103 | |
| 3104 | qp_init_attr->cap = qp_attr->cap; |
| 3105 | |
| 3106 | qp_init_attr->create_flags = 0; |
| 3107 | if (qp->flags & MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK) |
| 3108 | qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK; |
| 3109 | |
| 3110 | qp_init_attr->sq_sig_type = qp->sq_signal_bits & MLX5_WQE_CTRL_CQ_UPDATE ? |
| 3111 | IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; |
| 3112 | |
| 3113 | out_free: |
| 3114 | kfree(outb); |
| 3115 | |
| 3116 | out: |
| 3117 | mutex_unlock(&qp->mutex); |
| 3118 | return err; |
| 3119 | } |
| 3120 | |
| 3121 | struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev, |
| 3122 | struct ib_ucontext *context, |
| 3123 | struct ib_udata *udata) |
| 3124 | { |
| 3125 | struct mlx5_ib_dev *dev = to_mdev(ibdev); |
| 3126 | struct mlx5_ib_xrcd *xrcd; |
| 3127 | int err; |
| 3128 | |
Saeed Mahameed | 938fe83 | 2015-05-28 22:28:41 +0300 | [diff] [blame] | 3129 | if (!MLX5_CAP_GEN(dev->mdev, xrc)) |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 3130 | return ERR_PTR(-ENOSYS); |
| 3131 | |
| 3132 | xrcd = kmalloc(sizeof(*xrcd), GFP_KERNEL); |
| 3133 | if (!xrcd) |
| 3134 | return ERR_PTR(-ENOMEM); |
| 3135 | |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 3136 | err = mlx5_core_xrcd_alloc(dev->mdev, &xrcd->xrcdn); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 3137 | if (err) { |
| 3138 | kfree(xrcd); |
| 3139 | return ERR_PTR(-ENOMEM); |
| 3140 | } |
| 3141 | |
| 3142 | return &xrcd->ibxrcd; |
| 3143 | } |
| 3144 | |
| 3145 | int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd) |
| 3146 | { |
| 3147 | struct mlx5_ib_dev *dev = to_mdev(xrcd->device); |
| 3148 | u32 xrcdn = to_mxrcd(xrcd)->xrcdn; |
| 3149 | int err; |
| 3150 | |
Jack Morgenstein | 9603b61 | 2014-07-28 23:30:22 +0300 | [diff] [blame] | 3151 | err = mlx5_core_xrcd_dealloc(dev->mdev, xrcdn); |
Eli Cohen | e126ba9 | 2013-07-07 17:25:49 +0300 | [diff] [blame] | 3152 | if (err) { |
| 3153 | mlx5_ib_warn(dev, "failed to dealloc xrcdn 0x%x\n", xrcdn); |
| 3154 | return err; |
| 3155 | } |
| 3156 | |
| 3157 | kfree(xrcd); |
| 3158 | |
| 3159 | return 0; |
| 3160 | } |