blob: 86d07670bfeb3a9c421cc097dce792a00fe98497 [file] [log] [blame]
Eli Cohene126ba92013-07-07 17:25:49 +03001/*
Saeed Mahameed6cf0a152015-04-02 17:07:30 +03002 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
Eli Cohene126ba92013-07-07 17:25:49 +03003 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#ifndef MLX5_IB_H
34#define MLX5_IB_H
35
36#include <linux/kernel.h>
37#include <linux/sched.h>
38#include <rdma/ib_verbs.h>
39#include <rdma/ib_smi.h>
40#include <linux/mlx5/driver.h>
41#include <linux/mlx5/cq.h>
42#include <linux/mlx5/qp.h>
43#include <linux/mlx5/srq.h>
44#include <linux/types.h>
majd@mellanox.com146d2f12016-01-14 19:13:02 +020045#include <linux/mlx5/transobj.h>
Matan Barakd2370e02016-02-29 18:05:30 +020046#include <rdma/ib_user_verbs.h>
Leon Romanovsky3085e292016-09-22 17:31:11 +030047#include <rdma/mlx5-abi.h>
Eli Cohene126ba92013-07-07 17:25:49 +030048
49#define mlx5_ib_dbg(dev, format, arg...) \
50pr_debug("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
51 __LINE__, current->pid, ##arg)
52
53#define mlx5_ib_err(dev, format, arg...) \
54pr_err("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
55 __LINE__, current->pid, ##arg)
56
57#define mlx5_ib_warn(dev, format, arg...) \
58pr_warn("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
59 __LINE__, current->pid, ##arg)
60
Matan Barakb368d7c2015-12-15 20:30:12 +020061#define field_avail(type, fld, sz) (offsetof(type, fld) + \
62 sizeof(((type *)0)->fld) <= (sz))
Haggai Abramovskycfb5e082016-01-14 19:12:57 +020063#define MLX5_IB_DEFAULT_UIDX 0xffffff
64#define MLX5_USER_ASSIGNED_UIDX_MASK __mlx5_mask(qpc, user_index)
Matan Barakb368d7c2015-12-15 20:30:12 +020065
Majd Dibbiny762f8992016-10-27 16:36:47 +030066#define MLX5_MKEY_PAGE_SHIFT_MASK __mlx5_mask(mkc, log_page_size)
67
Eli Cohene126ba92013-07-07 17:25:49 +030068enum {
69 MLX5_IB_MMAP_CMD_SHIFT = 8,
70 MLX5_IB_MMAP_CMD_MASK = 0xff,
71};
72
Eli Cohene126ba92013-07-07 17:25:49 +030073enum {
74 MLX5_RES_SCAT_DATA32_CQE = 0x1,
75 MLX5_RES_SCAT_DATA64_CQE = 0x2,
76 MLX5_REQ_SCAT_DATA32_CQE = 0x11,
77 MLX5_REQ_SCAT_DATA64_CQE = 0x22,
78};
79
80enum mlx5_ib_latency_class {
81 MLX5_IB_LATENCY_CLASS_LOW,
82 MLX5_IB_LATENCY_CLASS_MEDIUM,
83 MLX5_IB_LATENCY_CLASS_HIGH,
Eli Cohene126ba92013-07-07 17:25:49 +030084};
85
86enum mlx5_ib_mad_ifc_flags {
87 MLX5_MAD_IFC_IGNORE_MKEY = 1,
88 MLX5_MAD_IFC_IGNORE_BKEY = 2,
89 MLX5_MAD_IFC_NET_VIEW = 4,
90};
91
Leon Romanovsky051f2632015-12-20 12:16:11 +020092enum {
Eli Cohen2f5ff262017-01-03 23:55:21 +020093 MLX5_CROSS_CHANNEL_BFREG = 0,
Leon Romanovsky051f2632015-12-20 12:16:11 +020094};
95
Haggai Abramovskycfb5e082016-01-14 19:12:57 +020096enum {
97 MLX5_CQE_VERSION_V0,
98 MLX5_CQE_VERSION_V1,
99};
100
Artemy Kovalyoveb761892017-08-17 15:52:09 +0300101enum {
102 MLX5_TM_MAX_RNDV_MSG_SIZE = 64,
103 MLX5_TM_MAX_SGE = 1,
104};
105
Yishai Hadas4ed131d2017-12-24 16:31:35 +0200106enum {
107 MLX5_IB_INVALID_UAR_INDEX = BIT(31),
Yishai Hadas1ee47ab2017-12-24 16:31:36 +0200108 MLX5_IB_INVALID_BFREG = BIT(31),
Yishai Hadas4ed131d2017-12-24 16:31:35 +0200109};
110
Maor Gottlieb7c2344c2016-06-17 14:56:44 +0300111struct mlx5_ib_vma_private_data {
112 struct list_head list;
113 struct vm_area_struct *vma;
Majd Dibbinyad9a3662017-12-24 13:54:56 +0200114 /* protect vma_private_list add/del */
115 struct mutex *vma_private_list_mutex;
Maor Gottlieb7c2344c2016-06-17 14:56:44 +0300116};
117
Eli Cohene126ba92013-07-07 17:25:49 +0300118struct mlx5_ib_ucontext {
119 struct ib_ucontext ibucontext;
120 struct list_head db_page_list;
121
122 /* protect doorbell record alloc/free
123 */
124 struct mutex db_page_mutex;
Eli Cohen2f5ff262017-01-03 23:55:21 +0200125 struct mlx5_bfreg_info bfregi;
Haggai Abramovskycfb5e082016-01-14 19:12:57 +0200126 u8 cqe_version;
majd@mellanox.com146d2f12016-01-14 19:13:02 +0200127 /* Transport Domain number */
128 u32 tdn;
Maor Gottlieb7c2344c2016-06-17 14:56:44 +0300129 struct list_head vma_private_list;
Majd Dibbinyad9a3662017-12-24 13:54:56 +0200130 /* protect vma_private_list add/del */
131 struct mutex vma_private_list_mutex;
Artemy Kovalyov7d0cc6e2017-01-02 11:37:44 +0200132
133 unsigned long upd_xlt_page;
134 /* protect ODP/KSM */
135 struct mutex upd_xlt_page_mutex;
Eli Cohenb037c292017-01-03 23:55:26 +0200136 u64 lib_caps;
Eli Cohene126ba92013-07-07 17:25:49 +0300137};
138
139static inline struct mlx5_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext)
140{
141 return container_of(ibucontext, struct mlx5_ib_ucontext, ibucontext);
142}
143
144struct mlx5_ib_pd {
145 struct ib_pd ibpd;
146 u32 pdn;
Eli Cohene126ba92013-07-07 17:25:49 +0300147};
148
Maor Gottlieb038d2ef2016-01-11 10:26:07 +0200149#define MLX5_IB_FLOW_MCAST_PRIO (MLX5_BY_PASS_NUM_PRIOS - 1)
Maor Gottlieb35d190112016-03-07 18:51:47 +0200150#define MLX5_IB_FLOW_LAST_PRIO (MLX5_BY_PASS_NUM_REGULAR_PRIOS - 1)
Maor Gottlieb038d2ef2016-01-11 10:26:07 +0200151#if (MLX5_IB_FLOW_LAST_PRIO <= 0)
152#error "Invalid number of bypass priorities"
153#endif
154#define MLX5_IB_FLOW_LEFTOVERS_PRIO (MLX5_IB_FLOW_MCAST_PRIO + 1)
155
156#define MLX5_IB_NUM_FLOW_FT (MLX5_IB_FLOW_LEFTOVERS_PRIO + 1)
Maor Gottliebcc0e5d42016-08-28 14:16:34 +0300157#define MLX5_IB_NUM_SNIFFER_FTS 2
Maor Gottlieb038d2ef2016-01-11 10:26:07 +0200158struct mlx5_ib_flow_prio {
159 struct mlx5_flow_table *flow_table;
160 unsigned int refcount;
161};
162
163struct mlx5_ib_flow_handler {
164 struct list_head list;
165 struct ib_flow ibflow;
Maor Gottlieb5497adc2016-08-28 14:16:31 +0300166 struct mlx5_ib_flow_prio *prio;
Mark Bloch74491de2016-08-31 11:24:25 +0000167 struct mlx5_flow_handle *rule;
Maor Gottlieb038d2ef2016-01-11 10:26:07 +0200168};
169
170struct mlx5_ib_flow_db {
171 struct mlx5_ib_flow_prio prios[MLX5_IB_NUM_FLOW_FT];
Maor Gottliebcc0e5d42016-08-28 14:16:34 +0300172 struct mlx5_ib_flow_prio sniffer[MLX5_IB_NUM_SNIFFER_FTS];
Aviv Heller9ef9c642016-09-18 20:48:01 +0300173 struct mlx5_flow_table *lag_demux_ft;
Maor Gottlieb038d2ef2016-01-11 10:26:07 +0200174 /* Protect flow steering bypass flow tables
175 * when add/del flow rules.
176 * only single add/removal of flow steering rule could be done
177 * simultaneously.
178 */
179 struct mutex lock;
180};
181
Eli Cohene126ba92013-07-07 17:25:49 +0300182/* Use macros here so that don't have to duplicate
183 * enum ib_send_flags and enum ib_qp_type for low-level driver
184 */
185
Artemy Kovalyov31616252017-01-02 11:37:42 +0200186#define MLX5_IB_SEND_UMR_ENABLE_MR (IB_SEND_RESERVED_START << 0)
187#define MLX5_IB_SEND_UMR_DISABLE_MR (IB_SEND_RESERVED_START << 1)
188#define MLX5_IB_SEND_UMR_FAIL_IF_FREE (IB_SEND_RESERVED_START << 2)
189#define MLX5_IB_SEND_UMR_UPDATE_XLT (IB_SEND_RESERVED_START << 3)
190#define MLX5_IB_SEND_UMR_UPDATE_TRANSLATION (IB_SEND_RESERVED_START << 4)
191#define MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS IB_SEND_RESERVED_END
Noa Osherovich56e11d62016-02-29 16:46:51 +0200192
Eli Cohene126ba92013-07-07 17:25:49 +0300193#define MLX5_IB_QPT_REG_UMR IB_QPT_RESERVED1
Haggai Erand16e91d2016-02-29 15:45:05 +0200194/*
195 * IB_QPT_GSI creates the software wrapper around GSI, and MLX5_IB_QPT_HW_GSI
196 * creates the actual hardware QP.
197 */
198#define MLX5_IB_QPT_HW_GSI IB_QPT_RESERVED2
Moni Shouab4aaa1f2018-01-02 16:19:31 +0200199#define MLX5_IB_QPT_DCI IB_QPT_RESERVED3
200#define MLX5_IB_QPT_DCT IB_QPT_RESERVED4
Eli Cohene126ba92013-07-07 17:25:49 +0300201#define MLX5_IB_WR_UMR IB_WR_RESERVED1
202
Artemy Kovalyov31616252017-01-02 11:37:42 +0200203#define MLX5_IB_UMR_OCTOWORD 16
204#define MLX5_IB_UMR_XLT_ALIGNMENT 64
205
Artemy Kovalyov7d0cc6e2017-01-02 11:37:44 +0200206#define MLX5_IB_UPD_XLT_ZAP BIT(0)
207#define MLX5_IB_UPD_XLT_ENABLE BIT(1)
208#define MLX5_IB_UPD_XLT_ATOMIC BIT(2)
209#define MLX5_IB_UPD_XLT_ADDR BIT(3)
210#define MLX5_IB_UPD_XLT_PD BIT(4)
211#define MLX5_IB_UPD_XLT_ACCESS BIT(5)
Artemy Kovalyov81713d32017-01-18 16:58:11 +0200212#define MLX5_IB_UPD_XLT_INDIRECT BIT(6)
Artemy Kovalyov7d0cc6e2017-01-02 11:37:44 +0200213
Haggai Eranb11a4f92016-02-29 15:45:03 +0200214/* Private QP creation flags to be passed in ib_qp_init_attr.create_flags.
215 *
216 * These flags are intended for internal use by the mlx5_ib driver, and they
217 * rely on the range reserved for that use in the ib_qp_create_flags enum.
218 */
219
220/* Create a UD QP whose source QP number is 1 */
221static inline enum ib_qp_create_flags mlx5_ib_create_qp_sqpn_qp1(void)
222{
223 return IB_QP_CREATE_RESERVED_START;
224}
225
Eli Cohene126ba92013-07-07 17:25:49 +0300226struct wr_list {
227 u16 opcode;
228 u16 next;
229};
230
Noa Osheroviche4cc4fa2017-01-18 15:40:03 +0200231enum mlx5_ib_rq_flags {
232 MLX5_IB_RQ_CVLAN_STRIPPING = 1 << 0,
Noa Osherovichb1383aa2017-10-29 13:59:45 +0200233 MLX5_IB_RQ_PCI_WRITE_END_PADDING = 1 << 1,
Noa Osheroviche4cc4fa2017-01-18 15:40:03 +0200234};
235
Eli Cohene126ba92013-07-07 17:25:49 +0300236struct mlx5_ib_wq {
237 u64 *wrid;
238 u32 *wr_data;
239 struct wr_list *w_list;
240 unsigned *wqe_head;
241 u16 unsig_count;
242
243 /* serialize post to the work queue
244 */
245 spinlock_t lock;
246 int wqe_cnt;
247 int max_post;
248 int max_gs;
249 int offset;
250 int wqe_shift;
251 unsigned head;
252 unsigned tail;
253 u16 cur_post;
254 u16 last_poll;
255 void *qend;
256};
257
Maor Gottlieb03404e82017-05-30 10:29:13 +0300258enum mlx5_ib_wq_flags {
259 MLX5_IB_WQ_FLAGS_DELAY_DROP = 0x1,
Noa Osherovichccc87082017-10-17 18:01:13 +0300260 MLX5_IB_WQ_FLAGS_STRIDING_RQ = 0x2,
Maor Gottlieb03404e82017-05-30 10:29:13 +0300261};
262
Noa Osherovichb4f34592017-10-17 18:01:12 +0300263#define MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES 9
264#define MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES 16
265#define MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES 6
266#define MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES 13
267
Yishai Hadas79b20a62016-05-23 15:20:50 +0300268struct mlx5_ib_rwq {
269 struct ib_wq ibwq;
Yishai Hadas350d0e42016-08-28 14:58:18 +0300270 struct mlx5_core_qp core_qp;
Yishai Hadas79b20a62016-05-23 15:20:50 +0300271 u32 rq_num_pas;
272 u32 log_rq_stride;
273 u32 log_rq_size;
274 u32 rq_page_offset;
275 u32 log_page_size;
Noa Osherovichccc87082017-10-17 18:01:13 +0300276 u32 log_num_strides;
277 u32 two_byte_shift_en;
278 u32 single_stride_log_num_of_bytes;
Yishai Hadas79b20a62016-05-23 15:20:50 +0300279 struct ib_umem *umem;
280 size_t buf_size;
281 unsigned int page_shift;
282 int create_type;
283 struct mlx5_db db;
284 u32 user_index;
285 u32 wqe_count;
286 u32 wqe_shift;
287 int wq_sig;
Maor Gottlieb03404e82017-05-30 10:29:13 +0300288 u32 create_flags; /* Use enum mlx5_ib_wq_flags */
Yishai Hadas79b20a62016-05-23 15:20:50 +0300289};
290
Eli Cohene126ba92013-07-07 17:25:49 +0300291enum {
292 MLX5_QP_USER,
293 MLX5_QP_KERNEL,
294 MLX5_QP_EMPTY
295};
296
Yishai Hadas79b20a62016-05-23 15:20:50 +0300297enum {
298 MLX5_WQ_USER,
299 MLX5_WQ_KERNEL
300};
301
Yishai Hadasc5f90922016-05-23 15:20:53 +0300302struct mlx5_ib_rwq_ind_table {
303 struct ib_rwq_ind_table ib_rwq_ind_tbl;
304 u32 rqtn;
305};
306
majd@mellanox.com19098df2016-01-14 19:13:03 +0200307struct mlx5_ib_ubuffer {
308 struct ib_umem *umem;
309 int buf_size;
310 u64 buf_addr;
311};
312
313struct mlx5_ib_qp_base {
314 struct mlx5_ib_qp *container_mibqp;
315 struct mlx5_core_qp mqp;
316 struct mlx5_ib_ubuffer ubuffer;
317};
318
319struct mlx5_ib_qp_trans {
320 struct mlx5_ib_qp_base base;
321 u16 xrcdn;
322 u8 alt_port;
323 u8 atomic_rd_en;
324 u8 resp_depth;
325};
326
Yishai Hadas28d61372016-05-23 15:20:56 +0300327struct mlx5_ib_rss_qp {
328 u32 tirn;
329};
330
Maor Gottlieb038d2ef2016-01-11 10:26:07 +0200331struct mlx5_ib_rq {
majd@mellanox.com0fb2ed62016-01-14 19:13:04 +0200332 struct mlx5_ib_qp_base base;
333 struct mlx5_ib_wq *rq;
334 struct mlx5_ib_ubuffer ubuffer;
335 struct mlx5_db *doorbell;
Maor Gottlieb038d2ef2016-01-11 10:26:07 +0200336 u32 tirn;
majd@mellanox.com0fb2ed62016-01-14 19:13:04 +0200337 u8 state;
Noa Osheroviche4cc4fa2017-01-18 15:40:03 +0200338 u32 flags;
majd@mellanox.com0fb2ed62016-01-14 19:13:04 +0200339};
340
341struct mlx5_ib_sq {
342 struct mlx5_ib_qp_base base;
343 struct mlx5_ib_wq *sq;
344 struct mlx5_ib_ubuffer ubuffer;
345 struct mlx5_db *doorbell;
Mark Blochb96c9dd2018-01-29 10:40:37 +0000346 struct mlx5_flow_handle *flow_rule;
majd@mellanox.com0fb2ed62016-01-14 19:13:04 +0200347 u32 tisn;
348 u8 state;
Maor Gottlieb038d2ef2016-01-11 10:26:07 +0200349};
350
351struct mlx5_ib_raw_packet_qp {
majd@mellanox.com0fb2ed62016-01-14 19:13:04 +0200352 struct mlx5_ib_sq sq;
Maor Gottlieb038d2ef2016-01-11 10:26:07 +0200353 struct mlx5_ib_rq rq;
354};
355
Eli Cohen5fe9dec2017-01-03 23:55:25 +0200356struct mlx5_bf {
357 int buf_size;
358 unsigned long offset;
359 struct mlx5_sq_bfreg *bfreg;
360};
361
Moni Shouab4aaa1f2018-01-02 16:19:31 +0200362struct mlx5_ib_dct {
363 struct mlx5_core_dct mdct;
364 u32 *in;
365};
366
Eli Cohene126ba92013-07-07 17:25:49 +0300367struct mlx5_ib_qp {
368 struct ib_qp ibqp;
Maor Gottlieb038d2ef2016-01-11 10:26:07 +0200369 union {
majd@mellanox.com0fb2ed62016-01-14 19:13:04 +0200370 struct mlx5_ib_qp_trans trans_qp;
371 struct mlx5_ib_raw_packet_qp raw_packet_qp;
Yishai Hadas28d61372016-05-23 15:20:56 +0300372 struct mlx5_ib_rss_qp rss_qp;
Moni Shouab4aaa1f2018-01-02 16:19:31 +0200373 struct mlx5_ib_dct dct;
Maor Gottlieb038d2ef2016-01-11 10:26:07 +0200374 };
Yonatan Cohen388ca8b2018-01-02 16:08:06 +0200375 struct mlx5_frag_buf buf;
Eli Cohene126ba92013-07-07 17:25:49 +0300376
377 struct mlx5_db db;
378 struct mlx5_ib_wq rq;
379
Eli Cohene126ba92013-07-07 17:25:49 +0300380 u8 sq_signal_bits;
Max Gurtovoy6e8484c2017-05-28 10:53:11 +0300381 u8 next_fence;
Eli Cohene126ba92013-07-07 17:25:49 +0300382 struct mlx5_ib_wq sq;
383
Eli Cohene126ba92013-07-07 17:25:49 +0300384 /* serialize qp state modifications
385 */
386 struct mutex mutex;
Eli Cohene126ba92013-07-07 17:25:49 +0300387 u32 flags;
388 u8 port;
Eli Cohene126ba92013-07-07 17:25:49 +0300389 u8 state;
Eli Cohene126ba92013-07-07 17:25:49 +0300390 int wq_sig;
391 int scat_cqe;
392 int max_inline_data;
Eli Cohen5fe9dec2017-01-03 23:55:25 +0200393 struct mlx5_bf bf;
Eli Cohene126ba92013-07-07 17:25:49 +0300394 int has_rq;
395
396 /* only for user space QPs. For kernel
397 * we have it from the bf object
398 */
Eli Cohen2f5ff262017-01-03 23:55:21 +0200399 int bfregn;
Eli Cohene126ba92013-07-07 17:25:49 +0300400
401 int create_type;
Sagi Grimberge1e66cc2014-02-23 14:19:07 +0200402
403 /* Store signature errors */
404 bool signature_en;
Haggai Eran6aec21f2014-12-11 17:04:23 +0200405
Maor Gottlieb89ea94a72016-06-17 15:01:38 +0300406 struct list_head qps_list;
407 struct list_head cq_recv_list;
408 struct list_head cq_send_list;
Bodong Wang7d29f342016-12-01 13:43:16 +0200409 u32 rate_limit;
Yishai Hadasc2e53b22017-06-08 16:15:08 +0300410 u32 underlay_qpn;
Maor Gottliebf95ef6c2017-10-19 08:25:55 +0300411 bool tunnel_offload_en;
Moni Shouab4aaa1f2018-01-02 16:19:31 +0200412 /* storage for qp sub type when core qp type is IB_QPT_DRIVER */
413 enum ib_qp_type qp_sub_type;
Eli Cohene126ba92013-07-07 17:25:49 +0300414};
415
416struct mlx5_ib_cq_buf {
Yonatan Cohen388ca8b2018-01-02 16:08:06 +0200417 struct mlx5_frag_buf_ctrl fbc;
Eli Cohene126ba92013-07-07 17:25:49 +0300418 struct ib_umem *umem;
419 int cqe_size;
Eli Cohenbde51582014-01-14 17:45:18 +0200420 int nent;
Eli Cohene126ba92013-07-07 17:25:49 +0300421};
422
423enum mlx5_ib_qp_flags {
Erez Shitritf0313962016-02-21 16:27:17 +0200424 MLX5_IB_QP_LSO = IB_QP_CREATE_IPOIB_UD_LSO,
425 MLX5_IB_QP_BLOCK_MULTICAST_LOOPBACK = IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
426 MLX5_IB_QP_CROSS_CHANNEL = IB_QP_CREATE_CROSS_CHANNEL,
427 MLX5_IB_QP_MANAGED_SEND = IB_QP_CREATE_MANAGED_SEND,
428 MLX5_IB_QP_MANAGED_RECV = IB_QP_CREATE_MANAGED_RECV,
429 MLX5_IB_QP_SIGNATURE_HANDLING = 1 << 5,
Haggai Eranb11a4f92016-02-29 15:45:03 +0200430 /* QP uses 1 as its source QP number */
431 MLX5_IB_QP_SQPN_QP1 = 1 << 6,
Majd Dibbiny358e42e2016-04-17 17:19:37 +0300432 MLX5_IB_QP_CAP_SCATTER_FCS = 1 << 7,
Yishai Hadasd9f88e52016-08-28 10:58:37 +0300433 MLX5_IB_QP_RSS = 1 << 8,
Noa Osheroviche4cc4fa2017-01-18 15:40:03 +0200434 MLX5_IB_QP_CVLAN_STRIPPING = 1 << 9,
Yishai Hadasc2e53b22017-06-08 16:15:08 +0300435 MLX5_IB_QP_UNDERLAY = 1 << 10,
Noa Osherovichb1383aa2017-10-29 13:59:45 +0200436 MLX5_IB_QP_PCI_WRITE_END_PADDING = 1 << 11,
Maor Gottliebf95ef6c2017-10-19 08:25:55 +0300437 MLX5_IB_QP_TUNNEL_OFFLOAD = 1 << 12,
Eli Cohene126ba92013-07-07 17:25:49 +0300438};
439
Haggai Eran968e78d2014-12-11 17:04:11 +0200440struct mlx5_umr_wr {
Christoph Hellwige622f2f2015-10-08 09:16:33 +0100441 struct ib_send_wr wr;
Artemy Kovalyov31616252017-01-02 11:37:42 +0200442 u64 virt_addr;
443 u64 offset;
Haggai Eran968e78d2014-12-11 17:04:11 +0200444 struct ib_pd *pd;
445 unsigned int page_shift;
Artemy Kovalyov31616252017-01-02 11:37:42 +0200446 unsigned int xlt_size;
Maor Gottliebb216af42016-11-27 15:18:22 +0200447 u64 length;
Haggai Eran968e78d2014-12-11 17:04:11 +0200448 int access_flags;
449 u32 mkey;
450};
451
Christoph Hellwige622f2f2015-10-08 09:16:33 +0100452static inline struct mlx5_umr_wr *umr_wr(struct ib_send_wr *wr)
453{
454 return container_of(wr, struct mlx5_umr_wr, wr);
455}
456
Eli Cohene126ba92013-07-07 17:25:49 +0300457struct mlx5_shared_mr_info {
458 int mr_id;
459 struct ib_umem *umem;
460};
461
Guy Levi7a0c8f42017-10-19 08:25:53 +0300462enum mlx5_ib_cq_pr_flags {
463 MLX5_IB_CQ_PR_FLAGS_CQE_128_PAD = 1 << 0,
464};
465
Eli Cohene126ba92013-07-07 17:25:49 +0300466struct mlx5_ib_cq {
467 struct ib_cq ibcq;
468 struct mlx5_core_cq mcq;
469 struct mlx5_ib_cq_buf buf;
470 struct mlx5_db db;
471
472 /* serialize access to the CQ
473 */
474 spinlock_t lock;
475
476 /* protect resize cq
477 */
478 struct mutex resize_mutex;
Eli Cohenbde51582014-01-14 17:45:18 +0200479 struct mlx5_ib_cq_buf *resize_buf;
Eli Cohene126ba92013-07-07 17:25:49 +0300480 struct ib_umem *resize_umem;
481 int cqe_size;
Maor Gottlieb89ea94a72016-06-17 15:01:38 +0300482 struct list_head list_send_qp;
483 struct list_head list_recv_qp;
Leon Romanovsky051f2632015-12-20 12:16:11 +0200484 u32 create_flags;
Haggai Eran25361e02016-02-29 15:45:08 +0200485 struct list_head wc_list;
486 enum ib_cq_notify_flags notify_flags;
487 struct work_struct notify_work;
Guy Levi7a0c8f42017-10-19 08:25:53 +0300488 u16 private_flags; /* Use mlx5_ib_cq_pr_flags */
Haggai Eran25361e02016-02-29 15:45:08 +0200489};
490
491struct mlx5_ib_wc {
492 struct ib_wc wc;
493 struct list_head list;
Eli Cohene126ba92013-07-07 17:25:49 +0300494};
495
496struct mlx5_ib_srq {
497 struct ib_srq ibsrq;
498 struct mlx5_core_srq msrq;
Yonatan Cohen388ca8b2018-01-02 16:08:06 +0200499 struct mlx5_frag_buf buf;
Eli Cohene126ba92013-07-07 17:25:49 +0300500 struct mlx5_db db;
501 u64 *wrid;
502 /* protect SRQ hanlding
503 */
504 spinlock_t lock;
505 int head;
506 int tail;
507 u16 wqe_ctr;
508 struct ib_umem *umem;
509 /* serialize arming a SRQ
510 */
511 struct mutex mutex;
512 int wq_sig;
513};
514
515struct mlx5_ib_xrcd {
516 struct ib_xrcd ibxrcd;
517 u32 xrcdn;
518};
519
Haggai Erancc149f752014-12-11 17:04:21 +0200520enum mlx5_ib_mtt_access_flags {
521 MLX5_IB_MTT_READ = (1 << 0),
522 MLX5_IB_MTT_WRITE = (1 << 1),
523};
524
525#define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)
526
Eli Cohene126ba92013-07-07 17:25:49 +0300527struct mlx5_ib_mr {
528 struct ib_mr ibmr;
Sagi Grimberg8a187ee2015-10-13 19:11:26 +0300529 void *descs;
530 dma_addr_t desc_map;
531 int ndescs;
532 int max_descs;
533 int desc_size;
Sagi Grimbergb005d312016-02-29 19:07:33 +0200534 int access_mode;
Matan Baraka606b0f2016-02-29 18:05:28 +0200535 struct mlx5_core_mkey mmkey;
Eli Cohene126ba92013-07-07 17:25:49 +0300536 struct ib_umem *umem;
537 struct mlx5_shared_mr_info *smr_info;
538 struct list_head list;
539 int order;
Ilya Lesokhin8b7ff7f2017-08-17 15:52:29 +0300540 bool allocated_from_cache;
Eli Cohene126ba92013-07-07 17:25:49 +0300541 int npages;
Eli Cohen746b5582013-10-23 09:53:14 +0300542 struct mlx5_ib_dev *dev;
Saeed Mahameedec22eb52016-07-16 06:28:36 +0300543 u32 out[MLX5_ST_SZ_DW(create_mkey_out)];
Sagi Grimberg3121e3c2014-02-23 14:19:06 +0200544 struct mlx5_core_sig_ctx *sig;
Haggai Eranb4cfe442014-12-11 17:04:26 +0200545 int live;
Sagi Grimberg8a187ee2015-10-13 19:11:26 +0300546 void *descs_alloc;
Noa Osherovich56e11d62016-02-29 16:46:51 +0200547 int access_flags; /* Needed for rereg MR */
Artemy Kovalyov81713d32017-01-18 16:58:11 +0200548
549 struct mlx5_ib_mr *parent;
550 atomic_t num_leaf_free;
551 wait_queue_head_t q_leaf_free;
Eli Cohene126ba92013-07-07 17:25:49 +0300552};
553
Matan Barakd2370e02016-02-29 18:05:30 +0200554struct mlx5_ib_mw {
555 struct ib_mw ibmw;
556 struct mlx5_core_mkey mmkey;
Artemy Kovalyovdb570d72017-04-05 09:23:59 +0300557 int ndescs;
Eli Cohene126ba92013-07-07 17:25:49 +0300558};
559
Shachar Raindela74d2412014-05-22 14:50:12 +0300560struct mlx5_ib_umr_context {
Christoph Hellwigadd08d72016-03-03 09:38:22 +0100561 struct ib_cqe cqe;
Shachar Raindela74d2412014-05-22 14:50:12 +0300562 enum ib_wc_status status;
563 struct completion done;
564};
565
Eli Cohene126ba92013-07-07 17:25:49 +0300566struct umr_common {
567 struct ib_pd *pd;
568 struct ib_cq *cq;
569 struct ib_qp *qp;
Eli Cohene126ba92013-07-07 17:25:49 +0300570 /* control access to UMR QP
571 */
572 struct semaphore sem;
573};
574
575enum {
576 MLX5_FMR_INVALID,
577 MLX5_FMR_VALID,
578 MLX5_FMR_BUSY,
579};
580
Eli Cohene126ba92013-07-07 17:25:49 +0300581struct mlx5_cache_ent {
582 struct list_head head;
583 /* sync access to the cahce entry
584 */
585 spinlock_t lock;
586
587
588 struct dentry *dir;
589 char name[4];
590 u32 order;
Artemy Kovalyov49780d42017-01-18 16:58:10 +0200591 u32 xlt;
592 u32 access_mode;
593 u32 page;
594
Eli Cohene126ba92013-07-07 17:25:49 +0300595 u32 size;
596 u32 cur;
597 u32 miss;
598 u32 limit;
599
600 struct dentry *fsize;
601 struct dentry *fcur;
602 struct dentry *fmiss;
603 struct dentry *flimit;
604
605 struct mlx5_ib_dev *dev;
606 struct work_struct work;
607 struct delayed_work dwork;
Eli Cohen746b5582013-10-23 09:53:14 +0300608 int pending;
Artemy Kovalyov49780d42017-01-18 16:58:10 +0200609 struct completion compl;
Eli Cohene126ba92013-07-07 17:25:49 +0300610};
611
612struct mlx5_mr_cache {
613 struct workqueue_struct *wq;
614 struct mlx5_cache_ent ent[MAX_MR_CACHE_ENTRIES];
615 int stopped;
616 struct dentry *root;
617 unsigned long last_add;
618};
619
Haggai Erand16e91d2016-02-29 15:45:05 +0200620struct mlx5_ib_gsi_qp;
621
622struct mlx5_ib_port_resources {
Haggai Eran7722f472016-02-29 15:45:07 +0200623 struct mlx5_ib_resources *devr;
Haggai Erand16e91d2016-02-29 15:45:05 +0200624 struct mlx5_ib_gsi_qp *gsi;
Haggai Eran7722f472016-02-29 15:45:07 +0200625 struct work_struct pkey_change_work;
Haggai Erand16e91d2016-02-29 15:45:05 +0200626};
627
Eli Cohene126ba92013-07-07 17:25:49 +0300628struct mlx5_ib_resources {
629 struct ib_cq *c0;
630 struct ib_xrcd *x0;
631 struct ib_xrcd *x1;
632 struct ib_pd *p0;
633 struct ib_srq *s0;
Haggai Abramonvsky4aa17b22015-06-04 19:30:48 +0300634 struct ib_srq *s1;
Haggai Erand16e91d2016-02-29 15:45:05 +0200635 struct mlx5_ib_port_resources ports[2];
636 /* Protects changes to the port resources */
637 struct mutex mutex;
Eli Cohene126ba92013-07-07 17:25:49 +0300638};
639
Parav Pandite1f24a72017-04-16 07:29:29 +0300640struct mlx5_ib_counters {
Kamal Heib7c16f472017-01-18 15:25:09 +0200641 const char **names;
642 size_t *offsets;
Parav Pandite1f24a72017-04-16 07:29:29 +0300643 u32 num_q_counters;
644 u32 num_cong_counters;
Kamal Heib7c16f472017-01-18 15:25:09 +0200645 u16 set_id;
Daniel Jurgensaac44922018-01-04 17:25:40 +0200646 bool set_id_valid;
Kamal Heib7c16f472017-01-18 15:25:09 +0200647};
648
Daniel Jurgens32f69e42018-01-04 17:25:36 +0200649struct mlx5_ib_multiport_info;
650
651struct mlx5_ib_multiport {
652 struct mlx5_ib_multiport_info *mpi;
653 /* To be held when accessing the multiport info */
654 spinlock_t mpi_lock;
655};
656
Mark Bloch0837e862016-06-17 15:10:55 +0300657struct mlx5_ib_port {
Parav Pandite1f24a72017-04-16 07:29:29 +0300658 struct mlx5_ib_counters cnts;
Daniel Jurgens32f69e42018-01-04 17:25:36 +0200659 struct mlx5_ib_multiport mp;
Parav Pandita9e546e2018-01-04 17:25:39 +0200660 struct mlx5_ib_dbg_cc_params *dbg_cc_params;
Mark Bloch0837e862016-06-17 15:10:55 +0300661};
662
Achiad Shochatfc24fc52015-12-23 18:47:17 +0200663struct mlx5_roce {
664 /* Protect mlx5_ib_get_netdev from invoking dev_hold() with a NULL
665 * netdev pointer
666 */
667 rwlock_t netdev_lock;
668 struct net_device *netdev;
669 struct notifier_block nb;
Aviv Heller13eab212016-09-18 20:48:04 +0300670 atomic_t next_port;
Moni Shouafd65f1b2017-05-30 09:56:05 +0300671 enum ib_port_state last_port_state;
Daniel Jurgens7fd8aef2018-01-04 17:25:35 +0200672 struct mlx5_ib_dev *dev;
673 u8 native_port_num;
Achiad Shochatfc24fc52015-12-23 18:47:17 +0200674};
675
Parav Pandit4a2da0b2017-05-30 10:05:15 +0300676struct mlx5_ib_dbg_param {
677 int offset;
678 struct mlx5_ib_dev *dev;
679 struct dentry *dentry;
Parav Pandita9e546e2018-01-04 17:25:39 +0200680 u8 port_num;
Parav Pandit4a2da0b2017-05-30 10:05:15 +0300681};
682
683enum mlx5_ib_dbg_cc_types {
684 MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE,
685 MLX5_IB_DBG_CC_RP_CLAMP_TGT_RATE_ATI,
686 MLX5_IB_DBG_CC_RP_TIME_RESET,
687 MLX5_IB_DBG_CC_RP_BYTE_RESET,
688 MLX5_IB_DBG_CC_RP_THRESHOLD,
689 MLX5_IB_DBG_CC_RP_AI_RATE,
690 MLX5_IB_DBG_CC_RP_HAI_RATE,
691 MLX5_IB_DBG_CC_RP_MIN_DEC_FAC,
692 MLX5_IB_DBG_CC_RP_MIN_RATE,
693 MLX5_IB_DBG_CC_RP_RATE_TO_SET_ON_FIRST_CNP,
694 MLX5_IB_DBG_CC_RP_DCE_TCP_G,
695 MLX5_IB_DBG_CC_RP_DCE_TCP_RTT,
696 MLX5_IB_DBG_CC_RP_RATE_REDUCE_MONITOR_PERIOD,
697 MLX5_IB_DBG_CC_RP_INITIAL_ALPHA_VALUE,
698 MLX5_IB_DBG_CC_RP_GD,
699 MLX5_IB_DBG_CC_NP_CNP_DSCP,
700 MLX5_IB_DBG_CC_NP_CNP_PRIO_MODE,
701 MLX5_IB_DBG_CC_NP_CNP_PRIO,
702 MLX5_IB_DBG_CC_MAX,
703};
704
705struct mlx5_ib_dbg_cc_params {
706 struct dentry *root;
707 struct mlx5_ib_dbg_param params[MLX5_IB_DBG_CC_MAX];
708};
709
Maor Gottlieb03404e82017-05-30 10:29:13 +0300710enum {
711 MLX5_MAX_DELAY_DROP_TIMEOUT_MS = 100,
712};
713
Maor Gottliebfe248c32017-05-30 10:29:14 +0300714struct mlx5_ib_dbg_delay_drop {
715 struct dentry *dir_debugfs;
716 struct dentry *rqs_cnt_debugfs;
717 struct dentry *events_cnt_debugfs;
718 struct dentry *timeout_debugfs;
719};
720
Maor Gottlieb03404e82017-05-30 10:29:13 +0300721struct mlx5_ib_delay_drop {
722 struct mlx5_ib_dev *dev;
723 struct work_struct delay_drop_work;
724 /* serialize setting of delay drop */
725 struct mutex lock;
726 u32 timeout;
727 bool activate;
Maor Gottliebfe248c32017-05-30 10:29:14 +0300728 atomic_t events_cnt;
729 atomic_t rqs_cnt;
730 struct mlx5_ib_dbg_delay_drop *dbg;
Maor Gottlieb03404e82017-05-30 10:29:13 +0300731};
732
Mark Bloch16c19752018-01-01 13:06:58 +0200733enum mlx5_ib_stages {
734 MLX5_IB_STAGE_INIT,
Mark Bloch9a4ca382018-01-16 14:42:35 +0000735 MLX5_IB_STAGE_FLOW_DB,
Mark Bloch16c19752018-01-01 13:06:58 +0200736 MLX5_IB_STAGE_CAPS,
Mark Bloch8e6efa32017-11-06 12:22:13 +0000737 MLX5_IB_STAGE_NON_DEFAULT_CB,
Mark Bloch16c19752018-01-01 13:06:58 +0200738 MLX5_IB_STAGE_ROCE,
739 MLX5_IB_STAGE_DEVICE_RESOURCES,
740 MLX5_IB_STAGE_ODP,
741 MLX5_IB_STAGE_COUNTERS,
742 MLX5_IB_STAGE_CONG_DEBUGFS,
743 MLX5_IB_STAGE_UAR,
744 MLX5_IB_STAGE_BFREG,
745 MLX5_IB_STAGE_IB_REG,
746 MLX5_IB_STAGE_UMR_RESOURCES,
747 MLX5_IB_STAGE_DELAY_DROP,
748 MLX5_IB_STAGE_CLASS_ATTR,
Mark Blochfc385b7a2018-01-16 14:34:48 +0000749 MLX5_IB_STAGE_REP_REG,
Mark Bloch16c19752018-01-01 13:06:58 +0200750 MLX5_IB_STAGE_MAX,
751};
752
753struct mlx5_ib_stage {
754 int (*init)(struct mlx5_ib_dev *dev);
755 void (*cleanup)(struct mlx5_ib_dev *dev);
756};
757
758#define STAGE_CREATE(_stage, _init, _cleanup) \
759 .stage[_stage] = {.init = _init, .cleanup = _cleanup}
760
761struct mlx5_ib_profile {
762 struct mlx5_ib_stage stage[MLX5_IB_STAGE_MAX];
763};
764
Daniel Jurgens32f69e42018-01-04 17:25:36 +0200765struct mlx5_ib_multiport_info {
766 struct list_head list;
767 struct mlx5_ib_dev *ibdev;
768 struct mlx5_core_dev *mdev;
769 struct completion unref_comp;
770 u64 sys_image_guid;
771 u32 mdev_refcnt;
772 bool is_master;
773 bool unaffiliate;
774};
775
Eli Cohene126ba92013-07-07 17:25:49 +0300776struct mlx5_ib_dev {
777 struct ib_device ib_dev;
Jack Morgenstein9603b612014-07-28 23:30:22 +0300778 struct mlx5_core_dev *mdev;
Daniel Jurgens7fd8aef2018-01-04 17:25:35 +0200779 struct mlx5_roce roce[MLX5_MAX_PORTS];
Eli Cohene126ba92013-07-07 17:25:49 +0300780 int num_ports;
Eli Cohene126ba92013-07-07 17:25:49 +0300781 /* serialize update of capability mask
782 */
783 struct mutex cap_mask_mutex;
784 bool ib_active;
785 struct umr_common umrc;
786 /* sync used page count stats
787 */
Eli Cohene126ba92013-07-07 17:25:49 +0300788 struct mlx5_ib_resources devr;
789 struct mlx5_mr_cache cache;
Eli Cohen746b5582013-10-23 09:53:14 +0300790 struct timer_list delay_timer;
Moshe Lazer6bc1a652016-10-27 16:36:42 +0300791 /* Prevents soft lock on massive reg MRs */
792 struct mutex slow_path_mutex;
Eli Cohen746b5582013-10-23 09:53:14 +0300793 int fill_delay;
Haggai Eran8cdd3122014-12-11 17:04:20 +0200794#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
795 struct ib_odp_caps odp_caps;
Artemy Kovalyovc438fde2017-01-02 11:37:43 +0200796 u64 odp_max_size;
Haggai Eran6aec21f2014-12-11 17:04:23 +0200797 /*
798 * Sleepable RCU that prevents destruction of MRs while they are still
799 * being used by a page fault handler.
800 */
801 struct srcu_struct mr_srcu;
Artemy Kovalyov81713d32017-01-18 16:58:11 +0200802 u32 null_mkey;
Haggai Eran8cdd3122014-12-11 17:04:20 +0200803#endif
Mark Bloch9a4ca382018-01-16 14:42:35 +0000804 struct mlx5_ib_flow_db *flow_db;
Maor Gottlieb89ea94a72016-06-17 15:01:38 +0300805 /* protect resources needed as part of reset flow */
806 spinlock_t reset_flow_resource_lock;
807 struct list_head qp_list;
Mark Bloch0837e862016-06-17 15:10:55 +0300808 /* Array with num_ports elements */
809 struct mlx5_ib_port *port;
Huy Nguyenc85023e2017-05-30 09:42:54 +0300810 struct mlx5_sq_bfreg bfreg;
811 struct mlx5_sq_bfreg fp_bfreg;
Maor Gottlieb03404e82017-05-30 10:29:13 +0300812 struct mlx5_ib_delay_drop delay_drop;
Mark Bloch16c19752018-01-01 13:06:58 +0200813 const struct mlx5_ib_profile *profile;
Mark Blochfc385b7a2018-01-16 14:34:48 +0000814 struct mlx5_eswitch_rep *rep;
Huy Nguyenc85023e2017-05-30 09:42:54 +0300815
816 /* protect the user_td */
817 struct mutex lb_mutex;
818 u32 user_td;
819 u8 umr_fence;
Daniel Jurgens32f69e42018-01-04 17:25:36 +0200820 struct list_head ib_dev_list;
821 u64 sys_image_guid;
Eli Cohene126ba92013-07-07 17:25:49 +0300822};
823
824static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq)
825{
826 return container_of(mcq, struct mlx5_ib_cq, mcq);
827}
828
829static inline struct mlx5_ib_xrcd *to_mxrcd(struct ib_xrcd *ibxrcd)
830{
831 return container_of(ibxrcd, struct mlx5_ib_xrcd, ibxrcd);
832}
833
834static inline struct mlx5_ib_dev *to_mdev(struct ib_device *ibdev)
835{
836 return container_of(ibdev, struct mlx5_ib_dev, ib_dev);
837}
838
Eli Cohene126ba92013-07-07 17:25:49 +0300839static inline struct mlx5_ib_cq *to_mcq(struct ib_cq *ibcq)
840{
841 return container_of(ibcq, struct mlx5_ib_cq, ibcq);
842}
843
844static inline struct mlx5_ib_qp *to_mibqp(struct mlx5_core_qp *mqp)
845{
majd@mellanox.com19098df2016-01-14 19:13:03 +0200846 return container_of(mqp, struct mlx5_ib_qp_base, mqp)->container_mibqp;
Eli Cohene126ba92013-07-07 17:25:49 +0300847}
848
Yishai Hadas350d0e42016-08-28 14:58:18 +0300849static inline struct mlx5_ib_rwq *to_mibrwq(struct mlx5_core_qp *core_qp)
850{
851 return container_of(core_qp, struct mlx5_ib_rwq, core_qp);
852}
853
Matan Baraka606b0f2016-02-29 18:05:28 +0200854static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mkey *mmkey)
Sagi Grimbergd5436ba2014-02-23 14:19:12 +0200855{
Matan Baraka606b0f2016-02-29 18:05:28 +0200856 return container_of(mmkey, struct mlx5_ib_mr, mmkey);
Sagi Grimbergd5436ba2014-02-23 14:19:12 +0200857}
858
Eli Cohene126ba92013-07-07 17:25:49 +0300859static inline struct mlx5_ib_pd *to_mpd(struct ib_pd *ibpd)
860{
861 return container_of(ibpd, struct mlx5_ib_pd, ibpd);
862}
863
864static inline struct mlx5_ib_srq *to_msrq(struct ib_srq *ibsrq)
865{
866 return container_of(ibsrq, struct mlx5_ib_srq, ibsrq);
867}
868
869static inline struct mlx5_ib_qp *to_mqp(struct ib_qp *ibqp)
870{
871 return container_of(ibqp, struct mlx5_ib_qp, ibqp);
872}
873
Yishai Hadas79b20a62016-05-23 15:20:50 +0300874static inline struct mlx5_ib_rwq *to_mrwq(struct ib_wq *ibwq)
875{
876 return container_of(ibwq, struct mlx5_ib_rwq, ibwq);
877}
878
Yishai Hadasc5f90922016-05-23 15:20:53 +0300879static inline struct mlx5_ib_rwq_ind_table *to_mrwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
880{
881 return container_of(ib_rwq_ind_tbl, struct mlx5_ib_rwq_ind_table, ib_rwq_ind_tbl);
882}
883
Eli Cohene126ba92013-07-07 17:25:49 +0300884static inline struct mlx5_ib_srq *to_mibsrq(struct mlx5_core_srq *msrq)
885{
886 return container_of(msrq, struct mlx5_ib_srq, msrq);
887}
888
889static inline struct mlx5_ib_mr *to_mmr(struct ib_mr *ibmr)
890{
891 return container_of(ibmr, struct mlx5_ib_mr, ibmr);
892}
893
Matan Barakd2370e02016-02-29 18:05:30 +0200894static inline struct mlx5_ib_mw *to_mmw(struct ib_mw *ibmw)
895{
896 return container_of(ibmw, struct mlx5_ib_mw, ibmw);
897}
898
Eli Cohene126ba92013-07-07 17:25:49 +0300899int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,
900 struct mlx5_db *db);
901void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db);
902void __mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
903void mlx5_ib_cq_clean(struct mlx5_ib_cq *cq, u32 qpn, struct mlx5_ib_srq *srq);
904void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index);
905int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey, int ignore_bkey,
Ira Weinya97e2d82015-05-31 17:15:30 -0400906 u8 port, const struct ib_wc *in_wc, const struct ib_grh *in_grh,
907 const void *in_mad, void *response_mad);
Dasaratharaman Chandramouli90898852017-04-29 14:41:18 -0400908struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
Moni Shoua477864c2016-11-23 08:23:24 +0200909 struct ib_udata *udata);
Dasaratharaman Chandramouli90898852017-04-29 14:41:18 -0400910int mlx5_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
Eli Cohene126ba92013-07-07 17:25:49 +0300911int mlx5_ib_destroy_ah(struct ib_ah *ah);
912struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
913 struct ib_srq_init_attr *init_attr,
914 struct ib_udata *udata);
915int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
916 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
917int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr);
918int mlx5_ib_destroy_srq(struct ib_srq *srq);
919int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
920 struct ib_recv_wr **bad_wr);
921struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
922 struct ib_qp_init_attr *init_attr,
923 struct ib_udata *udata);
924int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
925 int attr_mask, struct ib_udata *udata);
926int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
927 struct ib_qp_init_attr *qp_init_attr);
928int mlx5_ib_destroy_qp(struct ib_qp *qp);
929int mlx5_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
930 struct ib_send_wr **bad_wr);
931int mlx5_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
932 struct ib_recv_wr **bad_wr);
933void *mlx5_get_send_wqe(struct mlx5_ib_qp *qp, int n);
Haggai Eranc1395a22014-12-11 17:04:14 +0200934int mlx5_ib_read_user_wqe(struct mlx5_ib_qp *qp, int send, int wqe_index,
majd@mellanox.com19098df2016-01-14 19:13:03 +0200935 void *buffer, u32 length,
936 struct mlx5_ib_qp_base *base);
Matan Barakbcf4c1e2015-06-11 16:35:20 +0300937struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
938 const struct ib_cq_init_attr *attr,
939 struct ib_ucontext *context,
Eli Cohene126ba92013-07-07 17:25:49 +0300940 struct ib_udata *udata);
941int mlx5_ib_destroy_cq(struct ib_cq *cq);
942int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
943int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
944int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
945int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);
946struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc);
947struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
948 u64 virt_addr, int access_flags,
949 struct ib_udata *udata);
Matan Barakd2370e02016-02-29 18:05:30 +0200950struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
951 struct ib_udata *udata);
952int mlx5_ib_dealloc_mw(struct ib_mw *mw);
Artemy Kovalyov7d0cc6e2017-01-02 11:37:44 +0200953int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
954 int page_shift, int flags);
Artemy Kovalyov81713d32017-01-18 16:58:11 +0200955struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,
956 int access_flags);
957void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *mr);
Noa Osherovich56e11d62016-02-29 16:46:51 +0200958int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
959 u64 length, u64 virt_addr, int access_flags,
960 struct ib_pd *pd, struct ib_udata *udata);
Eli Cohene126ba92013-07-07 17:25:49 +0300961int mlx5_ib_dereg_mr(struct ib_mr *ibmr);
Sagi Grimberg9bee1782015-07-30 10:32:35 +0300962struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
963 enum ib_mr_type mr_type,
964 u32 max_num_sg);
Christoph Hellwigff2ba992016-05-03 18:01:04 +0200965int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
Bart Van Assche9aa8b322016-05-12 10:49:15 -0700966 unsigned int *sg_offset);
Eli Cohene126ba92013-07-07 17:25:49 +0300967int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
Ira Weinya97e2d82015-05-31 17:15:30 -0400968 const struct ib_wc *in_wc, const struct ib_grh *in_grh,
Ira Weiny4cd7c942015-06-06 14:38:31 -0400969 const struct ib_mad_hdr *in, size_t in_mad_size,
970 struct ib_mad_hdr *out, size_t *out_mad_size,
971 u16 *out_mad_pkey_index);
Eli Cohene126ba92013-07-07 17:25:49 +0300972struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
973 struct ib_ucontext *context,
974 struct ib_udata *udata);
975int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd);
Eli Cohene126ba92013-07-07 17:25:49 +0300976int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset);
977int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);
Majd Dibbiny1b5daf12015-06-04 19:30:46 +0300978int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,
979 struct ib_smp *out_mad);
980int mlx5_query_mad_ifc_system_image_guid(struct ib_device *ibdev,
981 __be64 *sys_image_guid);
982int mlx5_query_mad_ifc_max_pkeys(struct ib_device *ibdev,
983 u16 *max_pkeys);
984int mlx5_query_mad_ifc_vendor_id(struct ib_device *ibdev,
985 u32 *vendor_id);
986int mlx5_query_mad_ifc_node_desc(struct mlx5_ib_dev *dev, char *node_desc);
987int mlx5_query_mad_ifc_node_guid(struct mlx5_ib_dev *dev, __be64 *node_guid);
988int mlx5_query_mad_ifc_pkey(struct ib_device *ibdev, u8 port, u16 index,
989 u16 *pkey);
990int mlx5_query_mad_ifc_gids(struct ib_device *ibdev, u8 port, int index,
991 union ib_gid *gid);
992int mlx5_query_mad_ifc_port(struct ib_device *ibdev, u8 port,
993 struct ib_port_attr *props);
Eli Cohene126ba92013-07-07 17:25:49 +0300994int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
995 struct ib_port_attr *props);
996int mlx5_ib_init_fmr(struct mlx5_ib_dev *dev);
997void mlx5_ib_cleanup_fmr(struct mlx5_ib_dev *dev);
Majd Dibbiny762f8992016-10-27 16:36:47 +0300998void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
999 unsigned long max_page_shift,
1000 int *count, int *shift,
Eli Cohene126ba92013-07-07 17:25:49 +03001001 int *ncont, int *order);
Haggai Eran832a6b02014-12-11 17:04:22 +02001002void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
1003 int page_shift, size_t offset, size_t num_pages,
1004 __be64 *pas, int access_flags);
Eli Cohene126ba92013-07-07 17:25:49 +03001005void mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem,
Haggai Erancc149f752014-12-11 17:04:21 +02001006 int page_shift, __be64 *pas, int access_flags);
Eli Cohene126ba92013-07-07 17:25:49 +03001007void mlx5_ib_copy_pas(u64 *old, u64 *new, int step, int num);
1008int mlx5_ib_get_cqe_size(struct mlx5_ib_dev *dev, struct ib_cq *ibcq);
1009int mlx5_mr_cache_init(struct mlx5_ib_dev *dev);
1010int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev);
Artemy Kovalyov49780d42017-01-18 16:58:10 +02001011
1012struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry);
1013void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr);
Sagi Grimbergd5436ba2014-02-23 14:19:12 +02001014int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
1015 struct ib_mr_status *mr_status);
Yishai Hadas79b20a62016-05-23 15:20:50 +03001016struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
1017 struct ib_wq_init_attr *init_attr,
1018 struct ib_udata *udata);
1019int mlx5_ib_destroy_wq(struct ib_wq *wq);
1020int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
1021 u32 wq_attr_mask, struct ib_udata *udata);
Yishai Hadasc5f90922016-05-23 15:20:53 +03001022struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
1023 struct ib_rwq_ind_table_init_attr *init_attr,
1024 struct ib_udata *udata);
1025int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *wq_ind_table);
Moni Shoua776a3902018-01-02 16:19:33 +02001026bool mlx5_ib_dc_atomic_is_supported(struct mlx5_ib_dev *dev);
1027
Eli Cohene126ba92013-07-07 17:25:49 +03001028
Haggai Eran8cdd3122014-12-11 17:04:20 +02001029#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
Saeed Mahameed938fe832015-05-28 22:28:41 +03001030void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev);
Artemy Kovalyovd9aaed82017-01-02 11:37:46 +02001031void mlx5_ib_pfault(struct mlx5_core_dev *mdev, void *context,
1032 struct mlx5_pagefault *pfault);
Haggai Eran6aec21f2014-12-11 17:04:23 +02001033int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev);
Haggai Eran6aec21f2014-12-11 17:04:23 +02001034int __init mlx5_ib_odp_init(void);
1035void mlx5_ib_odp_cleanup(void);
Haggai Eranb4cfe442014-12-11 17:04:26 +02001036void mlx5_ib_invalidate_range(struct ib_umem *umem, unsigned long start,
1037 unsigned long end);
Artemy Kovalyov81713d32017-01-18 16:58:11 +02001038void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent);
1039void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset,
1040 size_t nentries, struct mlx5_ib_mr *mr, int flags);
Haggai Eran6aec21f2014-12-11 17:04:23 +02001041#else /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
Saeed Mahameed938fe832015-05-28 22:28:41 +03001042static inline void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev)
Haggai Eran8cdd3122014-12-11 17:04:20 +02001043{
Saeed Mahameed938fe832015-05-28 22:28:41 +03001044 return;
Haggai Eran8cdd3122014-12-11 17:04:20 +02001045}
Haggai Eran6aec21f2014-12-11 17:04:23 +02001046
Haggai Eran6aec21f2014-12-11 17:04:23 +02001047static inline int mlx5_ib_odp_init_one(struct mlx5_ib_dev *ibdev) { return 0; }
Haggai Eran6aec21f2014-12-11 17:04:23 +02001048static inline int mlx5_ib_odp_init(void) { return 0; }
Artemy Kovalyov81713d32017-01-18 16:58:11 +02001049static inline void mlx5_ib_odp_cleanup(void) {}
1050static inline void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent) {}
1051static inline void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset,
1052 size_t nentries, struct mlx5_ib_mr *mr,
1053 int flags) {}
Haggai Eran6aec21f2014-12-11 17:04:23 +02001054
Haggai Eran8cdd3122014-12-11 17:04:20 +02001055#endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
1056
Arnd Bergmann9967c702016-03-23 11:37:45 +01001057int mlx5_ib_get_vf_config(struct ib_device *device, int vf,
1058 u8 port, struct ifla_vf_info *info);
1059int mlx5_ib_set_vf_link_state(struct ib_device *device, int vf,
1060 u8 port, int state);
1061int mlx5_ib_get_vf_stats(struct ib_device *device, int vf,
1062 u8 port, struct ifla_vf_stats *stats);
1063int mlx5_ib_set_vf_guid(struct ib_device *device, int vf, u8 port,
1064 u64 guid, int type);
1065
Achiad Shochat2811ba52015-12-23 18:47:24 +02001066__be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev, u8 port_num,
1067 int index);
Majd Dibbinyed884512017-01-18 14:10:35 +02001068int mlx5_get_roce_gid_type(struct mlx5_ib_dev *dev, u8 port_num,
1069 int index, enum ib_gid_type *gid_type);
Achiad Shochat2811ba52015-12-23 18:47:24 +02001070
Parav Pandita9e546e2018-01-04 17:25:39 +02001071void mlx5_ib_cleanup_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num);
1072int mlx5_ib_init_cong_debugfs(struct mlx5_ib_dev *dev, u8 port_num);
Parav Pandit4a2da0b2017-05-30 10:05:15 +03001073
Haggai Erand16e91d2016-02-29 15:45:05 +02001074/* GSI QP helper functions */
1075struct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd,
1076 struct ib_qp_init_attr *init_attr);
1077int mlx5_ib_gsi_destroy_qp(struct ib_qp *qp);
1078int mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
1079 int attr_mask);
1080int mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
1081 int qp_attr_mask,
1082 struct ib_qp_init_attr *qp_init_attr);
1083int mlx5_ib_gsi_post_send(struct ib_qp *qp, struct ib_send_wr *wr,
1084 struct ib_send_wr **bad_wr);
1085int mlx5_ib_gsi_post_recv(struct ib_qp *qp, struct ib_recv_wr *wr,
1086 struct ib_recv_wr **bad_wr);
Haggai Eran7722f472016-02-29 15:45:07 +02001087void mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi);
Haggai Erand16e91d2016-02-29 15:45:05 +02001088
Haggai Eran25361e02016-02-29 15:45:08 +02001089int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc);
1090
Yishai Hadas4ed131d2017-12-24 16:31:35 +02001091void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi,
1092 int bfregn);
Daniel Jurgens32f69e42018-01-04 17:25:36 +02001093struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi);
1094struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *dev,
1095 u8 ib_port_num,
1096 u8 *native_port_num);
1097void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *dev,
1098 u8 port_num);
Yishai Hadas4ed131d2017-12-24 16:31:35 +02001099
Eli Cohene126ba92013-07-07 17:25:49 +03001100static inline void init_query_mad(struct ib_smp *mad)
1101{
1102 mad->base_version = 1;
1103 mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED;
1104 mad->class_version = 1;
1105 mad->method = IB_MGMT_METHOD_GET;
1106}
1107
1108static inline u8 convert_access(int acc)
1109{
1110 return (acc & IB_ACCESS_REMOTE_ATOMIC ? MLX5_PERM_ATOMIC : 0) |
1111 (acc & IB_ACCESS_REMOTE_WRITE ? MLX5_PERM_REMOTE_WRITE : 0) |
1112 (acc & IB_ACCESS_REMOTE_READ ? MLX5_PERM_REMOTE_READ : 0) |
1113 (acc & IB_ACCESS_LOCAL_WRITE ? MLX5_PERM_LOCAL_WRITE : 0) |
1114 MLX5_PERM_LOCAL_READ;
1115}
1116
Sagi Grimbergb6364012015-09-02 22:23:04 +03001117static inline int is_qp1(enum ib_qp_type qp_type)
1118{
Haggai Erand16e91d2016-02-29 15:45:05 +02001119 return qp_type == MLX5_IB_QPT_HW_GSI;
Sagi Grimbergb6364012015-09-02 22:23:04 +03001120}
1121
Haggai Erancc149f752014-12-11 17:04:21 +02001122#define MLX5_MAX_UMR_SHIFT 16
1123#define MLX5_MAX_UMR_PAGES (1 << MLX5_MAX_UMR_SHIFT)
1124
Leon Romanovsky051f2632015-12-20 12:16:11 +02001125static inline u32 check_cq_create_flags(u32 flags)
1126{
1127 /*
1128 * It returns non-zero value for unsupported CQ
1129 * create flags, otherwise it returns zero.
1130 */
Jason Gunthorpebeb801a2018-01-26 15:16:46 -07001131 return (flags & ~(IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN |
1132 IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION));
Leon Romanovsky051f2632015-12-20 12:16:11 +02001133}
Haggai Abramovskycfb5e082016-01-14 19:12:57 +02001134
1135static inline int verify_assign_uidx(u8 cqe_version, u32 cmd_uidx,
1136 u32 *user_index)
1137{
1138 if (cqe_version) {
1139 if ((cmd_uidx == MLX5_IB_DEFAULT_UIDX) ||
1140 (cmd_uidx & ~MLX5_USER_ASSIGNED_UIDX_MASK))
1141 return -EINVAL;
1142 *user_index = cmd_uidx;
1143 } else {
1144 *user_index = MLX5_IB_DEFAULT_UIDX;
1145 }
1146
1147 return 0;
1148}
Leon Romanovsky3085e292016-09-22 17:31:11 +03001149
1150static inline int get_qp_user_index(struct mlx5_ib_ucontext *ucontext,
1151 struct mlx5_ib_create_qp *ucmd,
1152 int inlen,
1153 u32 *user_index)
1154{
1155 u8 cqe_version = ucontext->cqe_version;
1156
1157 if (field_avail(struct mlx5_ib_create_qp, uidx, inlen) &&
1158 !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
1159 return 0;
1160
1161 if (!!(field_avail(struct mlx5_ib_create_qp, uidx, inlen) !=
1162 !!cqe_version))
1163 return -EINVAL;
1164
1165 return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
1166}
1167
1168static inline int get_srq_user_index(struct mlx5_ib_ucontext *ucontext,
1169 struct mlx5_ib_create_srq *ucmd,
1170 int inlen,
1171 u32 *user_index)
1172{
1173 u8 cqe_version = ucontext->cqe_version;
1174
1175 if (field_avail(struct mlx5_ib_create_srq, uidx, inlen) &&
1176 !cqe_version && (ucmd->uidx == MLX5_IB_DEFAULT_UIDX))
1177 return 0;
1178
1179 if (!!(field_avail(struct mlx5_ib_create_srq, uidx, inlen) !=
1180 !!cqe_version))
1181 return -EINVAL;
1182
1183 return verify_assign_uidx(cqe_version, ucmd->uidx, user_index);
1184}
Eli Cohenb037c292017-01-03 23:55:26 +02001185
1186static inline int get_uars_per_sys_page(struct mlx5_ib_dev *dev, bool lib_support)
1187{
1188 return lib_support && MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1189 MLX5_UARS_IN_PAGE : 1;
1190}
1191
Yishai Hadas31a78a52017-12-24 16:31:34 +02001192static inline int get_num_static_uars(struct mlx5_ib_dev *dev,
1193 struct mlx5_bfreg_info *bfregi)
Eli Cohenb037c292017-01-03 23:55:26 +02001194{
Yishai Hadas31a78a52017-12-24 16:31:34 +02001195 return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) * bfregi->num_static_sys_pages;
Eli Cohenb037c292017-01-03 23:55:26 +02001196}
1197
Eli Cohene126ba92013-07-07 17:25:49 +03001198#endif /* MLX5_IB_H */