blob: 04e90192a50dd7f825fe8f8e9df35d7cfb1456c3 [file] [log] [blame]
Dennis Dalessandro01946212016-01-06 09:50:24 -08001#ifndef DEF_RDMA_VT_H
2#define DEF_RDMA_VT_H
3
4/*
5 * Copyright(c) 2015 Intel Corporation.
6 *
7 * This file is provided under a dual BSD/GPLv2 license. When using or
8 * redistributing this file, you may do so under either license.
9 *
10 * GPL LICENSE SUMMARY
11 *
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of version 2 of the GNU General Public License as
14 * published by the Free Software Foundation.
15 *
16 * This program is distributed in the hope that it will be useful, but
17 * WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 * General Public License for more details.
20 *
21 * BSD LICENSE
22 *
23 * Redistribution and use in source and binary forms, with or without
24 * modification, are permitted provided that the following conditions
25 * are met:
26 *
27 * - Redistributions of source code must retain the above copyright
28 * notice, this list of conditions and the following disclaimer.
29 * - Redistributions in binary form must reproduce the above copyright
30 * notice, this list of conditions and the following disclaimer in
31 * the documentation and/or other materials provided with the
32 * distribution.
33 * - Neither the name of Intel Corporation nor the names of its
34 * contributors may be used to endorse or promote products derived
35 * from this software without specific prior written permission.
36 *
37 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
38 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
39 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
40 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
41 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
42 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
43 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
44 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
45 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
46 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
47 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
48 *
49 */
50
51/*
52 * Structure that low level drivers will populate in order to register with the
53 * rdmavt layer.
54 */
55
Dennis Dalessandrof3d01bb2016-01-06 10:04:13 -080056#include <linux/spinlock.h>
57#include <linux/list.h>
Dennis Dalessandro3b0b3fb2016-01-22 13:00:35 -080058#include <linux/hash.h>
Dennis Dalessandrob4e64392016-01-06 10:04:31 -080059#include <rdma/ib_verbs.h>
60#include <rdma/rdmavt_mr.h>
61#include <rdma/rdmavt_qp.h>
Kamal Heibf2f34212016-01-06 10:03:47 -080062
Dennis Dalessandro8afd32e2016-01-06 09:51:48 -080063/*
Dennis Dalessandro0b8a8aa2016-01-06 10:03:07 -080064 * For some of the IBTA objects there will likely be some
65 * initializations required. We need flags to determine whether it is OK
66 * for rdmavt to do this or not. This does not imply any functions of a
67 * partiuclar IBTA object are overridden.
68 */
69#define RVT_FLAG_MR_INIT_DRIVER BIT(1)
70#define RVT_FLAG_QP_INIT_DRIVER BIT(2)
71#define RVT_FLAG_CQ_INIT_DRIVER BIT(3)
72
Dennis Dalessandro38ce2c62016-01-06 10:05:12 -080073#define RVT_MAX_PKEY_VALUES 16
74
Dennis Dalessandrof3d01bb2016-01-06 10:04:13 -080075struct rvt_ibport {
76 struct rvt_qp __rcu *qp[2];
77 struct ib_mad_agent *send_agent; /* agent for SMI (traps) */
78 struct rb_root mcast_tree;
79 spinlock_t lock; /* protect changes in this struct */
80
81 /* non-zero when timer is set */
82 unsigned long mkey_lease_timeout;
83 unsigned long trap_timeout;
84 __be64 gid_prefix; /* in network order */
85 __be64 mkey;
86 u64 tid;
87 u32 port_cap_flags;
88 u32 pma_sample_start;
89 u32 pma_sample_interval;
90 __be16 pma_counter_select[5];
91 u16 pma_tag;
92 u16 mkey_lease_period;
93 u16 sm_lid;
94 u8 sm_sl;
95 u8 mkeyprot;
96 u8 subnet_timeout;
97 u8 vl_high_limit;
98
99 /*
100 * Driver is expected to keep these up to date. These
101 * counters are informational only and not required to be
102 * completely accurate.
103 */
104 u64 n_rc_resends;
105 u64 n_seq_naks;
106 u64 n_rdma_seq;
107 u64 n_rnr_naks;
108 u64 n_other_naks;
109 u64 n_loop_pkts;
110 u64 n_pkt_drops;
111 u64 n_vl15_dropped;
112 u64 n_rc_timeouts;
113 u64 n_dmawait;
114 u64 n_unaligned;
115 u64 n_rc_dupreq;
116 u64 n_rc_seqnak;
117 u16 pkey_violations;
118 u16 qkey_violations;
119 u16 mkey_violations;
120
121 /* Hot-path per CPU counters to avoid cacheline trading to update */
122 u64 z_rc_acks;
123 u64 z_rc_qacks;
124 u64 z_rc_delayed_comp;
125 u64 __percpu *rc_acks;
126 u64 __percpu *rc_qacks;
127 u64 __percpu *rc_delayed_comp;
128
129 void *priv; /* driver private data */
130
Dennis Dalessandro38ce2c62016-01-06 10:05:12 -0800131 /*
132 * The pkey table is allocated and maintained by the driver. Drivers
133 * need to have access to this before registering with rdmav. However
134 * rdmavt will need access to it so drivers need to proviee this during
135 * the attach port API call.
136 */
137 u16 *pkey_table;
138
Dennis Dalessandrof3d01bb2016-01-06 10:04:13 -0800139 /* TODO: Move sm_ah and smi_ah into here as well*/
140};
141
Dennis Dalessandro6f6387a2016-01-22 13:00:15 -0800142#define RVT_CQN_MAX 16 /* maximum length of cq name */
143
Dennis Dalessandroca889e82016-01-06 10:02:41 -0800144/*
Dennis Dalessandro8afd32e2016-01-06 09:51:48 -0800145 * Things that are driver specific, module parameters in hfi1 and qib
146 */
147struct rvt_driver_params {
Dennis Dalessandrob1070a72016-01-06 09:52:19 -0800148 /*
149 * driver required fields:
150 * node_guid
151 * phys_port_cnt
152 * dma_device
153 * owner
154 * driver optional fields (rvt will provide generic value if blank):
155 * name
156 * node_desc
157 * rvt fields, driver value ignored:
158 * uverbs_abi_ver
159 * node_type
160 * num_comp_vectors
161 * uverbs_cmd_mask
162 */
163 struct ib_device_attr props;
164
165 /*
166 * Drivers will need to support a number of notifications to rvt in
167 * accordance with certain events. This structure should contain a mask
168 * of the supported events. Such events that the rvt may need to know
169 * about include:
170 * port errors
171 * port active
172 * lid change
173 * sm change
174 * client reregister
175 * pkey change
176 *
177 * There may also be other events that the rvt layers needs to know
178 * about this is not an exhaustive list. Some events though rvt does not
179 * need to rely on the driver for such as completion queue error.
180 */
181 int rvt_signal_supported;
182
183 /*
184 * Anything driver specific that is not covered by props
185 * For instance special module parameters. Goes here.
186 */
Dennis Dalessandro7b1e2092016-01-06 10:03:31 -0800187 unsigned int lkey_table_size;
Dennis Dalessandro0acb0cc2016-01-06 10:04:46 -0800188 unsigned int qp_table_size;
189 int qpn_start;
190 int qpn_inc;
191 int qpn_res_start;
192 int qpn_res_end;
Dennis Dalessandrof3d01bb2016-01-06 10:04:13 -0800193 int nports;
Dennis Dalessandro38ce2c62016-01-06 10:05:12 -0800194 int npkeys;
Dennis Dalessandro0acb0cc2016-01-06 10:04:46 -0800195 u8 qos_shift;
Dennis Dalessandro6f6387a2016-01-22 13:00:15 -0800196 char cq_name[RVT_CQN_MAX];
197 int node;
Dennis Dalessandro3b0b3fb2016-01-22 13:00:35 -0800198 int max_rdma_atomic;
199 int psn_mask;
200 int psn_shift;
201 int psn_modify_mask;
Dennis Dalessandro8afd32e2016-01-06 09:51:48 -0800202};
203
204/* Protection domain */
205struct rvt_pd {
206 struct ib_pd ibpd;
207 int user; /* non-zero if created from user space */
208};
209
Kamal Heib119a8e72016-01-06 10:03:59 -0800210/* Address handle */
211struct rvt_ah {
212 struct ib_ah ibah;
213 struct ib_ah_attr attr;
214 atomic_t refcount;
Dennis Dalessandrob036db82016-01-06 10:04:23 -0800215 u8 vl;
216 u8 log_pmtu;
217};
218
219struct rvt_dev_info;
220struct rvt_driver_provided {
221 /*
222 * The work to create port files in /sys/class Infiniband is different
223 * depending on the driver. This should not be extracted away and
224 * instead drivers are responsible for setting the correct callback for
225 * this.
226 */
227
228 /* -------------------*/
229 /* Required functions */
230 /* -------------------*/
231 int (*port_callback)(struct ib_device *, u8, struct kobject *);
232 const char * (*get_card_name)(struct rvt_dev_info *rdi);
233 struct pci_dev * (*get_pci_dev)(struct rvt_dev_info *rdi);
Dennis Dalessandro515667f2016-01-22 12:50:17 -0800234 unsigned (*free_all_qps)(struct rvt_dev_info *rdi);
Mike Marciniszynd2b8d4d2016-01-22 12:50:43 -0800235 void * (*qp_priv_alloc)(struct rvt_dev_info *rdi, struct rvt_qp *qp,
236 gfp_t gfp);
Dennis Dalessandro515667f2016-01-22 12:50:17 -0800237 void (*qp_priv_free)(struct rvt_dev_info *rdi, struct rvt_qp *qp);
238 void (*notify_qp_reset)(struct rvt_qp *qp);
Dennis Dalessandrobfbac092016-01-22 13:00:22 -0800239 void (*schedule_send)(struct rvt_qp *qp);
240 void (*do_send)(struct rvt_qp *qp);
Dennis Dalessandro3b0b3fb2016-01-22 13:00:35 -0800241 int (*get_pmtu_from_attr)(struct rvt_dev_info *rdi, struct rvt_qp *qp,
242 struct ib_qp_attr *attr);
243 void (*flush_qp_waiters)(struct rvt_qp *qp);
244 void (*stop_send_queue)(struct rvt_qp *qp);
245 void (*quiesce_qp)(struct rvt_qp *qp);
246 void (*notify_error_qp)(struct rvt_qp *qp);
247 u32 (*mtu_from_qp)(struct rvt_dev_info *rdi, struct rvt_qp *qp,
248 u32 pmtu);
249 int (*mtu_to_path_mtu)(u32 mtu);
Dennis Dalessandrob036db82016-01-06 10:04:23 -0800250
251 /*--------------------*/
252 /* Optional functions */
253 /*--------------------*/
254 int (*check_ah)(struct ib_device *, struct ib_ah_attr *);
255 void (*notify_new_ah)(struct ib_device *, struct ib_ah_attr *,
256 struct rvt_ah *);
Dennis Dalessandro515667f2016-01-22 12:50:17 -0800257 int (*alloc_qpn)(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt,
Mike Marciniszynd2b8d4d2016-01-22 12:50:43 -0800258 enum ib_qp_type type, u8 port, gfp_t gfp);
Kamal Heib119a8e72016-01-06 10:03:59 -0800259};
260
Dennis Dalessandro01946212016-01-06 09:50:24 -0800261struct rvt_dev_info {
Dennis Dalessandro7b1e2092016-01-06 10:03:31 -0800262 struct ib_device ibdev; /* Keep this first. Nothing above here */
263
Dennis Dalessandrob1070a72016-01-06 09:52:19 -0800264 /*
265 * Prior to calling for registration the driver will be responsible for
266 * allocating space for this structure.
267 *
268 * The driver will also be responsible for filling in certain members of
Harish Chegondifeaeb6e2016-01-22 12:50:36 -0800269 * dparms.props. The driver needs to fill in dparms exactly as it would
270 * want values reported to a ULP. This will be returned to the caller
271 * in rdmavt's device. The driver should also therefore refrain from
272 * modifying this directly after registration with rdmavt.
Dennis Dalessandrob1070a72016-01-06 09:52:19 -0800273 */
Dennis Dalessandro8afd32e2016-01-06 09:51:48 -0800274
Dennis Dalessandrob1070a72016-01-06 09:52:19 -0800275 /* Driver specific properties */
Dennis Dalessandro8afd32e2016-01-06 09:51:48 -0800276 struct rvt_driver_params dparms;
Dennis Dalessandrob1070a72016-01-06 09:52:19 -0800277
Dennis Dalessandrob92a7562016-01-06 10:01:42 -0800278 struct rvt_mregion __rcu *dma_mr;
279 struct rvt_lkey_table lkey_table;
280
Dennis Dalessandroaec57782016-01-06 10:02:52 -0800281 /* Driver specific helper functions */
282 struct rvt_driver_provided driver_f;
Dennis Dalessandro01946212016-01-06 09:50:24 -0800283
Dennis Dalessandro8afd32e2016-01-06 09:51:48 -0800284 /* Internal use */
285 int n_pds_allocated;
286 spinlock_t n_pds_lock; /* Protect pd allocated count */
Dennis Dalessandro0b8a8aa2016-01-06 10:03:07 -0800287
Kamal Heib119a8e72016-01-06 10:03:59 -0800288 int n_ahs_allocated;
289 spinlock_t n_ahs_lock; /* Protect ah allocated count */
290
Dennis Dalessandro0b8a8aa2016-01-06 10:03:07 -0800291 int flags;
Dennis Dalessandrof3d01bb2016-01-06 10:04:13 -0800292 struct rvt_ibport **ports;
Dennis Dalessandro0acb0cc2016-01-06 10:04:46 -0800293
Dennis Dalessandro515667f2016-01-22 12:50:17 -0800294 /* QP */
Dennis Dalessandro0acb0cc2016-01-06 10:04:46 -0800295 struct rvt_qp_ibdev *qp_dev;
Dennis Dalessandro515667f2016-01-22 12:50:17 -0800296 u32 n_qps_allocated; /* number of QPs allocated for device */
297 spinlock_t n_qps_lock; /* keep track of number of qps */
Dennis Dalessandro822514d2016-01-06 10:04:57 -0800298
299 /* memory maps */
300 struct list_head pending_mmaps;
301 spinlock_t mmap_offset_lock; /* protect mmap_offset */
302 u32 mmap_offset;
303 spinlock_t pending_lock; /* protect pending mmap list */
Dennis Dalessandro6f6387a2016-01-22 13:00:15 -0800304
305 /* CQ */
306 struct kthread_worker *worker; /* per device cq worker */
307 u32 n_cqs_allocated; /* number of CQs allocated for device */
308 spinlock_t n_cqs_lock; /* protect count of in use cqs */
Dennis Dalessandro4e740802016-01-22 13:00:55 -0800309
310 /* Multicast */
311 u32 n_mcast_grps_allocated; /* number of mcast groups allocated */
312 spinlock_t n_mcast_grps_lock;
313
Dennis Dalessandro01946212016-01-06 09:50:24 -0800314};
315
Dennis Dalessandro8afd32e2016-01-06 09:51:48 -0800316static inline struct rvt_pd *ibpd_to_rvtpd(struct ib_pd *ibpd)
317{
318 return container_of(ibpd, struct rvt_pd, ibpd);
319}
320
Kamal Heib119a8e72016-01-06 10:03:59 -0800321static inline struct rvt_ah *ibah_to_rvtah(struct ib_ah *ibah)
322{
323 return container_of(ibah, struct rvt_ah, ibah);
324}
325
Dennis Dalessandro8afd32e2016-01-06 09:51:48 -0800326static inline struct rvt_dev_info *ib_to_rvt(struct ib_device *ibdev)
327{
328 return container_of(ibdev, struct rvt_dev_info, ibdev);
329}
330
Dennis Dalessandro70a1a352016-01-06 10:04:06 -0800331static inline struct rvt_srq *ibsrq_to_rvtsrq(struct ib_srq *ibsrq)
332{
333 return container_of(ibsrq, struct rvt_srq, ibsrq);
334}
335
Dennis Dalessandrobfbac092016-01-22 13:00:22 -0800336static inline struct rvt_qp *ibqp_to_rvtqp(struct ib_qp *ibqp)
337{
338 return container_of(ibqp, struct rvt_qp, ibqp);
339}
340
Dennis Dalessandro38ce2c62016-01-06 10:05:12 -0800341static inline unsigned rvt_get_npkeys(struct rvt_dev_info *rdi)
342{
343 /*
344 * All ports have same number of pkeys.
345 */
346 return rdi->dparms.npkeys;
347}
348
349/*
350 * Return the indexed PKEY from the port PKEY table.
351 */
352static inline u16 rvt_get_pkey(struct rvt_dev_info *rdi,
353 int port_index,
354 unsigned index)
355{
356 if (index >= rvt_get_npkeys(rdi))
357 return 0;
358 else
359 return rdi->ports[port_index]->pkey_table[index];
360}
361
Dennis Dalessandro3b0b3fb2016-01-22 13:00:35 -0800362/**
363 * rvt_lookup_qpn - return the QP with the given QPN
364 * @ibp: the ibport
365 * @qpn: the QP number to look up
366 *
367 * The caller must hold the rcu_read_lock(), and keep the lock until
368 * the returned qp is no longer in use.
369 */
370/* TODO: Remove this and put in rdmavt/qp.h when no longer needed by drivers */
371static inline struct rvt_qp *rvt_lookup_qpn(struct rvt_dev_info *rdi,
372 struct rvt_ibport *rvp,
373 u32 qpn) __must_hold(RCU)
374{
375 struct rvt_qp *qp = NULL;
376
377 if (unlikely(qpn <= 1)) {
378 qp = rcu_dereference(rvp->qp[qpn]);
379 } else {
380 u32 n = hash_32(qpn, rdi->qp_dev->qp_table_bits);
381
382 for (qp = rcu_dereference(rdi->qp_dev->qp_table[n]); qp;
383 qp = rcu_dereference(qp->next))
384 if (qp->ibqp.qp_num == qpn)
385 break;
386 }
387 return qp;
388}
389
Dennis Dalessandro01946212016-01-06 09:50:24 -0800390int rvt_register_device(struct rvt_dev_info *rvd);
391void rvt_unregister_device(struct rvt_dev_info *rvd);
Kamal Heib119a8e72016-01-06 10:03:59 -0800392int rvt_check_ah(struct ib_device *ibdev, struct ib_ah_attr *ah_attr);
Dennis Dalessandro38ce2c62016-01-06 10:05:12 -0800393int rvt_init_port(struct rvt_dev_info *rdi, struct rvt_ibport *port,
394 int portnum, u16 *pkey_table);
Dennis Dalessandro7b1e2092016-01-06 10:03:31 -0800395int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge,
396 u32 len, u64 vaddr, u32 rkey, int acc);
397int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
398 struct rvt_sge *isge, struct ib_sge *sge, int acc);
Dennis Dalessandro822514d2016-01-06 10:04:57 -0800399int rvt_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
400void rvt_release_mmap_info(struct kref *ref);
401struct rvt_mmap_info *rvt_create_mmap_info(struct rvt_dev_info *rdi,
402 u32 size,
403 struct ib_ucontext *context,
404 void *obj);
405void rvt_update_mmap_info(struct rvt_dev_info *rdi, struct rvt_mmap_info *ip,
406 u32 size, void *obj);
Dennis Dalessandro4e740802016-01-22 13:00:55 -0800407int rvt_reg_mr(struct rvt_qp *qp, struct ib_reg_wr *wr);
408struct rvt_mcast *rvt_mcast_find(struct rvt_ibport *ibp, union ib_gid *mgid);
Dennis Dalessandro822514d2016-01-06 10:04:57 -0800409
Dennis Dalessandro5a9cf6f2016-01-22 12:50:24 -0800410/* Temporary export */
411void rvt_reset_qp(struct rvt_dev_info *rdi, struct rvt_qp *qp,
412 enum ib_qp_type type);
Dennis Dalessandro4e740802016-01-22 13:00:55 -0800413
Dennis Dalessandro01946212016-01-06 09:50:24 -0800414#endif /* DEF_RDMA_VT_H */