blob: 8dc8415d152d00ca550f9cfa993d0da0bacb27c2 [file] [log] [blame]
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001#include <linux/socket.h>
2#include <linux/in.h>
3#include <linux/in6.h>
4#include <rdma/ib_verbs.h>
5#include <rdma/rdma_cm.h>
6
Sagi Grimberg24f412d2014-12-07 13:12:02 +02007#define DRV_NAME "isert"
8#define PFX DRV_NAME ": "
9
10#define isert_dbg(fmt, arg...) \
11 do { \
12 if (unlikely(isert_debug_level > 2)) \
13 printk(KERN_DEBUG PFX "%s: " fmt,\
14 __func__ , ## arg); \
15 } while (0)
16
17#define isert_warn(fmt, arg...) \
18 do { \
19 if (unlikely(isert_debug_level > 0)) \
20 pr_warn(PFX "%s: " fmt, \
21 __func__ , ## arg); \
22 } while (0)
23
24#define isert_info(fmt, arg...) \
25 do { \
26 if (unlikely(isert_debug_level > 1)) \
27 pr_info(PFX "%s: " fmt, \
28 __func__ , ## arg); \
29 } while (0)
30
31#define isert_err(fmt, arg...) \
32 pr_err(PFX "%s: " fmt, __func__ , ## arg)
33
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -080034#define ISERT_RDMA_LISTEN_BACKLOG 10
Vu Pham59464ef2013-08-28 23:23:35 +030035#define ISCSI_ISER_SG_TABLESIZE 256
Nicholas Bellinger9bb4ca62014-02-27 07:02:48 -080036#define ISER_FASTREG_LI_WRID 0xffffffffffffffffULL
Sagi Grimbergbdf20e72014-12-02 16:57:43 +020037#define ISER_BEACON_WRID 0xfffffffffffffffeULL
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -080038
39enum isert_desc_type {
40 ISCSI_TX_CONTROL,
41 ISCSI_TX_DATAIN
42};
43
44enum iser_ib_op_code {
45 ISER_IB_RECV,
46 ISER_IB_SEND,
47 ISER_IB_RDMA_WRITE,
48 ISER_IB_RDMA_READ,
49};
50
51enum iser_conn_state {
52 ISER_CONN_INIT,
53 ISER_CONN_UP,
Sagi Grimberg128e9cc2014-12-02 16:57:20 +020054 ISER_CONN_FULL_FEATURE,
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -080055 ISER_CONN_TERMINATING,
56 ISER_CONN_DOWN,
57};
58
59struct iser_rx_desc {
60 struct iser_hdr iser_header;
61 struct iscsi_hdr iscsi_header;
62 char data[ISER_RECV_DATA_SEG_LEN];
63 u64 dma_addr;
64 struct ib_sge rx_sg;
65 char pad[ISER_RX_PAD_SIZE];
66} __packed;
67
68struct iser_tx_desc {
69 struct iser_hdr iser_header;
70 struct iscsi_hdr iscsi_header;
71 enum isert_desc_type type;
72 u64 dma_addr;
73 struct ib_sge tx_sg[2];
74 int num_sge;
75 struct isert_cmd *isert_cmd;
76 struct ib_send_wr send_wr;
77} __packed;
78
Sagi Grimbergd3e125d2014-02-19 17:50:23 +020079enum isert_indicator {
80 ISERT_PROTECTED = 1 << 0,
81 ISERT_DATA_KEY_VALID = 1 << 1,
82 ISERT_PROT_KEY_VALID = 1 << 2,
83 ISERT_SIG_KEY_VALID = 1 << 3,
84};
85
86struct pi_context {
87 struct ib_mr *prot_mr;
88 struct ib_fast_reg_page_list *prot_frpl;
89 struct ib_mr *sig_mr;
90};
91
Vu Pham59464ef2013-08-28 23:23:35 +030092struct fast_reg_descriptor {
Sagi Grimbergd3e125d2014-02-19 17:50:23 +020093 struct list_head list;
94 struct ib_mr *data_mr;
95 struct ib_fast_reg_page_list *data_frpl;
96 u8 ind;
97 struct pi_context *pi_ctx;
Vu Pham59464ef2013-08-28 23:23:35 +030098};
99
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +0200100struct isert_data_buf {
101 struct scatterlist *sg;
102 int nents;
103 u32 sg_off;
104 u32 len; /* cur_rdma_length */
105 u32 offset;
106 unsigned int dma_nents;
107 enum dma_data_direction dma_dir;
108};
109
Sagi Grimberg570db172014-12-02 16:57:31 +0200110enum {
111 DATA = 0,
112 PROT = 1,
113 SIG = 2,
114};
115
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800116struct isert_rdma_wr {
117 struct list_head wr_list;
118 struct isert_cmd *isert_cmd;
119 enum iser_ib_op_code iser_ib_op;
120 struct ib_sge *ib_sge;
Vu Pham59464ef2013-08-28 23:23:35 +0300121 struct ib_sge s_ib_sge;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800122 int send_wr_num;
123 struct ib_send_wr *send_wr;
Vu Pham59464ef2013-08-28 23:23:35 +0300124 struct ib_send_wr s_send_wr;
Sagi Grimberg570db172014-12-02 16:57:31 +0200125 struct ib_sge ib_sg[3];
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +0200126 struct isert_data_buf data;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +0200127 struct isert_data_buf prot;
Vu Pham59464ef2013-08-28 23:23:35 +0300128 struct fast_reg_descriptor *fr_desc;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800129};
130
131struct isert_cmd {
132 uint32_t read_stag;
133 uint32_t write_stag;
134 uint64_t read_va;
135 uint64_t write_va;
Nicholas Bellingerdbbc5d12013-07-03 19:39:37 -0700136 u64 pdu_buf_dma;
137 u32 pdu_buf_len;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800138 u32 read_va_off;
139 u32 write_va_off;
140 u32 rdma_wr_num;
141 struct isert_conn *conn;
Nicholas Bellingerd703ce22013-08-17 14:27:56 -0700142 struct iscsi_cmd *iscsi_cmd;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800143 struct iser_tx_desc tx_desc;
144 struct isert_rdma_wr rdma_wr;
145 struct work_struct comp_work;
146};
147
148struct isert_device;
149
150struct isert_conn {
151 enum iser_conn_state state;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800152 int post_recv_buf_count;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800153 u32 responder_resources;
154 u32 initiator_depth;
Sagi Grimberg23a548e2014-12-02 16:57:35 +0200155 bool pi_support;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800156 u32 max_sge;
157 char *login_buf;
158 char *login_req_buf;
159 char *login_rsp_buf;
160 u64 login_req_dma;
Sagi Grimberg2371e5d2014-12-02 16:57:21 +0200161 int login_req_len;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800162 u64 login_rsp_dma;
163 unsigned int conn_rx_desc_head;
164 struct iser_rx_desc *conn_rx_descs;
165 struct ib_recv_wr conn_rx_wr[ISERT_MIN_POSTED_RX];
166 struct iscsi_conn *conn;
167 struct list_head conn_accept_node;
168 struct completion conn_login_comp;
Sagi Grimberg2371e5d2014-12-02 16:57:21 +0200169 struct completion login_req_comp;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800170 struct iser_tx_desc conn_login_tx_desc;
171 struct rdma_cm_id *conn_cm_id;
172 struct ib_pd *conn_pd;
173 struct ib_mr *conn_mr;
174 struct ib_qp *conn_qp;
175 struct isert_device *conn_device;
Nicholas Bellingerb2cb9642013-07-03 03:05:37 -0700176 struct mutex conn_mutex;
Nicholas Bellingerdefd8842014-02-03 12:54:39 -0800177 struct completion conn_wait;
178 struct completion conn_wait_comp_err;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800179 struct kref conn_kref;
Sagi Grimberga3a5a822014-01-09 18:40:50 +0200180 struct list_head conn_fr_pool;
181 int conn_fr_pool_size;
182 /* lock to protect fastreg pool */
Vu Pham59464ef2013-08-28 23:23:35 +0300183 spinlock_t conn_lock;
Sagi Grimbergb02efbf2014-12-02 16:57:29 +0200184 struct work_struct release_work;
Sagi Grimbergbdf20e72014-12-02 16:57:43 +0200185 struct ib_recv_wr beacon;
Sagi Grimberg991bb762014-12-07 13:12:01 +0200186 bool logout_posted;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800187};
188
189#define ISERT_MAX_CQ 64
190
Sagi Grimberg4a295ba2014-12-02 16:57:40 +0200191/**
192 * struct isert_comp - iSER completion context
193 *
194 * @device: pointer to device handle
Sagi Grimberg6f0fae32014-12-02 16:57:41 +0200195 * @cq: completion queue
Sagi Grimberg36ea63b2014-12-02 16:57:45 +0200196 * @wcs: work completion array
Sagi Grimberg4a295ba2014-12-02 16:57:40 +0200197 * @active_qps: Number of active QPs attached
198 * to completion context
Sagi Grimberg6f0fae32014-12-02 16:57:41 +0200199 * @work: completion work handle
Sagi Grimberg4a295ba2014-12-02 16:57:40 +0200200 */
201struct isert_comp {
Sagi Grimberg6f0fae32014-12-02 16:57:41 +0200202 struct isert_device *device;
203 struct ib_cq *cq;
Sagi Grimberg36ea63b2014-12-02 16:57:45 +0200204 struct ib_wc wcs[16];
Sagi Grimberg4a295ba2014-12-02 16:57:40 +0200205 int active_qps;
Sagi Grimberg6f0fae32014-12-02 16:57:41 +0200206 struct work_struct work;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800207};
208
209struct isert_device {
Sagi Grimberga3a5a822014-01-09 18:40:50 +0200210 int use_fastreg;
Sagi Grimbergd3e125d2014-02-19 17:50:23 +0200211 bool pi_capable;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800212 int refcount;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800213 struct ib_device *ib_device;
Sagi Grimberg4a295ba2014-12-02 16:57:40 +0200214 struct isert_comp *comps;
215 int comps_used;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800216 struct list_head dev_node;
Vu Pham59464ef2013-08-28 23:23:35 +0300217 struct ib_device_attr dev_attr;
Vu Phamd40945d2013-08-28 23:23:34 +0300218 int (*reg_rdma_mem)(struct iscsi_conn *conn,
219 struct iscsi_cmd *cmd,
220 struct isert_rdma_wr *wr);
221 void (*unreg_rdma_mem)(struct isert_cmd *isert_cmd,
222 struct isert_conn *isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800223};
224
225struct isert_np {
Sagi Grimbergca6c1d82014-12-02 16:57:27 +0200226 struct iscsi_np *np;
Sagi Grimberg531b7bf2014-04-29 13:13:45 +0300227 struct semaphore np_sem;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800228 struct rdma_cm_id *np_cm_id;
229 struct mutex np_accept_mutex;
230 struct list_head np_accept_list;
231 struct completion np_login_comp;
232};