blob: dfbbbb28090b2301c7742fd28b145b62054a76af [file] [log] [blame]
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001/*******************************************************************************
2 * This file contains iSCSI extentions for RDMA (iSER) Verbs
3 *
Nicholas Bellinger4c762512013-09-05 15:29:12 -07004 * (c) Copyright 2013 Datera, Inc.
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08005 *
6 * Nicholas A. Bellinger <nab@linux-iscsi.org>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 ****************************************************************************/
18
19#include <linux/string.h>
20#include <linux/module.h>
21#include <linux/scatterlist.h>
22#include <linux/socket.h>
23#include <linux/in.h>
24#include <linux/in6.h>
25#include <rdma/ib_verbs.h>
26#include <rdma/rdma_cm.h>
27#include <target/target_core_base.h>
28#include <target/target_core_fabric.h>
29#include <target/iscsi/iscsi_transport.h>
Sagi Grimberg531b7bf2014-04-29 13:13:45 +030030#include <linux/semaphore.h>
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -080031
32#include "isert_proto.h"
33#include "ib_isert.h"
34
35#define ISERT_MAX_CONN 8
36#define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
37#define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN)
Sagi Grimbergbdf20e72014-12-02 16:57:43 +020038#define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \
39 ISERT_MAX_CONN)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -080040
Sagi Grimberg45678b62015-01-25 19:11:19 +020041static int isert_debug_level;
Sagi Grimberg24f412d2014-12-07 13:12:02 +020042module_param_named(debug_level, isert_debug_level, int, 0644);
43MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)");
44
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -080045static DEFINE_MUTEX(device_list_mutex);
46static LIST_HEAD(device_list);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -080047static struct workqueue_struct *isert_comp_wq;
Sagi Grimbergb02efbf2014-12-02 16:57:29 +020048static struct workqueue_struct *isert_release_wq;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -080049
50static void
Vu Phamd40945d2013-08-28 23:23:34 +030051isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn);
52static int
53isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
54 struct isert_rdma_wr *wr);
Vu Pham59464ef2013-08-28 23:23:35 +030055static void
Sagi Grimberga3a5a822014-01-09 18:40:50 +020056isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn);
Vu Pham59464ef2013-08-28 23:23:35 +030057static int
Sagi Grimberga3a5a822014-01-09 18:40:50 +020058isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
59 struct isert_rdma_wr *wr);
Sagi Grimbergf93f3a72014-02-19 17:50:24 +020060static int
61isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd);
Sagi Grimberg2371e5d2014-12-02 16:57:21 +020062static int
63isert_rdma_post_recvl(struct isert_conn *isert_conn);
64static int
65isert_rdma_accept(struct isert_conn *isert_conn);
Sagi Grimbergca6c1d82014-12-02 16:57:27 +020066struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np);
Vu Phamd40945d2013-08-28 23:23:34 +030067
Sagi Grimberg2f1b6b72015-06-04 19:49:20 +030068static void isert_release_work(struct work_struct *work);
69
Sagi Grimberg302cc7c2014-12-02 16:57:34 +020070static inline bool
71isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd)
72{
Sagi Grimberg23a548e2014-12-02 16:57:35 +020073 return (conn->pi_support &&
Sagi Grimberg302cc7c2014-12-02 16:57:34 +020074 cmd->prot_op != TARGET_PROT_NORMAL);
75}
76
77
Vu Phamd40945d2013-08-28 23:23:34 +030078static void
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -080079isert_qp_event_callback(struct ib_event *e, void *context)
80{
Sagi Grimberg67004252015-03-29 15:52:09 +030081 struct isert_conn *isert_conn = context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -080082
Sagi Grimbergea8a1612015-05-18 13:40:31 +030083 isert_err("%s (%d): conn %p\n",
84 ib_event_msg(e->event), e->event, isert_conn);
85
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -080086 switch (e->event) {
87 case IB_EVENT_COMM_EST:
Sagi Grimbergdac6ab32015-03-29 15:52:19 +030088 rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -080089 break;
90 case IB_EVENT_QP_LAST_WQE_REACHED:
Sagi Grimberg4c22e072014-12-07 13:12:03 +020091 isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -080092 break;
93 default:
94 break;
95 }
96}
97
98static int
99isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr)
100{
101 int ret;
102
103 ret = ib_query_device(ib_dev, devattr);
104 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200105 isert_err("ib_query_device() failed: %d\n", ret);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800106 return ret;
107 }
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200108 isert_dbg("devattr->max_sge: %d\n", devattr->max_sge);
109 isert_dbg("devattr->max_sge_rd: %d\n", devattr->max_sge_rd);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800110
111 return 0;
112}
113
Sagi Grimberg40fc0692015-03-29 15:52:10 +0300114static struct isert_comp *
115isert_comp_get(struct isert_conn *isert_conn)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800116{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300117 struct isert_device *device = isert_conn->device;
Sagi Grimberg4a295ba2014-12-02 16:57:40 +0200118 struct isert_comp *comp;
Sagi Grimberg40fc0692015-03-29 15:52:10 +0300119 int i, min = 0;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800120
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800121 mutex_lock(&device_list_mutex);
Sagi Grimberg4a295ba2014-12-02 16:57:40 +0200122 for (i = 0; i < device->comps_used; i++)
123 if (device->comps[i].active_qps <
124 device->comps[min].active_qps)
125 min = i;
126 comp = &device->comps[min];
127 comp->active_qps++;
Sagi Grimberg40fc0692015-03-29 15:52:10 +0300128 mutex_unlock(&device_list_mutex);
129
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200130 isert_info("conn %p, using comp %p min_index: %d\n",
Sagi Grimberg4a295ba2014-12-02 16:57:40 +0200131 isert_conn, comp, min);
Sagi Grimberg40fc0692015-03-29 15:52:10 +0300132
133 return comp;
134}
135
136static void
137isert_comp_put(struct isert_comp *comp)
138{
139 mutex_lock(&device_list_mutex);
140 comp->active_qps--;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800141 mutex_unlock(&device_list_mutex);
Sagi Grimberg40fc0692015-03-29 15:52:10 +0300142}
143
144static struct ib_qp *
145isert_create_qp(struct isert_conn *isert_conn,
146 struct isert_comp *comp,
147 struct rdma_cm_id *cma_id)
148{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300149 struct isert_device *device = isert_conn->device;
Sagi Grimberg40fc0692015-03-29 15:52:10 +0300150 struct ib_qp_init_attr attr;
151 int ret;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800152
153 memset(&attr, 0, sizeof(struct ib_qp_init_attr));
154 attr.event_handler = isert_qp_event_callback;
155 attr.qp_context = isert_conn;
Sagi Grimberg6f0fae32014-12-02 16:57:41 +0200156 attr.send_cq = comp->cq;
157 attr.recv_cq = comp->cq;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800158 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS;
Sagi Grimbergbdf20e72014-12-02 16:57:43 +0200159 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800160 /*
161 * FIXME: Use devattr.max_sge - 2 for max_send_sge as
Or Gerlitzf57915c2014-10-22 14:55:49 -0700162 * work-around for RDMA_READs with ConnectX-2.
163 *
164 * Also, still make sure to have at least two SGEs for
165 * outgoing control PDU responses.
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800166 */
Or Gerlitzf57915c2014-10-22 14:55:49 -0700167 attr.cap.max_send_sge = max(2, device->dev_attr.max_sge - 2);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800168 isert_conn->max_sge = attr.cap.max_send_sge;
169
170 attr.cap.max_recv_sge = 1;
171 attr.sq_sig_type = IB_SIGNAL_REQ_WR;
172 attr.qp_type = IB_QPT_RC;
Sagi Grimberg570db172014-12-02 16:57:31 +0200173 if (device->pi_capable)
Sagi Grimbergd3e125d2014-02-19 17:50:23 +0200174 attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800175
Sagi Grimberg67cb3942015-03-29 15:52:05 +0300176 ret = rdma_create_qp(cma_id, device->pd, &attr);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800177 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200178 isert_err("rdma_create_qp failed for cma_id %d\n", ret);
Sagi Grimberg40fc0692015-03-29 15:52:10 +0300179 return ERR_PTR(ret);
180 }
181
182 return cma_id->qp;
183}
184
185static int
186isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
187{
188 struct isert_comp *comp;
189 int ret;
190
191 comp = isert_comp_get(isert_conn);
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300192 isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id);
193 if (IS_ERR(isert_conn->qp)) {
194 ret = PTR_ERR(isert_conn->qp);
Sagi Grimberg19e20902014-12-02 16:57:26 +0200195 goto err;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800196 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800197
198 return 0;
Sagi Grimberg19e20902014-12-02 16:57:26 +0200199err:
Sagi Grimberg40fc0692015-03-29 15:52:10 +0300200 isert_comp_put(comp);
Sagi Grimberg19e20902014-12-02 16:57:26 +0200201 return ret;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800202}
203
204static void
205isert_cq_event_callback(struct ib_event *e, void *context)
206{
Sagi Grimberg4c22e072014-12-07 13:12:03 +0200207 isert_dbg("event: %d\n", e->event);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800208}
209
210static int
211isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
212{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300213 struct isert_device *device = isert_conn->device;
Sagi Grimberg67cb3942015-03-29 15:52:05 +0300214 struct ib_device *ib_dev = device->ib_device;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800215 struct iser_rx_desc *rx_desc;
216 struct ib_sge *rx_sg;
217 u64 dma_addr;
218 int i, j;
219
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300220 isert_conn->rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS *
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800221 sizeof(struct iser_rx_desc), GFP_KERNEL);
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300222 if (!isert_conn->rx_descs)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800223 goto fail;
224
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300225 rx_desc = isert_conn->rx_descs;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800226
227 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) {
228 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc,
229 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
230 if (ib_dma_mapping_error(ib_dev, dma_addr))
231 goto dma_map_fail;
232
233 rx_desc->dma_addr = dma_addr;
234
235 rx_sg = &rx_desc->rx_sg;
236 rx_sg->addr = rx_desc->dma_addr;
237 rx_sg->length = ISER_RX_PAYLOAD_SIZE;
Jason Gunthorpe34efc7d2015-07-30 17:22:21 -0600238 rx_sg->lkey = device->pd->local_dma_lkey;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800239 }
240
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800241 return 0;
242
243dma_map_fail:
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300244 rx_desc = isert_conn->rx_descs;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800245 for (j = 0; j < i; j++, rx_desc++) {
246 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
247 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
248 }
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300249 kfree(isert_conn->rx_descs);
250 isert_conn->rx_descs = NULL;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800251fail:
Sagi Grimberg4c22e072014-12-07 13:12:03 +0200252 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn);
253
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800254 return -ENOMEM;
255}
256
257static void
258isert_free_rx_descriptors(struct isert_conn *isert_conn)
259{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300260 struct ib_device *ib_dev = isert_conn->device->ib_device;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800261 struct iser_rx_desc *rx_desc;
262 int i;
263
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300264 if (!isert_conn->rx_descs)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800265 return;
266
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300267 rx_desc = isert_conn->rx_descs;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800268 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) {
269 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
270 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
271 }
272
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300273 kfree(isert_conn->rx_descs);
274 isert_conn->rx_descs = NULL;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800275}
276
Sagi Grimberg6f0fae32014-12-02 16:57:41 +0200277static void isert_cq_work(struct work_struct *);
278static void isert_cq_callback(struct ib_cq *, void *);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800279
Sagi Grimberg172369c2015-03-29 15:52:11 +0300280static void
281isert_free_comps(struct isert_device *device)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800282{
Sagi Grimberg172369c2015-03-29 15:52:11 +0300283 int i;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800284
Sagi Grimberg172369c2015-03-29 15:52:11 +0300285 for (i = 0; i < device->comps_used; i++) {
286 struct isert_comp *comp = &device->comps[i];
Vu Pham59464ef2013-08-28 23:23:35 +0300287
Sagi Grimberg172369c2015-03-29 15:52:11 +0300288 if (comp->cq) {
289 cancel_work_sync(&comp->work);
290 ib_destroy_cq(comp->cq);
291 }
Vu Pham59464ef2013-08-28 23:23:35 +0300292 }
Sagi Grimberg172369c2015-03-29 15:52:11 +0300293 kfree(device->comps);
294}
Vu Phamd40945d2013-08-28 23:23:34 +0300295
Sagi Grimberg172369c2015-03-29 15:52:11 +0300296static int
297isert_alloc_comps(struct isert_device *device,
298 struct ib_device_attr *attr)
299{
300 int i, max_cqe, ret = 0;
Sagi Grimbergd3e125d2014-02-19 17:50:23 +0200301
Sagi Grimberg4a295ba2014-12-02 16:57:40 +0200302 device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(),
Sagi Grimberg172369c2015-03-29 15:52:11 +0300303 device->ib_device->num_comp_vectors));
304
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200305 isert_info("Using %d CQs, %s supports %d vectors support "
Sagi Grimberg4a295ba2014-12-02 16:57:40 +0200306 "Fast registration %d pi_capable %d\n",
307 device->comps_used, device->ib_device->name,
308 device->ib_device->num_comp_vectors, device->use_fastreg,
309 device->pi_capable);
310
311 device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp),
312 GFP_KERNEL);
313 if (!device->comps) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200314 isert_err("Unable to allocate completion contexts\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800315 return -ENOMEM;
316 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800317
Sagi Grimberg172369c2015-03-29 15:52:11 +0300318 max_cqe = min(ISER_MAX_CQ_LEN, attr->max_cqe);
319
Sagi Grimberg4a295ba2014-12-02 16:57:40 +0200320 for (i = 0; i < device->comps_used; i++) {
Matan Barak8e372102015-06-11 16:35:21 +0300321 struct ib_cq_init_attr cq_attr = {};
Sagi Grimberg4a295ba2014-12-02 16:57:40 +0200322 struct isert_comp *comp = &device->comps[i];
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800323
Sagi Grimberg4a295ba2014-12-02 16:57:40 +0200324 comp->device = device;
Sagi Grimberg6f0fae32014-12-02 16:57:41 +0200325 INIT_WORK(&comp->work, isert_cq_work);
Matan Barak8e372102015-06-11 16:35:21 +0300326 cq_attr.cqe = max_cqe;
327 cq_attr.comp_vector = i;
Sagi Grimberg6f0fae32014-12-02 16:57:41 +0200328 comp->cq = ib_create_cq(device->ib_device,
329 isert_cq_callback,
330 isert_cq_event_callback,
331 (void *)comp,
Matan Barak8e372102015-06-11 16:35:21 +0300332 &cq_attr);
Sagi Grimberg6f0fae32014-12-02 16:57:41 +0200333 if (IS_ERR(comp->cq)) {
Sagi Grimberg172369c2015-03-29 15:52:11 +0300334 isert_err("Unable to allocate cq\n");
Sagi Grimberg6f0fae32014-12-02 16:57:41 +0200335 ret = PTR_ERR(comp->cq);
336 comp->cq = NULL;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800337 goto out_cq;
Wei Yongjun94a71112013-10-29 09:56:34 +0800338 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800339
Sagi Grimberg6f0fae32014-12-02 16:57:41 +0200340 ret = ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP);
Wei Yongjun94a71112013-10-29 09:56:34 +0800341 if (ret)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800342 goto out_cq;
343 }
344
Sagi Grimberg172369c2015-03-29 15:52:11 +0300345 return 0;
346out_cq:
347 isert_free_comps(device);
348 return ret;
349}
350
351static int
352isert_create_device_ib_res(struct isert_device *device)
353{
Sagi Grimberg172369c2015-03-29 15:52:11 +0300354 struct ib_device_attr *dev_attr;
Sagi Grimbergfd8205e2015-03-29 15:52:12 +0300355 int ret;
Sagi Grimberg172369c2015-03-29 15:52:11 +0300356
357 dev_attr = &device->dev_attr;
Sagi Grimbergfd8205e2015-03-29 15:52:12 +0300358 ret = isert_query_device(device->ib_device, dev_attr);
Sagi Grimberg172369c2015-03-29 15:52:11 +0300359 if (ret)
360 return ret;
361
362 /* asign function handlers */
363 if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS &&
364 dev_attr->device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) {
365 device->use_fastreg = 1;
366 device->reg_rdma_mem = isert_reg_rdma;
367 device->unreg_rdma_mem = isert_unreg_rdma;
368 } else {
369 device->use_fastreg = 0;
370 device->reg_rdma_mem = isert_map_rdma;
371 device->unreg_rdma_mem = isert_unmap_cmd;
372 }
373
374 ret = isert_alloc_comps(device, dev_attr);
375 if (ret)
376 return ret;
377
Sagi Grimberg67cb3942015-03-29 15:52:05 +0300378 device->pd = ib_alloc_pd(device->ib_device);
379 if (IS_ERR(device->pd)) {
380 ret = PTR_ERR(device->pd);
381 isert_err("failed to allocate pd, device %p, ret=%d\n",
382 device, ret);
383 goto out_cq;
384 }
385
Sagi Grimberg172369c2015-03-29 15:52:11 +0300386 /* Check signature cap */
387 device->pi_capable = dev_attr->device_cap_flags &
388 IB_DEVICE_SIGNATURE_HANDOVER ? true : false;
Sagi Grimberg67cb3942015-03-29 15:52:05 +0300389
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800390 return 0;
391
392out_cq:
Sagi Grimberg172369c2015-03-29 15:52:11 +0300393 isert_free_comps(device);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800394 return ret;
395}
396
397static void
398isert_free_device_ib_res(struct isert_device *device)
399{
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200400 isert_info("device %p\n", device);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800401
Sagi Grimberg67cb3942015-03-29 15:52:05 +0300402 ib_dealloc_pd(device->pd);
Sagi Grimberg172369c2015-03-29 15:52:11 +0300403 isert_free_comps(device);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800404}
405
406static void
Sagi Grimbergcf8ae952015-03-29 15:52:14 +0300407isert_device_put(struct isert_device *device)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800408{
409 mutex_lock(&device_list_mutex);
410 device->refcount--;
Sagi Grimberg4c22e072014-12-07 13:12:03 +0200411 isert_info("device %p refcount %d\n", device, device->refcount);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800412 if (!device->refcount) {
413 isert_free_device_ib_res(device);
414 list_del(&device->dev_node);
415 kfree(device);
416 }
417 mutex_unlock(&device_list_mutex);
418}
419
420static struct isert_device *
Sagi Grimbergcf8ae952015-03-29 15:52:14 +0300421isert_device_get(struct rdma_cm_id *cma_id)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800422{
423 struct isert_device *device;
424 int ret;
425
426 mutex_lock(&device_list_mutex);
427 list_for_each_entry(device, &device_list, dev_node) {
428 if (device->ib_device->node_guid == cma_id->device->node_guid) {
429 device->refcount++;
Sagi Grimberg4c22e072014-12-07 13:12:03 +0200430 isert_info("Found iser device %p refcount %d\n",
431 device, device->refcount);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800432 mutex_unlock(&device_list_mutex);
433 return device;
434 }
435 }
436
437 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL);
438 if (!device) {
439 mutex_unlock(&device_list_mutex);
440 return ERR_PTR(-ENOMEM);
441 }
442
443 INIT_LIST_HEAD(&device->dev_node);
444
445 device->ib_device = cma_id->device;
446 ret = isert_create_device_ib_res(device);
447 if (ret) {
448 kfree(device);
449 mutex_unlock(&device_list_mutex);
450 return ERR_PTR(ret);
451 }
452
453 device->refcount++;
454 list_add_tail(&device->dev_node, &device_list);
Sagi Grimberg4c22e072014-12-07 13:12:03 +0200455 isert_info("Created a new iser device %p refcount %d\n",
456 device, device->refcount);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800457 mutex_unlock(&device_list_mutex);
458
459 return device;
460}
461
Vu Pham59464ef2013-08-28 23:23:35 +0300462static void
Sagi Grimberga3a5a822014-01-09 18:40:50 +0200463isert_conn_free_fastreg_pool(struct isert_conn *isert_conn)
Vu Pham59464ef2013-08-28 23:23:35 +0300464{
465 struct fast_reg_descriptor *fr_desc, *tmp;
466 int i = 0;
467
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300468 if (list_empty(&isert_conn->fr_pool))
Vu Pham59464ef2013-08-28 23:23:35 +0300469 return;
470
Sagi Grimberg4c22e072014-12-07 13:12:03 +0200471 isert_info("Freeing conn %p fastreg pool", isert_conn);
Vu Pham59464ef2013-08-28 23:23:35 +0300472
473 list_for_each_entry_safe(fr_desc, tmp,
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300474 &isert_conn->fr_pool, list) {
Vu Pham59464ef2013-08-28 23:23:35 +0300475 list_del(&fr_desc->list);
Vu Pham59464ef2013-08-28 23:23:35 +0300476 ib_dereg_mr(fr_desc->data_mr);
Sagi Grimbergd3e125d2014-02-19 17:50:23 +0200477 if (fr_desc->pi_ctx) {
Sagi Grimbergd3e125d2014-02-19 17:50:23 +0200478 ib_dereg_mr(fr_desc->pi_ctx->prot_mr);
Sagi Grimberg8b91ffc2015-07-30 10:32:34 +0300479 ib_dereg_mr(fr_desc->pi_ctx->sig_mr);
Sagi Grimbergd3e125d2014-02-19 17:50:23 +0200480 kfree(fr_desc->pi_ctx);
481 }
Vu Pham59464ef2013-08-28 23:23:35 +0300482 kfree(fr_desc);
483 ++i;
484 }
485
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300486 if (i < isert_conn->fr_pool_size)
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200487 isert_warn("Pool still has %d regions registered\n",
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300488 isert_conn->fr_pool_size - i);
Vu Pham59464ef2013-08-28 23:23:35 +0300489}
490
491static int
Sagi Grimberg570db172014-12-02 16:57:31 +0200492isert_create_pi_ctx(struct fast_reg_descriptor *desc,
493 struct ib_device *device,
494 struct ib_pd *pd)
495{
Sagi Grimberg570db172014-12-02 16:57:31 +0200496 struct pi_context *pi_ctx;
497 int ret;
498
499 pi_ctx = kzalloc(sizeof(*desc->pi_ctx), GFP_KERNEL);
500 if (!pi_ctx) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200501 isert_err("Failed to allocate pi context\n");
Sagi Grimberg570db172014-12-02 16:57:31 +0200502 return -ENOMEM;
503 }
504
Sagi Grimberga89be2c2015-07-30 10:32:37 +0300505 pi_ctx->prot_mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG,
506 ISCSI_ISER_SG_TABLESIZE);
Sagi Grimberg570db172014-12-02 16:57:31 +0200507 if (IS_ERR(pi_ctx->prot_mr)) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200508 isert_err("Failed to allocate prot frmr err=%ld\n",
Sagi Grimberg570db172014-12-02 16:57:31 +0200509 PTR_ERR(pi_ctx->prot_mr));
510 ret = PTR_ERR(pi_ctx->prot_mr);
Sagi Grimberg16c2d702015-10-13 19:11:34 +0300511 goto err_pi_ctx;
Sagi Grimberg570db172014-12-02 16:57:31 +0200512 }
513 desc->ind |= ISERT_PROT_KEY_VALID;
514
Sagi Grimberg9bee1782015-07-30 10:32:35 +0300515 pi_ctx->sig_mr = ib_alloc_mr(pd, IB_MR_TYPE_SIGNATURE, 2);
Sagi Grimberg570db172014-12-02 16:57:31 +0200516 if (IS_ERR(pi_ctx->sig_mr)) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200517 isert_err("Failed to allocate signature enabled mr err=%ld\n",
Sagi Grimberg570db172014-12-02 16:57:31 +0200518 PTR_ERR(pi_ctx->sig_mr));
519 ret = PTR_ERR(pi_ctx->sig_mr);
520 goto err_prot_mr;
521 }
522
523 desc->pi_ctx = pi_ctx;
524 desc->ind |= ISERT_SIG_KEY_VALID;
525 desc->ind &= ~ISERT_PROTECTED;
526
527 return 0;
528
529err_prot_mr:
Roland Dreierb2feda42015-05-29 23:12:10 -0700530 ib_dereg_mr(pi_ctx->prot_mr);
Sagi Grimberg570db172014-12-02 16:57:31 +0200531err_pi_ctx:
Roland Dreierb2feda42015-05-29 23:12:10 -0700532 kfree(pi_ctx);
Sagi Grimberg570db172014-12-02 16:57:31 +0200533
534 return ret;
535}
536
537static int
Sagi Grimbergdc87a902014-01-09 18:40:51 +0200538isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd,
Sagi Grimberg570db172014-12-02 16:57:31 +0200539 struct fast_reg_descriptor *fr_desc)
Sagi Grimbergdc87a902014-01-09 18:40:51 +0200540{
Sagi Grimberga89be2c2015-07-30 10:32:37 +0300541 fr_desc->data_mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG,
542 ISCSI_ISER_SG_TABLESIZE);
Sagi Grimbergdc87a902014-01-09 18:40:51 +0200543 if (IS_ERR(fr_desc->data_mr)) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200544 isert_err("Failed to allocate data frmr err=%ld\n",
Sagi Grimberg4c22e072014-12-07 13:12:03 +0200545 PTR_ERR(fr_desc->data_mr));
Sagi Grimberg16c2d702015-10-13 19:11:34 +0300546 return PTR_ERR(fr_desc->data_mr);
Sagi Grimbergdc87a902014-01-09 18:40:51 +0200547 }
Sagi Grimbergd3e125d2014-02-19 17:50:23 +0200548 fr_desc->ind |= ISERT_DATA_KEY_VALID;
Sagi Grimbergdc87a902014-01-09 18:40:51 +0200549
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200550 isert_dbg("Created fr_desc %p\n", fr_desc);
Sagi Grimbergdc87a902014-01-09 18:40:51 +0200551
552 return 0;
553}
554
555static int
Sagi Grimberg570db172014-12-02 16:57:31 +0200556isert_conn_create_fastreg_pool(struct isert_conn *isert_conn)
Vu Pham59464ef2013-08-28 23:23:35 +0300557{
558 struct fast_reg_descriptor *fr_desc;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300559 struct isert_device *device = isert_conn->device;
Nicholas Bellingerf46d6a82014-03-21 18:10:25 -0700560 struct se_session *se_sess = isert_conn->conn->sess->se_sess;
561 struct se_node_acl *se_nacl = se_sess->se_node_acl;
562 int i, ret, tag_num;
563 /*
564 * Setup the number of FRMRs based upon the number of tags
565 * available to session in iscsi_target_locate_portal().
566 */
567 tag_num = max_t(u32, ISCSIT_MIN_TAGS, se_nacl->queue_depth);
568 tag_num = (tag_num * 2) + ISCSIT_EXTRA_TAGS;
Vu Pham59464ef2013-08-28 23:23:35 +0300569
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300570 isert_conn->fr_pool_size = 0;
Nicholas Bellingerf46d6a82014-03-21 18:10:25 -0700571 for (i = 0; i < tag_num; i++) {
Vu Pham59464ef2013-08-28 23:23:35 +0300572 fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL);
573 if (!fr_desc) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200574 isert_err("Failed to allocate fast_reg descriptor\n");
Vu Pham59464ef2013-08-28 23:23:35 +0300575 ret = -ENOMEM;
576 goto err;
577 }
578
Sagi Grimbergdc87a902014-01-09 18:40:51 +0200579 ret = isert_create_fr_desc(device->ib_device,
Sagi Grimberg67cb3942015-03-29 15:52:05 +0300580 device->pd, fr_desc);
Sagi Grimbergdc87a902014-01-09 18:40:51 +0200581 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200582 isert_err("Failed to create fastreg descriptor err=%d\n",
Sagi Grimbergdc87a902014-01-09 18:40:51 +0200583 ret);
Nicholas Bellingera80e21b2014-02-03 12:59:56 -0800584 kfree(fr_desc);
Vu Pham59464ef2013-08-28 23:23:35 +0300585 goto err;
586 }
587
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300588 list_add_tail(&fr_desc->list, &isert_conn->fr_pool);
589 isert_conn->fr_pool_size++;
Vu Pham59464ef2013-08-28 23:23:35 +0300590 }
591
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200592 isert_dbg("Creating conn %p fastreg pool size=%d",
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300593 isert_conn, isert_conn->fr_pool_size);
Vu Pham59464ef2013-08-28 23:23:35 +0300594
595 return 0;
596
597err:
Sagi Grimberga3a5a822014-01-09 18:40:50 +0200598 isert_conn_free_fastreg_pool(isert_conn);
Vu Pham59464ef2013-08-28 23:23:35 +0300599 return ret;
600}
601
Sagi Grimbergae9ea9e2015-03-29 15:52:15 +0300602static void
603isert_init_conn(struct isert_conn *isert_conn)
604{
605 isert_conn->state = ISER_CONN_INIT;
Jenny Derzhavetzbd379222015-09-06 14:52:24 +0300606 INIT_LIST_HEAD(&isert_conn->node);
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300607 init_completion(&isert_conn->login_comp);
Sagi Grimbergae9ea9e2015-03-29 15:52:15 +0300608 init_completion(&isert_conn->login_req_comp);
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300609 init_completion(&isert_conn->wait);
610 kref_init(&isert_conn->kref);
611 mutex_init(&isert_conn->mutex);
612 spin_lock_init(&isert_conn->pool_lock);
613 INIT_LIST_HEAD(&isert_conn->fr_pool);
Sagi Grimberg2f1b6b72015-06-04 19:49:20 +0300614 INIT_WORK(&isert_conn->release_work, isert_release_work);
Sagi Grimbergae9ea9e2015-03-29 15:52:15 +0300615}
616
617static void
618isert_free_login_buf(struct isert_conn *isert_conn)
619{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300620 struct ib_device *ib_dev = isert_conn->device->ib_device;
Sagi Grimbergae9ea9e2015-03-29 15:52:15 +0300621
622 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
623 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
624 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
625 ISCSI_DEF_MAX_RECV_SEG_LEN,
626 DMA_FROM_DEVICE);
627 kfree(isert_conn->login_buf);
628}
629
630static int
631isert_alloc_login_buf(struct isert_conn *isert_conn,
632 struct ib_device *ib_dev)
633{
634 int ret;
635
636 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN +
637 ISER_RX_LOGIN_SIZE, GFP_KERNEL);
638 if (!isert_conn->login_buf) {
639 isert_err("Unable to allocate isert_conn->login_buf\n");
640 return -ENOMEM;
641 }
642
643 isert_conn->login_req_buf = isert_conn->login_buf;
644 isert_conn->login_rsp_buf = isert_conn->login_buf +
645 ISCSI_DEF_MAX_RECV_SEG_LEN;
646
647 isert_dbg("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n",
648 isert_conn->login_buf, isert_conn->login_req_buf,
649 isert_conn->login_rsp_buf);
650
651 isert_conn->login_req_dma = ib_dma_map_single(ib_dev,
652 (void *)isert_conn->login_req_buf,
653 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);
654
655 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma);
656 if (ret) {
657 isert_err("login_req_dma mapping error: %d\n", ret);
658 isert_conn->login_req_dma = 0;
659 goto out_login_buf;
660 }
661
662 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev,
663 (void *)isert_conn->login_rsp_buf,
664 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE);
665
666 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma);
667 if (ret) {
668 isert_err("login_rsp_dma mapping error: %d\n", ret);
669 isert_conn->login_rsp_dma = 0;
670 goto out_req_dma_map;
671 }
672
673 return 0;
674
675out_req_dma_map:
676 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
677 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE);
678out_login_buf:
679 kfree(isert_conn->login_buf);
680 return ret;
681}
682
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800683static int
684isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
685{
Sagi Grimbergca6c1d82014-12-02 16:57:27 +0200686 struct isert_np *isert_np = cma_id->context;
687 struct iscsi_np *np = isert_np->np;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800688 struct isert_conn *isert_conn;
689 struct isert_device *device;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800690 int ret = 0;
Sagi Grimberg14f4b542014-04-29 13:13:47 +0300691
692 spin_lock_bh(&np->np_thread_lock);
693 if (!np->enabled) {
694 spin_unlock_bh(&np->np_thread_lock);
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200695 isert_dbg("iscsi_np is not enabled, reject connect request\n");
Sagi Grimberg14f4b542014-04-29 13:13:47 +0300696 return rdma_reject(cma_id, NULL, 0);
697 }
698 spin_unlock_bh(&np->np_thread_lock);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800699
Sagi Grimberg4c22e072014-12-07 13:12:03 +0200700 isert_dbg("cma_id: %p, portal: %p\n",
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800701 cma_id, cma_id->context);
702
703 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL);
Sagi Grimbergae9ea9e2015-03-29 15:52:15 +0300704 if (!isert_conn)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800705 return -ENOMEM;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800706
Sagi Grimbergae9ea9e2015-03-29 15:52:15 +0300707 isert_init_conn(isert_conn);
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300708 isert_conn->cm_id = cma_id;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800709
Sagi Grimbergae9ea9e2015-03-29 15:52:15 +0300710 ret = isert_alloc_login_buf(isert_conn, cma_id->device);
711 if (ret)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800712 goto out;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800713
Sagi Grimbergcf8ae952015-03-29 15:52:14 +0300714 device = isert_device_get(cma_id);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800715 if (IS_ERR(device)) {
716 ret = PTR_ERR(device);
717 goto out_rsp_dma_map;
718 }
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300719 isert_conn->device = device;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800720
Sagi Grimberg1a92e172014-06-19 13:54:19 +0300721 /* Set max inflight RDMA READ requests */
722 isert_conn->initiator_depth = min_t(u8,
723 event->param.conn.initiator_depth,
724 device->dev_attr.max_qp_init_rd_atom);
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200725 isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth);
Sagi Grimberg1a92e172014-06-19 13:54:19 +0300726
Sagi Grimberg570db172014-12-02 16:57:31 +0200727 ret = isert_conn_setup_qp(isert_conn, cma_id);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800728 if (ret)
729 goto out_conn_dev;
730
Sagi Grimberg2371e5d2014-12-02 16:57:21 +0200731 ret = isert_rdma_post_recvl(isert_conn);
732 if (ret)
733 goto out_conn_dev;
734
735 ret = isert_rdma_accept(isert_conn);
736 if (ret)
737 goto out_conn_dev;
738
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +0300739 mutex_lock(&isert_np->mutex);
Jenny Derzhavetzbd379222015-09-06 14:52:24 +0300740 list_add_tail(&isert_conn->node, &isert_np->accepted);
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +0300741 mutex_unlock(&isert_np->mutex);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800742
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800743 return 0;
744
745out_conn_dev:
Sagi Grimbergcf8ae952015-03-29 15:52:14 +0300746 isert_device_put(device);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800747out_rsp_dma_map:
Sagi Grimbergae9ea9e2015-03-29 15:52:15 +0300748 isert_free_login_buf(isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800749out:
750 kfree(isert_conn);
Sagi Grimberg2371e5d2014-12-02 16:57:21 +0200751 rdma_reject(cma_id, NULL, 0);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800752 return ret;
753}
754
755static void
756isert_connect_release(struct isert_conn *isert_conn)
757{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300758 struct isert_device *device = isert_conn->device;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800759
Sagi Grimberg4c22e072014-12-07 13:12:03 +0200760 isert_dbg("conn %p\n", isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800761
Sagi Grimberg57df81e2015-03-29 15:52:17 +0300762 BUG_ON(!device);
763
764 if (device->use_fastreg)
Sagi Grimberga3a5a822014-01-09 18:40:50 +0200765 isert_conn_free_fastreg_pool(isert_conn);
Vu Pham59464ef2013-08-28 23:23:35 +0300766
Sagi Grimberg19e20902014-12-02 16:57:26 +0200767 isert_free_rx_descriptors(isert_conn);
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300768 if (isert_conn->cm_id)
769 rdma_destroy_id(isert_conn->cm_id);
Sagi Grimberg19e20902014-12-02 16:57:26 +0200770
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300771 if (isert_conn->qp) {
772 struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context;
Sagi Grimberg4a295ba2014-12-02 16:57:40 +0200773
Sagi Grimberg40fc0692015-03-29 15:52:10 +0300774 isert_comp_put(comp);
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300775 ib_destroy_qp(isert_conn->qp);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800776 }
777
Sagi Grimbergae9ea9e2015-03-29 15:52:15 +0300778 if (isert_conn->login_buf)
779 isert_free_login_buf(isert_conn);
780
Sagi Grimberg57df81e2015-03-29 15:52:17 +0300781 isert_device_put(device);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800782
Sagi Grimberg57df81e2015-03-29 15:52:17 +0300783 kfree(isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800784}
785
786static void
787isert_connected_handler(struct rdma_cm_id *cma_id)
788{
Sagi Grimberg19e20902014-12-02 16:57:26 +0200789 struct isert_conn *isert_conn = cma_id->qp->qp_context;
Jenny Derzhavetzbd379222015-09-06 14:52:24 +0300790 struct isert_np *isert_np = cma_id->context;
Sagi Grimbergc2f88b12014-07-02 16:19:24 +0300791
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200792 isert_info("conn %p\n", isert_conn);
Sagi Grimberg128e9cc2014-12-02 16:57:20 +0200793
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300794 mutex_lock(&isert_conn->mutex);
Jenny Derzhavetzbd379222015-09-06 14:52:24 +0300795 isert_conn->state = ISER_CONN_UP;
796 kref_get(&isert_conn->kref);
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300797 mutex_unlock(&isert_conn->mutex);
Jenny Derzhavetzbd379222015-09-06 14:52:24 +0300798
799 mutex_lock(&isert_np->mutex);
800 list_move_tail(&isert_conn->node, &isert_np->pending);
801 mutex_unlock(&isert_np->mutex);
802
803 isert_info("np %p: Allow accept_np to continue\n", isert_np);
804 up(&isert_np->sem);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800805}
806
807static void
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300808isert_release_kref(struct kref *kref)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800809{
810 struct isert_conn *isert_conn = container_of(kref,
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300811 struct isert_conn, kref);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800812
Sagi Grimberg4c22e072014-12-07 13:12:03 +0200813 isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm,
814 current->pid);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800815
816 isert_connect_release(isert_conn);
817}
818
819static void
820isert_put_conn(struct isert_conn *isert_conn)
821{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300822 kref_put(&isert_conn->kref, isert_release_kref);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800823}
824
Sagi Grimberg954f2372014-12-02 16:57:17 +0200825/**
826 * isert_conn_terminate() - Initiate connection termination
827 * @isert_conn: isert connection struct
828 *
829 * Notes:
Sagi Grimberg128e9cc2014-12-02 16:57:20 +0200830 * In case the connection state is FULL_FEATURE, move state
Sagi Grimberg954f2372014-12-02 16:57:17 +0200831 * to TEMINATING and start teardown sequence (rdma_disconnect).
Sagi Grimberg128e9cc2014-12-02 16:57:20 +0200832 * In case the connection state is UP, complete flush as well.
Sagi Grimberg954f2372014-12-02 16:57:17 +0200833 *
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300834 * This routine must be called with mutex held. Thus it is
Sagi Grimberg954f2372014-12-02 16:57:17 +0200835 * safe to call multiple times.
836 */
837static void
838isert_conn_terminate(struct isert_conn *isert_conn)
839{
840 int err;
841
Sagi Grimberg128e9cc2014-12-02 16:57:20 +0200842 switch (isert_conn->state) {
843 case ISER_CONN_TERMINATING:
844 break;
845 case ISER_CONN_UP:
Sagi Grimberg128e9cc2014-12-02 16:57:20 +0200846 case ISER_CONN_FULL_FEATURE: /* FALLTHRU */
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200847 isert_info("Terminating conn %p state %d\n",
Sagi Grimberg954f2372014-12-02 16:57:17 +0200848 isert_conn, isert_conn->state);
Sagi Grimberg128e9cc2014-12-02 16:57:20 +0200849 isert_conn->state = ISER_CONN_TERMINATING;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300850 err = rdma_disconnect(isert_conn->cm_id);
Sagi Grimberg954f2372014-12-02 16:57:17 +0200851 if (err)
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200852 isert_warn("Failed rdma_disconnect isert_conn %p\n",
Sagi Grimberg954f2372014-12-02 16:57:17 +0200853 isert_conn);
Sagi Grimberg128e9cc2014-12-02 16:57:20 +0200854 break;
855 default:
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200856 isert_warn("conn %p teminating in state %d\n",
Sagi Grimberg128e9cc2014-12-02 16:57:20 +0200857 isert_conn, isert_conn->state);
Sagi Grimberg954f2372014-12-02 16:57:17 +0200858 }
859}
860
Sagi Grimberg3b726ae2014-10-28 13:45:03 -0700861static int
Sagi Grimbergca6c1d82014-12-02 16:57:27 +0200862isert_np_cma_handler(struct isert_np *isert_np,
863 enum rdma_cm_event_type event)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800864{
Sagi Grimbergea8a1612015-05-18 13:40:31 +0300865 isert_dbg("%s (%d): isert np %p\n",
866 rdma_event_msg(event), event, isert_np);
Sagi Grimbergca6c1d82014-12-02 16:57:27 +0200867
868 switch (event) {
869 case RDMA_CM_EVENT_DEVICE_REMOVAL:
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +0300870 isert_np->cm_id = NULL;
Sagi Grimbergca6c1d82014-12-02 16:57:27 +0200871 break;
872 case RDMA_CM_EVENT_ADDR_CHANGE:
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +0300873 isert_np->cm_id = isert_setup_id(isert_np);
874 if (IS_ERR(isert_np->cm_id)) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200875 isert_err("isert np %p setup id failed: %ld\n",
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +0300876 isert_np, PTR_ERR(isert_np->cm_id));
877 isert_np->cm_id = NULL;
Sagi Grimbergca6c1d82014-12-02 16:57:27 +0200878 }
879 break;
880 default:
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200881 isert_err("isert np %p Unexpected event %d\n",
Sagi Grimbergca6c1d82014-12-02 16:57:27 +0200882 isert_np, event);
883 }
884
885 return -1;
886}
887
888static int
889isert_disconnected_handler(struct rdma_cm_id *cma_id,
890 enum rdma_cm_event_type event)
891{
892 struct isert_np *isert_np = cma_id->context;
Sagi Grimberg3b726ae2014-10-28 13:45:03 -0700893 struct isert_conn *isert_conn;
Sagi Grimberg2f1b6b72015-06-04 19:49:20 +0300894 bool terminating = false;
Sagi Grimberg3b726ae2014-10-28 13:45:03 -0700895
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +0300896 if (isert_np->cm_id == cma_id)
Sagi Grimbergca6c1d82014-12-02 16:57:27 +0200897 return isert_np_cma_handler(cma_id->context, event);
Sagi Grimberg3b726ae2014-10-28 13:45:03 -0700898
Sagi Grimberg19e20902014-12-02 16:57:26 +0200899 isert_conn = cma_id->qp->qp_context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800900
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300901 mutex_lock(&isert_conn->mutex);
Sagi Grimberg2f1b6b72015-06-04 19:49:20 +0300902 terminating = (isert_conn->state == ISER_CONN_TERMINATING);
Sagi Grimberg128e9cc2014-12-02 16:57:20 +0200903 isert_conn_terminate(isert_conn);
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300904 mutex_unlock(&isert_conn->mutex);
Sagi Grimberg128e9cc2014-12-02 16:57:20 +0200905
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300906 isert_info("conn %p completing wait\n", isert_conn);
907 complete(&isert_conn->wait);
Sagi Grimberg3b726ae2014-10-28 13:45:03 -0700908
Sagi Grimberg2f1b6b72015-06-04 19:49:20 +0300909 if (terminating)
910 goto out;
911
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +0300912 mutex_lock(&isert_np->mutex);
Jenny Derzhavetzbd379222015-09-06 14:52:24 +0300913 if (!list_empty(&isert_conn->node)) {
914 list_del_init(&isert_conn->node);
Sagi Grimberg2f1b6b72015-06-04 19:49:20 +0300915 isert_put_conn(isert_conn);
916 queue_work(isert_release_wq, &isert_conn->release_work);
917 }
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +0300918 mutex_unlock(&isert_np->mutex);
Sagi Grimberg2f1b6b72015-06-04 19:49:20 +0300919
920out:
Sagi Grimberg3b726ae2014-10-28 13:45:03 -0700921 return 0;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800922}
923
Sagi Grimberg4a579da2015-03-29 15:52:04 +0300924static int
Sagi Grimberg954f2372014-12-02 16:57:17 +0200925isert_connect_error(struct rdma_cm_id *cma_id)
926{
Sagi Grimberg19e20902014-12-02 16:57:26 +0200927 struct isert_conn *isert_conn = cma_id->qp->qp_context;
Sagi Grimberg954f2372014-12-02 16:57:17 +0200928
Jenny Derzhavetzbd379222015-09-06 14:52:24 +0300929 list_del_init(&isert_conn->node);
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300930 isert_conn->cm_id = NULL;
Sagi Grimberg954f2372014-12-02 16:57:17 +0200931 isert_put_conn(isert_conn);
Sagi Grimberg4a579da2015-03-29 15:52:04 +0300932
933 return -1;
Sagi Grimberg954f2372014-12-02 16:57:17 +0200934}
935
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800936static int
937isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
938{
939 int ret = 0;
940
Sagi Grimbergea8a1612015-05-18 13:40:31 +0300941 isert_info("%s (%d): status %d id %p np %p\n",
942 rdma_event_msg(event->event), event->event,
Sagi Grimberg4c22e072014-12-07 13:12:03 +0200943 event->status, cma_id, cma_id->context);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800944
945 switch (event->event) {
946 case RDMA_CM_EVENT_CONNECT_REQUEST:
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800947 ret = isert_connect_request(cma_id, event);
Sagi Grimberg3b726ae2014-10-28 13:45:03 -0700948 if (ret)
Sagi Grimberg4c22e072014-12-07 13:12:03 +0200949 isert_err("failed handle connect request %d\n", ret);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800950 break;
951 case RDMA_CM_EVENT_ESTABLISHED:
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800952 isert_connected_handler(cma_id);
953 break;
Sagi Grimberg88c40152014-05-19 17:44:24 +0300954 case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */
955 case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */
956 case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */
Sagi Grimberg88c40152014-05-19 17:44:24 +0300957 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */
Sagi Grimbergca6c1d82014-12-02 16:57:27 +0200958 ret = isert_disconnected_handler(cma_id, event->event);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800959 break;
Sagi Grimberg954f2372014-12-02 16:57:17 +0200960 case RDMA_CM_EVENT_REJECTED: /* FALLTHRU */
961 case RDMA_CM_EVENT_UNREACHABLE: /* FALLTHRU */
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800962 case RDMA_CM_EVENT_CONNECT_ERROR:
Sagi Grimberg4a579da2015-03-29 15:52:04 +0300963 ret = isert_connect_error(cma_id);
Sagi Grimberg954f2372014-12-02 16:57:17 +0200964 break;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800965 default:
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200966 isert_err("Unhandled RDMA CMA event: %d\n", event->event);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800967 break;
968 }
969
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800970 return ret;
971}
972
973static int
Jenny Derzhavetz4366b192015-09-06 14:52:25 +0300974isert_post_recvm(struct isert_conn *isert_conn, u32 count)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800975{
976 struct ib_recv_wr *rx_wr, *rx_wr_failed;
977 int i, ret;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800978 struct iser_rx_desc *rx_desc;
979
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300980 for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) {
Jenny Derzhavetz4366b192015-09-06 14:52:25 +0300981 rx_desc = &isert_conn->rx_descs[i];
982 rx_wr->wr_id = (uintptr_t)rx_desc;
983 rx_wr->sg_list = &rx_desc->rx_sg;
984 rx_wr->num_sge = 1;
985 rx_wr->next = rx_wr + 1;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800986 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800987 rx_wr--;
988 rx_wr->next = NULL; /* mark end of work requests list */
989
990 isert_conn->post_recv_buf_count += count;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +0300991 ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr,
Jenny Derzhavetz4366b192015-09-06 14:52:25 +0300992 &rx_wr_failed);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800993 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +0200994 isert_err("ib_post_recv() failed with ret: %d\n", ret);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800995 isert_conn->post_recv_buf_count -= count;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -0800996 }
Jenny Derzhavetz4366b192015-09-06 14:52:25 +0300997
998 return ret;
999}
1000
1001static int
1002isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc)
1003{
1004 struct ib_recv_wr *rx_wr_failed, rx_wr;
1005 int ret;
1006
1007 rx_wr.wr_id = (uintptr_t)rx_desc;
1008 rx_wr.sg_list = &rx_desc->rx_sg;
1009 rx_wr.num_sge = 1;
1010 rx_wr.next = NULL;
1011
1012 isert_conn->post_recv_buf_count++;
1013 ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_failed);
1014 if (ret) {
1015 isert_err("ib_post_recv() failed with ret: %d\n", ret);
1016 isert_conn->post_recv_buf_count--;
1017 }
1018
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001019 return ret;
1020}
1021
1022static int
1023isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc)
1024{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001025 struct ib_device *ib_dev = isert_conn->cm_id->device;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001026 struct ib_send_wr send_wr, *send_wr_failed;
1027 int ret;
1028
1029 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr,
1030 ISER_HEADERS_LEN, DMA_TO_DEVICE);
1031
1032 send_wr.next = NULL;
Sagi Grimbergb0a191e2014-12-02 16:57:39 +02001033 send_wr.wr_id = (uintptr_t)tx_desc;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001034 send_wr.sg_list = tx_desc->tx_sg;
1035 send_wr.num_sge = tx_desc->num_sge;
1036 send_wr.opcode = IB_WR_SEND;
1037 send_wr.send_flags = IB_SEND_SIGNALED;
1038
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001039 ret = ib_post_send(isert_conn->qp, &send_wr, &send_wr_failed);
Sagi Grimbergbdf20e72014-12-02 16:57:43 +02001040 if (ret)
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001041 isert_err("ib_post_send() failed, ret: %d\n", ret);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001042
1043 return ret;
1044}
1045
1046static void
1047isert_create_send_desc(struct isert_conn *isert_conn,
1048 struct isert_cmd *isert_cmd,
1049 struct iser_tx_desc *tx_desc)
1050{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001051 struct isert_device *device = isert_conn->device;
Sagi Grimberg67cb3942015-03-29 15:52:05 +03001052 struct ib_device *ib_dev = device->ib_device;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001053
1054 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
1055 ISER_HEADERS_LEN, DMA_TO_DEVICE);
1056
1057 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
1058 tx_desc->iser_header.flags = ISER_VER;
1059
1060 tx_desc->num_sge = 1;
1061 tx_desc->isert_cmd = isert_cmd;
1062
Jason Gunthorpe34efc7d2015-07-30 17:22:21 -06001063 if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) {
1064 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey;
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001065 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001066 }
1067}
1068
1069static int
1070isert_init_tx_hdrs(struct isert_conn *isert_conn,
1071 struct iser_tx_desc *tx_desc)
1072{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001073 struct isert_device *device = isert_conn->device;
Sagi Grimberg67cb3942015-03-29 15:52:05 +03001074 struct ib_device *ib_dev = device->ib_device;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001075 u64 dma_addr;
1076
1077 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc,
1078 ISER_HEADERS_LEN, DMA_TO_DEVICE);
1079 if (ib_dma_mapping_error(ib_dev, dma_addr)) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001080 isert_err("ib_dma_mapping_error() failed\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001081 return -ENOMEM;
1082 }
1083
1084 tx_desc->dma_addr = dma_addr;
1085 tx_desc->tx_sg[0].addr = tx_desc->dma_addr;
1086 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
Jason Gunthorpe34efc7d2015-07-30 17:22:21 -06001087 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001088
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001089 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n",
1090 tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length,
1091 tx_desc->tx_sg[0].lkey);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001092
1093 return 0;
1094}
1095
1096static void
Nicholas Bellinger95b60f02013-11-05 13:16:12 -08001097isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
Sagi Grimberg68a86de2014-12-02 16:57:37 +02001098 struct ib_send_wr *send_wr)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001099{
Nicholas Bellinger95b60f02013-11-05 13:16:12 -08001100 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc;
1101
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001102 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND;
Sagi Grimbergb0a191e2014-12-02 16:57:39 +02001103 send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001104 send_wr->opcode = IB_WR_SEND;
Nicholas Bellinger95b60f02013-11-05 13:16:12 -08001105 send_wr->sg_list = &tx_desc->tx_sg[0];
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001106 send_wr->num_sge = isert_cmd->tx_desc.num_sge;
Nicholas Bellinger95b60f02013-11-05 13:16:12 -08001107 send_wr->send_flags = IB_SEND_SIGNALED;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001108}
1109
1110static int
1111isert_rdma_post_recvl(struct isert_conn *isert_conn)
1112{
1113 struct ib_recv_wr rx_wr, *rx_wr_fail;
1114 struct ib_sge sge;
1115 int ret;
1116
1117 memset(&sge, 0, sizeof(struct ib_sge));
1118 sge.addr = isert_conn->login_req_dma;
1119 sge.length = ISER_RX_LOGIN_SIZE;
Jason Gunthorpe34efc7d2015-07-30 17:22:21 -06001120 sge.lkey = isert_conn->device->pd->local_dma_lkey;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001121
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001122 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n",
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001123 sge.addr, sge.length, sge.lkey);
1124
1125 memset(&rx_wr, 0, sizeof(struct ib_recv_wr));
Sagi Grimbergb0a191e2014-12-02 16:57:39 +02001126 rx_wr.wr_id = (uintptr_t)isert_conn->login_req_buf;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001127 rx_wr.sg_list = &sge;
1128 rx_wr.num_sge = 1;
1129
1130 isert_conn->post_recv_buf_count++;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001131 ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_fail);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001132 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001133 isert_err("ib_post_recv() failed: %d\n", ret);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001134 isert_conn->post_recv_buf_count--;
1135 }
1136
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001137 return ret;
1138}
1139
1140static int
1141isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
1142 u32 length)
1143{
1144 struct isert_conn *isert_conn = conn->context;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001145 struct isert_device *device = isert_conn->device;
Sagi Grimberg67cb3942015-03-29 15:52:05 +03001146 struct ib_device *ib_dev = device->ib_device;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001147 struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001148 int ret;
1149
1150 isert_create_send_desc(isert_conn, NULL, tx_desc);
1151
1152 memcpy(&tx_desc->iscsi_header, &login->rsp[0],
1153 sizeof(struct iscsi_hdr));
1154
1155 isert_init_tx_hdrs(isert_conn, tx_desc);
1156
1157 if (length > 0) {
1158 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1];
1159
1160 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma,
1161 length, DMA_TO_DEVICE);
1162
1163 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length);
1164
1165 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma,
1166 length, DMA_TO_DEVICE);
1167
1168 tx_dsg->addr = isert_conn->login_rsp_dma;
1169 tx_dsg->length = length;
Jason Gunthorpe34efc7d2015-07-30 17:22:21 -06001170 tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001171 tx_desc->num_sge = 2;
1172 }
1173 if (!login->login_failed) {
1174 if (login->login_complete) {
Sagi Grimberge0546fc2014-06-10 13:41:41 +03001175 if (!conn->sess->sess_ops->SessionType &&
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001176 isert_conn->device->use_fastreg) {
Sagi Grimberg570db172014-12-02 16:57:31 +02001177 ret = isert_conn_create_fastreg_pool(isert_conn);
Nicholas Bellingerf46d6a82014-03-21 18:10:25 -07001178 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001179 isert_err("Conn: %p failed to create"
Nicholas Bellingerf46d6a82014-03-21 18:10:25 -07001180 " fastreg pool\n", isert_conn);
1181 return ret;
1182 }
1183 }
1184
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001185 ret = isert_alloc_rx_descriptors(isert_conn);
1186 if (ret)
1187 return ret;
1188
Jenny Derzhavetz4366b192015-09-06 14:52:25 +03001189 ret = isert_post_recvm(isert_conn,
1190 ISERT_QP_MAX_RECV_DTOS);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001191 if (ret)
1192 return ret;
1193
Sagi Grimberg128e9cc2014-12-02 16:57:20 +02001194 /* Now we are in FULL_FEATURE phase */
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001195 mutex_lock(&isert_conn->mutex);
Sagi Grimberg128e9cc2014-12-02 16:57:20 +02001196 isert_conn->state = ISER_CONN_FULL_FEATURE;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001197 mutex_unlock(&isert_conn->mutex);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001198 goto post_send;
1199 }
1200
1201 ret = isert_rdma_post_recvl(isert_conn);
1202 if (ret)
1203 return ret;
1204 }
1205post_send:
1206 ret = isert_post_send(isert_conn, tx_desc);
1207 if (ret)
1208 return ret;
1209
1210 return 0;
1211}
1212
1213static void
Sagi Grimberg2371e5d2014-12-02 16:57:21 +02001214isert_rx_login_req(struct isert_conn *isert_conn)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001215{
Sagi Grimberg2371e5d2014-12-02 16:57:21 +02001216 struct iser_rx_desc *rx_desc = (void *)isert_conn->login_req_buf;
1217 int rx_buflen = isert_conn->login_req_len;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001218 struct iscsi_conn *conn = isert_conn->conn;
1219 struct iscsi_login *login = conn->conn_login;
1220 int size;
1221
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001222 isert_info("conn %p\n", isert_conn);
Sagi Grimberg2371e5d2014-12-02 16:57:21 +02001223
1224 WARN_ON_ONCE(!login);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001225
1226 if (login->first_request) {
1227 struct iscsi_login_req *login_req =
1228 (struct iscsi_login_req *)&rx_desc->iscsi_header;
1229 /*
1230 * Setup the initial iscsi_login values from the leading
1231 * login request PDU.
1232 */
1233 login->leading_connection = (!login_req->tsih) ? 1 : 0;
1234 login->current_stage =
1235 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK)
1236 >> 2;
1237 login->version_min = login_req->min_version;
1238 login->version_max = login_req->max_version;
1239 memcpy(login->isid, login_req->isid, 6);
1240 login->cmd_sn = be32_to_cpu(login_req->cmdsn);
1241 login->init_task_tag = login_req->itt;
1242 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1243 login->cid = be16_to_cpu(login_req->cid);
1244 login->tsih = be16_to_cpu(login_req->tsih);
1245 }
1246
1247 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN);
1248
1249 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS);
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001250 isert_dbg("Using login payload size: %d, rx_buflen: %d "
1251 "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen,
1252 MAX_KEY_VALUE_PAIRS);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001253 memcpy(login->req_buf, &rx_desc->data[0], size);
1254
Nicholas Bellinger6faaa852013-08-18 16:35:46 -07001255 if (login->first_request) {
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001256 complete(&isert_conn->login_comp);
Nicholas Bellinger6faaa852013-08-18 16:35:46 -07001257 return;
1258 }
1259 schedule_delayed_work(&conn->login_work, 0);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001260}
1261
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001262static struct iscsi_cmd
Jenny Derzhavetz4366b192015-09-06 14:52:25 +03001263*isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001264{
Sagi Grimberg67004252015-03-29 15:52:09 +03001265 struct isert_conn *isert_conn = conn->context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001266 struct isert_cmd *isert_cmd;
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001267 struct iscsi_cmd *cmd;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001268
Nicholas Bellinger676687c2014-01-20 03:36:44 +00001269 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001270 if (!cmd) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001271 isert_err("Unable to allocate iscsi_cmd + isert_cmd\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001272 return NULL;
1273 }
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001274 isert_cmd = iscsit_priv_cmd(cmd);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001275 isert_cmd->conn = isert_conn;
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001276 isert_cmd->iscsi_cmd = cmd;
Jenny Derzhavetz4366b192015-09-06 14:52:25 +03001277 isert_cmd->rx_desc = rx_desc;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001278
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001279 return cmd;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001280}
1281
1282static int
1283isert_handle_scsi_cmd(struct isert_conn *isert_conn,
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001284 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd,
1285 struct iser_rx_desc *rx_desc, unsigned char *buf)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001286{
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001287 struct iscsi_conn *conn = isert_conn->conn;
1288 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001289 int imm_data, imm_data_len, unsol_data, sg_nents, rc;
1290 bool dump_payload = false;
Jenny Derzhavetz9fd60082015-09-06 14:52:26 +03001291 unsigned int data_len;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001292
1293 rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
1294 if (rc < 0)
1295 return rc;
1296
1297 imm_data = cmd->immediate_data;
1298 imm_data_len = cmd->first_burst_len;
1299 unsol_data = cmd->unsolicited_data;
Jenny Derzhavetz9fd60082015-09-06 14:52:26 +03001300 data_len = cmd->se_cmd.data_length;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001301
Jenny Derzhavetz9fd60082015-09-06 14:52:26 +03001302 if (imm_data && imm_data_len == data_len)
1303 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001304 rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
1305 if (rc < 0) {
1306 return 0;
1307 } else if (rc > 0) {
1308 dump_payload = true;
1309 goto sequence_cmd;
1310 }
1311
1312 if (!imm_data)
1313 return 0;
1314
Jenny Derzhavetz9fd60082015-09-06 14:52:26 +03001315 if (imm_data_len != data_len) {
1316 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE));
1317 sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents,
1318 &rx_desc->data[0], imm_data_len);
1319 isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n",
1320 sg_nents, imm_data_len);
1321 } else {
1322 sg_init_table(&isert_cmd->sg, 1);
1323 cmd->se_cmd.t_data_sg = &isert_cmd->sg;
1324 cmd->se_cmd.t_data_nents = 1;
1325 sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len);
1326 isert_dbg("Transfer Immediate imm_data_len: %d\n",
1327 imm_data_len);
1328 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001329
1330 cmd->write_data_done += imm_data_len;
1331
1332 if (cmd->write_data_done == cmd->se_cmd.data_length) {
1333 spin_lock_bh(&cmd->istate_lock);
1334 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1335 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1336 spin_unlock_bh(&cmd->istate_lock);
1337 }
1338
1339sequence_cmd:
Nicholas Bellinger561bf152013-07-03 03:58:58 -07001340 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001341
1342 if (!rc && dump_payload == false && unsol_data)
1343 iscsit_set_unsoliticed_dataout(cmd);
Nicholas Bellinger6cc44a62014-05-23 00:48:35 -07001344 else if (dump_payload && imm_data)
Bart Van Asscheafc16602015-04-27 13:52:36 +02001345 target_put_sess_cmd(&cmd->se_cmd);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001346
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001347 return 0;
1348}
1349
1350static int
1351isert_handle_iscsi_dataout(struct isert_conn *isert_conn,
1352 struct iser_rx_desc *rx_desc, unsigned char *buf)
1353{
1354 struct scatterlist *sg_start;
1355 struct iscsi_conn *conn = isert_conn->conn;
1356 struct iscsi_cmd *cmd = NULL;
1357 struct iscsi_data *hdr = (struct iscsi_data *)buf;
1358 u32 unsol_data_len = ntoh24(hdr->dlength);
1359 int rc, sg_nents, sg_off, page_off;
1360
1361 rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
1362 if (rc < 0)
1363 return rc;
1364 else if (!cmd)
1365 return 0;
1366 /*
1367 * FIXME: Unexpected unsolicited_data out
1368 */
1369 if (!cmd->unsolicited_data) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001370 isert_err("Received unexpected solicited data payload\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001371 dump_stack();
1372 return -1;
1373 }
1374
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001375 isert_dbg("Unsolicited DataOut unsol_data_len: %u, "
1376 "write_data_done: %u, data_length: %u\n",
1377 unsol_data_len, cmd->write_data_done,
1378 cmd->se_cmd.data_length);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001379
1380 sg_off = cmd->write_data_done / PAGE_SIZE;
1381 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1382 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE));
1383 page_off = cmd->write_data_done % PAGE_SIZE;
1384 /*
1385 * FIXME: Non page-aligned unsolicited_data out
1386 */
1387 if (page_off) {
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001388 isert_err("unexpected non-page aligned data payload\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001389 dump_stack();
1390 return -1;
1391 }
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001392 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u "
1393 "sg_nents: %u from %p %u\n", sg_start, sg_off,
1394 sg_nents, &rx_desc->data[0], unsol_data_len);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001395
1396 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0],
1397 unsol_data_len);
1398
1399 rc = iscsit_check_dataout_payload(cmd, hdr, false);
1400 if (rc < 0)
1401 return rc;
1402
Jenny Derzhavetz4366b192015-09-06 14:52:25 +03001403 /*
1404 * multiple data-outs on the same command can arrive -
1405 * so post the buffer before hand
1406 */
1407 rc = isert_post_recv(isert_conn, rx_desc);
1408 if (rc) {
1409 isert_err("ib_post_recv failed with %d\n", rc);
1410 return rc;
1411 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001412 return 0;
1413}
1414
1415static int
Nicholas Bellinger778de362013-06-14 16:07:47 -07001416isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001417 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc,
1418 unsigned char *buf)
Nicholas Bellinger778de362013-06-14 16:07:47 -07001419{
Nicholas Bellinger778de362013-06-14 16:07:47 -07001420 struct iscsi_conn *conn = isert_conn->conn;
1421 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf;
1422 int rc;
1423
1424 rc = iscsit_setup_nop_out(conn, cmd, hdr);
1425 if (rc < 0)
1426 return rc;
1427 /*
1428 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload
1429 */
1430
1431 return iscsit_process_nop_out(conn, cmd, hdr);
1432}
1433
1434static int
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07001435isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001436 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc,
1437 struct iscsi_text *hdr)
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07001438{
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07001439 struct iscsi_conn *conn = isert_conn->conn;
1440 u32 payload_length = ntoh24(hdr->dlength);
1441 int rc;
Sagi Grimbergb44a2b62015-01-26 12:49:08 +02001442 unsigned char *text_in = NULL;
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07001443
1444 rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1445 if (rc < 0)
1446 return rc;
1447
Sagi Grimbergb44a2b62015-01-26 12:49:08 +02001448 if (payload_length) {
1449 text_in = kzalloc(payload_length, GFP_KERNEL);
1450 if (!text_in) {
1451 isert_err("Unable to allocate text_in of payload_length: %u\n",
1452 payload_length);
1453 return -ENOMEM;
1454 }
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07001455 }
1456 cmd->text_in_ptr = text_in;
1457
1458 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length);
1459
1460 return iscsit_process_text_cmd(conn, cmd, hdr);
1461}
1462
1463static int
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001464isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
1465 uint32_t read_stag, uint64_t read_va,
1466 uint32_t write_stag, uint64_t write_va)
1467{
1468 struct iscsi_hdr *hdr = &rx_desc->iscsi_header;
1469 struct iscsi_conn *conn = isert_conn->conn;
1470 struct iscsi_cmd *cmd;
1471 struct isert_cmd *isert_cmd;
1472 int ret = -EINVAL;
1473 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1474
Sagi Grimbergfb140272015-03-29 15:52:08 +03001475 if (conn->sess->sess_ops->SessionType &&
Nicholas Bellingerca40d242013-07-07 17:45:08 -07001476 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001477 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery,"
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001478 " ignoring\n", opcode);
Nicholas Bellingerca40d242013-07-07 17:45:08 -07001479 return 0;
1480 }
1481
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001482 switch (opcode) {
1483 case ISCSI_OP_SCSI_CMD:
Jenny Derzhavetz4366b192015-09-06 14:52:25 +03001484 cmd = isert_allocate_cmd(conn, rx_desc);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001485 if (!cmd)
1486 break;
1487
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001488 isert_cmd = iscsit_priv_cmd(cmd);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001489 isert_cmd->read_stag = read_stag;
1490 isert_cmd->read_va = read_va;
1491 isert_cmd->write_stag = write_stag;
1492 isert_cmd->write_va = write_va;
1493
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001494 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd,
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001495 rx_desc, (unsigned char *)hdr);
1496 break;
1497 case ISCSI_OP_NOOP_OUT:
Jenny Derzhavetz4366b192015-09-06 14:52:25 +03001498 cmd = isert_allocate_cmd(conn, rx_desc);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001499 if (!cmd)
1500 break;
1501
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001502 isert_cmd = iscsit_priv_cmd(cmd);
1503 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd,
Nicholas Bellinger778de362013-06-14 16:07:47 -07001504 rx_desc, (unsigned char *)hdr);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001505 break;
1506 case ISCSI_OP_SCSI_DATA_OUT:
1507 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc,
1508 (unsigned char *)hdr);
1509 break;
1510 case ISCSI_OP_SCSI_TMFUNC:
Jenny Derzhavetz4366b192015-09-06 14:52:25 +03001511 cmd = isert_allocate_cmd(conn, rx_desc);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001512 if (!cmd)
1513 break;
1514
1515 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1516 (unsigned char *)hdr);
1517 break;
1518 case ISCSI_OP_LOGOUT:
Jenny Derzhavetz4366b192015-09-06 14:52:25 +03001519 cmd = isert_allocate_cmd(conn, rx_desc);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001520 if (!cmd)
1521 break;
1522
1523 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001524 break;
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07001525 case ISCSI_OP_TEXT:
Jenny Derzhavetz4366b192015-09-06 14:52:25 +03001526 if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF)
Sagi Grimberge4f4e802015-02-09 18:07:25 +02001527 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
Jenny Derzhavetz4366b192015-09-06 14:52:25 +03001528 else
1529 cmd = isert_allocate_cmd(conn, rx_desc);
1530
1531 if (!cmd)
1532 break;
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07001533
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001534 isert_cmd = iscsit_priv_cmd(cmd);
1535 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd,
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07001536 rx_desc, (struct iscsi_text *)hdr);
1537 break;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001538 default:
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001539 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001540 dump_stack();
1541 break;
1542 }
1543
1544 return ret;
1545}
1546
1547static void
1548isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn)
1549{
1550 struct iser_hdr *iser_hdr = &rx_desc->iser_header;
1551 uint64_t read_va = 0, write_va = 0;
1552 uint32_t read_stag = 0, write_stag = 0;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001553
1554 switch (iser_hdr->flags & 0xF0) {
1555 case ISCSI_CTRL:
1556 if (iser_hdr->flags & ISER_RSV) {
1557 read_stag = be32_to_cpu(iser_hdr->read_stag);
1558 read_va = be64_to_cpu(iser_hdr->read_va);
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001559 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n",
1560 read_stag, (unsigned long long)read_va);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001561 }
1562 if (iser_hdr->flags & ISER_WSV) {
1563 write_stag = be32_to_cpu(iser_hdr->write_stag);
1564 write_va = be64_to_cpu(iser_hdr->write_va);
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001565 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n",
1566 write_stag, (unsigned long long)write_va);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001567 }
1568
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001569 isert_dbg("ISER ISCSI_CTRL PDU\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001570 break;
1571 case ISER_HELLO:
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001572 isert_err("iSER Hello message\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001573 break;
1574 default:
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001575 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001576 break;
1577 }
1578
Bart Van Assche6c760b32015-10-22 11:14:18 -07001579 isert_rx_opcode(isert_conn, rx_desc,
1580 read_stag, read_va, write_stag, write_va);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001581}
1582
1583static void
Sagi Grimberg77486812015-03-29 15:52:13 +03001584isert_rcv_completion(struct iser_rx_desc *desc,
1585 struct isert_conn *isert_conn,
1586 u32 xfer_len)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001587{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001588 struct ib_device *ib_dev = isert_conn->cm_id->device;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001589 struct iscsi_hdr *hdr;
1590 u64 rx_dma;
Jenny Derzhavetz4366b192015-09-06 14:52:25 +03001591 int rx_buflen;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001592
1593 if ((char *)desc == isert_conn->login_req_buf) {
1594 rx_dma = isert_conn->login_req_dma;
1595 rx_buflen = ISER_RX_LOGIN_SIZE;
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001596 isert_dbg("login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001597 rx_dma, rx_buflen);
1598 } else {
1599 rx_dma = desc->dma_addr;
1600 rx_buflen = ISER_RX_PAYLOAD_SIZE;
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001601 isert_dbg("req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n",
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001602 rx_dma, rx_buflen);
1603 }
1604
1605 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE);
1606
1607 hdr = &desc->iscsi_header;
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001608 isert_dbg("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n",
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001609 hdr->opcode, hdr->itt, hdr->flags,
1610 (int)(xfer_len - ISER_HEADERS_LEN));
1611
Sagi Grimberg2371e5d2014-12-02 16:57:21 +02001612 if ((char *)desc == isert_conn->login_req_buf) {
1613 isert_conn->login_req_len = xfer_len - ISER_HEADERS_LEN;
1614 if (isert_conn->conn) {
1615 struct iscsi_login *login = isert_conn->conn->conn_login;
1616
1617 if (login && !login->first_request)
1618 isert_rx_login_req(isert_conn);
1619 }
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001620 mutex_lock(&isert_conn->mutex);
Sagi Grimberg2371e5d2014-12-02 16:57:21 +02001621 complete(&isert_conn->login_req_comp);
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001622 mutex_unlock(&isert_conn->mutex);
Sagi Grimberg2371e5d2014-12-02 16:57:21 +02001623 } else {
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001624 isert_rx_do_work(desc, isert_conn);
Sagi Grimberg2371e5d2014-12-02 16:57:21 +02001625 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001626
1627 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen,
1628 DMA_FROM_DEVICE);
1629
1630 isert_conn->post_recv_buf_count--;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001631}
1632
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02001633static int
1634isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
1635 struct scatterlist *sg, u32 nents, u32 length, u32 offset,
1636 enum iser_ib_op_code op, struct isert_data_buf *data)
1637{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001638 struct ib_device *ib_dev = isert_conn->cm_id->device;
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02001639
1640 data->dma_dir = op == ISER_IB_RDMA_WRITE ?
1641 DMA_TO_DEVICE : DMA_FROM_DEVICE;
1642
1643 data->len = length - offset;
1644 data->offset = offset;
1645 data->sg_off = data->offset / PAGE_SIZE;
1646
1647 data->sg = &sg[data->sg_off];
1648 data->nents = min_t(unsigned int, nents - data->sg_off,
1649 ISCSI_ISER_SG_TABLESIZE);
1650 data->len = min_t(unsigned int, data->len, ISCSI_ISER_SG_TABLESIZE *
1651 PAGE_SIZE);
1652
1653 data->dma_nents = ib_dma_map_sg(ib_dev, data->sg, data->nents,
1654 data->dma_dir);
1655 if (unlikely(!data->dma_nents)) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001656 isert_err("Cmd: unable to dma map SGs %p\n", sg);
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02001657 return -EINVAL;
1658 }
1659
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001660 isert_dbg("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n",
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001661 isert_cmd, data->dma_nents, data->sg, data->nents, data->len);
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02001662
1663 return 0;
1664}
1665
1666static void
1667isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data)
1668{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001669 struct ib_device *ib_dev = isert_conn->cm_id->device;
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02001670
1671 ib_dma_unmap_sg(ib_dev, data->sg, data->nents, data->dma_dir);
1672 memset(data, 0, sizeof(*data));
1673}
1674
1675
1676
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001677static void
1678isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn)
1679{
1680 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001681
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001682 isert_dbg("Cmd %p\n", isert_cmd);
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02001683
1684 if (wr->data.sg) {
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001685 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd);
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02001686 isert_unmap_data_buf(isert_conn, &wr->data);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001687 }
1688
Christoph Hellwige622f2f2015-10-08 09:16:33 +01001689 if (wr->rdma_wr) {
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001690 isert_dbg("Cmd %p free send_wr\n", isert_cmd);
Christoph Hellwige622f2f2015-10-08 09:16:33 +01001691 kfree(wr->rdma_wr);
1692 wr->rdma_wr = NULL;
Vu Pham90ecc6e2013-08-28 23:23:33 +03001693 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001694
Vu Pham90ecc6e2013-08-28 23:23:33 +03001695 if (wr->ib_sge) {
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001696 isert_dbg("Cmd %p free ib_sge\n", isert_cmd);
Vu Pham90ecc6e2013-08-28 23:23:33 +03001697 kfree(wr->ib_sge);
1698 wr->ib_sge = NULL;
1699 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001700}
1701
1702static void
Sagi Grimberga3a5a822014-01-09 18:40:50 +02001703isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn)
Vu Pham59464ef2013-08-28 23:23:35 +03001704{
1705 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
Vu Pham59464ef2013-08-28 23:23:35 +03001706
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001707 isert_dbg("Cmd %p\n", isert_cmd);
Vu Pham59464ef2013-08-28 23:23:35 +03001708
1709 if (wr->fr_desc) {
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001710 isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd, wr->fr_desc);
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02001711 if (wr->fr_desc->ind & ISERT_PROTECTED) {
1712 isert_unmap_data_buf(isert_conn, &wr->prot);
1713 wr->fr_desc->ind &= ~ISERT_PROTECTED;
1714 }
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001715 spin_lock_bh(&isert_conn->pool_lock);
1716 list_add_tail(&wr->fr_desc->list, &isert_conn->fr_pool);
1717 spin_unlock_bh(&isert_conn->pool_lock);
Vu Pham59464ef2013-08-28 23:23:35 +03001718 wr->fr_desc = NULL;
1719 }
1720
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02001721 if (wr->data.sg) {
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001722 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd);
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02001723 isert_unmap_data_buf(isert_conn, &wr->data);
Vu Pham59464ef2013-08-28 23:23:35 +03001724 }
1725
1726 wr->ib_sge = NULL;
Christoph Hellwige622f2f2015-10-08 09:16:33 +01001727 wr->rdma_wr = NULL;
Vu Pham59464ef2013-08-28 23:23:35 +03001728}
1729
1730static void
Nicholas Bellinger03e78482014-03-30 15:50:03 -07001731isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001732{
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001733 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001734 struct isert_conn *isert_conn = isert_cmd->conn;
Nicholas Bellinger186a9642013-07-03 03:11:48 -07001735 struct iscsi_conn *conn = isert_conn->conn;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001736 struct isert_device *device = isert_conn->device;
Sagi Grimberge4f4e802015-02-09 18:07:25 +02001737 struct iscsi_text_rsp *hdr;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001738
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001739 isert_dbg("Cmd %p\n", isert_cmd);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001740
1741 switch (cmd->iscsi_opcode) {
1742 case ISCSI_OP_SCSI_CMD:
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001743 spin_lock_bh(&conn->cmd_lock);
1744 if (!list_empty(&cmd->i_conn_node))
Nicholas Bellinger5159d762014-02-03 12:53:51 -08001745 list_del_init(&cmd->i_conn_node);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001746 spin_unlock_bh(&conn->cmd_lock);
1747
Nicholas Bellinger03e78482014-03-30 15:50:03 -07001748 if (cmd->data_direction == DMA_TO_DEVICE) {
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001749 iscsit_stop_dataout_timer(cmd);
Nicholas Bellinger03e78482014-03-30 15:50:03 -07001750 /*
1751 * Check for special case during comp_err where
1752 * WRITE_PENDING has been handed off from core,
1753 * but requires an extra target_put_sess_cmd()
1754 * before transport_generic_free_cmd() below.
1755 */
1756 if (comp_err &&
1757 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) {
1758 struct se_cmd *se_cmd = &cmd->se_cmd;
1759
Bart Van Asscheafc16602015-04-27 13:52:36 +02001760 target_put_sess_cmd(se_cmd);
Nicholas Bellinger03e78482014-03-30 15:50:03 -07001761 }
1762 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001763
Vu Phamd40945d2013-08-28 23:23:34 +03001764 device->unreg_rdma_mem(isert_cmd, isert_conn);
Nicholas Bellinger186a9642013-07-03 03:11:48 -07001765 transport_generic_free_cmd(&cmd->se_cmd, 0);
1766 break;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001767 case ISCSI_OP_SCSI_TMFUNC:
Nicholas Bellinger186a9642013-07-03 03:11:48 -07001768 spin_lock_bh(&conn->cmd_lock);
1769 if (!list_empty(&cmd->i_conn_node))
Nicholas Bellinger5159d762014-02-03 12:53:51 -08001770 list_del_init(&cmd->i_conn_node);
Nicholas Bellinger186a9642013-07-03 03:11:48 -07001771 spin_unlock_bh(&conn->cmd_lock);
1772
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001773 transport_generic_free_cmd(&cmd->se_cmd, 0);
1774 break;
1775 case ISCSI_OP_REJECT:
1776 case ISCSI_OP_NOOP_OUT:
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07001777 case ISCSI_OP_TEXT:
Sagi Grimberge4f4e802015-02-09 18:07:25 +02001778 hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;
1779 /* If the continue bit is on, keep the command alive */
1780 if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)
1781 break;
1782
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001783 spin_lock_bh(&conn->cmd_lock);
1784 if (!list_empty(&cmd->i_conn_node))
Nicholas Bellinger5159d762014-02-03 12:53:51 -08001785 list_del_init(&cmd->i_conn_node);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001786 spin_unlock_bh(&conn->cmd_lock);
1787
1788 /*
1789 * Handle special case for REJECT when iscsi_add_reject*() has
1790 * overwritten the original iscsi_opcode assignment, and the
1791 * associated cmd->se_cmd needs to be released.
1792 */
1793 if (cmd->se_cmd.se_tfo != NULL) {
Rasmus Villemoes11378cd2015-02-06 01:09:05 +01001794 isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n",
Nicholas Bellinger3df8f682013-06-26 02:31:42 -07001795 cmd->iscsi_opcode);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001796 transport_generic_free_cmd(&cmd->se_cmd, 0);
1797 break;
1798 }
1799 /*
1800 * Fall-through
1801 */
1802 default:
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001803 iscsit_release_cmd(cmd);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001804 break;
1805 }
1806}
1807
1808static void
1809isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev)
1810{
1811 if (tx_desc->dma_addr != 0) {
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001812 isert_dbg("unmap single for tx_desc->dma_addr\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001813 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr,
1814 ISER_HEADERS_LEN, DMA_TO_DEVICE);
1815 tx_desc->dma_addr = 0;
1816 }
1817}
1818
1819static void
1820isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,
Nicholas Bellinger03e78482014-03-30 15:50:03 -07001821 struct ib_device *ib_dev, bool comp_err)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001822{
Nicholas Bellingerdbbc5d12013-07-03 19:39:37 -07001823 if (isert_cmd->pdu_buf_dma != 0) {
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001824 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n");
Nicholas Bellingerdbbc5d12013-07-03 19:39:37 -07001825 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma,
1826 isert_cmd->pdu_buf_len, DMA_TO_DEVICE);
1827 isert_cmd->pdu_buf_dma = 0;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001828 }
1829
1830 isert_unmap_tx_desc(tx_desc, ib_dev);
Nicholas Bellinger03e78482014-03-30 15:50:03 -07001831 isert_put_cmd(isert_cmd, comp_err);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001832}
1833
Sagi Grimberg96b79732014-03-17 12:52:18 +02001834static int
1835isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr)
1836{
1837 struct ib_mr_status mr_status;
1838 int ret;
1839
1840 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status);
1841 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001842 isert_err("ib_check_mr_status failed, ret %d\n", ret);
Sagi Grimberg96b79732014-03-17 12:52:18 +02001843 goto fail_mr_status;
1844 }
1845
1846 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) {
1847 u64 sec_offset_err;
1848 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8;
1849
1850 switch (mr_status.sig_err.err_type) {
1851 case IB_SIG_BAD_GUARD:
1852 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED;
1853 break;
1854 case IB_SIG_BAD_REFTAG:
1855 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED;
1856 break;
1857 case IB_SIG_BAD_APPTAG:
1858 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED;
1859 break;
1860 }
1861 sec_offset_err = mr_status.sig_err.sig_err_offset;
1862 do_div(sec_offset_err, block_size);
1863 se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba;
1864
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001865 isert_err("PI error found type %d at sector 0x%llx "
1866 "expected 0x%x vs actual 0x%x\n",
1867 mr_status.sig_err.err_type,
1868 (unsigned long long)se_cmd->bad_sector,
1869 mr_status.sig_err.expected,
1870 mr_status.sig_err.actual);
Sagi Grimberg96b79732014-03-17 12:52:18 +02001871 ret = 1;
1872 }
1873
1874fail_mr_status:
1875 return ret;
1876}
1877
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001878static void
Sagi Grimbergf93f3a72014-02-19 17:50:24 +02001879isert_completion_rdma_write(struct iser_tx_desc *tx_desc,
1880 struct isert_cmd *isert_cmd)
1881{
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02001882 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
Sagi Grimbergf93f3a72014-02-19 17:50:24 +02001883 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02001884 struct se_cmd *se_cmd = &cmd->se_cmd;
Sagi Grimbergf93f3a72014-02-19 17:50:24 +02001885 struct isert_conn *isert_conn = isert_cmd->conn;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001886 struct isert_device *device = isert_conn->device;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02001887 int ret = 0;
Sagi Grimbergf93f3a72014-02-19 17:50:24 +02001888
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02001889 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) {
Sagi Grimberg96b79732014-03-17 12:52:18 +02001890 ret = isert_check_pi_status(se_cmd,
1891 wr->fr_desc->pi_ctx->sig_mr);
1892 wr->fr_desc->ind &= ~ISERT_PROTECTED;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02001893 }
1894
Sagi Grimbergf93f3a72014-02-19 17:50:24 +02001895 device->unreg_rdma_mem(isert_cmd, isert_conn);
Christoph Hellwige622f2f2015-10-08 09:16:33 +01001896 wr->rdma_wr_num = 0;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02001897 if (ret)
1898 transport_send_check_condition_and_sense(se_cmd,
1899 se_cmd->pi_err, 0);
1900 else
1901 isert_put_response(isert_conn->conn, cmd);
Sagi Grimbergf93f3a72014-02-19 17:50:24 +02001902}
1903
1904static void
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001905isert_completion_rdma_read(struct iser_tx_desc *tx_desc,
1906 struct isert_cmd *isert_cmd)
1907{
1908 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001909 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001910 struct se_cmd *se_cmd = &cmd->se_cmd;
Vu Pham90ecc6e2013-08-28 23:23:33 +03001911 struct isert_conn *isert_conn = isert_cmd->conn;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001912 struct isert_device *device = isert_conn->device;
Sagi Grimberg5bac4b12014-03-18 14:58:27 +02001913 int ret = 0;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001914
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02001915 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) {
Sagi Grimberg96b79732014-03-17 12:52:18 +02001916 ret = isert_check_pi_status(se_cmd,
1917 wr->fr_desc->pi_ctx->sig_mr);
1918 wr->fr_desc->ind &= ~ISERT_PROTECTED;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02001919 }
1920
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001921 iscsit_stop_dataout_timer(cmd);
Vu Phamd40945d2013-08-28 23:23:34 +03001922 device->unreg_rdma_mem(isert_cmd, isert_conn);
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02001923 cmd->write_data_done = wr->data.len;
Christoph Hellwige622f2f2015-10-08 09:16:33 +01001924 wr->rdma_wr_num = 0;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001925
Sagi Grimberg24f412d2014-12-07 13:12:02 +02001926 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001927 spin_lock_bh(&cmd->istate_lock);
1928 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1929 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1930 spin_unlock_bh(&cmd->istate_lock);
1931
Sagi Grimberg364189f2015-03-29 15:52:03 +03001932 if (ret) {
Bart Van Asscheafc16602015-04-27 13:52:36 +02001933 target_put_sess_cmd(se_cmd);
Sagi Grimberg5bac4b12014-03-18 14:58:27 +02001934 transport_send_check_condition_and_sense(se_cmd,
1935 se_cmd->pi_err, 0);
Sagi Grimberg364189f2015-03-29 15:52:03 +03001936 } else {
Sagi Grimberg5bac4b12014-03-18 14:58:27 +02001937 target_execute_cmd(se_cmd);
Sagi Grimberg364189f2015-03-29 15:52:03 +03001938 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001939}
1940
1941static void
1942isert_do_control_comp(struct work_struct *work)
1943{
1944 struct isert_cmd *isert_cmd = container_of(work,
1945 struct isert_cmd, comp_work);
1946 struct isert_conn *isert_conn = isert_cmd->conn;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001947 struct ib_device *ib_dev = isert_conn->cm_id->device;
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001948 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001949
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001950 isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state);
1951
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001952 switch (cmd->i_state) {
1953 case ISTATE_SEND_TASKMGTRSP:
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001954 iscsit_tmr_post_handler(cmd, cmd->conn);
Sagi Grimberg10633c32014-12-07 13:12:04 +02001955 case ISTATE_SEND_REJECT: /* FALLTHRU */
1956 case ISTATE_SEND_TEXTRSP: /* FALLTHRU */
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001957 cmd->i_state = ISTATE_SENT_STATUS;
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001958 isert_completion_put(&isert_cmd->tx_desc, isert_cmd,
1959 ib_dev, false);
Nicholas Bellinger3df8f682013-06-26 02:31:42 -07001960 break;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001961 case ISTATE_SEND_LOGOUTRSP:
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001962 iscsit_logout_post_handler(cmd, cmd->conn);
1963 break;
1964 default:
Sagi Grimberg4c22e072014-12-07 13:12:03 +02001965 isert_err("Unknown i_state %d\n", cmd->i_state);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001966 dump_stack();
1967 break;
1968 }
1969}
1970
1971static void
1972isert_response_completion(struct iser_tx_desc *tx_desc,
1973 struct isert_cmd *isert_cmd,
1974 struct isert_conn *isert_conn,
1975 struct ib_device *ib_dev)
1976{
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07001977 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001978
1979 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP ||
Nicholas Bellinger3df8f682013-06-26 02:31:42 -07001980 cmd->i_state == ISTATE_SEND_LOGOUTRSP ||
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07001981 cmd->i_state == ISTATE_SEND_REJECT ||
1982 cmd->i_state == ISTATE_SEND_TEXTRSP) {
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001983 isert_unmap_tx_desc(tx_desc, ib_dev);
1984
1985 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);
1986 queue_work(isert_comp_wq, &isert_cmd->comp_work);
1987 return;
1988 }
Sagi Grimberg897bb2c2014-03-17 12:52:17 +02001989
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001990 cmd->i_state = ISTATE_SENT_STATUS;
Nicholas Bellinger03e78482014-03-30 15:50:03 -07001991 isert_completion_put(tx_desc, isert_cmd, ib_dev, false);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001992}
1993
1994static void
Sagi Grimberg77486812015-03-29 15:52:13 +03001995isert_snd_completion(struct iser_tx_desc *tx_desc,
Sagi Grimberg68a86de2014-12-02 16:57:37 +02001996 struct isert_conn *isert_conn)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001997{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03001998 struct ib_device *ib_dev = isert_conn->cm_id->device;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08001999 struct isert_cmd *isert_cmd = tx_desc->isert_cmd;
2000 struct isert_rdma_wr *wr;
2001
2002 if (!isert_cmd) {
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002003 isert_unmap_tx_desc(tx_desc, ib_dev);
2004 return;
2005 }
2006 wr = &isert_cmd->rdma_wr;
2007
Sagi Grimberg4c22e072014-12-07 13:12:03 +02002008 isert_dbg("Cmd %p iser_ib_op %d\n", isert_cmd, wr->iser_ib_op);
2009
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002010 switch (wr->iser_ib_op) {
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002011 case ISER_IB_SEND:
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002012 isert_response_completion(tx_desc, isert_cmd,
2013 isert_conn, ib_dev);
2014 break;
2015 case ISER_IB_RDMA_WRITE:
Sagi Grimbergf93f3a72014-02-19 17:50:24 +02002016 isert_completion_rdma_write(tx_desc, isert_cmd);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002017 break;
2018 case ISER_IB_RDMA_READ:
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002019 isert_completion_rdma_read(tx_desc, isert_cmd);
2020 break;
2021 default:
Sagi Grimberg4c22e072014-12-07 13:12:03 +02002022 isert_err("Unknown wr->iser_ib_op: 0x%x\n", wr->iser_ib_op);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002023 dump_stack();
2024 break;
2025 }
2026}
2027
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002028/**
2029 * is_isert_tx_desc() - Indicate if the completion wr_id
2030 * is a TX descriptor or not.
2031 * @isert_conn: iser connection
2032 * @wr_id: completion WR identifier
2033 *
2034 * Since we cannot rely on wc opcode in FLUSH errors
2035 * we must work around it by checking if the wr_id address
2036 * falls in the iser connection rx_descs buffer. If so
2037 * it is an RX descriptor, otherwize it is a TX.
2038 */
2039static inline bool
2040is_isert_tx_desc(struct isert_conn *isert_conn, void *wr_id)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002041{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002042 void *start = isert_conn->rx_descs;
2043 int len = ISERT_QP_MAX_RECV_DTOS * sizeof(*isert_conn->rx_descs);
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002044
2045 if (wr_id >= start && wr_id < start + len)
2046 return false;
2047
2048 return true;
2049}
2050
2051static void
2052isert_cq_comp_err(struct isert_conn *isert_conn, struct ib_wc *wc)
2053{
Sagi Grimbergbdf20e72014-12-02 16:57:43 +02002054 if (wc->wr_id == ISER_BEACON_WRID) {
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002055 isert_info("conn %p completing wait_comp_err\n",
Sagi Grimbergbdf20e72014-12-02 16:57:43 +02002056 isert_conn);
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002057 complete(&isert_conn->wait_comp_err);
Nicholas Bellingered4520a2014-12-07 13:12:05 +02002058 } else if (is_isert_tx_desc(isert_conn, (void *)(uintptr_t)wc->wr_id)) {
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002059 struct ib_device *ib_dev = isert_conn->cm_id->device;
Sagi Grimbergdf43deb2014-12-02 16:57:38 +02002060 struct isert_cmd *isert_cmd;
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002061 struct iser_tx_desc *desc;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002062
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002063 desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id;
2064 isert_cmd = desc->isert_cmd;
Sagi Grimbergdf43deb2014-12-02 16:57:38 +02002065 if (!isert_cmd)
2066 isert_unmap_tx_desc(desc, ib_dev);
2067 else
2068 isert_completion_put(desc, isert_cmd, ib_dev, true);
Sagi Grimbergdf43deb2014-12-02 16:57:38 +02002069 } else {
2070 isert_conn->post_recv_buf_count--;
Sagi Grimbergbdf20e72014-12-02 16:57:43 +02002071 if (!isert_conn->post_recv_buf_count)
2072 iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
Sagi Grimbergdf43deb2014-12-02 16:57:38 +02002073 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002074}
2075
2076static void
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002077isert_handle_wc(struct ib_wc *wc)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002078{
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002079 struct isert_conn *isert_conn;
2080 struct iser_tx_desc *tx_desc;
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002081 struct iser_rx_desc *rx_desc;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002082
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002083 isert_conn = wc->qp->qp_context;
2084 if (likely(wc->status == IB_WC_SUCCESS)) {
2085 if (wc->opcode == IB_WC_RECV) {
2086 rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id;
Sagi Grimberg77486812015-03-29 15:52:13 +03002087 isert_rcv_completion(rx_desc, isert_conn, wc->byte_len);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002088 } else {
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002089 tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id;
Sagi Grimberg77486812015-03-29 15:52:13 +03002090 isert_snd_completion(tx_desc, isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002091 }
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002092 } else {
2093 if (wc->status != IB_WC_WR_FLUSH_ERR)
Sagi Grimbergea8a1612015-05-18 13:40:31 +03002094 isert_err("%s (%d): wr id %llx vend_err %x\n",
2095 ib_wc_status_msg(wc->status), wc->status,
2096 wc->wr_id, wc->vendor_err);
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002097 else
Sagi Grimbergea8a1612015-05-18 13:40:31 +03002098 isert_dbg("%s (%d): wr id %llx\n",
2099 ib_wc_status_msg(wc->status), wc->status,
2100 wc->wr_id);
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002101
2102 if (wc->wr_id != ISER_FASTREG_LI_WRID)
2103 isert_cq_comp_err(isert_conn, wc);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002104 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002105}
2106
2107static void
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002108isert_cq_work(struct work_struct *work)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002109{
Sagi Grimberg37d9fe82014-12-02 16:57:44 +02002110 enum { isert_poll_budget = 65536 };
Sagi Grimberg4a295ba2014-12-02 16:57:40 +02002111 struct isert_comp *comp = container_of(work, struct isert_comp,
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002112 work);
Sagi Grimberg36ea63b2014-12-02 16:57:45 +02002113 struct ib_wc *const wcs = comp->wcs;
2114 int i, n, completed = 0;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002115
Sagi Grimberg36ea63b2014-12-02 16:57:45 +02002116 while ((n = ib_poll_cq(comp->cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) {
2117 for (i = 0; i < n; i++)
2118 isert_handle_wc(&wcs[i]);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002119
Sagi Grimberg36ea63b2014-12-02 16:57:45 +02002120 completed += n;
2121 if (completed >= isert_poll_budget)
Sagi Grimberg37d9fe82014-12-02 16:57:44 +02002122 break;
2123 }
2124
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002125 ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002126}
2127
2128static void
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002129isert_cq_callback(struct ib_cq *cq, void *context)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002130{
Sagi Grimberg4a295ba2014-12-02 16:57:40 +02002131 struct isert_comp *comp = context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002132
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02002133 queue_work(isert_comp_wq, &comp->work);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002134}
2135
2136static int
2137isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)
2138{
2139 struct ib_send_wr *wr_failed;
2140 int ret;
2141
Jenny Derzhavetz4366b192015-09-06 14:52:25 +03002142 ret = isert_post_recv(isert_conn, isert_cmd->rx_desc);
2143 if (ret) {
2144 isert_err("ib_post_recv failed with %d\n", ret);
2145 return ret;
2146 }
2147
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002148 ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr,
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002149 &wr_failed);
2150 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002151 isert_err("ib_post_send failed with %d\n", ret);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002152 return ret;
2153 }
2154 return ret;
2155}
2156
2157static int
2158isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
2159{
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07002160 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
Sagi Grimberg67004252015-03-29 15:52:09 +03002161 struct isert_conn *isert_conn = conn->context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002162 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
2163 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)
2164 &isert_cmd->tx_desc.iscsi_header;
2165
2166 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
2167 iscsit_build_rsp_pdu(cmd, conn, true, hdr);
2168 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
2169 /*
2170 * Attach SENSE DATA payload to iSCSI Response PDU
2171 */
2172 if (cmd->se_cmd.sense_buffer &&
2173 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
2174 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002175 struct isert_device *device = isert_conn->device;
Sagi Grimberg67cb3942015-03-29 15:52:05 +03002176 struct ib_device *ib_dev = device->ib_device;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002177 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
Nicholas Bellingerdbbc5d12013-07-03 19:39:37 -07002178 u32 padding, pdu_len;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002179
2180 put_unaligned_be16(cmd->se_cmd.scsi_sense_length,
2181 cmd->sense_buffer);
2182 cmd->se_cmd.scsi_sense_length += sizeof(__be16);
2183
2184 padding = -(cmd->se_cmd.scsi_sense_length) & 3;
2185 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
Nicholas Bellingerdbbc5d12013-07-03 19:39:37 -07002186 pdu_len = cmd->se_cmd.scsi_sense_length + padding;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002187
Nicholas Bellingerdbbc5d12013-07-03 19:39:37 -07002188 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
2189 (void *)cmd->sense_buffer, pdu_len,
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002190 DMA_TO_DEVICE);
2191
Nicholas Bellingerdbbc5d12013-07-03 19:39:37 -07002192 isert_cmd->pdu_buf_len = pdu_len;
2193 tx_dsg->addr = isert_cmd->pdu_buf_dma;
2194 tx_dsg->length = pdu_len;
Jason Gunthorpe34efc7d2015-07-30 17:22:21 -06002195 tx_dsg->lkey = device->pd->local_dma_lkey;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002196 isert_cmd->tx_desc.num_sge = 2;
2197 }
2198
Sagi Grimberg68a86de2014-12-02 16:57:37 +02002199 isert_init_send_wr(isert_conn, isert_cmd, send_wr);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002200
Sagi Grimberg4c22e072014-12-07 13:12:03 +02002201 isert_dbg("Posting SCSI Response\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002202
2203 return isert_post_response(isert_conn, isert_cmd);
2204}
2205
Nicholas Bellinger131e6ab2014-03-22 14:55:56 -07002206static void
2207isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
2208{
2209 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
Sagi Grimberg67004252015-03-29 15:52:09 +03002210 struct isert_conn *isert_conn = conn->context;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002211 struct isert_device *device = isert_conn->device;
Nicholas Bellinger131e6ab2014-03-22 14:55:56 -07002212
2213 spin_lock_bh(&conn->cmd_lock);
2214 if (!list_empty(&cmd->i_conn_node))
2215 list_del_init(&cmd->i_conn_node);
2216 spin_unlock_bh(&conn->cmd_lock);
2217
2218 if (cmd->data_direction == DMA_TO_DEVICE)
2219 iscsit_stop_dataout_timer(cmd);
2220
2221 device->unreg_rdma_mem(isert_cmd, isert_conn);
2222}
2223
Nicholas Bellingere70beee2014-04-02 12:52:38 -07002224static enum target_prot_op
2225isert_get_sup_prot_ops(struct iscsi_conn *conn)
2226{
Sagi Grimberg67004252015-03-29 15:52:09 +03002227 struct isert_conn *isert_conn = conn->context;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002228 struct isert_device *device = isert_conn->device;
Nicholas Bellingere70beee2014-04-02 12:52:38 -07002229
Sagi Grimberg23a548e2014-12-02 16:57:35 +02002230 if (conn->tpg->tpg_attrib.t10_pi) {
2231 if (device->pi_capable) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002232 isert_info("conn %p PI offload enabled\n", isert_conn);
Sagi Grimberg23a548e2014-12-02 16:57:35 +02002233 isert_conn->pi_support = true;
2234 return TARGET_PROT_ALL;
2235 }
2236 }
2237
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002238 isert_info("conn %p PI offload disabled\n", isert_conn);
Sagi Grimberg23a548e2014-12-02 16:57:35 +02002239 isert_conn->pi_support = false;
Nicholas Bellingere70beee2014-04-02 12:52:38 -07002240
2241 return TARGET_PROT_NORMAL;
2242}
2243
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002244static int
2245isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
2246 bool nopout_response)
2247{
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07002248 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
Sagi Grimberg67004252015-03-29 15:52:09 +03002249 struct isert_conn *isert_conn = conn->context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002250 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
2251
2252 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
2253 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *)
2254 &isert_cmd->tx_desc.iscsi_header,
2255 nopout_response);
2256 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
Sagi Grimberg68a86de2014-12-02 16:57:37 +02002257 isert_init_send_wr(isert_conn, isert_cmd, send_wr);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002258
Sagi Grimberg4c22e072014-12-07 13:12:03 +02002259 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002260
2261 return isert_post_response(isert_conn, isert_cmd);
2262}
2263
2264static int
2265isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2266{
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07002267 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
Sagi Grimberg67004252015-03-29 15:52:09 +03002268 struct isert_conn *isert_conn = conn->context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002269 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
2270
2271 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
2272 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *)
2273 &isert_cmd->tx_desc.iscsi_header);
2274 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
Sagi Grimberg68a86de2014-12-02 16:57:37 +02002275 isert_init_send_wr(isert_conn, isert_cmd, send_wr);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002276
Sagi Grimberg4c22e072014-12-07 13:12:03 +02002277 isert_dbg("conn %p Posting Logout Response\n", isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002278
2279 return isert_post_response(isert_conn, isert_cmd);
2280}
2281
2282static int
2283isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2284{
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07002285 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
Sagi Grimberg67004252015-03-29 15:52:09 +03002286 struct isert_conn *isert_conn = conn->context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002287 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
2288
2289 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
2290 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *)
2291 &isert_cmd->tx_desc.iscsi_header);
2292 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
Sagi Grimberg68a86de2014-12-02 16:57:37 +02002293 isert_init_send_wr(isert_conn, isert_cmd, send_wr);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002294
Sagi Grimberg4c22e072014-12-07 13:12:03 +02002295 isert_dbg("conn %p Posting Task Management Response\n", isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002296
2297 return isert_post_response(isert_conn, isert_cmd);
2298}
2299
2300static int
2301isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2302{
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07002303 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
Sagi Grimberg67004252015-03-29 15:52:09 +03002304 struct isert_conn *isert_conn = conn->context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002305 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002306 struct isert_device *device = isert_conn->device;
Sagi Grimberg67cb3942015-03-29 15:52:05 +03002307 struct ib_device *ib_dev = device->ib_device;
Nicholas Bellinger3df8f682013-06-26 02:31:42 -07002308 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
2309 struct iscsi_reject *hdr =
2310 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002311
2312 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
Nicholas Bellinger3df8f682013-06-26 02:31:42 -07002313 iscsit_build_reject(cmd, conn, hdr);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002314 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
Nicholas Bellinger3df8f682013-06-26 02:31:42 -07002315
2316 hton24(hdr->dlength, ISCSI_HDR_LEN);
Nicholas Bellingerdbbc5d12013-07-03 19:39:37 -07002317 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
Nicholas Bellinger3df8f682013-06-26 02:31:42 -07002318 (void *)cmd->buf_ptr, ISCSI_HDR_LEN,
2319 DMA_TO_DEVICE);
Nicholas Bellingerdbbc5d12013-07-03 19:39:37 -07002320 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN;
2321 tx_dsg->addr = isert_cmd->pdu_buf_dma;
Nicholas Bellinger3df8f682013-06-26 02:31:42 -07002322 tx_dsg->length = ISCSI_HDR_LEN;
Jason Gunthorpe34efc7d2015-07-30 17:22:21 -06002323 tx_dsg->lkey = device->pd->local_dma_lkey;
Nicholas Bellinger3df8f682013-06-26 02:31:42 -07002324 isert_cmd->tx_desc.num_sge = 2;
2325
Sagi Grimberg68a86de2014-12-02 16:57:37 +02002326 isert_init_send_wr(isert_conn, isert_cmd, send_wr);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002327
Sagi Grimberg4c22e072014-12-07 13:12:03 +02002328 isert_dbg("conn %p Posting Reject\n", isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002329
2330 return isert_post_response(isert_conn, isert_cmd);
2331}
2332
2333static int
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07002334isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2335{
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07002336 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
Sagi Grimberg67004252015-03-29 15:52:09 +03002337 struct isert_conn *isert_conn = conn->context;
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07002338 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
2339 struct iscsi_text_rsp *hdr =
2340 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;
2341 u32 txt_rsp_len;
2342 int rc;
2343
2344 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
Sagi Grimberg22c7aaa2014-06-10 18:27:59 +03002345 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND);
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07002346 if (rc < 0)
2347 return rc;
2348
2349 txt_rsp_len = rc;
2350 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
2351
2352 if (txt_rsp_len) {
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002353 struct isert_device *device = isert_conn->device;
Sagi Grimberg67cb3942015-03-29 15:52:05 +03002354 struct ib_device *ib_dev = device->ib_device;
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07002355 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
2356 void *txt_rsp_buf = cmd->buf_ptr;
2357
2358 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
2359 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE);
2360
2361 isert_cmd->pdu_buf_len = txt_rsp_len;
2362 tx_dsg->addr = isert_cmd->pdu_buf_dma;
2363 tx_dsg->length = txt_rsp_len;
Jason Gunthorpe34efc7d2015-07-30 17:22:21 -06002364 tx_dsg->lkey = device->pd->local_dma_lkey;
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07002365 isert_cmd->tx_desc.num_sge = 2;
2366 }
Sagi Grimberg68a86de2014-12-02 16:57:37 +02002367 isert_init_send_wr(isert_conn, isert_cmd, send_wr);
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07002368
Sagi Grimbergf64d2792015-01-25 19:11:20 +02002369 isert_dbg("conn %p Text Response\n", isert_conn);
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07002370
2371 return isert_post_response(isert_conn, isert_cmd);
2372}
2373
2374static int
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002375isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002376 struct ib_sge *ib_sge, struct ib_rdma_wr *rdma_wr,
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002377 u32 data_left, u32 offset)
2378{
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07002379 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002380 struct scatterlist *sg_start, *tmp_sg;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002381 struct isert_device *device = isert_conn->device;
Sagi Grimberg67cb3942015-03-29 15:52:05 +03002382 struct ib_device *ib_dev = device->ib_device;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002383 u32 sg_off, page_off;
2384 int i = 0, sg_nents;
2385
2386 sg_off = offset / PAGE_SIZE;
2387 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
2388 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge);
2389 page_off = offset % PAGE_SIZE;
2390
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002391 rdma_wr->wr.sg_list = ib_sge;
2392 rdma_wr->wr.wr_id = (uintptr_t)&isert_cmd->tx_desc;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002393 /*
2394 * Perform mapping of TCM scatterlist memory ib_sge dma_addr.
2395 */
2396 for_each_sg(sg_start, tmp_sg, sg_nents, i) {
Sagi Grimberg4c22e072014-12-07 13:12:03 +02002397 isert_dbg("RDMA from SGL dma_addr: 0x%llx dma_len: %u, "
2398 "page_off: %u\n",
2399 (unsigned long long)tmp_sg->dma_address,
2400 tmp_sg->length, page_off);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002401
2402 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off;
2403 ib_sge->length = min_t(u32, data_left,
2404 ib_sg_dma_len(ib_dev, tmp_sg) - page_off);
Jason Gunthorpe34efc7d2015-07-30 17:22:21 -06002405 ib_sge->lkey = device->pd->local_dma_lkey;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002406
Sagi Grimberg4c22e072014-12-07 13:12:03 +02002407 isert_dbg("RDMA ib_sge: addr: 0x%llx length: %u lkey: %x\n",
2408 ib_sge->addr, ib_sge->length, ib_sge->lkey);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002409 page_off = 0;
2410 data_left -= ib_sge->length;
Sagi Grimberg9253e662015-06-04 19:49:19 +03002411 if (!data_left)
2412 break;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002413 ib_sge++;
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002414 isert_dbg("Incrementing ib_sge pointer to %p\n", ib_sge);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002415 }
2416
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002417 rdma_wr->wr.num_sge = ++i;
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002418 isert_dbg("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n",
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002419 rdma_wr->wr.sg_list, rdma_wr->wr.num_sge);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002420
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002421 return rdma_wr->wr.num_sge;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002422}
2423
2424static int
Vu Pham90ecc6e2013-08-28 23:23:33 +03002425isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2426 struct isert_rdma_wr *wr)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002427{
2428 struct se_cmd *se_cmd = &cmd->se_cmd;
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07002429 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
Sagi Grimberg67004252015-03-29 15:52:09 +03002430 struct isert_conn *isert_conn = conn->context;
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002431 struct isert_data_buf *data = &wr->data;
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002432 struct ib_rdma_wr *rdma_wr;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002433 struct ib_sge *ib_sge;
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002434 u32 offset, data_len, data_left, rdma_write_max, va_offset = 0;
2435 int ret = 0, i, ib_sge_cnt;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002436
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002437 isert_cmd->tx_desc.isert_cmd = isert_cmd;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002438
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002439 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0;
2440 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg,
2441 se_cmd->t_data_nents, se_cmd->data_length,
2442 offset, wr->iser_ib_op, &wr->data);
2443 if (ret)
2444 return ret;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002445
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002446 data_left = data->len;
2447 offset = data->offset;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002448
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002449 ib_sge = kzalloc(sizeof(struct ib_sge) * data->nents, GFP_KERNEL);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002450 if (!ib_sge) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002451 isert_warn("Unable to allocate ib_sge\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002452 ret = -ENOMEM;
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002453 goto unmap_cmd;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002454 }
Vu Pham90ecc6e2013-08-28 23:23:33 +03002455 wr->ib_sge = ib_sge;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002456
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002457 wr->rdma_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge);
2458 wr->rdma_wr = kzalloc(sizeof(struct ib_rdma_wr) * wr->rdma_wr_num,
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002459 GFP_KERNEL);
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002460 if (!wr->rdma_wr) {
2461 isert_dbg("Unable to allocate wr->rdma_wr\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002462 ret = -ENOMEM;
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002463 goto unmap_cmd;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002464 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002465
2466 wr->isert_cmd = isert_cmd;
2467 rdma_write_max = isert_conn->max_sge * PAGE_SIZE;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002468
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002469 for (i = 0; i < wr->rdma_wr_num; i++) {
2470 rdma_wr = &isert_cmd->rdma_wr.rdma_wr[i];
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002471 data_len = min(data_left, rdma_write_max);
2472
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002473 rdma_wr->wr.send_flags = 0;
Vu Pham90ecc6e2013-08-28 23:23:33 +03002474 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) {
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002475 rdma_wr->wr.opcode = IB_WR_RDMA_WRITE;
2476 rdma_wr->remote_addr = isert_cmd->read_va + offset;
2477 rdma_wr->rkey = isert_cmd->read_stag;
2478 if (i + 1 == wr->rdma_wr_num)
2479 rdma_wr->wr.next = &isert_cmd->tx_desc.send_wr;
Vu Pham90ecc6e2013-08-28 23:23:33 +03002480 else
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002481 rdma_wr->wr.next = &wr->rdma_wr[i + 1].wr;
Vu Pham90ecc6e2013-08-28 23:23:33 +03002482 } else {
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002483 rdma_wr->wr.opcode = IB_WR_RDMA_READ;
2484 rdma_wr->remote_addr = isert_cmd->write_va + va_offset;
2485 rdma_wr->rkey = isert_cmd->write_stag;
2486 if (i + 1 == wr->rdma_wr_num)
2487 rdma_wr->wr.send_flags = IB_SEND_SIGNALED;
Vu Pham90ecc6e2013-08-28 23:23:33 +03002488 else
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002489 rdma_wr->wr.next = &wr->rdma_wr[i + 1].wr;
Vu Pham90ecc6e2013-08-28 23:23:33 +03002490 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002491
2492 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge,
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002493 rdma_wr, data_len, offset);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002494 ib_sge += ib_sge_cnt;
2495
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002496 offset += data_len;
Vu Pham90ecc6e2013-08-28 23:23:33 +03002497 va_offset += data_len;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002498 data_left -= data_len;
2499 }
Vu Pham90ecc6e2013-08-28 23:23:33 +03002500
2501 return 0;
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002502unmap_cmd:
2503 isert_unmap_data_buf(isert_conn, data);
2504
Vu Pham90ecc6e2013-08-28 23:23:33 +03002505 return ret;
2506}
2507
Sagi Grimberg10633c32014-12-07 13:12:04 +02002508static inline void
2509isert_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr)
2510{
2511 u32 rkey;
2512
2513 memset(inv_wr, 0, sizeof(*inv_wr));
2514 inv_wr->wr_id = ISER_FASTREG_LI_WRID;
2515 inv_wr->opcode = IB_WR_LOCAL_INV;
2516 inv_wr->ex.invalidate_rkey = mr->rkey;
2517
2518 /* Bump the key */
2519 rkey = ib_inc_rkey(mr->rkey);
2520 ib_update_fast_reg_key(mr, rkey);
2521}
2522
Vu Pham59464ef2013-08-28 23:23:35 +03002523static int
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002524isert_fast_reg_mr(struct isert_conn *isert_conn,
2525 struct fast_reg_descriptor *fr_desc,
2526 struct isert_data_buf *mem,
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002527 enum isert_indicator ind,
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002528 struct ib_sge *sge)
Vu Pham59464ef2013-08-28 23:23:35 +03002529{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002530 struct isert_device *device = isert_conn->device;
Sagi Grimberg67cb3942015-03-29 15:52:05 +03002531 struct ib_device *ib_dev = device->ib_device;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002532 struct ib_mr *mr;
Sagi Grimberg16c2d702015-10-13 19:11:34 +03002533 struct ib_reg_wr reg_wr;
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002534 struct ib_send_wr inv_wr, *bad_wr, *wr = NULL;
Sagi Grimberg16c2d702015-10-13 19:11:34 +03002535 int ret, n;
Vu Pham59464ef2013-08-28 23:23:35 +03002536
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002537 if (mem->dma_nents == 1) {
Jason Gunthorpe34efc7d2015-07-30 17:22:21 -06002538 sge->lkey = device->pd->local_dma_lkey;
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002539 sge->addr = ib_sg_dma_address(ib_dev, &mem->sg[0]);
2540 sge->length = ib_sg_dma_len(ib_dev, &mem->sg[0]);
Sagi Grimberg4c22e072014-12-07 13:12:03 +02002541 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n",
2542 sge->addr, sge->length, sge->lkey);
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002543 return 0;
2544 }
2545
Sagi Grimberg16c2d702015-10-13 19:11:34 +03002546 if (ind == ISERT_DATA_KEY_VALID)
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002547 /* Registering data buffer */
2548 mr = fr_desc->data_mr;
Sagi Grimberg16c2d702015-10-13 19:11:34 +03002549 else
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002550 /* Registering protection buffer */
2551 mr = fr_desc->pi_ctx->prot_mr;
Vu Pham59464ef2013-08-28 23:23:35 +03002552
Sagi Grimberg10633c32014-12-07 13:12:04 +02002553 if (!(fr_desc->ind & ind)) {
2554 isert_inv_rkey(&inv_wr, mr);
Vu Pham59464ef2013-08-28 23:23:35 +03002555 wr = &inv_wr;
Vu Pham59464ef2013-08-28 23:23:35 +03002556 }
2557
Sagi Grimberg16c2d702015-10-13 19:11:34 +03002558 n = ib_map_mr_sg(mr, mem->sg, mem->nents, PAGE_SIZE);
2559 if (unlikely(n != mem->nents)) {
2560 isert_err("failed to map mr sg (%d/%d)\n",
2561 n, mem->nents);
2562 return n < 0 ? n : -EINVAL;
2563 }
2564
2565 isert_dbg("Use fr_desc %p sg_nents %d offset %u\n",
2566 fr_desc, mem->nents, mem->offset);
2567
2568 reg_wr.wr.next = NULL;
2569 reg_wr.wr.opcode = IB_WR_REG_MR;
2570 reg_wr.wr.wr_id = ISER_FASTREG_LI_WRID;
2571 reg_wr.wr.send_flags = 0;
2572 reg_wr.wr.num_sge = 0;
2573 reg_wr.mr = mr;
2574 reg_wr.key = mr->lkey;
2575 reg_wr.access = IB_ACCESS_LOCAL_WRITE;
Vu Pham59464ef2013-08-28 23:23:35 +03002576
2577 if (!wr)
Sagi Grimberg16c2d702015-10-13 19:11:34 +03002578 wr = &reg_wr.wr;
Vu Pham59464ef2013-08-28 23:23:35 +03002579 else
Sagi Grimberg16c2d702015-10-13 19:11:34 +03002580 wr->next = &reg_wr.wr;
Vu Pham59464ef2013-08-28 23:23:35 +03002581
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002582 ret = ib_post_send(isert_conn->qp, wr, &bad_wr);
Vu Pham59464ef2013-08-28 23:23:35 +03002583 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002584 isert_err("fast registration failed, ret:%d\n", ret);
Vu Pham59464ef2013-08-28 23:23:35 +03002585 return ret;
2586 }
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002587 fr_desc->ind &= ~ind;
Vu Pham59464ef2013-08-28 23:23:35 +03002588
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002589 sge->lkey = mr->lkey;
Sagi Grimberg16c2d702015-10-13 19:11:34 +03002590 sge->addr = mr->iova;
2591 sge->length = mr->length;
Vu Pham59464ef2013-08-28 23:23:35 +03002592
Sagi Grimberg4c22e072014-12-07 13:12:03 +02002593 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n",
2594 sge->addr, sge->length, sge->lkey);
Vu Pham59464ef2013-08-28 23:23:35 +03002595
2596 return ret;
2597}
2598
Sagi Grimberg3d73cf12014-08-13 19:54:34 +03002599static inline void
2600isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs,
2601 struct ib_sig_domain *domain)
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002602{
Sagi Grimberg78eda2b2014-08-13 19:54:35 +03002603 domain->sig_type = IB_SIG_TYPE_T10_DIF;
Sagi Grimberg3d73cf12014-08-13 19:54:34 +03002604 domain->sig.dif.bg_type = IB_T10DIF_CRC;
2605 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size;
2606 domain->sig.dif.ref_tag = se_cmd->reftag_seed;
Sagi Grimberg78eda2b2014-08-13 19:54:35 +03002607 /*
2608 * At the moment we hard code those, but if in the future
2609 * the target core would like to use it, we will take it
2610 * from se_cmd.
2611 */
2612 domain->sig.dif.apptag_check_mask = 0xffff;
2613 domain->sig.dif.app_escape = true;
2614 domain->sig.dif.ref_escape = true;
2615 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT ||
2616 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT)
2617 domain->sig.dif.ref_remap = true;
Sagi Grimberg3d73cf12014-08-13 19:54:34 +03002618};
2619
2620static int
2621isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs)
2622{
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002623 switch (se_cmd->prot_op) {
2624 case TARGET_PROT_DIN_INSERT:
2625 case TARGET_PROT_DOUT_STRIP:
Sagi Grimberg78eda2b2014-08-13 19:54:35 +03002626 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;
Sagi Grimberg3d73cf12014-08-13 19:54:34 +03002627 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire);
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002628 break;
2629 case TARGET_PROT_DOUT_INSERT:
2630 case TARGET_PROT_DIN_STRIP:
Sagi Grimberg78eda2b2014-08-13 19:54:35 +03002631 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
Sagi Grimberg3d73cf12014-08-13 19:54:34 +03002632 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem);
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002633 break;
2634 case TARGET_PROT_DIN_PASS:
2635 case TARGET_PROT_DOUT_PASS:
Sagi Grimberg3d73cf12014-08-13 19:54:34 +03002636 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire);
2637 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem);
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002638 break;
2639 default:
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002640 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op);
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002641 return -EINVAL;
2642 }
2643
2644 return 0;
2645}
2646
2647static inline u8
2648isert_set_prot_checks(u8 prot_checks)
2649{
2650 return (prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) |
2651 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) |
2652 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0);
2653}
2654
2655static int
Sagi Grimberg570db172014-12-02 16:57:31 +02002656isert_reg_sig_mr(struct isert_conn *isert_conn,
2657 struct se_cmd *se_cmd,
2658 struct isert_rdma_wr *rdma_wr,
2659 struct fast_reg_descriptor *fr_desc)
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002660{
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002661 struct ib_sig_handover_wr sig_wr;
2662 struct ib_send_wr inv_wr, *bad_wr, *wr = NULL;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002663 struct pi_context *pi_ctx = fr_desc->pi_ctx;
2664 struct ib_sig_attrs sig_attrs;
2665 int ret;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002666
2667 memset(&sig_attrs, 0, sizeof(sig_attrs));
2668 ret = isert_set_sig_attrs(se_cmd, &sig_attrs);
2669 if (ret)
2670 goto err;
2671
2672 sig_attrs.check_mask = isert_set_prot_checks(se_cmd->prot_checks);
2673
2674 if (!(fr_desc->ind & ISERT_SIG_KEY_VALID)) {
Sagi Grimberg10633c32014-12-07 13:12:04 +02002675 isert_inv_rkey(&inv_wr, pi_ctx->sig_mr);
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002676 wr = &inv_wr;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002677 }
2678
2679 memset(&sig_wr, 0, sizeof(sig_wr));
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002680 sig_wr.wr.opcode = IB_WR_REG_SIG_MR;
2681 sig_wr.wr.wr_id = ISER_FASTREG_LI_WRID;
2682 sig_wr.wr.sg_list = &rdma_wr->ib_sg[DATA];
2683 sig_wr.wr.num_sge = 1;
2684 sig_wr.access_flags = IB_ACCESS_LOCAL_WRITE;
2685 sig_wr.sig_attrs = &sig_attrs;
2686 sig_wr.sig_mr = pi_ctx->sig_mr;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002687 if (se_cmd->t_prot_sg)
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002688 sig_wr.prot = &rdma_wr->ib_sg[PROT];
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002689
2690 if (!wr)
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002691 wr = &sig_wr.wr;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002692 else
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002693 wr->next = &sig_wr.wr;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002694
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002695 ret = ib_post_send(isert_conn->qp, wr, &bad_wr);
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002696 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002697 isert_err("fast registration failed, ret:%d\n", ret);
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002698 goto err;
2699 }
2700 fr_desc->ind &= ~ISERT_SIG_KEY_VALID;
2701
Sagi Grimberg570db172014-12-02 16:57:31 +02002702 rdma_wr->ib_sg[SIG].lkey = pi_ctx->sig_mr->lkey;
2703 rdma_wr->ib_sg[SIG].addr = 0;
2704 rdma_wr->ib_sg[SIG].length = se_cmd->data_length;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002705 if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP &&
2706 se_cmd->prot_op != TARGET_PROT_DOUT_INSERT)
2707 /*
2708 * We have protection guards on the wire
2709 * so we need to set a larget transfer
2710 */
Sagi Grimberg570db172014-12-02 16:57:31 +02002711 rdma_wr->ib_sg[SIG].length += se_cmd->prot_length;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002712
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002713 isert_dbg("sig_sge: addr: 0x%llx length: %u lkey: %x\n",
Sagi Grimberg570db172014-12-02 16:57:31 +02002714 rdma_wr->ib_sg[SIG].addr, rdma_wr->ib_sg[SIG].length,
2715 rdma_wr->ib_sg[SIG].lkey);
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002716err:
2717 return ret;
2718}
2719
Vu Pham59464ef2013-08-28 23:23:35 +03002720static int
Sagi Grimberg570db172014-12-02 16:57:31 +02002721isert_handle_prot_cmd(struct isert_conn *isert_conn,
2722 struct isert_cmd *isert_cmd,
2723 struct isert_rdma_wr *wr)
2724{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002725 struct isert_device *device = isert_conn->device;
Sagi Grimberg570db172014-12-02 16:57:31 +02002726 struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd;
2727 int ret;
2728
2729 if (!wr->fr_desc->pi_ctx) {
2730 ret = isert_create_pi_ctx(wr->fr_desc,
2731 device->ib_device,
Sagi Grimberg67cb3942015-03-29 15:52:05 +03002732 device->pd);
Sagi Grimberg570db172014-12-02 16:57:31 +02002733 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002734 isert_err("conn %p failed to allocate pi_ctx\n",
Sagi Grimberg570db172014-12-02 16:57:31 +02002735 isert_conn);
2736 return ret;
2737 }
2738 }
2739
2740 if (se_cmd->t_prot_sg) {
2741 ret = isert_map_data_buf(isert_conn, isert_cmd,
2742 se_cmd->t_prot_sg,
2743 se_cmd->t_prot_nents,
2744 se_cmd->prot_length,
2745 0, wr->iser_ib_op, &wr->prot);
2746 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002747 isert_err("conn %p failed to map protection buffer\n",
Sagi Grimberg570db172014-12-02 16:57:31 +02002748 isert_conn);
2749 return ret;
2750 }
2751
2752 memset(&wr->ib_sg[PROT], 0, sizeof(wr->ib_sg[PROT]));
2753 ret = isert_fast_reg_mr(isert_conn, wr->fr_desc, &wr->prot,
2754 ISERT_PROT_KEY_VALID, &wr->ib_sg[PROT]);
2755 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002756 isert_err("conn %p failed to fast reg mr\n",
Sagi Grimberg570db172014-12-02 16:57:31 +02002757 isert_conn);
2758 goto unmap_prot_cmd;
2759 }
2760 }
2761
2762 ret = isert_reg_sig_mr(isert_conn, se_cmd, wr, wr->fr_desc);
2763 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002764 isert_err("conn %p failed to fast reg mr\n",
Sagi Grimberg570db172014-12-02 16:57:31 +02002765 isert_conn);
2766 goto unmap_prot_cmd;
2767 }
2768 wr->fr_desc->ind |= ISERT_PROTECTED;
2769
2770 return 0;
2771
2772unmap_prot_cmd:
2773 if (se_cmd->t_prot_sg)
2774 isert_unmap_data_buf(isert_conn, &wr->prot);
2775
2776 return ret;
2777}
2778
2779static int
Sagi Grimberga3a5a822014-01-09 18:40:50 +02002780isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
2781 struct isert_rdma_wr *wr)
Vu Pham59464ef2013-08-28 23:23:35 +03002782{
2783 struct se_cmd *se_cmd = &cmd->se_cmd;
2784 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002785 struct isert_conn *isert_conn = conn->context;
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002786 struct fast_reg_descriptor *fr_desc = NULL;
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002787 struct ib_rdma_wr *rdma_wr;
Sagi Grimberg570db172014-12-02 16:57:31 +02002788 struct ib_sge *ib_sg;
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002789 u32 offset;
2790 int ret = 0;
Vu Pham59464ef2013-08-28 23:23:35 +03002791 unsigned long flags;
2792
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002793 isert_cmd->tx_desc.isert_cmd = isert_cmd;
2794
2795 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0;
2796 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg,
2797 se_cmd->t_data_nents, se_cmd->data_length,
2798 offset, wr->iser_ib_op, &wr->data);
2799 if (ret)
2800 return ret;
2801
Sagi Grimberg302cc7c2014-12-02 16:57:34 +02002802 if (wr->data.dma_nents != 1 || isert_prot_cmd(isert_conn, se_cmd)) {
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002803 spin_lock_irqsave(&isert_conn->pool_lock, flags);
2804 fr_desc = list_first_entry(&isert_conn->fr_pool,
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002805 struct fast_reg_descriptor, list);
2806 list_del(&fr_desc->list);
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002807 spin_unlock_irqrestore(&isert_conn->pool_lock, flags);
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002808 wr->fr_desc = fr_desc;
Vu Pham59464ef2013-08-28 23:23:35 +03002809 }
2810
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002811 ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data,
Sagi Grimberg570db172014-12-02 16:57:31 +02002812 ISERT_DATA_KEY_VALID, &wr->ib_sg[DATA]);
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002813 if (ret)
2814 goto unmap_cmd;
Vu Pham59464ef2013-08-28 23:23:35 +03002815
Sagi Grimberg302cc7c2014-12-02 16:57:34 +02002816 if (isert_prot_cmd(isert_conn, se_cmd)) {
Sagi Grimberg570db172014-12-02 16:57:31 +02002817 ret = isert_handle_prot_cmd(isert_conn, isert_cmd, wr);
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002818 if (ret)
Sagi Grimberg570db172014-12-02 16:57:31 +02002819 goto unmap_cmd;
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002820
Sagi Grimberg570db172014-12-02 16:57:31 +02002821 ib_sg = &wr->ib_sg[SIG];
2822 } else {
2823 ib_sg = &wr->ib_sg[DATA];
2824 }
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002825
Sagi Grimberg570db172014-12-02 16:57:31 +02002826 memcpy(&wr->s_ib_sge, ib_sg, sizeof(*ib_sg));
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002827 wr->ib_sge = &wr->s_ib_sge;
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002828 wr->rdma_wr_num = 1;
2829 memset(&wr->s_rdma_wr, 0, sizeof(wr->s_rdma_wr));
2830 wr->rdma_wr = &wr->s_rdma_wr;
Vu Pham59464ef2013-08-28 23:23:35 +03002831 wr->isert_cmd = isert_cmd;
Vu Pham59464ef2013-08-28 23:23:35 +03002832
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002833 rdma_wr = &isert_cmd->rdma_wr.s_rdma_wr;
2834 rdma_wr->wr.sg_list = &wr->s_ib_sge;
2835 rdma_wr->wr.num_sge = 1;
2836 rdma_wr->wr.wr_id = (uintptr_t)&isert_cmd->tx_desc;
Vu Pham59464ef2013-08-28 23:23:35 +03002837 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) {
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002838 rdma_wr->wr.opcode = IB_WR_RDMA_WRITE;
2839 rdma_wr->remote_addr = isert_cmd->read_va;
2840 rdma_wr->rkey = isert_cmd->read_stag;
2841 rdma_wr->wr.send_flags = !isert_prot_cmd(isert_conn, se_cmd) ?
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002842 0 : IB_SEND_SIGNALED;
Vu Pham59464ef2013-08-28 23:23:35 +03002843 } else {
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002844 rdma_wr->wr.opcode = IB_WR_RDMA_READ;
2845 rdma_wr->remote_addr = isert_cmd->write_va;
2846 rdma_wr->rkey = isert_cmd->write_stag;
2847 rdma_wr->wr.send_flags = IB_SEND_SIGNALED;
Vu Pham59464ef2013-08-28 23:23:35 +03002848 }
2849
Vu Pham59464ef2013-08-28 23:23:35 +03002850 return 0;
Sagi Grimberg570db172014-12-02 16:57:31 +02002851
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002852unmap_cmd:
2853 if (fr_desc) {
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002854 spin_lock_irqsave(&isert_conn->pool_lock, flags);
2855 list_add_tail(&fr_desc->list, &isert_conn->fr_pool);
2856 spin_unlock_irqrestore(&isert_conn->pool_lock, flags);
Sagi Grimberge3d7e4c2014-02-19 17:50:22 +02002857 }
2858 isert_unmap_data_buf(isert_conn, &wr->data);
Vu Pham59464ef2013-08-28 23:23:35 +03002859
Vu Pham59464ef2013-08-28 23:23:35 +03002860 return ret;
2861}
2862
2863static int
Vu Pham90ecc6e2013-08-28 23:23:33 +03002864isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
2865{
2866 struct se_cmd *se_cmd = &cmd->se_cmd;
Vu Pham59464ef2013-08-28 23:23:35 +03002867 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
Vu Pham90ecc6e2013-08-28 23:23:33 +03002868 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
Sagi Grimberg67004252015-03-29 15:52:09 +03002869 struct isert_conn *isert_conn = conn->context;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002870 struct isert_device *device = isert_conn->device;
Vu Pham90ecc6e2013-08-28 23:23:33 +03002871 struct ib_send_wr *wr_failed;
2872 int rc;
2873
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002874 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n",
Vu Pham90ecc6e2013-08-28 23:23:33 +03002875 isert_cmd, se_cmd->data_length);
Sagi Grimberg4c22e072014-12-07 13:12:03 +02002876
Vu Pham90ecc6e2013-08-28 23:23:33 +03002877 wr->iser_ib_op = ISER_IB_RDMA_WRITE;
Vu Phamd40945d2013-08-28 23:23:34 +03002878 rc = device->reg_rdma_mem(conn, cmd, wr);
Vu Pham90ecc6e2013-08-28 23:23:33 +03002879 if (rc) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002880 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd);
Vu Pham90ecc6e2013-08-28 23:23:33 +03002881 return rc;
2882 }
2883
Sagi Grimberg302cc7c2014-12-02 16:57:34 +02002884 if (!isert_prot_cmd(isert_conn, se_cmd)) {
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002885 /*
2886 * Build isert_conn->tx_desc for iSCSI response PDU and attach
2887 */
2888 isert_create_send_desc(isert_conn, isert_cmd,
2889 &isert_cmd->tx_desc);
2890 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *)
2891 &isert_cmd->tx_desc.iscsi_header);
2892 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
2893 isert_init_send_wr(isert_conn, isert_cmd,
Sagi Grimberg68a86de2014-12-02 16:57:37 +02002894 &isert_cmd->tx_desc.send_wr);
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002895 isert_cmd->rdma_wr.s_rdma_wr.wr.next = &isert_cmd->tx_desc.send_wr;
2896 wr->rdma_wr_num += 1;
Jenny Derzhavetz4366b192015-09-06 14:52:25 +03002897
2898 rc = isert_post_recv(isert_conn, isert_cmd->rx_desc);
2899 if (rc) {
2900 isert_err("ib_post_recv failed with %d\n", rc);
2901 return rc;
2902 }
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002903 }
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002904
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002905 rc = ib_post_send(isert_conn->qp, &wr->rdma_wr->wr, &wr_failed);
Sagi Grimbergbdf20e72014-12-02 16:57:43 +02002906 if (rc)
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002907 isert_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n");
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002908
Sagi Grimberg302cc7c2014-12-02 16:57:34 +02002909 if (!isert_prot_cmd(isert_conn, se_cmd))
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002910 isert_dbg("Cmd: %p posted RDMA_WRITE + Response for iSER Data "
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002911 "READ\n", isert_cmd);
2912 else
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002913 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ\n",
Sagi Grimberg9e961ae2014-02-19 17:50:25 +02002914 isert_cmd);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002915
Vu Pham90ecc6e2013-08-28 23:23:33 +03002916 return 1;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002917}
2918
2919static int
2920isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery)
2921{
2922 struct se_cmd *se_cmd = &cmd->se_cmd;
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07002923 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002924 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;
Sagi Grimberg67004252015-03-29 15:52:09 +03002925 struct isert_conn *isert_conn = conn->context;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03002926 struct isert_device *device = isert_conn->device;
Vu Pham90ecc6e2013-08-28 23:23:33 +03002927 struct ib_send_wr *wr_failed;
2928 int rc;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002929
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002930 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n",
Vu Pham90ecc6e2013-08-28 23:23:33 +03002931 isert_cmd, se_cmd->data_length, cmd->write_data_done);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002932 wr->iser_ib_op = ISER_IB_RDMA_READ;
Vu Phamd40945d2013-08-28 23:23:34 +03002933 rc = device->reg_rdma_mem(conn, cmd, wr);
Vu Pham90ecc6e2013-08-28 23:23:33 +03002934 if (rc) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002935 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd);
Vu Pham90ecc6e2013-08-28 23:23:33 +03002936 return rc;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002937 }
2938
Christoph Hellwige622f2f2015-10-08 09:16:33 +01002939 rc = ib_post_send(isert_conn->qp, &wr->rdma_wr->wr, &wr_failed);
Sagi Grimbergbdf20e72014-12-02 16:57:43 +02002940 if (rc)
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002941 isert_warn("ib_post_send() failed for IB_WR_RDMA_READ\n");
Sagi Grimbergbdf20e72014-12-02 16:57:43 +02002942
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002943 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n",
Vu Pham90ecc6e2013-08-28 23:23:33 +03002944 isert_cmd);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002945
Vu Pham90ecc6e2013-08-28 23:23:33 +03002946 return 0;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002947}
2948
2949static int
2950isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
2951{
Jenny Derzhavetza4c15cd2015-09-06 14:52:20 +03002952 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2953 int ret = 0;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002954
2955 switch (state) {
Jenny Derzhavetza4c15cd2015-09-06 14:52:20 +03002956 case ISTATE_REMOVE:
2957 spin_lock_bh(&conn->cmd_lock);
2958 list_del_init(&cmd->i_conn_node);
2959 spin_unlock_bh(&conn->cmd_lock);
2960 isert_put_cmd(isert_cmd, true);
2961 break;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002962 case ISTATE_SEND_NOPIN_WANT_RESPONSE:
2963 ret = isert_put_nopin(cmd, conn, false);
2964 break;
2965 default:
Sagi Grimberg24f412d2014-12-07 13:12:02 +02002966 isert_err("Unknown immediate state: 0x%02x\n", state);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002967 ret = -EINVAL;
2968 break;
2969 }
2970
2971 return ret;
2972}
2973
2974static int
2975isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
2976{
Sagi Grimberg991bb762014-12-07 13:12:01 +02002977 struct isert_conn *isert_conn = conn->context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002978 int ret;
2979
2980 switch (state) {
2981 case ISTATE_SEND_LOGOUTRSP:
2982 ret = isert_put_logout_rsp(cmd, conn);
Sagi Grimberg991bb762014-12-07 13:12:01 +02002983 if (!ret)
2984 isert_conn->logout_posted = true;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002985 break;
2986 case ISTATE_SEND_NOPIN:
2987 ret = isert_put_nopin(cmd, conn, true);
2988 break;
2989 case ISTATE_SEND_TASKMGTRSP:
2990 ret = isert_put_tm_rsp(cmd, conn);
2991 break;
2992 case ISTATE_SEND_REJECT:
2993 ret = isert_put_reject(cmd, conn);
2994 break;
Nicholas Bellingeradb54c22013-06-14 16:47:15 -07002995 case ISTATE_SEND_TEXTRSP:
2996 ret = isert_put_text_rsp(cmd, conn);
2997 break;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08002998 case ISTATE_SEND_STATUS:
2999 /*
3000 * Special case for sending non GOOD SCSI status from TX thread
3001 * context during pre se_cmd excecution failure.
3002 */
3003 ret = isert_put_response(conn, cmd);
3004 break;
3005 default:
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003006 isert_err("Unknown response state: 0x%02x\n", state);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003007 ret = -EINVAL;
3008 break;
3009 }
3010
3011 return ret;
3012}
3013
Sagi Grimbergca6c1d82014-12-02 16:57:27 +02003014struct rdma_cm_id *
3015isert_setup_id(struct isert_np *isert_np)
3016{
3017 struct iscsi_np *np = isert_np->np;
3018 struct rdma_cm_id *id;
3019 struct sockaddr *sa;
3020 int ret;
3021
3022 sa = (struct sockaddr *)&np->np_sockaddr;
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003023 isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa);
Sagi Grimbergca6c1d82014-12-02 16:57:27 +02003024
Guy Shapirofa201052015-10-22 15:20:10 +03003025 id = rdma_create_id(&init_net, isert_cma_handler, isert_np,
Sagi Grimbergca6c1d82014-12-02 16:57:27 +02003026 RDMA_PS_TCP, IB_QPT_RC);
3027 if (IS_ERR(id)) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003028 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id));
Sagi Grimbergca6c1d82014-12-02 16:57:27 +02003029 ret = PTR_ERR(id);
3030 goto out;
3031 }
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003032 isert_dbg("id %p context %p\n", id, id->context);
Sagi Grimbergca6c1d82014-12-02 16:57:27 +02003033
3034 ret = rdma_bind_addr(id, sa);
3035 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003036 isert_err("rdma_bind_addr() failed: %d\n", ret);
Sagi Grimbergca6c1d82014-12-02 16:57:27 +02003037 goto out_id;
3038 }
3039
Sagi Grimberg992607e2015-03-29 15:52:18 +03003040 ret = rdma_listen(id, 0);
Sagi Grimbergca6c1d82014-12-02 16:57:27 +02003041 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003042 isert_err("rdma_listen() failed: %d\n", ret);
Sagi Grimbergca6c1d82014-12-02 16:57:27 +02003043 goto out_id;
3044 }
3045
3046 return id;
3047out_id:
3048 rdma_destroy_id(id);
3049out:
3050 return ERR_PTR(ret);
3051}
3052
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003053static int
3054isert_setup_np(struct iscsi_np *np,
Andy Grover13a3cf02015-08-24 10:26:06 -07003055 struct sockaddr_storage *ksockaddr)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003056{
3057 struct isert_np *isert_np;
3058 struct rdma_cm_id *isert_lid;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003059 int ret;
3060
3061 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL);
3062 if (!isert_np) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003063 isert_err("Unable to allocate struct isert_np\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003064 return -ENOMEM;
3065 }
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +03003066 sema_init(&isert_np->sem, 0);
3067 mutex_init(&isert_np->mutex);
Jenny Derzhavetzbd379222015-09-06 14:52:24 +03003068 INIT_LIST_HEAD(&isert_np->accepted);
3069 INIT_LIST_HEAD(&isert_np->pending);
Sagi Grimbergca6c1d82014-12-02 16:57:27 +02003070 isert_np->np = np;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003071
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003072 /*
3073 * Setup the np->np_sockaddr from the passed sockaddr setup
3074 * in iscsi_target_configfs.c code..
3075 */
3076 memcpy(&np->np_sockaddr, ksockaddr,
Andy Grover13a3cf02015-08-24 10:26:06 -07003077 sizeof(struct sockaddr_storage));
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003078
Sagi Grimbergca6c1d82014-12-02 16:57:27 +02003079 isert_lid = isert_setup_id(isert_np);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003080 if (IS_ERR(isert_lid)) {
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003081 ret = PTR_ERR(isert_lid);
3082 goto out;
3083 }
3084
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +03003085 isert_np->cm_id = isert_lid;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003086 np->np_context = isert_np;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003087
3088 return 0;
3089
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003090out:
3091 kfree(isert_np);
Sagi Grimbergca6c1d82014-12-02 16:57:27 +02003092
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003093 return ret;
3094}
3095
3096static int
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003097isert_rdma_accept(struct isert_conn *isert_conn)
3098{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03003099 struct rdma_cm_id *cm_id = isert_conn->cm_id;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003100 struct rdma_conn_param cp;
3101 int ret;
3102
3103 memset(&cp, 0, sizeof(struct rdma_conn_param));
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003104 cp.initiator_depth = isert_conn->initiator_depth;
3105 cp.retry_count = 7;
3106 cp.rnr_retry_count = 7;
3107
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003108 ret = rdma_accept(cm_id, &cp);
3109 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003110 isert_err("rdma_accept() failed with: %d\n", ret);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003111 return ret;
3112 }
3113
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003114 return 0;
3115}
3116
3117static int
3118isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
3119{
Sagi Grimberg67004252015-03-29 15:52:09 +03003120 struct isert_conn *isert_conn = conn->context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003121 int ret;
3122
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003123 isert_info("before login_req comp conn: %p\n", isert_conn);
Sagi Grimberg2371e5d2014-12-02 16:57:21 +02003124 ret = wait_for_completion_interruptible(&isert_conn->login_req_comp);
3125 if (ret) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003126 isert_err("isert_conn %p interrupted before got login req\n",
Sagi Grimberg2371e5d2014-12-02 16:57:21 +02003127 isert_conn);
3128 return ret;
3129 }
3130 reinit_completion(&isert_conn->login_req_comp);
3131
Nicholas Bellinger6faaa852013-08-18 16:35:46 -07003132 /*
3133 * For login requests after the first PDU, isert_rx_login_req() will
3134 * kick schedule_delayed_work(&conn->login_work) as the packet is
3135 * received, which turns this callback from iscsi_target_do_login_rx()
3136 * into a NOP.
3137 */
3138 if (!login->first_request)
3139 return 0;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003140
Sagi Grimberg2371e5d2014-12-02 16:57:21 +02003141 isert_rx_login_req(isert_conn);
3142
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03003143 isert_info("before login_comp conn: %p\n", conn);
3144 ret = wait_for_completion_interruptible(&isert_conn->login_comp);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003145 if (ret)
3146 return ret;
3147
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003148 isert_info("processing login->req: %p\n", login->req);
Sagi Grimberg2371e5d2014-12-02 16:57:21 +02003149
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003150 return 0;
3151}
3152
3153static void
3154isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn,
3155 struct isert_conn *isert_conn)
3156{
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03003157 struct rdma_cm_id *cm_id = isert_conn->cm_id;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003158 struct rdma_route *cm_route = &cm_id->route;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003159
3160 conn->login_family = np->np_sockaddr.ss_family;
3161
Andy Groverdc58f762015-08-24 10:26:05 -07003162 conn->login_sockaddr = cm_route->addr.dst_addr;
3163 conn->local_sockaddr = cm_route->addr.src_addr;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003164}
3165
3166static int
3167isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn)
3168{
Sagi Grimberg67004252015-03-29 15:52:09 +03003169 struct isert_np *isert_np = np->np_context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003170 struct isert_conn *isert_conn;
Sagi Grimbergc6b8e912015-03-29 15:52:16 +03003171 int ret;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003172
3173accept_wait:
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +03003174 ret = down_interruptible(&isert_np->sem);
Sagi Grimbergc6b8e912015-03-29 15:52:16 +03003175 if (ret)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003176 return -ENODEV;
3177
3178 spin_lock_bh(&np->np_thread_lock);
Sagi Grimberge346ab32014-05-19 17:44:22 +03003179 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) {
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003180 spin_unlock_bh(&np->np_thread_lock);
Rasmus Villemoes11378cd2015-02-06 01:09:05 +01003181 isert_dbg("np_thread_state %d\n",
Sagi Grimberge346ab32014-05-19 17:44:22 +03003182 np->np_thread_state);
3183 /**
3184 * No point in stalling here when np_thread
3185 * is in state RESET/SHUTDOWN/EXIT - bail
3186 **/
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003187 return -ENODEV;
3188 }
3189 spin_unlock_bh(&np->np_thread_lock);
3190
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +03003191 mutex_lock(&isert_np->mutex);
Jenny Derzhavetzbd379222015-09-06 14:52:24 +03003192 if (list_empty(&isert_np->pending)) {
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +03003193 mutex_unlock(&isert_np->mutex);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003194 goto accept_wait;
3195 }
Jenny Derzhavetzbd379222015-09-06 14:52:24 +03003196 isert_conn = list_first_entry(&isert_np->pending,
3197 struct isert_conn, node);
3198 list_del_init(&isert_conn->node);
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +03003199 mutex_unlock(&isert_np->mutex);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003200
3201 conn->context = isert_conn;
3202 isert_conn->conn = conn;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003203
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003204 isert_set_conn_info(np, conn, isert_conn);
3205
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003206 isert_dbg("Processing isert_conn: %p\n", isert_conn);
Sagi Grimberg2371e5d2014-12-02 16:57:21 +02003207
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003208 return 0;
3209}
3210
3211static void
3212isert_free_np(struct iscsi_np *np)
3213{
Sagi Grimberg67004252015-03-29 15:52:09 +03003214 struct isert_np *isert_np = np->np_context;
Sagi Grimberg268e6812014-12-02 16:57:36 +02003215 struct isert_conn *isert_conn, *n;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003216
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +03003217 if (isert_np->cm_id)
3218 rdma_destroy_id(isert_np->cm_id);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003219
Sagi Grimberg268e6812014-12-02 16:57:36 +02003220 /*
3221 * FIXME: At this point we don't have a good way to insure
3222 * that at this point we don't have hanging connections that
3223 * completed RDMA establishment but didn't start iscsi login
3224 * process. So work-around this by cleaning up what ever piled
Jenny Derzhavetzbd379222015-09-06 14:52:24 +03003225 * up in accepted and pending lists.
Sagi Grimberg268e6812014-12-02 16:57:36 +02003226 */
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +03003227 mutex_lock(&isert_np->mutex);
Jenny Derzhavetzbd379222015-09-06 14:52:24 +03003228 if (!list_empty(&isert_np->pending)) {
3229 isert_info("Still have isert pending connections\n");
Sagi Grimberg268e6812014-12-02 16:57:36 +02003230 list_for_each_entry_safe(isert_conn, n,
Jenny Derzhavetzbd379222015-09-06 14:52:24 +03003231 &isert_np->pending,
3232 node) {
3233 isert_info("cleaning isert_conn %p state (%d)\n",
3234 isert_conn, isert_conn->state);
3235 isert_connect_release(isert_conn);
3236 }
3237 }
3238
3239 if (!list_empty(&isert_np->accepted)) {
3240 isert_info("Still have isert accepted connections\n");
3241 list_for_each_entry_safe(isert_conn, n,
3242 &isert_np->accepted,
3243 node) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003244 isert_info("cleaning isert_conn %p state (%d)\n",
Sagi Grimberg268e6812014-12-02 16:57:36 +02003245 isert_conn, isert_conn->state);
3246 isert_connect_release(isert_conn);
3247 }
3248 }
Jenny Derzhavetzed8cb0a2015-09-06 14:52:23 +03003249 mutex_unlock(&isert_np->mutex);
Sagi Grimberg268e6812014-12-02 16:57:36 +02003250
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003251 np->np_context = NULL;
3252 kfree(isert_np);
3253}
3254
Sagi Grimbergb02efbf2014-12-02 16:57:29 +02003255static void isert_release_work(struct work_struct *work)
3256{
3257 struct isert_conn *isert_conn = container_of(work,
3258 struct isert_conn,
3259 release_work);
3260
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003261 isert_info("Starting release conn %p\n", isert_conn);
Sagi Grimbergb02efbf2014-12-02 16:57:29 +02003262
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03003263 wait_for_completion(&isert_conn->wait);
Sagi Grimbergb02efbf2014-12-02 16:57:29 +02003264
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03003265 mutex_lock(&isert_conn->mutex);
Sagi Grimbergb02efbf2014-12-02 16:57:29 +02003266 isert_conn->state = ISER_CONN_DOWN;
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03003267 mutex_unlock(&isert_conn->mutex);
Sagi Grimbergb02efbf2014-12-02 16:57:29 +02003268
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003269 isert_info("Destroying conn %p\n", isert_conn);
Sagi Grimbergb02efbf2014-12-02 16:57:29 +02003270 isert_put_conn(isert_conn);
3271}
3272
Sagi Grimbergbdf20e72014-12-02 16:57:43 +02003273static void
Sagi Grimberg991bb762014-12-07 13:12:01 +02003274isert_wait4logout(struct isert_conn *isert_conn)
3275{
3276 struct iscsi_conn *conn = isert_conn->conn;
3277
Sagi Grimberg4c22e072014-12-07 13:12:03 +02003278 isert_info("conn %p\n", isert_conn);
3279
Sagi Grimberg991bb762014-12-07 13:12:01 +02003280 if (isert_conn->logout_posted) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003281 isert_info("conn %p wait for conn_logout_comp\n", isert_conn);
Sagi Grimberg991bb762014-12-07 13:12:01 +02003282 wait_for_completion_timeout(&conn->conn_logout_comp,
3283 SECONDS_FOR_LOGOUT_COMP * HZ);
3284 }
3285}
3286
3287static void
Sagi Grimbergc7e160e2014-12-02 16:57:46 +02003288isert_wait4cmds(struct iscsi_conn *conn)
3289{
Sagi Grimberg4c22e072014-12-07 13:12:03 +02003290 isert_info("iscsi_conn %p\n", conn);
3291
Sagi Grimbergc7e160e2014-12-02 16:57:46 +02003292 if (conn->sess) {
3293 target_sess_cmd_list_set_waiting(conn->sess->se_sess);
3294 target_wait_for_sess_cmds(conn->sess->se_sess);
3295 }
3296}
3297
3298static void
Sagi Grimbergbdf20e72014-12-02 16:57:43 +02003299isert_wait4flush(struct isert_conn *isert_conn)
3300{
3301 struct ib_recv_wr *bad_wr;
3302
Sagi Grimberg4c22e072014-12-07 13:12:03 +02003303 isert_info("conn %p\n", isert_conn);
3304
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03003305 init_completion(&isert_conn->wait_comp_err);
Sagi Grimbergbdf20e72014-12-02 16:57:43 +02003306 isert_conn->beacon.wr_id = ISER_BEACON_WRID;
3307 /* post an indication that all flush errors were consumed */
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03003308 if (ib_post_recv(isert_conn->qp, &isert_conn->beacon, &bad_wr)) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003309 isert_err("conn %p failed to post beacon", isert_conn);
Sagi Grimbergbdf20e72014-12-02 16:57:43 +02003310 return;
3311 }
3312
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03003313 wait_for_completion(&isert_conn->wait_comp_err);
Sagi Grimbergbdf20e72014-12-02 16:57:43 +02003314}
3315
Jenny Derzhavetz3e03c4b2015-09-06 14:52:21 +03003316/**
3317 * isert_put_unsol_pending_cmds() - Drop commands waiting for
3318 * unsolicitate dataout
3319 * @conn: iscsi connection
3320 *
3321 * We might still have commands that are waiting for unsolicited
3322 * dataouts messages. We must put the extra reference on those
3323 * before blocking on the target_wait_for_session_cmds
3324 */
3325static void
3326isert_put_unsol_pending_cmds(struct iscsi_conn *conn)
3327{
3328 struct iscsi_cmd *cmd, *tmp;
3329 static LIST_HEAD(drop_cmd_list);
3330
3331 spin_lock_bh(&conn->cmd_lock);
3332 list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) {
3333 if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) &&
3334 (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) &&
3335 (cmd->write_data_done < cmd->se_cmd.data_length))
3336 list_move_tail(&cmd->i_conn_node, &drop_cmd_list);
3337 }
3338 spin_unlock_bh(&conn->cmd_lock);
3339
3340 list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) {
3341 list_del_init(&cmd->i_conn_node);
3342 if (cmd->i_state != ISTATE_REMOVE) {
3343 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
3344
3345 isert_info("conn %p dropping cmd %p\n", conn, cmd);
3346 isert_put_cmd(isert_cmd, true);
3347 }
3348 }
3349}
3350
Nicholas Bellingerdefd8842014-02-03 12:54:39 -08003351static void isert_wait_conn(struct iscsi_conn *conn)
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003352{
3353 struct isert_conn *isert_conn = conn->context;
3354
Sagi Grimberg4c22e072014-12-07 13:12:03 +02003355 isert_info("Starting conn %p\n", isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003356
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03003357 mutex_lock(&isert_conn->mutex);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003358 /*
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03003359 * Only wait for wait_comp_err if the isert_conn made it
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003360 * into full feature phase..
3361 */
Nicholas Bellingerb2cb9642013-07-03 03:05:37 -07003362 if (isert_conn->state == ISER_CONN_INIT) {
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03003363 mutex_unlock(&isert_conn->mutex);
Nicholas Bellingerb2cb9642013-07-03 03:05:37 -07003364 return;
3365 }
Sagi Grimberg954f2372014-12-02 16:57:17 +02003366 isert_conn_terminate(isert_conn);
Sagi Grimbergdac6ab32015-03-29 15:52:19 +03003367 mutex_unlock(&isert_conn->mutex);
Nicholas Bellingerb2cb9642013-07-03 03:05:37 -07003368
Sagi Grimbergbdf20e72014-12-02 16:57:43 +02003369 isert_wait4flush(isert_conn);
Jenny Derzhavetz3e03c4b2015-09-06 14:52:21 +03003370 isert_put_unsol_pending_cmds(conn);
3371 isert_wait4cmds(conn);
Sagi Grimberg991bb762014-12-07 13:12:01 +02003372 isert_wait4logout(isert_conn);
Sagi Grimberg954f2372014-12-02 16:57:17 +02003373
Sagi Grimbergb02efbf2014-12-02 16:57:29 +02003374 queue_work(isert_release_wq, &isert_conn->release_work);
Nicholas Bellingerdefd8842014-02-03 12:54:39 -08003375}
3376
3377static void isert_free_conn(struct iscsi_conn *conn)
3378{
3379 struct isert_conn *isert_conn = conn->context;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003380
Sagi Grimberg524630d2015-06-04 19:49:21 +03003381 isert_wait4flush(isert_conn);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003382 isert_put_conn(isert_conn);
3383}
3384
3385static struct iscsit_transport iser_target_transport = {
3386 .name = "IB/iSER",
3387 .transport_type = ISCSI_INFINIBAND,
Nicholas Bellingerd703ce22013-08-17 14:27:56 -07003388 .priv_size = sizeof(struct isert_cmd),
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003389 .owner = THIS_MODULE,
3390 .iscsit_setup_np = isert_setup_np,
3391 .iscsit_accept_np = isert_accept_np,
3392 .iscsit_free_np = isert_free_np,
Nicholas Bellingerdefd8842014-02-03 12:54:39 -08003393 .iscsit_wait_conn = isert_wait_conn,
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003394 .iscsit_free_conn = isert_free_conn,
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003395 .iscsit_get_login_rx = isert_get_login_rx,
3396 .iscsit_put_login_tx = isert_put_login_tx,
3397 .iscsit_immediate_queue = isert_immediate_queue,
3398 .iscsit_response_queue = isert_response_queue,
3399 .iscsit_get_dataout = isert_get_dataout,
3400 .iscsit_queue_data_in = isert_put_datain,
3401 .iscsit_queue_status = isert_put_response,
Nicholas Bellinger131e6ab2014-03-22 14:55:56 -07003402 .iscsit_aborted_task = isert_aborted_task,
Nicholas Bellingere70beee2014-04-02 12:52:38 -07003403 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops,
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003404};
3405
3406static int __init isert_init(void)
3407{
3408 int ret;
3409
Sagi Grimberg631af552015-01-25 19:09:50 +02003410 isert_comp_wq = alloc_workqueue("isert_comp_wq",
3411 WQ_UNBOUND | WQ_HIGHPRI, 0);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003412 if (!isert_comp_wq) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003413 isert_err("Unable to allocate isert_comp_wq\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003414 ret = -ENOMEM;
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02003415 return -ENOMEM;
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003416 }
3417
Sagi Grimbergb02efbf2014-12-02 16:57:29 +02003418 isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND,
3419 WQ_UNBOUND_MAX_ACTIVE);
3420 if (!isert_release_wq) {
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003421 isert_err("Unable to allocate isert_release_wq\n");
Sagi Grimbergb02efbf2014-12-02 16:57:29 +02003422 ret = -ENOMEM;
3423 goto destroy_comp_wq;
3424 }
3425
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003426 iscsit_register_transport(&iser_target_transport);
Sagi Grimberg24f412d2014-12-07 13:12:02 +02003427 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n");
Sagi Grimbergb02efbf2014-12-02 16:57:29 +02003428
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003429 return 0;
3430
Sagi Grimbergb02efbf2014-12-02 16:57:29 +02003431destroy_comp_wq:
3432 destroy_workqueue(isert_comp_wq);
Sagi Grimberg6f0fae32014-12-02 16:57:41 +02003433
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003434 return ret;
3435}
3436
3437static void __exit isert_exit(void)
3438{
Sagi Grimbergf5ebec92014-05-19 17:44:25 +03003439 flush_scheduled_work();
Sagi Grimbergb02efbf2014-12-02 16:57:29 +02003440 destroy_workqueue(isert_release_wq);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003441 destroy_workqueue(isert_comp_wq);
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003442 iscsit_unregister_transport(&iser_target_transport);
Sagi Grimberg4c22e072014-12-07 13:12:03 +02003443 isert_info("iSER_TARGET[0] - Released iser_target_transport\n");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003444}
3445
3446MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure");
Sagi Grimberg9e35eff2015-03-29 15:52:20 +03003447MODULE_VERSION("1.0");
Nicholas Bellingerb8d26b32013-03-07 00:56:19 -08003448MODULE_AUTHOR("nab@Linux-iSCSI.org");
3449MODULE_LICENSE("GPL");
3450
3451module_init(isert_init);
3452module_exit(isert_exit);