blob: 6914cc18b4a1e585c9266decd7690c30dfc6027c [file] [log] [blame]
Bart Van Asschea42d9852011-10-14 01:30:46 +00001/*
2 * Copyright (c) 2006 - 2009 Mellanox Technology Inc. All rights reserved.
3 * Copyright (C) 2008 - 2011 Bart Van Assche <bvanassche@acm.org>.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 *
33 */
34
35#include <linux/module.h>
36#include <linux/init.h>
37#include <linux/slab.h>
38#include <linux/err.h>
39#include <linux/ctype.h>
40#include <linux/kthread.h>
41#include <linux/string.h>
42#include <linux/delay.h>
43#include <linux/atomic.h>
Bart Van Asscheba929992015-05-08 10:11:12 +020044#include <scsi/scsi_proto.h>
Bart Van Asschea42d9852011-10-14 01:30:46 +000045#include <scsi/scsi_tcq.h>
Bart Van Asschea42d9852011-10-14 01:30:46 +000046#include <target/target_core_base.h>
Bart Van Asschea42d9852011-10-14 01:30:46 +000047#include <target/target_core_fabric.h>
Bart Van Asschea42d9852011-10-14 01:30:46 +000048#include "ib_srpt.h"
49
50/* Name of this kernel module. */
51#define DRV_NAME "ib_srpt"
52#define DRV_VERSION "2.0.0"
53#define DRV_RELDATE "2011-02-14"
54
55#define SRPT_ID_STRING "Linux SRP target"
56
57#undef pr_fmt
58#define pr_fmt(fmt) DRV_NAME " " fmt
59
60MODULE_AUTHOR("Vu Pham and Bart Van Assche");
61MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
62 "v" DRV_VERSION " (" DRV_RELDATE ")");
63MODULE_LICENSE("Dual BSD/GPL");
64
65/*
66 * Global Variables
67 */
68
69static u64 srpt_service_guid;
Roland Dreier486d8b92012-02-02 12:55:58 -080070static DEFINE_SPINLOCK(srpt_dev_lock); /* Protects srpt_dev_list. */
71static LIST_HEAD(srpt_dev_list); /* List of srpt_device structures. */
Bart Van Asschea42d9852011-10-14 01:30:46 +000072
73static unsigned srp_max_req_size = DEFAULT_MAX_REQ_SIZE;
74module_param(srp_max_req_size, int, 0444);
75MODULE_PARM_DESC(srp_max_req_size,
76 "Maximum size of SRP request messages in bytes.");
77
78static int srpt_srq_size = DEFAULT_SRPT_SRQ_SIZE;
79module_param(srpt_srq_size, int, 0444);
80MODULE_PARM_DESC(srpt_srq_size,
81 "Shared receive queue (SRQ) size.");
82
83static int srpt_get_u64_x(char *buffer, struct kernel_param *kp)
84{
85 return sprintf(buffer, "0x%016llx", *(u64 *)kp->arg);
86}
87module_param_call(srpt_service_guid, NULL, srpt_get_u64_x, &srpt_service_guid,
88 0444);
89MODULE_PARM_DESC(srpt_service_guid,
90 "Using this value for ioc_guid, id_ext, and cm_listen_id"
91 " instead of using the node_guid of the first HCA.");
92
93static struct ib_client srpt_client;
Bart Van Assche2c7f37f2016-02-11 11:06:55 -080094static void srpt_release_cmd(struct se_cmd *se_cmd);
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -080095static void srpt_free_ch(struct kref *kref);
Bart Van Asschea42d9852011-10-14 01:30:46 +000096static int srpt_queue_status(struct se_cmd *cmd);
Christoph Hellwig59fae4d2015-09-29 13:00:44 +020097static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc);
98static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc);
Bart Van Assche387add42016-02-11 11:10:09 -080099static void srpt_process_wait_list(struct srpt_rdma_ch *ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000100
Bart Van Asschef130c222016-02-11 11:05:38 -0800101/*
102 * The only allowed channel state changes are those that change the channel
103 * state into a state with a higher numerical value. Hence the new > prev test.
Bart Van Asschea42d9852011-10-14 01:30:46 +0000104 */
Bart Van Asschef130c222016-02-11 11:05:38 -0800105static bool srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new)
Bart Van Asschea42d9852011-10-14 01:30:46 +0000106{
107 unsigned long flags;
108 enum rdma_ch_state prev;
Bart Van Asschef130c222016-02-11 11:05:38 -0800109 bool changed = false;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000110
111 spin_lock_irqsave(&ch->spinlock, flags);
112 prev = ch->state;
Bart Van Asschef130c222016-02-11 11:05:38 -0800113 if (new > prev) {
Bart Van Asschea42d9852011-10-14 01:30:46 +0000114 ch->state = new;
Bart Van Asschef130c222016-02-11 11:05:38 -0800115 changed = true;
116 }
Bart Van Asschea42d9852011-10-14 01:30:46 +0000117 spin_unlock_irqrestore(&ch->spinlock, flags);
Bart Van Asschef130c222016-02-11 11:05:38 -0800118
119 return changed;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000120}
121
122/**
123 * srpt_event_handler() - Asynchronous IB event callback function.
124 *
125 * Callback function called by the InfiniBand core when an asynchronous IB
126 * event occurs. This callback may occur in interrupt context. See also
127 * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
128 * Architecture Specification.
129 */
130static void srpt_event_handler(struct ib_event_handler *handler,
131 struct ib_event *event)
132{
133 struct srpt_device *sdev;
134 struct srpt_port *sport;
135
136 sdev = ib_get_client_data(event->device, &srpt_client);
137 if (!sdev || sdev->device != event->device)
138 return;
139
140 pr_debug("ASYNC event= %d on device= %s\n", event->event,
Bart Van Asschef68cba4e92016-02-11 11:04:20 -0800141 sdev->device->name);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000142
143 switch (event->event) {
144 case IB_EVENT_PORT_ERR:
145 if (event->element.port_num <= sdev->device->phys_port_cnt) {
146 sport = &sdev->port[event->element.port_num - 1];
147 sport->lid = 0;
148 sport->sm_lid = 0;
149 }
150 break;
151 case IB_EVENT_PORT_ACTIVE:
152 case IB_EVENT_LID_CHANGE:
153 case IB_EVENT_PKEY_CHANGE:
154 case IB_EVENT_SM_CHANGE:
155 case IB_EVENT_CLIENT_REREGISTER:
Doug Ledford2aa1cf62014-08-12 19:20:10 -0400156 case IB_EVENT_GID_CHANGE:
Bart Van Asschea42d9852011-10-14 01:30:46 +0000157 /* Refresh port data asynchronously. */
158 if (event->element.port_num <= sdev->device->phys_port_cnt) {
159 sport = &sdev->port[event->element.port_num - 1];
160 if (!sport->lid && !sport->sm_lid)
161 schedule_work(&sport->work);
162 }
163 break;
164 default:
Doug Ledford9f5d32a2014-10-20 18:25:15 -0400165 pr_err("received unrecognized IB event %d\n",
Bart Van Asschea42d9852011-10-14 01:30:46 +0000166 event->event);
167 break;
168 }
169}
170
171/**
172 * srpt_srq_event() - SRQ event callback function.
173 */
174static void srpt_srq_event(struct ib_event *event, void *ctx)
175{
Doug Ledford9f5d32a2014-10-20 18:25:15 -0400176 pr_info("SRQ event %d\n", event->event);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000177}
178
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -0800179static const char *get_ch_state_name(enum rdma_ch_state s)
180{
181 switch (s) {
182 case CH_CONNECTING:
183 return "connecting";
184 case CH_LIVE:
185 return "live";
186 case CH_DISCONNECTING:
187 return "disconnecting";
188 case CH_DRAINING:
189 return "draining";
190 case CH_DISCONNECTED:
191 return "disconnected";
192 }
193 return "???";
194}
195
Bart Van Asschea42d9852011-10-14 01:30:46 +0000196/**
197 * srpt_qp_event() - QP event callback function.
198 */
199static void srpt_qp_event(struct ib_event *event, struct srpt_rdma_ch *ch)
200{
201 pr_debug("QP event %d on cm_id=%p sess_name=%s state=%d\n",
Bart Van Assche33912d72016-02-11 11:04:43 -0800202 event->event, ch->cm_id, ch->sess_name, ch->state);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000203
204 switch (event->event) {
205 case IB_EVENT_COMM_EST:
206 ib_cm_notify(ch->cm_id, event->event);
207 break;
208 case IB_EVENT_QP_LAST_WQE_REACHED:
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -0800209 pr_debug("%s-%d, state %s: received Last WQE event.\n",
210 ch->sess_name, ch->qp->qp_num,
211 get_ch_state_name(ch->state));
Bart Van Asschea42d9852011-10-14 01:30:46 +0000212 break;
213 default:
Doug Ledford9f5d32a2014-10-20 18:25:15 -0400214 pr_err("received unrecognized IB QP event %d\n", event->event);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000215 break;
216 }
217}
218
219/**
220 * srpt_set_ioc() - Helper function for initializing an IOUnitInfo structure.
221 *
222 * @slot: one-based slot number.
223 * @value: four-bit value.
224 *
225 * Copies the lowest four bits of value in element slot of the array of four
226 * bit elements called c_list (controller list). The index slot is one-based.
227 */
228static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
229{
230 u16 id;
231 u8 tmp;
232
233 id = (slot - 1) / 2;
234 if (slot & 0x1) {
235 tmp = c_list[id] & 0xf;
236 c_list[id] = (value << 4) | tmp;
237 } else {
238 tmp = c_list[id] & 0xf0;
239 c_list[id] = (value & 0xf) | tmp;
240 }
241}
242
243/**
244 * srpt_get_class_port_info() - Copy ClassPortInfo to a management datagram.
245 *
246 * See also section 16.3.3.1 ClassPortInfo in the InfiniBand Architecture
247 * Specification.
248 */
249static void srpt_get_class_port_info(struct ib_dm_mad *mad)
250{
251 struct ib_class_port_info *cif;
252
253 cif = (struct ib_class_port_info *)mad->data;
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -0800254 memset(cif, 0, sizeof(*cif));
Bart Van Asschea42d9852011-10-14 01:30:46 +0000255 cif->base_version = 1;
256 cif->class_version = 1;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000257
Erez Shitrit507f6af2016-05-25 22:02:04 +0300258 ib_set_cpi_resp_time(cif, 20);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000259 mad->mad_hdr.status = 0;
260}
261
262/**
263 * srpt_get_iou() - Write IOUnitInfo to a management datagram.
264 *
265 * See also section 16.3.3.3 IOUnitInfo in the InfiniBand Architecture
266 * Specification. See also section B.7, table B.6 in the SRP r16a document.
267 */
268static void srpt_get_iou(struct ib_dm_mad *mad)
269{
270 struct ib_dm_iou_info *ioui;
271 u8 slot;
272 int i;
273
274 ioui = (struct ib_dm_iou_info *)mad->data;
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530275 ioui->change_id = cpu_to_be16(1);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000276 ioui->max_controllers = 16;
277
278 /* set present for slot 1 and empty for the rest */
279 srpt_set_ioc(ioui->controller_list, 1, 1);
280 for (i = 1, slot = 2; i < 16; i++, slot++)
281 srpt_set_ioc(ioui->controller_list, slot, 0);
282
283 mad->mad_hdr.status = 0;
284}
285
286/**
287 * srpt_get_ioc() - Write IOControllerprofile to a management datagram.
288 *
289 * See also section 16.3.3.4 IOControllerProfile in the InfiniBand
290 * Architecture Specification. See also section B.7, table B.7 in the SRP
291 * r16a document.
292 */
293static void srpt_get_ioc(struct srpt_port *sport, u32 slot,
294 struct ib_dm_mad *mad)
295{
296 struct srpt_device *sdev = sport->sdev;
297 struct ib_dm_ioc_profile *iocp;
298
299 iocp = (struct ib_dm_ioc_profile *)mad->data;
300
301 if (!slot || slot > 16) {
302 mad->mad_hdr.status
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530303 = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000304 return;
305 }
306
307 if (slot > 2) {
308 mad->mad_hdr.status
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530309 = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000310 return;
311 }
312
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -0800313 memset(iocp, 0, sizeof(*iocp));
Bart Van Asschea42d9852011-10-14 01:30:46 +0000314 strcpy(iocp->id_string, SRPT_ID_STRING);
315 iocp->guid = cpu_to_be64(srpt_service_guid);
Or Gerlitz4a061b22015-12-18 10:59:46 +0200316 iocp->vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
317 iocp->device_id = cpu_to_be32(sdev->device->attrs.vendor_part_id);
318 iocp->device_version = cpu_to_be16(sdev->device->attrs.hw_ver);
319 iocp->subsys_vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000320 iocp->subsys_device_id = 0x0;
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530321 iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
322 iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
323 iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
324 iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000325 iocp->send_queue_depth = cpu_to_be16(sdev->srq_size);
326 iocp->rdma_read_depth = 4;
327 iocp->send_size = cpu_to_be32(srp_max_req_size);
328 iocp->rdma_size = cpu_to_be32(min(sport->port_attrib.srp_max_rdma_size,
329 1U << 24));
330 iocp->num_svc_entries = 1;
331 iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
332 SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
333
334 mad->mad_hdr.status = 0;
335}
336
337/**
338 * srpt_get_svc_entries() - Write ServiceEntries to a management datagram.
339 *
340 * See also section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
341 * Specification. See also section B.7, table B.8 in the SRP r16a document.
342 */
343static void srpt_get_svc_entries(u64 ioc_guid,
344 u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
345{
346 struct ib_dm_svc_entries *svc_entries;
347
348 WARN_ON(!ioc_guid);
349
350 if (!slot || slot > 16) {
351 mad->mad_hdr.status
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530352 = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000353 return;
354 }
355
356 if (slot > 2 || lo > hi || hi > 1) {
357 mad->mad_hdr.status
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530358 = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000359 return;
360 }
361
362 svc_entries = (struct ib_dm_svc_entries *)mad->data;
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -0800363 memset(svc_entries, 0, sizeof(*svc_entries));
Bart Van Asschea42d9852011-10-14 01:30:46 +0000364 svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
365 snprintf(svc_entries->service_entries[0].name,
366 sizeof(svc_entries->service_entries[0].name),
367 "%s%016llx",
368 SRP_SERVICE_NAME_PREFIX,
369 ioc_guid);
370
371 mad->mad_hdr.status = 0;
372}
373
374/**
375 * srpt_mgmt_method_get() - Process a received management datagram.
376 * @sp: source port through which the MAD has been received.
377 * @rq_mad: received MAD.
378 * @rsp_mad: response MAD.
379 */
380static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
381 struct ib_dm_mad *rsp_mad)
382{
383 u16 attr_id;
384 u32 slot;
385 u8 hi, lo;
386
387 attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
388 switch (attr_id) {
389 case DM_ATTR_CLASS_PORT_INFO:
390 srpt_get_class_port_info(rsp_mad);
391 break;
392 case DM_ATTR_IOU_INFO:
393 srpt_get_iou(rsp_mad);
394 break;
395 case DM_ATTR_IOC_PROFILE:
396 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
397 srpt_get_ioc(sp, slot, rsp_mad);
398 break;
399 case DM_ATTR_SVC_ENTRIES:
400 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
401 hi = (u8) ((slot >> 8) & 0xff);
402 lo = (u8) (slot & 0xff);
403 slot = (u16) ((slot >> 16) & 0xffff);
404 srpt_get_svc_entries(srpt_service_guid,
405 slot, hi, lo, rsp_mad);
406 break;
407 default:
408 rsp_mad->mad_hdr.status =
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530409 cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000410 break;
411 }
412}
413
414/**
415 * srpt_mad_send_handler() - Post MAD-send callback function.
416 */
417static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
418 struct ib_mad_send_wc *mad_wc)
419{
420 ib_destroy_ah(mad_wc->send_buf->ah);
421 ib_free_send_mad(mad_wc->send_buf);
422}
423
424/**
425 * srpt_mad_recv_handler() - MAD reception callback function.
426 */
427static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
Christoph Hellwigca281262016-01-04 14:15:58 +0100428 struct ib_mad_send_buf *send_buf,
Bart Van Asschea42d9852011-10-14 01:30:46 +0000429 struct ib_mad_recv_wc *mad_wc)
430{
431 struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
432 struct ib_ah *ah;
433 struct ib_mad_send_buf *rsp;
434 struct ib_dm_mad *dm_mad;
435
436 if (!mad_wc || !mad_wc->recv_buf.mad)
437 return;
438
439 ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
440 mad_wc->recv_buf.grh, mad_agent->port_num);
441 if (IS_ERR(ah))
442 goto err;
443
444 BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
445
446 rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
447 mad_wc->wc->pkey_index, 0,
448 IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
Ira Weinyda2dfaa2015-06-06 14:38:28 -0400449 GFP_KERNEL,
450 IB_MGMT_BASE_VERSION);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000451 if (IS_ERR(rsp))
452 goto err_rsp;
453
454 rsp->ah = ah;
455
456 dm_mad = rsp->mad;
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -0800457 memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof(*dm_mad));
Bart Van Asschea42d9852011-10-14 01:30:46 +0000458 dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
459 dm_mad->mad_hdr.status = 0;
460
461 switch (mad_wc->recv_buf.mad->mad_hdr.method) {
462 case IB_MGMT_METHOD_GET:
463 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
464 break;
465 case IB_MGMT_METHOD_SET:
466 dm_mad->mad_hdr.status =
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530467 cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000468 break;
469 default:
470 dm_mad->mad_hdr.status =
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530471 cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000472 break;
473 }
474
475 if (!ib_post_send_mad(rsp, NULL)) {
476 ib_free_recv_mad(mad_wc);
477 /* will destroy_ah & free_send_mad in send completion */
478 return;
479 }
480
481 ib_free_send_mad(rsp);
482
483err_rsp:
484 ib_destroy_ah(ah);
485err:
486 ib_free_recv_mad(mad_wc);
487}
488
489/**
490 * srpt_refresh_port() - Configure a HCA port.
491 *
492 * Enable InfiniBand management datagram processing, update the cached sm_lid,
493 * lid and gid values, and register a callback function for processing MADs
494 * on the specified port.
495 *
496 * Note: It is safe to call this function more than once for the same port.
497 */
498static int srpt_refresh_port(struct srpt_port *sport)
499{
500 struct ib_mad_reg_req reg_req;
501 struct ib_port_modify port_modify;
502 struct ib_port_attr port_attr;
503 int ret;
504
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -0800505 memset(&port_modify, 0, sizeof(port_modify));
Bart Van Asschea42d9852011-10-14 01:30:46 +0000506 port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
507 port_modify.clr_port_cap_mask = 0;
508
509 ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
510 if (ret)
511 goto err_mod_port;
512
513 ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
514 if (ret)
515 goto err_query_port;
516
517 sport->sm_lid = port_attr.sm_lid;
518 sport->lid = port_attr.lid;
519
Matan Barak55ee3ab2015-10-15 18:38:45 +0300520 ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid,
521 NULL);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000522 if (ret)
523 goto err_query_port;
524
Doug Ledford716b076b2016-08-24 12:14:19 -0400525 snprintf(sport->port_guid, sizeof(sport->port_guid),
526 "0x%016llx%016llx",
527 be64_to_cpu(sport->gid.global.subnet_prefix),
528 be64_to_cpu(sport->gid.global.interface_id));
529
Bart Van Asschea42d9852011-10-14 01:30:46 +0000530 if (!sport->mad_agent) {
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -0800531 memset(&reg_req, 0, sizeof(reg_req));
Bart Van Asschea42d9852011-10-14 01:30:46 +0000532 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
533 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
534 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
535 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
536
537 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
538 sport->port,
539 IB_QPT_GSI,
540 &reg_req, 0,
541 srpt_mad_send_handler,
542 srpt_mad_recv_handler,
Ira Weiny0f29b462014-08-08 19:00:55 -0400543 sport, 0);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000544 if (IS_ERR(sport->mad_agent)) {
545 ret = PTR_ERR(sport->mad_agent);
546 sport->mad_agent = NULL;
547 goto err_query_port;
548 }
549 }
550
551 return 0;
552
553err_query_port:
554
555 port_modify.set_port_cap_mask = 0;
556 port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
557 ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
558
559err_mod_port:
560
561 return ret;
562}
563
564/**
565 * srpt_unregister_mad_agent() - Unregister MAD callback functions.
566 *
567 * Note: It is safe to call this function more than once for the same device.
568 */
569static void srpt_unregister_mad_agent(struct srpt_device *sdev)
570{
571 struct ib_port_modify port_modify = {
572 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
573 };
574 struct srpt_port *sport;
575 int i;
576
577 for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
578 sport = &sdev->port[i - 1];
579 WARN_ON(sport->port != i);
580 if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
Doug Ledford9f5d32a2014-10-20 18:25:15 -0400581 pr_err("disabling MAD processing failed.\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +0000582 if (sport->mad_agent) {
583 ib_unregister_mad_agent(sport->mad_agent);
584 sport->mad_agent = NULL;
585 }
586 }
587}
588
589/**
590 * srpt_alloc_ioctx() - Allocate an SRPT I/O context structure.
591 */
592static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev,
593 int ioctx_size, int dma_size,
594 enum dma_data_direction dir)
595{
596 struct srpt_ioctx *ioctx;
597
598 ioctx = kmalloc(ioctx_size, GFP_KERNEL);
599 if (!ioctx)
600 goto err;
601
602 ioctx->buf = kmalloc(dma_size, GFP_KERNEL);
603 if (!ioctx->buf)
604 goto err_free_ioctx;
605
606 ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf, dma_size, dir);
607 if (ib_dma_mapping_error(sdev->device, ioctx->dma))
608 goto err_free_buf;
609
610 return ioctx;
611
612err_free_buf:
613 kfree(ioctx->buf);
614err_free_ioctx:
615 kfree(ioctx);
616err:
617 return NULL;
618}
619
620/**
621 * srpt_free_ioctx() - Free an SRPT I/O context structure.
622 */
623static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx,
624 int dma_size, enum dma_data_direction dir)
625{
626 if (!ioctx)
627 return;
628
629 ib_dma_unmap_single(sdev->device, ioctx->dma, dma_size, dir);
630 kfree(ioctx->buf);
631 kfree(ioctx);
632}
633
634/**
635 * srpt_alloc_ioctx_ring() - Allocate a ring of SRPT I/O context structures.
636 * @sdev: Device to allocate the I/O context ring for.
637 * @ring_size: Number of elements in the I/O context ring.
638 * @ioctx_size: I/O context size.
639 * @dma_size: DMA buffer size.
640 * @dir: DMA data direction.
641 */
642static struct srpt_ioctx **srpt_alloc_ioctx_ring(struct srpt_device *sdev,
643 int ring_size, int ioctx_size,
644 int dma_size, enum dma_data_direction dir)
645{
646 struct srpt_ioctx **ring;
647 int i;
648
649 WARN_ON(ioctx_size != sizeof(struct srpt_recv_ioctx)
650 && ioctx_size != sizeof(struct srpt_send_ioctx));
651
652 ring = kmalloc(ring_size * sizeof(ring[0]), GFP_KERNEL);
653 if (!ring)
654 goto out;
655 for (i = 0; i < ring_size; ++i) {
656 ring[i] = srpt_alloc_ioctx(sdev, ioctx_size, dma_size, dir);
657 if (!ring[i])
658 goto err;
659 ring[i]->index = i;
660 }
661 goto out;
662
663err:
664 while (--i >= 0)
665 srpt_free_ioctx(sdev, ring[i], dma_size, dir);
666 kfree(ring);
Jesper Juhl715252d2012-02-04 23:49:40 +0100667 ring = NULL;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000668out:
669 return ring;
670}
671
672/**
673 * srpt_free_ioctx_ring() - Free the ring of SRPT I/O context structures.
674 */
675static void srpt_free_ioctx_ring(struct srpt_ioctx **ioctx_ring,
676 struct srpt_device *sdev, int ring_size,
677 int dma_size, enum dma_data_direction dir)
678{
679 int i;
680
681 for (i = 0; i < ring_size; ++i)
682 srpt_free_ioctx(sdev, ioctx_ring[i], dma_size, dir);
683 kfree(ioctx_ring);
684}
685
686/**
687 * srpt_get_cmd_state() - Get the state of a SCSI command.
688 */
689static enum srpt_command_state srpt_get_cmd_state(struct srpt_send_ioctx *ioctx)
690{
691 enum srpt_command_state state;
692 unsigned long flags;
693
694 BUG_ON(!ioctx);
695
696 spin_lock_irqsave(&ioctx->spinlock, flags);
697 state = ioctx->state;
698 spin_unlock_irqrestore(&ioctx->spinlock, flags);
699 return state;
700}
701
702/**
703 * srpt_set_cmd_state() - Set the state of a SCSI command.
704 *
705 * Does not modify the state of aborted commands. Returns the previous command
706 * state.
707 */
708static enum srpt_command_state srpt_set_cmd_state(struct srpt_send_ioctx *ioctx,
709 enum srpt_command_state new)
710{
711 enum srpt_command_state previous;
712 unsigned long flags;
713
714 BUG_ON(!ioctx);
715
716 spin_lock_irqsave(&ioctx->spinlock, flags);
717 previous = ioctx->state;
718 if (previous != SRPT_STATE_DONE)
719 ioctx->state = new;
720 spin_unlock_irqrestore(&ioctx->spinlock, flags);
721
722 return previous;
723}
724
725/**
726 * srpt_test_and_set_cmd_state() - Test and set the state of a command.
727 *
728 * Returns true if and only if the previous command state was equal to 'old'.
729 */
730static bool srpt_test_and_set_cmd_state(struct srpt_send_ioctx *ioctx,
731 enum srpt_command_state old,
732 enum srpt_command_state new)
733{
734 enum srpt_command_state previous;
735 unsigned long flags;
736
737 WARN_ON(!ioctx);
738 WARN_ON(old == SRPT_STATE_DONE);
739 WARN_ON(new == SRPT_STATE_NEW);
740
741 spin_lock_irqsave(&ioctx->spinlock, flags);
742 previous = ioctx->state;
743 if (previous == old)
744 ioctx->state = new;
745 spin_unlock_irqrestore(&ioctx->spinlock, flags);
746 return previous == old;
747}
748
749/**
750 * srpt_post_recv() - Post an IB receive request.
751 */
752static int srpt_post_recv(struct srpt_device *sdev,
753 struct srpt_recv_ioctx *ioctx)
754{
755 struct ib_sge list;
756 struct ib_recv_wr wr, *bad_wr;
757
758 BUG_ON(!sdev);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000759 list.addr = ioctx->ioctx.dma;
760 list.length = srp_max_req_size;
Jason Gunthorpe5a783952015-07-30 17:22:24 -0600761 list.lkey = sdev->pd->local_dma_lkey;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000762
Christoph Hellwig59fae4d2015-09-29 13:00:44 +0200763 ioctx->ioctx.cqe.done = srpt_recv_done;
764 wr.wr_cqe = &ioctx->ioctx.cqe;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000765 wr.next = NULL;
766 wr.sg_list = &list;
767 wr.num_sge = 1;
768
769 return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
770}
771
772/**
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -0800773 * srpt_zerolength_write() - Perform a zero-length RDMA write.
774 *
775 * A quote from the InfiniBand specification: C9-88: For an HCA responder
776 * using Reliable Connection service, for each zero-length RDMA READ or WRITE
777 * request, the R_Key shall not be validated, even if the request includes
778 * Immediate data.
779 */
780static int srpt_zerolength_write(struct srpt_rdma_ch *ch)
781{
782 struct ib_send_wr wr, *bad_wr;
783
784 memset(&wr, 0, sizeof(wr));
785 wr.opcode = IB_WR_RDMA_WRITE;
786 wr.wr_cqe = &ch->zw_cqe;
787 wr.send_flags = IB_SEND_SIGNALED;
788 return ib_post_send(ch->qp, &wr, &bad_wr);
789}
790
791static void srpt_zerolength_write_done(struct ib_cq *cq, struct ib_wc *wc)
792{
793 struct srpt_rdma_ch *ch = cq->cq_context;
794
Bart Van Assche387add42016-02-11 11:10:09 -0800795 if (wc->status == IB_WC_SUCCESS) {
796 srpt_process_wait_list(ch);
797 } else {
798 if (srpt_set_ch_state(ch, CH_DISCONNECTED))
799 schedule_work(&ch->release_work);
800 else
Dan Carpenter56586002016-03-18 08:41:59 +0300801 WARN_ONCE(1, "%s-%d\n", ch->sess_name, ch->qp->qp_num);
Bart Van Assche387add42016-02-11 11:10:09 -0800802 }
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -0800803}
804
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200805static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx,
806 struct srp_direct_buf *db, int nbufs, struct scatterlist **sg,
807 unsigned *sg_cnt)
808{
809 enum dma_data_direction dir = target_reverse_dma_direction(&ioctx->cmd);
810 struct srpt_rdma_ch *ch = ioctx->ch;
811 struct scatterlist *prev = NULL;
812 unsigned prev_nents;
813 int ret, i;
814
815 if (nbufs == 1) {
816 ioctx->rw_ctxs = &ioctx->s_rw_ctx;
817 } else {
818 ioctx->rw_ctxs = kmalloc_array(nbufs, sizeof(*ioctx->rw_ctxs),
819 GFP_KERNEL);
820 if (!ioctx->rw_ctxs)
821 return -ENOMEM;
822 }
823
824 for (i = ioctx->n_rw_ctx; i < nbufs; i++, db++) {
825 struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
826 u64 remote_addr = be64_to_cpu(db->va);
827 u32 size = be32_to_cpu(db->len);
828 u32 rkey = be32_to_cpu(db->key);
829
830 ret = target_alloc_sgl(&ctx->sg, &ctx->nents, size, false,
831 i < nbufs - 1);
832 if (ret)
833 goto unwind;
834
835 ret = rdma_rw_ctx_init(&ctx->rw, ch->qp, ch->sport->port,
836 ctx->sg, ctx->nents, 0, remote_addr, rkey, dir);
837 if (ret < 0) {
838 target_free_sgl(ctx->sg, ctx->nents);
839 goto unwind;
840 }
841
842 ioctx->n_rdma += ret;
843 ioctx->n_rw_ctx++;
844
845 if (prev) {
846 sg_unmark_end(&prev[prev_nents - 1]);
847 sg_chain(prev, prev_nents + 1, ctx->sg);
848 } else {
849 *sg = ctx->sg;
850 }
851
852 prev = ctx->sg;
853 prev_nents = ctx->nents;
854
855 *sg_cnt += ctx->nents;
856 }
857
858 return 0;
859
860unwind:
861 while (--i >= 0) {
862 struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
863
864 rdma_rw_ctx_destroy(&ctx->rw, ch->qp, ch->sport->port,
865 ctx->sg, ctx->nents, dir);
866 target_free_sgl(ctx->sg, ctx->nents);
867 }
868 if (ioctx->rw_ctxs != &ioctx->s_rw_ctx)
869 kfree(ioctx->rw_ctxs);
870 return ret;
871}
872
873static void srpt_free_rw_ctxs(struct srpt_rdma_ch *ch,
874 struct srpt_send_ioctx *ioctx)
875{
876 enum dma_data_direction dir = target_reverse_dma_direction(&ioctx->cmd);
877 int i;
878
879 for (i = 0; i < ioctx->n_rw_ctx; i++) {
880 struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
881
882 rdma_rw_ctx_destroy(&ctx->rw, ch->qp, ch->sport->port,
883 ctx->sg, ctx->nents, dir);
884 target_free_sgl(ctx->sg, ctx->nents);
885 }
886
887 if (ioctx->rw_ctxs != &ioctx->s_rw_ctx)
888 kfree(ioctx->rw_ctxs);
889}
890
891static inline void *srpt_get_desc_buf(struct srp_cmd *srp_cmd)
892{
893 /*
894 * The pointer computations below will only be compiled correctly
895 * if srp_cmd::add_data is declared as s8*, u8*, s8[] or u8[], so check
896 * whether srp_cmd::add_data has been declared as a byte pointer.
897 */
898 BUILD_BUG_ON(!__same_type(srp_cmd->add_data[0], (s8)0) &&
899 !__same_type(srp_cmd->add_data[0], (u8)0));
900
901 /*
902 * According to the SRP spec, the lower two bits of the 'ADDITIONAL
903 * CDB LENGTH' field are reserved and the size in bytes of this field
904 * is four times the value specified in bits 3..7. Hence the "& ~3".
905 */
906 return srp_cmd->add_data + (srp_cmd->add_cdb_len & ~3);
907}
908
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -0800909/**
Bart Van Asschea42d9852011-10-14 01:30:46 +0000910 * srpt_get_desc_tbl() - Parse the data descriptors of an SRP_CMD request.
911 * @ioctx: Pointer to the I/O context associated with the request.
912 * @srp_cmd: Pointer to the SRP_CMD request data.
913 * @dir: Pointer to the variable to which the transfer direction will be
914 * written.
915 * @data_len: Pointer to the variable to which the total data length of all
916 * descriptors in the SRP_CMD request will be written.
917 *
918 * This function initializes ioctx->nrbuf and ioctx->r_bufs.
919 *
920 * Returns -EINVAL when the SRP_CMD request contains inconsistent descriptors;
921 * -ENOMEM when memory allocation fails and zero upon success.
922 */
923static int srpt_get_desc_tbl(struct srpt_send_ioctx *ioctx,
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200924 struct srp_cmd *srp_cmd, enum dma_data_direction *dir,
925 struct scatterlist **sg, unsigned *sg_cnt, u64 *data_len)
Bart Van Asschea42d9852011-10-14 01:30:46 +0000926{
Bart Van Asschea42d9852011-10-14 01:30:46 +0000927 BUG_ON(!dir);
928 BUG_ON(!data_len);
929
Bart Van Asschea42d9852011-10-14 01:30:46 +0000930 /*
931 * The lower four bits of the buffer format field contain the DATA-IN
932 * buffer descriptor format, and the highest four bits contain the
933 * DATA-OUT buffer descriptor format.
934 */
Bart Van Asschea42d9852011-10-14 01:30:46 +0000935 if (srp_cmd->buf_fmt & 0xf)
936 /* DATA-IN: transfer data from target to initiator (read). */
937 *dir = DMA_FROM_DEVICE;
938 else if (srp_cmd->buf_fmt >> 4)
939 /* DATA-OUT: transfer data from initiator to target (write). */
940 *dir = DMA_TO_DEVICE;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200941 else
942 *dir = DMA_NONE;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000943
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200944 /* initialize data_direction early as srpt_alloc_rw_ctxs needs it */
945 ioctx->cmd.data_direction = *dir;
946
Bart Van Asschea42d9852011-10-14 01:30:46 +0000947 if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
948 ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200949 struct srp_direct_buf *db = srpt_get_desc_buf(srp_cmd);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000950
Bart Van Asschea42d9852011-10-14 01:30:46 +0000951 *data_len = be32_to_cpu(db->len);
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200952 return srpt_alloc_rw_ctxs(ioctx, db, 1, sg, sg_cnt);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000953 } else if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_INDIRECT) ||
954 ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_INDIRECT)) {
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200955 struct srp_indirect_buf *idb = srpt_get_desc_buf(srp_cmd);
956 int nbufs = be32_to_cpu(idb->table_desc.len) /
957 sizeof(struct srp_direct_buf);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000958
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200959 if (nbufs >
Bart Van Asschea42d9852011-10-14 01:30:46 +0000960 (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -0400961 pr_err("received unsupported SRP_CMD request"
Bart Van Asschea42d9852011-10-14 01:30:46 +0000962 " type (%u out + %u in != %u / %zu)\n",
963 srp_cmd->data_out_desc_cnt,
964 srp_cmd->data_in_desc_cnt,
965 be32_to_cpu(idb->table_desc.len),
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200966 sizeof(struct srp_direct_buf));
967 return -EINVAL;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000968 }
969
Bart Van Asschea42d9852011-10-14 01:30:46 +0000970 *data_len = be32_to_cpu(idb->len);
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200971 return srpt_alloc_rw_ctxs(ioctx, idb->desc_list, nbufs,
972 sg, sg_cnt);
973 } else {
974 *data_len = 0;
975 return 0;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000976 }
Bart Van Asschea42d9852011-10-14 01:30:46 +0000977}
978
979/**
980 * srpt_init_ch_qp() - Initialize queue pair attributes.
981 *
982 * Initialized the attributes of queue pair 'qp' by allowing local write,
983 * remote read and remote write. Also transitions 'qp' to state IB_QPS_INIT.
984 */
985static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
986{
987 struct ib_qp_attr *attr;
988 int ret;
989
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -0800990 attr = kzalloc(sizeof(*attr), GFP_KERNEL);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000991 if (!attr)
992 return -ENOMEM;
993
994 attr->qp_state = IB_QPS_INIT;
Bart Van Assche30191712018-01-03 13:39:15 -0800995 attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000996 attr->port_num = ch->sport->port;
997 attr->pkey_index = 0;
998
999 ret = ib_modify_qp(qp, attr,
1000 IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
1001 IB_QP_PKEY_INDEX);
1002
1003 kfree(attr);
1004 return ret;
1005}
1006
1007/**
1008 * srpt_ch_qp_rtr() - Change the state of a channel to 'ready to receive' (RTR).
1009 * @ch: channel of the queue pair.
1010 * @qp: queue pair to change the state of.
1011 *
1012 * Returns zero upon success and a negative value upon failure.
1013 *
1014 * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
1015 * If this structure ever becomes larger, it might be necessary to allocate
1016 * it dynamically instead of on the stack.
1017 */
1018static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1019{
1020 struct ib_qp_attr qp_attr;
1021 int attr_mask;
1022 int ret;
1023
1024 qp_attr.qp_state = IB_QPS_RTR;
1025 ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
1026 if (ret)
1027 goto out;
1028
1029 qp_attr.max_dest_rd_atomic = 4;
1030
1031 ret = ib_modify_qp(qp, &qp_attr, attr_mask);
1032
1033out:
1034 return ret;
1035}
1036
1037/**
1038 * srpt_ch_qp_rts() - Change the state of a channel to 'ready to send' (RTS).
1039 * @ch: channel of the queue pair.
1040 * @qp: queue pair to change the state of.
1041 *
1042 * Returns zero upon success and a negative value upon failure.
1043 *
1044 * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
1045 * If this structure ever becomes larger, it might be necessary to allocate
1046 * it dynamically instead of on the stack.
1047 */
1048static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1049{
1050 struct ib_qp_attr qp_attr;
1051 int attr_mask;
1052 int ret;
1053
1054 qp_attr.qp_state = IB_QPS_RTS;
1055 ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
1056 if (ret)
1057 goto out;
1058
1059 qp_attr.max_rd_atomic = 4;
1060
1061 ret = ib_modify_qp(qp, &qp_attr, attr_mask);
1062
1063out:
1064 return ret;
1065}
1066
1067/**
1068 * srpt_ch_qp_err() - Set the channel queue pair state to 'error'.
1069 */
1070static int srpt_ch_qp_err(struct srpt_rdma_ch *ch)
1071{
1072 struct ib_qp_attr qp_attr;
1073
1074 qp_attr.qp_state = IB_QPS_ERR;
1075 return ib_modify_qp(ch->qp, &qp_attr, IB_QP_STATE);
1076}
1077
1078/**
Bart Van Asschea42d9852011-10-14 01:30:46 +00001079 * srpt_get_send_ioctx() - Obtain an I/O context for sending to the initiator.
1080 */
1081static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch)
1082{
1083 struct srpt_send_ioctx *ioctx;
Bart Van Assche3c968882016-04-07 15:55:04 -07001084 unsigned long flags;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001085
1086 BUG_ON(!ch);
1087
Bart Van Assche3c968882016-04-07 15:55:04 -07001088 ioctx = NULL;
1089 spin_lock_irqsave(&ch->spinlock, flags);
1090 if (!list_empty(&ch->free_list)) {
1091 ioctx = list_first_entry(&ch->free_list,
1092 struct srpt_send_ioctx, free_list);
1093 list_del(&ioctx->free_list);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001094 }
Bart Van Assche3c968882016-04-07 15:55:04 -07001095 spin_unlock_irqrestore(&ch->spinlock, flags);
1096
1097 if (!ioctx)
1098 return ioctx;
1099
1100 BUG_ON(ioctx->ch != ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001101 spin_lock_init(&ioctx->spinlock);
1102 ioctx->state = SRPT_STATE_NEW;
Bart Van Assche3c968882016-04-07 15:55:04 -07001103 ioctx->n_rdma = 0;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001104 ioctx->n_rw_ctx = 0;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001105 init_completion(&ioctx->tx_done);
Bart Van Assche3c968882016-04-07 15:55:04 -07001106 ioctx->queue_status_only = false;
1107 /*
1108 * transport_init_se_cmd() does not initialize all fields, so do it
1109 * here.
1110 */
1111 memset(&ioctx->cmd, 0, sizeof(ioctx->cmd));
1112 memset(&ioctx->sense_data, 0, sizeof(ioctx->sense_data));
Bart Van Asschea42d9852011-10-14 01:30:46 +00001113
1114 return ioctx;
1115}
1116
1117/**
Bart Van Asschea42d9852011-10-14 01:30:46 +00001118 * srpt_abort_cmd() - Abort a SCSI command.
1119 * @ioctx: I/O context associated with the SCSI command.
1120 * @context: Preferred execution context.
1121 */
1122static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx)
1123{
1124 enum srpt_command_state state;
1125 unsigned long flags;
1126
1127 BUG_ON(!ioctx);
1128
1129 /*
1130 * If the command is in a state where the target core is waiting for
Bart Van Assche49f40162016-02-11 11:07:11 -08001131 * the ib_srpt driver, change the state to the next state.
Bart Van Asschea42d9852011-10-14 01:30:46 +00001132 */
1133
1134 spin_lock_irqsave(&ioctx->spinlock, flags);
1135 state = ioctx->state;
1136 switch (state) {
1137 case SRPT_STATE_NEED_DATA:
1138 ioctx->state = SRPT_STATE_DATA_IN;
1139 break;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001140 case SRPT_STATE_CMD_RSP_SENT:
1141 case SRPT_STATE_MGMT_RSP_SENT:
1142 ioctx->state = SRPT_STATE_DONE;
1143 break;
1144 default:
Bart Van Assche49f40162016-02-11 11:07:11 -08001145 WARN_ONCE(true, "%s: unexpected I/O context state %d\n",
1146 __func__, state);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001147 break;
1148 }
1149 spin_unlock_irqrestore(&ioctx->spinlock, flags);
1150
Bart Van Asschea42d9852011-10-14 01:30:46 +00001151 pr_debug("Aborting cmd with state %d and tag %lld\n", state,
Bart Van Assche649ee052015-04-14 13:26:44 +02001152 ioctx->cmd.tag);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001153
1154 switch (state) {
1155 case SRPT_STATE_NEW:
1156 case SRPT_STATE_DATA_IN:
1157 case SRPT_STATE_MGMT:
Bart Van Assche49f40162016-02-11 11:07:11 -08001158 case SRPT_STATE_DONE:
Bart Van Asschea42d9852011-10-14 01:30:46 +00001159 /*
1160 * Do nothing - defer abort processing until
1161 * srpt_queue_response() is invoked.
1162 */
Bart Van Asschea42d9852011-10-14 01:30:46 +00001163 break;
1164 case SRPT_STATE_NEED_DATA:
Bart Van Assche49f40162016-02-11 11:07:11 -08001165 pr_debug("tag %#llx: RDMA read error\n", ioctx->cmd.tag);
1166 transport_generic_request_failure(&ioctx->cmd,
1167 TCM_CHECK_CONDITION_ABORT_CMD);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001168 break;
1169 case SRPT_STATE_CMD_RSP_SENT:
1170 /*
1171 * SRP_RSP sending failed or the SRP_RSP send completion has
1172 * not been received in time.
1173 */
Bart Van Assche49f40162016-02-11 11:07:11 -08001174 transport_generic_free_cmd(&ioctx->cmd, 0);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001175 break;
1176 case SRPT_STATE_MGMT_RSP_SENT:
Bart Van Assche49f40162016-02-11 11:07:11 -08001177 transport_generic_free_cmd(&ioctx->cmd, 0);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001178 break;
1179 default:
Grant Grundler532ec6f2013-03-26 21:48:28 +00001180 WARN(1, "Unexpected command state (%d)", state);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001181 break;
1182 }
1183
Bart Van Asschea42d9852011-10-14 01:30:46 +00001184 return state;
1185}
1186
1187/**
Christoph Hellwige672a472012-07-08 15:58:43 -04001188 * XXX: what is now target_execute_cmd used to be asynchronous, and unmapping
1189 * the data that has been transferred via IB RDMA had to be postponed until the
Masanari Iida142ad5d2012-08-10 00:07:58 +00001190 * check_stop_free() callback. None of this is necessary anymore and needs to
Christoph Hellwige672a472012-07-08 15:58:43 -04001191 * be cleaned up.
Bart Van Asschea42d9852011-10-14 01:30:46 +00001192 */
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001193static void srpt_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001194{
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001195 struct srpt_rdma_ch *ch = cq->cq_context;
1196 struct srpt_send_ioctx *ioctx =
Bart Van Assche19f57292015-12-31 09:56:43 +01001197 container_of(wc->wr_cqe, struct srpt_send_ioctx, rdma_cqe);
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001198
Bart Van Asschea42d9852011-10-14 01:30:46 +00001199 WARN_ON(ioctx->n_rdma <= 0);
1200 atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001201 ioctx->n_rdma = 0;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001202
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001203 if (unlikely(wc->status != IB_WC_SUCCESS)) {
1204 pr_info("RDMA_READ for ioctx 0x%p failed with status %d\n",
1205 ioctx, wc->status);
1206 srpt_abort_cmd(ioctx);
1207 return;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001208 }
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001209
1210 if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA,
1211 SRPT_STATE_DATA_IN))
1212 target_execute_cmd(&ioctx->cmd);
1213 else
1214 pr_err("%s[%d]: wrong state = %d\n", __func__,
1215 __LINE__, srpt_get_cmd_state(ioctx));
Bart Van Asschea42d9852011-10-14 01:30:46 +00001216}
1217
Bart Van Asschea42d9852011-10-14 01:30:46 +00001218/**
1219 * srpt_build_cmd_rsp() - Build an SRP_RSP response.
1220 * @ch: RDMA channel through which the request has been received.
1221 * @ioctx: I/O context associated with the SRP_CMD request. The response will
1222 * be built in the buffer ioctx->buf points at and hence this function will
1223 * overwrite the request data.
1224 * @tag: tag of the request for which this response is being generated.
1225 * @status: value for the STATUS field of the SRP_RSP information unit.
1226 *
1227 * Returns the size in bytes of the SRP_RSP response.
1228 *
1229 * An SRP_RSP response contains a SCSI status or service response. See also
1230 * section 6.9 in the SRP r16a document for the format of an SRP_RSP
1231 * response. See also SPC-2 for more information about sense data.
1232 */
1233static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1234 struct srpt_send_ioctx *ioctx, u64 tag,
1235 int status)
1236{
1237 struct srp_rsp *srp_rsp;
1238 const u8 *sense_data;
1239 int sense_data_len, max_sense_len;
1240
1241 /*
1242 * The lowest bit of all SAM-3 status codes is zero (see also
1243 * paragraph 5.3 in SAM-3).
1244 */
1245 WARN_ON(status & 1);
1246
1247 srp_rsp = ioctx->ioctx.buf;
1248 BUG_ON(!srp_rsp);
1249
1250 sense_data = ioctx->sense_data;
1251 sense_data_len = ioctx->cmd.scsi_sense_length;
1252 WARN_ON(sense_data_len > sizeof(ioctx->sense_data));
1253
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08001254 memset(srp_rsp, 0, sizeof(*srp_rsp));
Bart Van Asschea42d9852011-10-14 01:30:46 +00001255 srp_rsp->opcode = SRP_RSP;
1256 srp_rsp->req_lim_delta =
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301257 cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0));
Bart Van Asschea42d9852011-10-14 01:30:46 +00001258 srp_rsp->tag = tag;
1259 srp_rsp->status = status;
1260
1261 if (sense_data_len) {
1262 BUILD_BUG_ON(MIN_MAX_RSP_SIZE <= sizeof(*srp_rsp));
1263 max_sense_len = ch->max_ti_iu_len - sizeof(*srp_rsp);
1264 if (sense_data_len > max_sense_len) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001265 pr_warn("truncated sense data from %d to %d"
1266 " bytes\n", sense_data_len, max_sense_len);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001267 sense_data_len = max_sense_len;
1268 }
1269
1270 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1271 srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1272 memcpy(srp_rsp + 1, sense_data, sense_data_len);
1273 }
1274
1275 return sizeof(*srp_rsp) + sense_data_len;
1276}
1277
1278/**
1279 * srpt_build_tskmgmt_rsp() - Build a task management response.
1280 * @ch: RDMA channel through which the request has been received.
1281 * @ioctx: I/O context in which the SRP_RSP response will be built.
1282 * @rsp_code: RSP_CODE that will be stored in the response.
1283 * @tag: Tag of the request for which this response is being generated.
1284 *
1285 * Returns the size in bytes of the SRP_RSP response.
1286 *
1287 * An SRP_RSP response contains a SCSI status or service response. See also
1288 * section 6.9 in the SRP r16a document for the format of an SRP_RSP
1289 * response.
1290 */
1291static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1292 struct srpt_send_ioctx *ioctx,
1293 u8 rsp_code, u64 tag)
1294{
1295 struct srp_rsp *srp_rsp;
1296 int resp_data_len;
1297 int resp_len;
1298
Jack Wangc807f642013-09-30 10:09:05 +02001299 resp_data_len = 4;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001300 resp_len = sizeof(*srp_rsp) + resp_data_len;
1301
1302 srp_rsp = ioctx->ioctx.buf;
1303 BUG_ON(!srp_rsp);
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08001304 memset(srp_rsp, 0, sizeof(*srp_rsp));
Bart Van Asschea42d9852011-10-14 01:30:46 +00001305
1306 srp_rsp->opcode = SRP_RSP;
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301307 srp_rsp->req_lim_delta =
1308 cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0));
Bart Van Asschea42d9852011-10-14 01:30:46 +00001309 srp_rsp->tag = tag;
1310
Jack Wangc807f642013-09-30 10:09:05 +02001311 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1312 srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1313 srp_rsp->data[3] = rsp_code;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001314
1315 return resp_len;
1316}
1317
Bart Van Asschea42d9852011-10-14 01:30:46 +00001318static int srpt_check_stop_free(struct se_cmd *cmd)
1319{
Nicholas Bellinger9474b042012-11-27 23:55:57 -08001320 struct srpt_send_ioctx *ioctx = container_of(cmd,
1321 struct srpt_send_ioctx, cmd);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001322
Bart Van Asscheafc16602015-04-27 13:52:36 +02001323 return target_put_sess_cmd(&ioctx->cmd);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001324}
1325
1326/**
1327 * srpt_handle_cmd() - Process SRP_CMD.
1328 */
Bart Van Assche2c7f37f2016-02-11 11:06:55 -08001329static void srpt_handle_cmd(struct srpt_rdma_ch *ch,
1330 struct srpt_recv_ioctx *recv_ioctx,
1331 struct srpt_send_ioctx *send_ioctx)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001332{
1333 struct se_cmd *cmd;
1334 struct srp_cmd *srp_cmd;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001335 struct scatterlist *sg = NULL;
1336 unsigned sg_cnt = 0;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001337 u64 data_len;
1338 enum dma_data_direction dir;
Nicholas Bellinger9474b042012-11-27 23:55:57 -08001339 int rc;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001340
1341 BUG_ON(!send_ioctx);
1342
1343 srp_cmd = recv_ioctx->ioctx.buf;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001344 cmd = &send_ioctx->cmd;
Bart Van Assche649ee052015-04-14 13:26:44 +02001345 cmd->tag = srp_cmd->tag;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001346
1347 switch (srp_cmd->task_attr) {
1348 case SRP_CMD_SIMPLE_Q:
Christoph Hellwig68d81f42014-11-24 07:07:25 -08001349 cmd->sam_task_attr = TCM_SIMPLE_TAG;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001350 break;
1351 case SRP_CMD_ORDERED_Q:
1352 default:
Christoph Hellwig68d81f42014-11-24 07:07:25 -08001353 cmd->sam_task_attr = TCM_ORDERED_TAG;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001354 break;
1355 case SRP_CMD_HEAD_OF_Q:
Christoph Hellwig68d81f42014-11-24 07:07:25 -08001356 cmd->sam_task_attr = TCM_HEAD_TAG;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001357 break;
1358 case SRP_CMD_ACA:
Christoph Hellwig68d81f42014-11-24 07:07:25 -08001359 cmd->sam_task_attr = TCM_ACA_TAG;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001360 break;
1361 }
1362
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001363 rc = srpt_get_desc_tbl(send_ioctx, srp_cmd, &dir, &sg, &sg_cnt,
1364 &data_len);
1365 if (rc) {
1366 if (rc != -EAGAIN) {
1367 pr_err("0x%llx: parsing SRP descriptor table failed.\n",
1368 srp_cmd->tag);
1369 }
Bart Van Assche2c7f37f2016-02-11 11:06:55 -08001370 goto release_ioctx;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001371 }
1372
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001373 rc = target_submit_cmd_map_sgls(cmd, ch->sess, srp_cmd->cdb,
Bart Van Asschee1dd4132016-02-11 11:05:19 -08001374 &send_ioctx->sense_data[0],
1375 scsilun_to_int(&srp_cmd->lun), data_len,
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001376 TCM_SIMPLE_TAG, dir, TARGET_SCF_ACK_KREF,
1377 sg, sg_cnt, NULL, 0, NULL, 0);
Nicholas Bellinger9474b042012-11-27 23:55:57 -08001378 if (rc != 0) {
Bart Van Assche2c7f37f2016-02-11 11:06:55 -08001379 pr_debug("target_submit_cmd() returned %d for tag %#llx\n", rc,
1380 srp_cmd->tag);
1381 goto release_ioctx;
Nicholas Bellinger187e70a2012-03-17 20:12:36 -07001382 }
Bart Van Assche2c7f37f2016-02-11 11:06:55 -08001383 return;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001384
Bart Van Assche2c7f37f2016-02-11 11:06:55 -08001385release_ioctx:
1386 send_ioctx->state = SRPT_STATE_DONE;
1387 srpt_release_cmd(cmd);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001388}
1389
Bart Van Asschea42d9852011-10-14 01:30:46 +00001390static int srp_tmr_to_tcm(int fn)
1391{
1392 switch (fn) {
1393 case SRP_TSK_ABORT_TASK:
1394 return TMR_ABORT_TASK;
1395 case SRP_TSK_ABORT_TASK_SET:
1396 return TMR_ABORT_TASK_SET;
1397 case SRP_TSK_CLEAR_TASK_SET:
1398 return TMR_CLEAR_TASK_SET;
1399 case SRP_TSK_LUN_RESET:
1400 return TMR_LUN_RESET;
1401 case SRP_TSK_CLEAR_ACA:
1402 return TMR_CLEAR_ACA;
1403 default:
1404 return -1;
1405 }
1406}
1407
1408/**
1409 * srpt_handle_tsk_mgmt() - Process an SRP_TSK_MGMT information unit.
1410 *
1411 * Returns 0 if and only if the request will be processed by the target core.
1412 *
1413 * For more information about SRP_TSK_MGMT information units, see also section
1414 * 6.7 in the SRP r16a document.
1415 */
1416static void srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1417 struct srpt_recv_ioctx *recv_ioctx,
1418 struct srpt_send_ioctx *send_ioctx)
1419{
1420 struct srp_tsk_mgmt *srp_tsk;
1421 struct se_cmd *cmd;
Nicholas Bellinger3e4f5742012-11-28 01:38:04 -08001422 struct se_session *sess = ch->sess;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001423 int tcm_tmr;
Nicholas Bellinger3e4f5742012-11-28 01:38:04 -08001424 int rc;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001425
1426 BUG_ON(!send_ioctx);
1427
1428 srp_tsk = recv_ioctx->ioctx.buf;
1429 cmd = &send_ioctx->cmd;
1430
1431 pr_debug("recv tsk_mgmt fn %d for task_tag %lld and cmd tag %lld"
1432 " cm_id %p sess %p\n", srp_tsk->tsk_mgmt_func,
1433 srp_tsk->task_tag, srp_tsk->tag, ch->cm_id, ch->sess);
1434
1435 srpt_set_cmd_state(send_ioctx, SRPT_STATE_MGMT);
Bart Van Assche649ee052015-04-14 13:26:44 +02001436 send_ioctx->cmd.tag = srp_tsk->tag;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001437 tcm_tmr = srp_tmr_to_tcm(srp_tsk->tsk_mgmt_func);
Bart Van Asschee1dd4132016-02-11 11:05:19 -08001438 rc = target_submit_tmr(&send_ioctx->cmd, sess, NULL,
1439 scsilun_to_int(&srp_tsk->lun), srp_tsk, tcm_tmr,
1440 GFP_KERNEL, srp_tsk->task_tag,
1441 TARGET_SCF_ACK_KREF);
Nicholas Bellinger3e4f5742012-11-28 01:38:04 -08001442 if (rc != 0) {
Bart Van Asschea42d9852011-10-14 01:30:46 +00001443 send_ioctx->cmd.se_tmr_req->response = TMR_FUNCTION_REJECTED;
Christoph Hellwigde103c92012-11-06 12:24:09 -08001444 goto fail;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001445 }
Christoph Hellwigde103c92012-11-06 12:24:09 -08001446 return;
1447fail:
Christoph Hellwigde103c92012-11-06 12:24:09 -08001448 transport_send_check_condition_and_sense(cmd, 0, 0); // XXX:
Bart Van Asschea42d9852011-10-14 01:30:46 +00001449}
1450
1451/**
1452 * srpt_handle_new_iu() - Process a newly received information unit.
1453 * @ch: RDMA channel through which the information unit has been received.
1454 * @ioctx: SRPT I/O context associated with the information unit.
1455 */
1456static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1457 struct srpt_recv_ioctx *recv_ioctx,
1458 struct srpt_send_ioctx *send_ioctx)
1459{
1460 struct srp_cmd *srp_cmd;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001461
1462 BUG_ON(!ch);
1463 BUG_ON(!recv_ioctx);
1464
1465 ib_dma_sync_single_for_cpu(ch->sport->sdev->device,
1466 recv_ioctx->ioctx.dma, srp_max_req_size,
1467 DMA_FROM_DEVICE);
1468
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001469 if (unlikely(ch->state == CH_CONNECTING))
1470 goto out_wait;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001471
Bart Van Assche33912d72016-02-11 11:04:43 -08001472 if (unlikely(ch->state != CH_LIVE))
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001473 return;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001474
1475 srp_cmd = recv_ioctx->ioctx.buf;
1476 if (srp_cmd->opcode == SRP_CMD || srp_cmd->opcode == SRP_TSK_MGMT) {
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001477 if (!send_ioctx) {
1478 if (!list_empty(&ch->cmd_wait_list))
1479 goto out_wait;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001480 send_ioctx = srpt_get_send_ioctx(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001481 }
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001482 if (unlikely(!send_ioctx))
1483 goto out_wait;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001484 }
1485
Bart Van Asschea42d9852011-10-14 01:30:46 +00001486 switch (srp_cmd->opcode) {
1487 case SRP_CMD:
1488 srpt_handle_cmd(ch, recv_ioctx, send_ioctx);
1489 break;
1490 case SRP_TSK_MGMT:
1491 srpt_handle_tsk_mgmt(ch, recv_ioctx, send_ioctx);
1492 break;
1493 case SRP_I_LOGOUT:
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001494 pr_err("Not yet implemented: SRP_I_LOGOUT\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +00001495 break;
1496 case SRP_CRED_RSP:
1497 pr_debug("received SRP_CRED_RSP\n");
1498 break;
1499 case SRP_AER_RSP:
1500 pr_debug("received SRP_AER_RSP\n");
1501 break;
1502 case SRP_RSP:
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001503 pr_err("Received SRP_RSP\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +00001504 break;
1505 default:
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001506 pr_err("received IU with unknown opcode 0x%x\n",
Bart Van Asschea42d9852011-10-14 01:30:46 +00001507 srp_cmd->opcode);
1508 break;
1509 }
1510
1511 srpt_post_recv(ch->sport->sdev, recv_ioctx);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001512 return;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001513
1514out_wait:
1515 list_add_tail(&recv_ioctx->wait_list, &ch->cmd_wait_list);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001516}
1517
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001518static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001519{
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001520 struct srpt_rdma_ch *ch = cq->cq_context;
1521 struct srpt_recv_ioctx *ioctx =
1522 container_of(wc->wr_cqe, struct srpt_recv_ioctx, ioctx.cqe);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001523
Bart Van Asschea42d9852011-10-14 01:30:46 +00001524 if (wc->status == IB_WC_SUCCESS) {
1525 int req_lim;
1526
1527 req_lim = atomic_dec_return(&ch->req_lim);
1528 if (unlikely(req_lim < 0))
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001529 pr_err("req_lim = %d < 0\n", req_lim);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001530 srpt_handle_new_iu(ch, ioctx, NULL);
1531 } else {
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001532 pr_info("receiving failed for ioctx %p with status %d\n",
1533 ioctx, wc->status);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001534 }
1535}
1536
Bart Van Assche539b3242016-02-11 11:09:50 -08001537/*
1538 * This function must be called from the context in which RDMA completions are
1539 * processed because it accesses the wait list without protection against
1540 * access from other threads.
1541 */
1542static void srpt_process_wait_list(struct srpt_rdma_ch *ch)
1543{
1544 struct srpt_send_ioctx *ioctx;
1545
1546 while (!list_empty(&ch->cmd_wait_list) &&
1547 ch->state >= CH_LIVE &&
1548 (ioctx = srpt_get_send_ioctx(ch)) != NULL) {
1549 struct srpt_recv_ioctx *recv_ioctx;
1550
1551 recv_ioctx = list_first_entry(&ch->cmd_wait_list,
1552 struct srpt_recv_ioctx,
1553 wait_list);
1554 list_del(&recv_ioctx->wait_list);
1555 srpt_handle_new_iu(ch, recv_ioctx, ioctx);
1556 }
1557}
1558
Bart Van Asschea42d9852011-10-14 01:30:46 +00001559/**
Bart Van Asschea42d9852011-10-14 01:30:46 +00001560 * Note: Although this has not yet been observed during tests, at least in
1561 * theory it is possible that the srpt_get_send_ioctx() call invoked by
1562 * srpt_handle_new_iu() fails. This is possible because the req_lim_delta
1563 * value in each response is set to one, and it is possible that this response
1564 * makes the initiator send a new request before the send completion for that
1565 * response has been processed. This could e.g. happen if the call to
1566 * srpt_put_send_iotcx() is delayed because of a higher priority interrupt or
1567 * if IB retransmission causes generation of the send completion to be
1568 * delayed. Incoming information units for which srpt_get_send_ioctx() fails
1569 * are queued on cmd_wait_list. The code below processes these delayed
1570 * requests one at a time.
1571 */
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001572static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001573{
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001574 struct srpt_rdma_ch *ch = cq->cq_context;
1575 struct srpt_send_ioctx *ioctx =
1576 container_of(wc->wr_cqe, struct srpt_send_ioctx, ioctx.cqe);
1577 enum srpt_command_state state;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001578
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001579 state = srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
1580
1581 WARN_ON(state != SRPT_STATE_CMD_RSP_SENT &&
1582 state != SRPT_STATE_MGMT_RSP_SENT);
1583
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001584 atomic_add(1 + ioctx->n_rdma, &ch->sq_wr_avail);
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001585
Bart Van Assche49f40162016-02-11 11:07:11 -08001586 if (wc->status != IB_WC_SUCCESS)
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001587 pr_info("sending response for ioctx 0x%p failed"
1588 " with status %d\n", ioctx, wc->status);
1589
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001590 if (state != SRPT_STATE_DONE) {
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001591 transport_generic_free_cmd(&ioctx->cmd, 0);
1592 } else {
1593 pr_err("IB completion has been received too late for"
1594 " wr_id = %u.\n", ioctx->ioctx.index);
1595 }
1596
Bart Van Assche539b3242016-02-11 11:09:50 -08001597 srpt_process_wait_list(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001598}
1599
Bart Van Asschea42d9852011-10-14 01:30:46 +00001600/**
1601 * srpt_create_ch_ib() - Create receive and send completion queues.
1602 */
1603static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1604{
1605 struct ib_qp_init_attr *qp_init;
1606 struct srpt_port *sport = ch->sport;
1607 struct srpt_device *sdev = sport->sdev;
Bart Van Assche30c6d872016-07-21 13:03:47 -07001608 const struct ib_device_attr *attrs = &sdev->device->attrs;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001609 u32 srp_sq_size = sport->port_attrib.srp_sq_size;
1610 int ret;
1611
1612 WARN_ON(ch->rq_size < 1);
1613
1614 ret = -ENOMEM;
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08001615 qp_init = kzalloc(sizeof(*qp_init), GFP_KERNEL);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001616 if (!qp_init)
1617 goto out;
1618
Bart Van Asscheab477c12014-10-19 18:05:33 +03001619retry:
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001620 ch->cq = ib_alloc_cq(sdev->device, ch, ch->rq_size + srp_sq_size,
1621 0 /* XXX: spread CQs */, IB_POLL_WORKQUEUE);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001622 if (IS_ERR(ch->cq)) {
1623 ret = PTR_ERR(ch->cq);
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001624 pr_err("failed to create CQ cqe= %d ret= %d\n",
Bart Van Asschea42d9852011-10-14 01:30:46 +00001625 ch->rq_size + srp_sq_size, ret);
1626 goto out;
1627 }
1628
1629 qp_init->qp_context = (void *)ch;
1630 qp_init->event_handler
1631 = (void(*)(struct ib_event *, void*))srpt_qp_event;
1632 qp_init->send_cq = ch->cq;
1633 qp_init->recv_cq = ch->cq;
1634 qp_init->srq = sdev->srq;
1635 qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1636 qp_init->qp_type = IB_QPT_RC;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001637 /*
1638 * We divide up our send queue size into half SEND WRs to send the
1639 * completions, and half R/W contexts to actually do the RDMA
1640 * READ/WRITE transfers. Note that we need to allocate CQ slots for
1641 * both both, as RDMA contexts will also post completions for the
1642 * RDMA READ case.
1643 */
1644 qp_init->cap.max_send_wr = srp_sq_size / 2;
1645 qp_init->cap.max_rdma_ctxs = srp_sq_size / 2;
Bart Van Assche30c6d872016-07-21 13:03:47 -07001646 qp_init->cap.max_send_sge = min(attrs->max_sge, SRPT_MAX_SG_PER_WQE);
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001647 qp_init->port_num = ch->sport->port;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001648
1649 ch->qp = ib_create_qp(sdev->pd, qp_init);
1650 if (IS_ERR(ch->qp)) {
1651 ret = PTR_ERR(ch->qp);
Bart Van Asscheab477c12014-10-19 18:05:33 +03001652 if (ret == -ENOMEM) {
1653 srp_sq_size /= 2;
1654 if (srp_sq_size >= MIN_SRPT_SQ_SIZE) {
1655 ib_destroy_cq(ch->cq);
1656 goto retry;
1657 }
1658 }
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001659 pr_err("failed to create_qp ret= %d\n", ret);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001660 goto err_destroy_cq;
1661 }
1662
1663 atomic_set(&ch->sq_wr_avail, qp_init->cap.max_send_wr);
1664
1665 pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n",
1666 __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1667 qp_init->cap.max_send_wr, ch->cm_id);
1668
1669 ret = srpt_init_ch_qp(ch, ch->qp);
1670 if (ret)
1671 goto err_destroy_qp;
1672
Bart Van Asschea42d9852011-10-14 01:30:46 +00001673out:
1674 kfree(qp_init);
1675 return ret;
1676
1677err_destroy_qp:
1678 ib_destroy_qp(ch->qp);
1679err_destroy_cq:
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001680 ib_free_cq(ch->cq);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001681 goto out;
1682}
1683
1684static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch)
1685{
Bart Van Asschea42d9852011-10-14 01:30:46 +00001686 ib_destroy_qp(ch->qp);
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001687 ib_free_cq(ch->cq);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001688}
1689
1690/**
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001691 * srpt_close_ch() - Close an RDMA channel.
Bart Van Asschea42d9852011-10-14 01:30:46 +00001692 *
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001693 * Make sure all resources associated with the channel will be deallocated at
1694 * an appropriate time.
Bart Van Asschea42d9852011-10-14 01:30:46 +00001695 *
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001696 * Returns true if and only if the channel state has been modified into
1697 * CH_DRAINING.
Bart Van Asschea42d9852011-10-14 01:30:46 +00001698 */
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001699static bool srpt_close_ch(struct srpt_rdma_ch *ch)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001700{
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001701 int ret;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001702
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001703 if (!srpt_set_ch_state(ch, CH_DRAINING)) {
Bart Van Assche2c3c2842018-07-02 14:08:18 -07001704 pr_debug("%s: already closed\n", ch->sess_name);
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001705 return false;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001706 }
Bart Van Asschea42d9852011-10-14 01:30:46 +00001707
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001708 kref_get(&ch->kref);
1709
1710 ret = srpt_ch_qp_err(ch);
1711 if (ret < 0)
1712 pr_err("%s-%d: changing queue pair into error state failed: %d\n",
1713 ch->sess_name, ch->qp->qp_num, ret);
1714
1715 pr_debug("%s-%d: queued zerolength write\n", ch->sess_name,
1716 ch->qp->qp_num);
1717 ret = srpt_zerolength_write(ch);
1718 if (ret < 0) {
1719 pr_err("%s-%d: queuing zero-length write failed: %d\n",
1720 ch->sess_name, ch->qp->qp_num, ret);
1721 if (srpt_set_ch_state(ch, CH_DISCONNECTED))
1722 schedule_work(&ch->release_work);
1723 else
1724 WARN_ON_ONCE(true);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001725 }
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001726
1727 kref_put(&ch->kref, srpt_free_ch);
1728
1729 return true;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001730}
1731
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001732/*
1733 * Change the channel state into CH_DISCONNECTING. If a channel has not yet
1734 * reached the connected state, close it. If a channel is in the connected
1735 * state, send a DREQ. If a DREQ has been received, send a DREP. Note: it is
1736 * the responsibility of the caller to ensure that this function is not
1737 * invoked concurrently with the code that accepts a connection. This means
1738 * that this function must either be invoked from inside a CM callback
1739 * function or that it must be invoked with the srpt_port.mutex held.
Bart Van Asschea42d9852011-10-14 01:30:46 +00001740 */
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001741static int srpt_disconnect_ch(struct srpt_rdma_ch *ch)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001742{
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001743 int ret;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001744
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001745 if (!srpt_set_ch_state(ch, CH_DISCONNECTING))
1746 return -ENOTCONN;
1747
1748 ret = ib_send_cm_dreq(ch->cm_id, NULL, 0);
1749 if (ret < 0)
1750 ret = ib_send_cm_drep(ch->cm_id, NULL, 0);
1751
1752 if (ret < 0 && srpt_close_ch(ch))
1753 ret = 0;
1754
1755 return ret;
1756}
1757
1758static void __srpt_close_all_ch(struct srpt_device *sdev)
1759{
1760 struct srpt_rdma_ch *ch;
1761
1762 lockdep_assert_held(&sdev->mutex);
1763
1764 list_for_each_entry(ch, &sdev->rch_list, list) {
1765 if (srpt_disconnect_ch(ch) >= 0)
Bart Van Assche6f590582018-07-02 14:08:45 -07001766 pr_info("Closing channel %s because target %s has been disabled\n",
1767 ch->sess_name,
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001768 sdev->device->name);
1769 srpt_close_ch(ch);
1770 }
Bart Van Asschea42d9852011-10-14 01:30:46 +00001771}
1772
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001773static void srpt_free_ch(struct kref *kref)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001774{
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001775 struct srpt_rdma_ch *ch = container_of(kref, struct srpt_rdma_ch, kref);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001776
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001777 kfree(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001778}
1779
1780static void srpt_release_channel_work(struct work_struct *w)
1781{
1782 struct srpt_rdma_ch *ch;
1783 struct srpt_device *sdev;
Nicholas Bellinger9474b042012-11-27 23:55:57 -08001784 struct se_session *se_sess;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001785
1786 ch = container_of(w, struct srpt_rdma_ch, release_work);
Bart Van Asschef108f0f2016-02-11 11:06:14 -08001787 pr_debug("%s: %s-%d; release_done = %p\n", __func__, ch->sess_name,
1788 ch->qp->qp_num, ch->release_done);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001789
1790 sdev = ch->sport->sdev;
1791 BUG_ON(!sdev);
1792
Nicholas Bellinger9474b042012-11-27 23:55:57 -08001793 se_sess = ch->sess;
1794 BUG_ON(!se_sess);
1795
Bart Van Assche88936252016-02-11 11:05:58 -08001796 target_sess_cmd_list_set_waiting(se_sess);
Joern Engelbe646c2d2013-05-15 00:44:07 -07001797 target_wait_for_sess_cmds(se_sess);
Nicholas Bellinger9474b042012-11-27 23:55:57 -08001798
1799 transport_deregister_session_configfs(se_sess);
1800 transport_deregister_session(se_sess);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001801 ch->sess = NULL;
1802
Nicholas Bellinger0b41d6c2013-09-18 12:48:27 -07001803 ib_destroy_cm_id(ch->cm_id);
1804
Bart Van Asschea42d9852011-10-14 01:30:46 +00001805 srpt_destroy_ch_ib(ch);
1806
1807 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
1808 ch->sport->sdev, ch->rq_size,
1809 ch->rsp_size, DMA_TO_DEVICE);
1810
Bart Van Assche86289912016-02-11 11:08:34 -08001811 mutex_lock(&sdev->mutex);
Bart Van Asschef108f0f2016-02-11 11:06:14 -08001812 list_del_init(&ch->list);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001813 if (ch->release_done)
1814 complete(ch->release_done);
Bart Van Assche86289912016-02-11 11:08:34 -08001815 mutex_unlock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001816
1817 wake_up(&sdev->ch_releaseQ);
1818
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001819 kref_put(&ch->kref, srpt_free_ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001820}
1821
Bart Van Asschea42d9852011-10-14 01:30:46 +00001822/**
1823 * srpt_cm_req_recv() - Process the event IB_CM_REQ_RECEIVED.
1824 *
1825 * Ownership of the cm_id is transferred to the target session if this
1826 * functions returns zero. Otherwise the caller remains the owner of cm_id.
1827 */
1828static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1829 struct ib_cm_req_event_param *param,
1830 void *private_data)
1831{
1832 struct srpt_device *sdev = cm_id->context;
1833 struct srpt_port *sport = &sdev->port[param->port - 1];
1834 struct srp_login_req *req;
1835 struct srp_login_rsp *rsp;
1836 struct srp_login_rej *rej;
1837 struct ib_cm_rep_param *rep_param;
1838 struct srpt_rdma_ch *ch, *tmp_ch;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001839 u32 it_iu_len;
Bart Van Assche3c968882016-04-07 15:55:04 -07001840 int i, ret = 0;
Nicholas Bellingerf246c942016-01-07 22:19:21 -08001841 unsigned char *p;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001842
1843 WARN_ON_ONCE(irqs_disabled());
1844
1845 if (WARN_ON(!sdev || !private_data))
1846 return -EINVAL;
1847
1848 req = (struct srp_login_req *)private_data;
1849
1850 it_iu_len = be32_to_cpu(req->req_it_iu_len);
1851
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001852 pr_info("Received SRP_LOGIN_REQ with i_port_id 0x%llx:0x%llx,"
1853 " t_port_id 0x%llx:0x%llx and it_iu_len %d on port %d"
1854 " (guid=0x%llx:0x%llx)\n",
1855 be64_to_cpu(*(__be64 *)&req->initiator_port_id[0]),
1856 be64_to_cpu(*(__be64 *)&req->initiator_port_id[8]),
1857 be64_to_cpu(*(__be64 *)&req->target_port_id[0]),
1858 be64_to_cpu(*(__be64 *)&req->target_port_id[8]),
1859 it_iu_len,
1860 param->port,
1861 be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[0]),
1862 be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[8]));
Bart Van Asschea42d9852011-10-14 01:30:46 +00001863
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08001864 rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
1865 rej = kzalloc(sizeof(*rej), GFP_KERNEL);
1866 rep_param = kzalloc(sizeof(*rep_param), GFP_KERNEL);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001867
1868 if (!rsp || !rej || !rep_param) {
1869 ret = -ENOMEM;
1870 goto out;
1871 }
1872
1873 if (it_iu_len > srp_max_req_size || it_iu_len < 64) {
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301874 rej->reason = cpu_to_be32(
1875 SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001876 ret = -EINVAL;
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001877 pr_err("rejected SRP_LOGIN_REQ because its"
Bart Van Asschea42d9852011-10-14 01:30:46 +00001878 " length (%d bytes) is out of range (%d .. %d)\n",
1879 it_iu_len, 64, srp_max_req_size);
1880 goto reject;
1881 }
1882
1883 if (!sport->enabled) {
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301884 rej->reason = cpu_to_be32(
1885 SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001886 ret = -EINVAL;
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001887 pr_err("rejected SRP_LOGIN_REQ because the target port"
Bart Van Asschea42d9852011-10-14 01:30:46 +00001888 " has not yet been enabled\n");
1889 goto reject;
1890 }
1891
1892 if ((req->req_flags & SRP_MTCH_ACTION) == SRP_MULTICHAN_SINGLE) {
1893 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1894
Bart Van Assche86289912016-02-11 11:08:34 -08001895 mutex_lock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001896
1897 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1898 if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1899 && !memcmp(ch->t_port_id, req->target_port_id, 16)
1900 && param->port == ch->sport->port
1901 && param->listen_id == ch->sport->sdev->cm_id
1902 && ch->cm_id) {
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001903 if (srpt_disconnect_ch(ch) < 0)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001904 continue;
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001905 pr_info("Relogin - closed existing channel %s\n",
1906 ch->sess_name);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001907 rsp->rsp_flags =
1908 SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1909 }
1910 }
1911
Bart Van Assche86289912016-02-11 11:08:34 -08001912 mutex_unlock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001913
1914 } else
1915 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1916
1917 if (*(__be64 *)req->target_port_id != cpu_to_be64(srpt_service_guid)
1918 || *(__be64 *)(req->target_port_id + 8) !=
1919 cpu_to_be64(srpt_service_guid)) {
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301920 rej->reason = cpu_to_be32(
1921 SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001922 ret = -ENOMEM;
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001923 pr_err("rejected SRP_LOGIN_REQ because it"
Bart Van Asschea42d9852011-10-14 01:30:46 +00001924 " has an invalid target port identifier.\n");
1925 goto reject;
1926 }
1927
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08001928 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001929 if (!ch) {
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301930 rej->reason = cpu_to_be32(
1931 SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001932 pr_err("rejected SRP_LOGIN_REQ because no memory.\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +00001933 ret = -ENOMEM;
1934 goto reject;
1935 }
1936
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001937 kref_init(&ch->kref);
1938 ch->zw_cqe.done = srpt_zerolength_write_done;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001939 INIT_WORK(&ch->release_work, srpt_release_channel_work);
1940 memcpy(ch->i_port_id, req->initiator_port_id, 16);
1941 memcpy(ch->t_port_id, req->target_port_id, 16);
1942 ch->sport = &sdev->port[param->port - 1];
1943 ch->cm_id = cm_id;
Bart Van Assche2739b592016-02-11 11:07:49 -08001944 cm_id->context = ch;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001945 /*
1946 * Avoid QUEUE_FULL conditions by limiting the number of buffers used
1947 * for the SRP protocol to the command queue size.
1948 */
1949 ch->rq_size = SRPT_RQ_SIZE;
1950 spin_lock_init(&ch->spinlock);
1951 ch->state = CH_CONNECTING;
1952 INIT_LIST_HEAD(&ch->cmd_wait_list);
1953 ch->rsp_size = ch->sport->port_attrib.srp_max_rsp_size;
1954
1955 ch->ioctx_ring = (struct srpt_send_ioctx **)
1956 srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size,
1957 sizeof(*ch->ioctx_ring[0]),
1958 ch->rsp_size, DMA_TO_DEVICE);
1959 if (!ch->ioctx_ring)
1960 goto free_ch;
1961
Bart Van Assche3c968882016-04-07 15:55:04 -07001962 INIT_LIST_HEAD(&ch->free_list);
1963 for (i = 0; i < ch->rq_size; i++) {
1964 ch->ioctx_ring[i]->ch = ch;
1965 list_add_tail(&ch->ioctx_ring[i]->free_list, &ch->free_list);
1966 }
1967
Bart Van Asschea42d9852011-10-14 01:30:46 +00001968 ret = srpt_create_ch_ib(ch);
1969 if (ret) {
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301970 rej->reason = cpu_to_be32(
1971 SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001972 pr_err("rejected SRP_LOGIN_REQ because creating"
Bart Van Asschea42d9852011-10-14 01:30:46 +00001973 " a new RDMA channel failed.\n");
1974 goto free_ring;
1975 }
1976
1977 ret = srpt_ch_qp_rtr(ch, ch->qp);
1978 if (ret) {
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301979 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001980 pr_err("rejected SRP_LOGIN_REQ because enabling"
Bart Van Asschea42d9852011-10-14 01:30:46 +00001981 " RTR failed (error code = %d)\n", ret);
1982 goto destroy_ib;
1983 }
Nicholas Bellingerf246c942016-01-07 22:19:21 -08001984
Bart Van Asschea42d9852011-10-14 01:30:46 +00001985 /*
Nicholas Bellingerf246c942016-01-07 22:19:21 -08001986 * Use the initator port identifier as the session name, when
1987 * checking against se_node_acl->initiatorname[] this can be
1988 * with or without preceeding '0x'.
Bart Van Asschea42d9852011-10-14 01:30:46 +00001989 */
1990 snprintf(ch->sess_name, sizeof(ch->sess_name), "0x%016llx%016llx",
1991 be64_to_cpu(*(__be64 *)ch->i_port_id),
1992 be64_to_cpu(*(__be64 *)(ch->i_port_id + 8)));
1993
1994 pr_debug("registering session %s\n", ch->sess_name);
Nicholas Bellingerf246c942016-01-07 22:19:21 -08001995 p = &ch->sess_name[0];
Bart Van Asschea42d9852011-10-14 01:30:46 +00001996
Nicholas Bellingerf246c942016-01-07 22:19:21 -08001997try_again:
Bart Van Assche3c968882016-04-07 15:55:04 -07001998 ch->sess = target_alloc_session(&sport->port_tpg_1, 0, 0,
Nicholas Bellingerb42057a2016-01-09 20:42:43 -08001999 TARGET_PROT_NORMAL, p, ch, NULL);
2000 if (IS_ERR(ch->sess)) {
Nicholas Bellingerf246c942016-01-07 22:19:21 -08002001 pr_info("Rejected login because no ACL has been"
Nicholas Bellingerb42057a2016-01-09 20:42:43 -08002002 " configured yet for initiator %s.\n", p);
Nicholas Bellingerf246c942016-01-07 22:19:21 -08002003 /*
2004 * XXX: Hack to retry of ch->i_port_id without leading '0x'
2005 */
2006 if (p == &ch->sess_name[0]) {
2007 p += 2;
2008 goto try_again;
2009 }
Nicholas Bellingerb42057a2016-01-09 20:42:43 -08002010 rej->reason = cpu_to_be32((PTR_ERR(ch->sess) == -ENOMEM) ?
2011 SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES :
Nicholas Bellingerf246c942016-01-07 22:19:21 -08002012 SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED);
Nicholas Bellingerf246c942016-01-07 22:19:21 -08002013 goto destroy_ib;
2014 }
Bart Van Asschea42d9852011-10-14 01:30:46 +00002015
2016 pr_debug("Establish connection sess=%p name=%s cm_id=%p\n", ch->sess,
2017 ch->sess_name, ch->cm_id);
2018
2019 /* create srp_login_response */
2020 rsp->opcode = SRP_LOGIN_RSP;
2021 rsp->tag = req->tag;
2022 rsp->max_it_iu_len = req->req_it_iu_len;
2023 rsp->max_ti_iu_len = req->req_it_iu_len;
2024 ch->max_ti_iu_len = it_iu_len;
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05302025 rsp->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT
2026 | SRP_BUF_FORMAT_INDIRECT);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002027 rsp->req_lim_delta = cpu_to_be32(ch->rq_size);
2028 atomic_set(&ch->req_lim, ch->rq_size);
2029 atomic_set(&ch->req_lim_delta, 0);
2030
2031 /* create cm reply */
2032 rep_param->qp_num = ch->qp->qp_num;
2033 rep_param->private_data = (void *)rsp;
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08002034 rep_param->private_data_len = sizeof(*rsp);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002035 rep_param->rnr_retry_count = 7;
2036 rep_param->flow_control = 1;
2037 rep_param->failover_accepted = 0;
2038 rep_param->srq = 1;
2039 rep_param->responder_resources = 4;
2040 rep_param->initiator_depth = 4;
2041
2042 ret = ib_send_cm_rep(cm_id, rep_param);
2043 if (ret) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002044 pr_err("sending SRP_LOGIN_REQ response failed"
Bart Van Asschea42d9852011-10-14 01:30:46 +00002045 " (error code = %d)\n", ret);
2046 goto release_channel;
2047 }
2048
Bart Van Assche86289912016-02-11 11:08:34 -08002049 mutex_lock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002050 list_add_tail(&ch->list, &sdev->rch_list);
Bart Van Assche86289912016-02-11 11:08:34 -08002051 mutex_unlock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002052
2053 goto out;
2054
2055release_channel:
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08002056 srpt_disconnect_ch(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002057 transport_deregister_session_configfs(ch->sess);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002058 transport_deregister_session(ch->sess);
2059 ch->sess = NULL;
2060
2061destroy_ib:
2062 srpt_destroy_ch_ib(ch);
2063
2064free_ring:
2065 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
2066 ch->sport->sdev, ch->rq_size,
2067 ch->rsp_size, DMA_TO_DEVICE);
2068free_ch:
2069 kfree(ch);
2070
2071reject:
2072 rej->opcode = SRP_LOGIN_REJ;
2073 rej->tag = req->tag;
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05302074 rej->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT
2075 | SRP_BUF_FORMAT_INDIRECT);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002076
2077 ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08002078 (void *)rej, sizeof(*rej));
Bart Van Asschea42d9852011-10-14 01:30:46 +00002079
2080out:
2081 kfree(rep_param);
2082 kfree(rsp);
2083 kfree(rej);
2084
2085 return ret;
2086}
2087
Bart Van Assche2739b592016-02-11 11:07:49 -08002088static void srpt_cm_rej_recv(struct srpt_rdma_ch *ch,
2089 enum ib_cm_rej_reason reason,
2090 const u8 *private_data,
2091 u8 private_data_len)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002092{
Bart Van Asschec13c90e2016-02-11 11:08:12 -08002093 char *priv = NULL;
2094 int i;
2095
2096 if (private_data_len && (priv = kmalloc(private_data_len * 3 + 1,
2097 GFP_KERNEL))) {
2098 for (i = 0; i < private_data_len; i++)
2099 sprintf(priv + 3 * i, " %02x", private_data[i]);
2100 }
2101 pr_info("Received CM REJ for ch %s-%d; reason %d%s%s.\n",
2102 ch->sess_name, ch->qp->qp_num, reason, private_data_len ?
2103 "; private data" : "", priv ? priv : " (?)");
2104 kfree(priv);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002105}
2106
2107/**
2108 * srpt_cm_rtu_recv() - Process an IB_CM_RTU_RECEIVED or USER_ESTABLISHED event.
2109 *
2110 * An IB_CM_RTU_RECEIVED message indicates that the connection is established
2111 * and that the recipient may begin transmitting (RTU = ready to use).
2112 */
Bart Van Assche2739b592016-02-11 11:07:49 -08002113static void srpt_cm_rtu_recv(struct srpt_rdma_ch *ch)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002114{
Bart Van Asschea42d9852011-10-14 01:30:46 +00002115 int ret;
2116
Bart Van Asschef130c222016-02-11 11:05:38 -08002117 if (srpt_set_ch_state(ch, CH_LIVE)) {
Bart Van Asschea42d9852011-10-14 01:30:46 +00002118 ret = srpt_ch_qp_rts(ch, ch->qp);
2119
Bart Van Assche387add42016-02-11 11:10:09 -08002120 if (ret == 0) {
2121 /* Trigger wait list processing. */
2122 ret = srpt_zerolength_write(ch);
2123 WARN_ONCE(ret < 0, "%d\n", ret);
2124 } else {
Bart Van Asschea42d9852011-10-14 01:30:46 +00002125 srpt_close_ch(ch);
Bart Van Assche387add42016-02-11 11:10:09 -08002126 }
Bart Van Asschea42d9852011-10-14 01:30:46 +00002127 }
2128}
2129
Bart Van Asschea42d9852011-10-14 01:30:46 +00002130/**
Bart Van Asschea42d9852011-10-14 01:30:46 +00002131 * srpt_cm_handler() - IB connection manager callback function.
2132 *
2133 * A non-zero return value will cause the caller destroy the CM ID.
2134 *
2135 * Note: srpt_cm_handler() must only return a non-zero value when transferring
2136 * ownership of the cm_id to a channel by srpt_cm_req_recv() failed. Returning
2137 * a non-zero value in any other case will trigger a race with the
2138 * ib_destroy_cm_id() call in srpt_release_channel().
2139 */
2140static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2141{
Bart Van Assche2739b592016-02-11 11:07:49 -08002142 struct srpt_rdma_ch *ch = cm_id->context;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002143 int ret;
2144
2145 ret = 0;
2146 switch (event->event) {
2147 case IB_CM_REQ_RECEIVED:
2148 ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
2149 event->private_data);
2150 break;
2151 case IB_CM_REJ_RECEIVED:
Bart Van Assche2739b592016-02-11 11:07:49 -08002152 srpt_cm_rej_recv(ch, event->param.rej_rcvd.reason,
2153 event->private_data,
2154 IB_CM_REJ_PRIVATE_DATA_SIZE);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002155 break;
2156 case IB_CM_RTU_RECEIVED:
2157 case IB_CM_USER_ESTABLISHED:
Bart Van Assche2739b592016-02-11 11:07:49 -08002158 srpt_cm_rtu_recv(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002159 break;
2160 case IB_CM_DREQ_RECEIVED:
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08002161 srpt_disconnect_ch(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002162 break;
2163 case IB_CM_DREP_RECEIVED:
Bart Van Assche2739b592016-02-11 11:07:49 -08002164 pr_info("Received CM DREP message for ch %s-%d.\n",
2165 ch->sess_name, ch->qp->qp_num);
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08002166 srpt_close_ch(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002167 break;
2168 case IB_CM_TIMEWAIT_EXIT:
Bart Van Assche2739b592016-02-11 11:07:49 -08002169 pr_info("Received CM TimeWait exit for ch %s-%d.\n",
2170 ch->sess_name, ch->qp->qp_num);
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08002171 srpt_close_ch(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002172 break;
2173 case IB_CM_REP_ERROR:
Bart Van Assche2739b592016-02-11 11:07:49 -08002174 pr_info("Received CM REP error for ch %s-%d.\n", ch->sess_name,
2175 ch->qp->qp_num);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002176 break;
2177 case IB_CM_DREQ_ERROR:
Bart Van Assche1e20a2a2016-02-11 11:07:29 -08002178 pr_info("Received CM DREQ ERROR event.\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +00002179 break;
2180 case IB_CM_MRA_RECEIVED:
Bart Van Assche1e20a2a2016-02-11 11:07:29 -08002181 pr_info("Received CM MRA event\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +00002182 break;
2183 default:
Bart Van Assche1e20a2a2016-02-11 11:07:29 -08002184 pr_err("received unrecognized CM event %d\n", event->event);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002185 break;
2186 }
2187
2188 return ret;
2189}
2190
Bart Van Asschea42d9852011-10-14 01:30:46 +00002191static int srpt_write_pending_status(struct se_cmd *se_cmd)
2192{
2193 struct srpt_send_ioctx *ioctx;
2194
2195 ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
2196 return srpt_get_cmd_state(ioctx) == SRPT_STATE_NEED_DATA;
2197}
2198
2199/*
2200 * srpt_write_pending() - Start data transfer from initiator to target (write).
2201 */
2202static int srpt_write_pending(struct se_cmd *se_cmd)
2203{
Bart Van Asschefc3af582016-02-11 11:09:10 -08002204 struct srpt_send_ioctx *ioctx =
2205 container_of(se_cmd, struct srpt_send_ioctx, cmd);
2206 struct srpt_rdma_ch *ch = ioctx->ch;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002207 struct ib_send_wr *first_wr = NULL, *bad_wr;
2208 struct ib_cqe *cqe = &ioctx->rdma_cqe;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002209 enum srpt_command_state new_state;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002210 int ret, i;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002211
2212 new_state = srpt_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA);
2213 WARN_ON(new_state == SRPT_STATE_DONE);
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002214
2215 if (atomic_sub_return(ioctx->n_rdma, &ch->sq_wr_avail) < 0) {
2216 pr_warn("%s: IB send queue full (needed %d)\n",
2217 __func__, ioctx->n_rdma);
2218 ret = -ENOMEM;
2219 goto out_undo;
2220 }
2221
2222 cqe->done = srpt_rdma_read_done;
2223 for (i = ioctx->n_rw_ctx - 1; i >= 0; i--) {
2224 struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
2225
2226 first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp, ch->sport->port,
2227 cqe, first_wr);
2228 cqe = NULL;
2229 }
2230
2231 ret = ib_post_send(ch->qp, first_wr, &bad_wr);
2232 if (ret) {
2233 pr_err("%s: ib_post_send() returned %d for %d (avail: %d)\n",
2234 __func__, ret, ioctx->n_rdma,
2235 atomic_read(&ch->sq_wr_avail));
2236 goto out_undo;
2237 }
2238
2239 return 0;
2240out_undo:
2241 atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);
2242 return ret;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002243}
2244
2245static u8 tcm_to_srp_tsk_mgmt_status(const int tcm_mgmt_status)
2246{
2247 switch (tcm_mgmt_status) {
2248 case TMR_FUNCTION_COMPLETE:
2249 return SRP_TSK_MGMT_SUCCESS;
2250 case TMR_FUNCTION_REJECTED:
2251 return SRP_TSK_MGMT_FUNC_NOT_SUPP;
2252 }
2253 return SRP_TSK_MGMT_FAILED;
2254}
2255
2256/**
2257 * srpt_queue_response() - Transmits the response to a SCSI command.
2258 *
2259 * Callback function called by the TCM core. Must not block since it can be
2260 * invoked on the context of the IB completion handler.
2261 */
Joern Engelb79fafa2013-07-03 11:22:17 -04002262static void srpt_queue_response(struct se_cmd *cmd)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002263{
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002264 struct srpt_send_ioctx *ioctx =
2265 container_of(cmd, struct srpt_send_ioctx, cmd);
2266 struct srpt_rdma_ch *ch = ioctx->ch;
2267 struct srpt_device *sdev = ch->sport->sdev;
Bart Van Assche10fce582016-07-21 13:04:06 -07002268 struct ib_send_wr send_wr, *first_wr = &send_wr, *bad_wr;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002269 struct ib_sge sge;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002270 enum srpt_command_state state;
2271 unsigned long flags;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002272 int resp_len, ret, i;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002273 u8 srp_tm_status;
2274
Bart Van Asschea42d9852011-10-14 01:30:46 +00002275 BUG_ON(!ch);
2276
2277 spin_lock_irqsave(&ioctx->spinlock, flags);
2278 state = ioctx->state;
2279 switch (state) {
2280 case SRPT_STATE_NEW:
2281 case SRPT_STATE_DATA_IN:
2282 ioctx->state = SRPT_STATE_CMD_RSP_SENT;
2283 break;
2284 case SRPT_STATE_MGMT:
2285 ioctx->state = SRPT_STATE_MGMT_RSP_SENT;
2286 break;
2287 default:
2288 WARN(true, "ch %p; cmd %d: unexpected command state %d\n",
2289 ch, ioctx->ioctx.index, ioctx->state);
2290 break;
2291 }
2292 spin_unlock_irqrestore(&ioctx->spinlock, flags);
2293
Bart Van Asschef9800a12017-05-04 15:50:53 -07002294 if (unlikely(WARN_ON_ONCE(state == SRPT_STATE_CMD_RSP_SENT)))
Joern Engelb79fafa2013-07-03 11:22:17 -04002295 return;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002296
Bart Van Asschea42d9852011-10-14 01:30:46 +00002297 /* For read commands, transfer the data to the initiator. */
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002298 if (ioctx->cmd.data_direction == DMA_FROM_DEVICE &&
2299 ioctx->cmd.data_length &&
Bart Van Asschea42d9852011-10-14 01:30:46 +00002300 !ioctx->queue_status_only) {
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002301 for (i = ioctx->n_rw_ctx - 1; i >= 0; i--) {
2302 struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
2303
2304 first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp,
Bart Van Assche10fce582016-07-21 13:04:06 -07002305 ch->sport->port, NULL, first_wr);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002306 }
2307 }
2308
2309 if (state != SRPT_STATE_MGMT)
Bart Van Assche649ee052015-04-14 13:26:44 +02002310 resp_len = srpt_build_cmd_rsp(ch, ioctx, ioctx->cmd.tag,
Bart Van Asschea42d9852011-10-14 01:30:46 +00002311 cmd->scsi_status);
2312 else {
2313 srp_tm_status
2314 = tcm_to_srp_tsk_mgmt_status(cmd->se_tmr_req->response);
2315 resp_len = srpt_build_tskmgmt_rsp(ch, ioctx, srp_tm_status,
Bart Van Assche649ee052015-04-14 13:26:44 +02002316 ioctx->cmd.tag);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002317 }
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002318
2319 atomic_inc(&ch->req_lim);
2320
2321 if (unlikely(atomic_sub_return(1 + ioctx->n_rdma,
2322 &ch->sq_wr_avail) < 0)) {
2323 pr_warn("%s: IB send queue full (needed %d)\n",
2324 __func__, ioctx->n_rdma);
2325 ret = -ENOMEM;
2326 goto out;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002327 }
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002328
2329 ib_dma_sync_single_for_device(sdev->device, ioctx->ioctx.dma, resp_len,
2330 DMA_TO_DEVICE);
2331
2332 sge.addr = ioctx->ioctx.dma;
2333 sge.length = resp_len;
2334 sge.lkey = sdev->pd->local_dma_lkey;
2335
2336 ioctx->ioctx.cqe.done = srpt_send_done;
2337 send_wr.next = NULL;
2338 send_wr.wr_cqe = &ioctx->ioctx.cqe;
2339 send_wr.sg_list = &sge;
2340 send_wr.num_sge = 1;
2341 send_wr.opcode = IB_WR_SEND;
2342 send_wr.send_flags = IB_SEND_SIGNALED;
2343
2344 ret = ib_post_send(ch->qp, first_wr, &bad_wr);
2345 if (ret < 0) {
2346 pr_err("%s: sending cmd response failed for tag %llu (%d)\n",
2347 __func__, ioctx->cmd.tag, ret);
2348 goto out;
2349 }
2350
2351 return;
2352
2353out:
2354 atomic_add(1 + ioctx->n_rdma, &ch->sq_wr_avail);
2355 atomic_dec(&ch->req_lim);
2356 srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
2357 target_put_sess_cmd(&ioctx->cmd);
Joern Engelb79fafa2013-07-03 11:22:17 -04002358}
Bart Van Asschea42d9852011-10-14 01:30:46 +00002359
Joern Engelb79fafa2013-07-03 11:22:17 -04002360static int srpt_queue_data_in(struct se_cmd *cmd)
2361{
2362 srpt_queue_response(cmd);
2363 return 0;
2364}
2365
2366static void srpt_queue_tm_rsp(struct se_cmd *cmd)
2367{
2368 srpt_queue_response(cmd);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002369}
2370
Bart Van Asscheea3a2342019-01-25 10:34:51 -08002371/*
2372 * This function is called for aborted commands if no response is sent to the
2373 * initiator. Make sure that the credits freed by aborting a command are
2374 * returned to the initiator the next time a response is sent by incrementing
2375 * ch->req_lim_delta.
2376 */
Nicholas Bellinger131e6ab2014-03-22 14:55:56 -07002377static void srpt_aborted_task(struct se_cmd *cmd)
2378{
Bart Van Asscheea3a2342019-01-25 10:34:51 -08002379 struct srpt_send_ioctx *ioctx = container_of(cmd,
2380 struct srpt_send_ioctx, cmd);
2381 struct srpt_rdma_ch *ch = ioctx->ch;
2382
2383 atomic_inc(&ch->req_lim_delta);
Nicholas Bellinger131e6ab2014-03-22 14:55:56 -07002384}
2385
Bart Van Asschea42d9852011-10-14 01:30:46 +00002386static int srpt_queue_status(struct se_cmd *cmd)
2387{
2388 struct srpt_send_ioctx *ioctx;
2389
2390 ioctx = container_of(cmd, struct srpt_send_ioctx, cmd);
2391 BUG_ON(ioctx->sense_data != cmd->sense_buffer);
2392 if (cmd->se_cmd_flags &
2393 (SCF_TRANSPORT_TASK_SENSE | SCF_EMULATED_TASK_SENSE))
2394 WARN_ON(cmd->scsi_status != SAM_STAT_CHECK_CONDITION);
2395 ioctx->queue_status_only = true;
Joern Engelb79fafa2013-07-03 11:22:17 -04002396 srpt_queue_response(cmd);
2397 return 0;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002398}
2399
2400static void srpt_refresh_port_work(struct work_struct *work)
2401{
2402 struct srpt_port *sport = container_of(work, struct srpt_port, work);
2403
2404 srpt_refresh_port(sport);
2405}
2406
Bart Van Asschea42d9852011-10-14 01:30:46 +00002407/**
2408 * srpt_release_sdev() - Free the channel resources associated with a target.
2409 */
2410static int srpt_release_sdev(struct srpt_device *sdev)
2411{
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08002412 int i, res;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002413
2414 WARN_ON_ONCE(irqs_disabled());
2415
2416 BUG_ON(!sdev);
2417
Bart Van Assche86289912016-02-11 11:08:34 -08002418 mutex_lock(&sdev->mutex);
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08002419 for (i = 0; i < ARRAY_SIZE(sdev->port); i++)
2420 sdev->port[i].enabled = false;
2421 __srpt_close_all_ch(sdev);
Bart Van Assche86289912016-02-11 11:08:34 -08002422 mutex_unlock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002423
2424 res = wait_event_interruptible(sdev->ch_releaseQ,
Bart Van Assche86289912016-02-11 11:08:34 -08002425 list_empty_careful(&sdev->rch_list));
Bart Van Asschea42d9852011-10-14 01:30:46 +00002426 if (res)
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002427 pr_err("%s: interrupted.\n", __func__);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002428
2429 return 0;
2430}
2431
2432static struct srpt_port *__srpt_lookup_port(const char *name)
2433{
2434 struct ib_device *dev;
2435 struct srpt_device *sdev;
2436 struct srpt_port *sport;
2437 int i;
2438
2439 list_for_each_entry(sdev, &srpt_dev_list, list) {
2440 dev = sdev->device;
2441 if (!dev)
2442 continue;
2443
2444 for (i = 0; i < dev->phys_port_cnt; i++) {
2445 sport = &sdev->port[i];
2446
2447 if (!strcmp(sport->port_guid, name))
2448 return sport;
2449 }
2450 }
2451
2452 return NULL;
2453}
2454
2455static struct srpt_port *srpt_lookup_port(const char *name)
2456{
2457 struct srpt_port *sport;
2458
2459 spin_lock(&srpt_dev_lock);
2460 sport = __srpt_lookup_port(name);
2461 spin_unlock(&srpt_dev_lock);
2462
2463 return sport;
2464}
2465
2466/**
2467 * srpt_add_one() - Infiniband device addition callback function.
2468 */
2469static void srpt_add_one(struct ib_device *device)
2470{
2471 struct srpt_device *sdev;
2472 struct srpt_port *sport;
2473 struct ib_srq_init_attr srq_attr;
2474 int i;
2475
2476 pr_debug("device = %p, device->dma_ops = %p\n", device,
2477 device->dma_ops);
2478
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08002479 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002480 if (!sdev)
2481 goto err;
2482
2483 sdev->device = device;
2484 INIT_LIST_HEAD(&sdev->rch_list);
2485 init_waitqueue_head(&sdev->ch_releaseQ);
Bart Van Assche86289912016-02-11 11:08:34 -08002486 mutex_init(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002487
Christoph Hellwiged082d32016-09-05 12:56:17 +02002488 sdev->pd = ib_alloc_pd(device, 0);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002489 if (IS_ERR(sdev->pd))
2490 goto free_dev;
2491
Or Gerlitz4a061b22015-12-18 10:59:46 +02002492 sdev->srq_size = min(srpt_srq_size, sdev->device->attrs.max_srq_wr);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002493
2494 srq_attr.event_handler = srpt_srq_event;
2495 srq_attr.srq_context = (void *)sdev;
2496 srq_attr.attr.max_wr = sdev->srq_size;
2497 srq_attr.attr.max_sge = 1;
2498 srq_attr.attr.srq_limit = 0;
Roland Dreier6f360332012-04-12 07:51:08 -07002499 srq_attr.srq_type = IB_SRQT_BASIC;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002500
2501 sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
2502 if (IS_ERR(sdev->srq))
Jason Gunthorpe5a783952015-07-30 17:22:24 -06002503 goto err_pd;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002504
2505 pr_debug("%s: create SRQ #wr= %d max_allow=%d dev= %s\n",
Or Gerlitz4a061b22015-12-18 10:59:46 +02002506 __func__, sdev->srq_size, sdev->device->attrs.max_srq_wr,
Bart Van Asschea42d9852011-10-14 01:30:46 +00002507 device->name);
2508
2509 if (!srpt_service_guid)
2510 srpt_service_guid = be64_to_cpu(device->node_guid);
2511
2512 sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
2513 if (IS_ERR(sdev->cm_id))
2514 goto err_srq;
2515
2516 /* print out target login information */
2517 pr_debug("Target login info: id_ext=%016llx,ioc_guid=%016llx,"
2518 "pkey=ffff,service_id=%016llx\n", srpt_service_guid,
2519 srpt_service_guid, srpt_service_guid);
2520
2521 /*
2522 * We do not have a consistent service_id (ie. also id_ext of target_id)
2523 * to identify this target. We currently use the guid of the first HCA
2524 * in the system as service_id; therefore, the target_id will change
2525 * if this HCA is gone bad and replaced by different HCA
2526 */
Haggai Eran73fec7f2015-07-30 17:50:26 +03002527 if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0))
Bart Van Asschea42d9852011-10-14 01:30:46 +00002528 goto err_cm;
2529
2530 INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
2531 srpt_event_handler);
2532 if (ib_register_event_handler(&sdev->event_handler))
2533 goto err_cm;
2534
2535 sdev->ioctx_ring = (struct srpt_recv_ioctx **)
2536 srpt_alloc_ioctx_ring(sdev, sdev->srq_size,
2537 sizeof(*sdev->ioctx_ring[0]),
2538 srp_max_req_size, DMA_FROM_DEVICE);
2539 if (!sdev->ioctx_ring)
2540 goto err_event;
2541
2542 for (i = 0; i < sdev->srq_size; ++i)
2543 srpt_post_recv(sdev, sdev->ioctx_ring[i]);
2544
Roland Dreierf2250662012-02-02 12:55:58 -08002545 WARN_ON(sdev->device->phys_port_cnt > ARRAY_SIZE(sdev->port));
Bart Van Asschea42d9852011-10-14 01:30:46 +00002546
2547 for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
2548 sport = &sdev->port[i - 1];
2549 sport->sdev = sdev;
2550 sport->port = i;
2551 sport->port_attrib.srp_max_rdma_size = DEFAULT_MAX_RDMA_SIZE;
2552 sport->port_attrib.srp_max_rsp_size = DEFAULT_MAX_RSP_SIZE;
2553 sport->port_attrib.srp_sq_size = DEF_SRPT_SQ_SIZE;
2554 INIT_WORK(&sport->work, srpt_refresh_port_work);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002555
2556 if (srpt_refresh_port(sport)) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002557 pr_err("MAD registration failed for %s-%d.\n",
Bart Van Asschef68cba4e92016-02-11 11:04:20 -08002558 sdev->device->name, i);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002559 goto err_ring;
2560 }
Bart Van Asschea42d9852011-10-14 01:30:46 +00002561 }
2562
2563 spin_lock(&srpt_dev_lock);
2564 list_add_tail(&sdev->list, &srpt_dev_list);
2565 spin_unlock(&srpt_dev_lock);
2566
2567out:
2568 ib_set_client_data(device, &srpt_client, sdev);
2569 pr_debug("added %s.\n", device->name);
2570 return;
2571
2572err_ring:
2573 srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,
2574 sdev->srq_size, srp_max_req_size,
2575 DMA_FROM_DEVICE);
2576err_event:
2577 ib_unregister_event_handler(&sdev->event_handler);
2578err_cm:
2579 ib_destroy_cm_id(sdev->cm_id);
2580err_srq:
2581 ib_destroy_srq(sdev->srq);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002582err_pd:
2583 ib_dealloc_pd(sdev->pd);
2584free_dev:
2585 kfree(sdev);
2586err:
2587 sdev = NULL;
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002588 pr_info("%s(%s) failed.\n", __func__, device->name);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002589 goto out;
2590}
2591
2592/**
2593 * srpt_remove_one() - InfiniBand device removal callback function.
2594 */
Haggai Eran7c1eb452015-07-30 17:50:14 +03002595static void srpt_remove_one(struct ib_device *device, void *client_data)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002596{
Haggai Eran7c1eb452015-07-30 17:50:14 +03002597 struct srpt_device *sdev = client_data;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002598 int i;
2599
Bart Van Asschea42d9852011-10-14 01:30:46 +00002600 if (!sdev) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002601 pr_info("%s(%s): nothing to do.\n", __func__, device->name);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002602 return;
2603 }
2604
2605 srpt_unregister_mad_agent(sdev);
2606
2607 ib_unregister_event_handler(&sdev->event_handler);
2608
2609 /* Cancel any work queued by the just unregistered IB event handler. */
2610 for (i = 0; i < sdev->device->phys_port_cnt; i++)
2611 cancel_work_sync(&sdev->port[i].work);
2612
2613 ib_destroy_cm_id(sdev->cm_id);
2614
2615 /*
2616 * Unregistering a target must happen after destroying sdev->cm_id
2617 * such that no new SRP_LOGIN_REQ information units can arrive while
2618 * destroying the target.
2619 */
2620 spin_lock(&srpt_dev_lock);
2621 list_del(&sdev->list);
2622 spin_unlock(&srpt_dev_lock);
2623 srpt_release_sdev(sdev);
2624
2625 ib_destroy_srq(sdev->srq);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002626 ib_dealloc_pd(sdev->pd);
2627
2628 srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,
2629 sdev->srq_size, srp_max_req_size, DMA_FROM_DEVICE);
2630 sdev->ioctx_ring = NULL;
2631 kfree(sdev);
2632}
2633
2634static struct ib_client srpt_client = {
2635 .name = DRV_NAME,
2636 .add = srpt_add_one,
2637 .remove = srpt_remove_one
2638};
2639
2640static int srpt_check_true(struct se_portal_group *se_tpg)
2641{
2642 return 1;
2643}
2644
2645static int srpt_check_false(struct se_portal_group *se_tpg)
2646{
2647 return 0;
2648}
2649
2650static char *srpt_get_fabric_name(void)
2651{
2652 return "srpt";
2653}
2654
Bart Van Asschea42d9852011-10-14 01:30:46 +00002655static char *srpt_get_fabric_wwn(struct se_portal_group *tpg)
2656{
2657 struct srpt_port *sport = container_of(tpg, struct srpt_port, port_tpg_1);
2658
2659 return sport->port_guid;
2660}
2661
2662static u16 srpt_get_tag(struct se_portal_group *tpg)
2663{
2664 return 1;
2665}
2666
Bart Van Asschea42d9852011-10-14 01:30:46 +00002667static u32 srpt_tpg_get_inst_index(struct se_portal_group *se_tpg)
2668{
2669 return 1;
2670}
2671
2672static void srpt_release_cmd(struct se_cmd *se_cmd)
2673{
Nicholas Bellinger9474b042012-11-27 23:55:57 -08002674 struct srpt_send_ioctx *ioctx = container_of(se_cmd,
2675 struct srpt_send_ioctx, cmd);
2676 struct srpt_rdma_ch *ch = ioctx->ch;
Bart Van Assche3c968882016-04-07 15:55:04 -07002677 unsigned long flags;
Nicholas Bellinger9474b042012-11-27 23:55:57 -08002678
Bart Van Assche84284962017-05-04 15:50:54 -07002679 WARN_ON_ONCE(ioctx->state != SRPT_STATE_DONE &&
2680 !(ioctx->cmd.transport_state & CMD_T_ABORTED));
Nicholas Bellinger9474b042012-11-27 23:55:57 -08002681
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002682 if (ioctx->n_rw_ctx) {
2683 srpt_free_rw_ctxs(ch, ioctx);
2684 ioctx->n_rw_ctx = 0;
Nicholas Bellinger9474b042012-11-27 23:55:57 -08002685 }
2686
Bart Van Assche3c968882016-04-07 15:55:04 -07002687 spin_lock_irqsave(&ch->spinlock, flags);
2688 list_add(&ioctx->free_list, &ch->free_list);
2689 spin_unlock_irqrestore(&ch->spinlock, flags);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002690}
2691
2692/**
Bart Van Asschea42d9852011-10-14 01:30:46 +00002693 * srpt_close_session() - Forcibly close a session.
2694 *
2695 * Callback function invoked by the TCM core to clean up sessions associated
2696 * with a node ACL when the user invokes
2697 * rmdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id
2698 */
2699static void srpt_close_session(struct se_session *se_sess)
2700{
2701 DECLARE_COMPLETION_ONSTACK(release_done);
Bart Van Asschef108f0f2016-02-11 11:06:14 -08002702 struct srpt_rdma_ch *ch = se_sess->fabric_sess_ptr;
2703 struct srpt_device *sdev = ch->sport->sdev;
2704 bool wait;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002705
Bart Van Asschef108f0f2016-02-11 11:06:14 -08002706 pr_debug("ch %s-%d state %d\n", ch->sess_name, ch->qp->qp_num,
2707 ch->state);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002708
Bart Van Assche86289912016-02-11 11:08:34 -08002709 mutex_lock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002710 BUG_ON(ch->release_done);
2711 ch->release_done = &release_done;
Bart Van Asschef108f0f2016-02-11 11:06:14 -08002712 wait = !list_empty(&ch->list);
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08002713 srpt_disconnect_ch(ch);
Bart Van Assche86289912016-02-11 11:08:34 -08002714 mutex_unlock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002715
Bart Van Asschef108f0f2016-02-11 11:06:14 -08002716 if (!wait)
2717 return;
2718
2719 while (wait_for_completion_timeout(&release_done, 180 * HZ) == 0)
2720 pr_info("%s(%s-%d state %d): still waiting ...\n", __func__,
2721 ch->sess_name, ch->qp->qp_num, ch->state);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002722}
2723
2724/**
Bart Van Asschea42d9852011-10-14 01:30:46 +00002725 * srpt_sess_get_index() - Return the value of scsiAttIntrPortIndex (SCSI-MIB).
2726 *
2727 * A quote from RFC 4455 (SCSI-MIB) about this MIB object:
2728 * This object represents an arbitrary integer used to uniquely identify a
2729 * particular attached remote initiator port to a particular SCSI target port
2730 * within a particular SCSI target device within a particular SCSI instance.
2731 */
2732static u32 srpt_sess_get_index(struct se_session *se_sess)
2733{
2734 return 0;
2735}
2736
2737static void srpt_set_default_node_attrs(struct se_node_acl *nacl)
2738{
2739}
2740
Bart Van Asschea42d9852011-10-14 01:30:46 +00002741/* Note: only used from inside debug printk's by the TCM core. */
2742static int srpt_get_tcm_cmd_state(struct se_cmd *se_cmd)
2743{
2744 struct srpt_send_ioctx *ioctx;
2745
2746 ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
2747 return srpt_get_cmd_state(ioctx);
2748}
2749
Bart Van Asschea42d9852011-10-14 01:30:46 +00002750/**
2751 * srpt_parse_i_port_id() - Parse an initiator port ID.
2752 * @name: ASCII representation of a 128-bit initiator port ID.
2753 * @i_port_id: Binary 128-bit port ID.
2754 */
2755static int srpt_parse_i_port_id(u8 i_port_id[16], const char *name)
2756{
2757 const char *p;
2758 unsigned len, count, leading_zero_bytes;
Bart Van Assche8105ea32017-10-11 10:27:22 -07002759 int ret;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002760
2761 p = name;
Rasmus Villemoesb60459f2014-10-13 15:54:46 -07002762 if (strncasecmp(p, "0x", 2) == 0)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002763 p += 2;
2764 ret = -EINVAL;
2765 len = strlen(p);
2766 if (len % 2)
2767 goto out;
2768 count = min(len / 2, 16U);
2769 leading_zero_bytes = 16 - count;
2770 memset(i_port_id, 0, leading_zero_bytes);
Bart Van Assche8105ea32017-10-11 10:27:22 -07002771 ret = hex2bin(i_port_id + leading_zero_bytes, p, count);
2772 if (ret < 0)
2773 pr_debug("hex2bin failed for srpt_parse_i_port_id: %d\n", ret);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002774out:
2775 return ret;
2776}
2777
2778/*
2779 * configfs callback function invoked for
2780 * mkdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id
2781 */
Christoph Hellwigc7d6a802015-04-13 19:51:14 +02002782static int srpt_init_nodeacl(struct se_node_acl *se_nacl, const char *name)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002783{
Bart Van Asschea42d9852011-10-14 01:30:46 +00002784 u8 i_port_id[16];
2785
2786 if (srpt_parse_i_port_id(i_port_id, name) < 0) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002787 pr_err("invalid initiator port ID %s\n", name);
Christoph Hellwigc7d6a802015-04-13 19:51:14 +02002788 return -EINVAL;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002789 }
Christoph Hellwigc7d6a802015-04-13 19:51:14 +02002790 return 0;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002791}
2792
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002793static ssize_t srpt_tpg_attrib_srp_max_rdma_size_show(struct config_item *item,
2794 char *page)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002795{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002796 struct se_portal_group *se_tpg = attrib_to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002797 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2798
2799 return sprintf(page, "%u\n", sport->port_attrib.srp_max_rdma_size);
2800}
2801
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002802static ssize_t srpt_tpg_attrib_srp_max_rdma_size_store(struct config_item *item,
2803 const char *page, size_t count)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002804{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002805 struct se_portal_group *se_tpg = attrib_to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002806 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2807 unsigned long val;
2808 int ret;
2809
Jingoo Han9d8abf42014-02-05 11:22:05 +09002810 ret = kstrtoul(page, 0, &val);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002811 if (ret < 0) {
Jingoo Han9d8abf42014-02-05 11:22:05 +09002812 pr_err("kstrtoul() failed with ret: %d\n", ret);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002813 return -EINVAL;
2814 }
2815 if (val > MAX_SRPT_RDMA_SIZE) {
2816 pr_err("val: %lu exceeds MAX_SRPT_RDMA_SIZE: %d\n", val,
2817 MAX_SRPT_RDMA_SIZE);
2818 return -EINVAL;
2819 }
2820 if (val < DEFAULT_MAX_RDMA_SIZE) {
2821 pr_err("val: %lu smaller than DEFAULT_MAX_RDMA_SIZE: %d\n",
2822 val, DEFAULT_MAX_RDMA_SIZE);
2823 return -EINVAL;
2824 }
2825 sport->port_attrib.srp_max_rdma_size = val;
2826
2827 return count;
2828}
2829
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002830static ssize_t srpt_tpg_attrib_srp_max_rsp_size_show(struct config_item *item,
2831 char *page)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002832{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002833 struct se_portal_group *se_tpg = attrib_to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002834 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2835
2836 return sprintf(page, "%u\n", sport->port_attrib.srp_max_rsp_size);
2837}
2838
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002839static ssize_t srpt_tpg_attrib_srp_max_rsp_size_store(struct config_item *item,
2840 const char *page, size_t count)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002841{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002842 struct se_portal_group *se_tpg = attrib_to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002843 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2844 unsigned long val;
2845 int ret;
2846
Jingoo Han9d8abf42014-02-05 11:22:05 +09002847 ret = kstrtoul(page, 0, &val);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002848 if (ret < 0) {
Jingoo Han9d8abf42014-02-05 11:22:05 +09002849 pr_err("kstrtoul() failed with ret: %d\n", ret);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002850 return -EINVAL;
2851 }
2852 if (val > MAX_SRPT_RSP_SIZE) {
2853 pr_err("val: %lu exceeds MAX_SRPT_RSP_SIZE: %d\n", val,
2854 MAX_SRPT_RSP_SIZE);
2855 return -EINVAL;
2856 }
2857 if (val < MIN_MAX_RSP_SIZE) {
2858 pr_err("val: %lu smaller than MIN_MAX_RSP_SIZE: %d\n", val,
2859 MIN_MAX_RSP_SIZE);
2860 return -EINVAL;
2861 }
2862 sport->port_attrib.srp_max_rsp_size = val;
2863
2864 return count;
2865}
2866
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002867static ssize_t srpt_tpg_attrib_srp_sq_size_show(struct config_item *item,
2868 char *page)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002869{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002870 struct se_portal_group *se_tpg = attrib_to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002871 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2872
2873 return sprintf(page, "%u\n", sport->port_attrib.srp_sq_size);
2874}
2875
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002876static ssize_t srpt_tpg_attrib_srp_sq_size_store(struct config_item *item,
2877 const char *page, size_t count)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002878{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002879 struct se_portal_group *se_tpg = attrib_to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002880 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2881 unsigned long val;
2882 int ret;
2883
Jingoo Han9d8abf42014-02-05 11:22:05 +09002884 ret = kstrtoul(page, 0, &val);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002885 if (ret < 0) {
Jingoo Han9d8abf42014-02-05 11:22:05 +09002886 pr_err("kstrtoul() failed with ret: %d\n", ret);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002887 return -EINVAL;
2888 }
2889 if (val > MAX_SRPT_SRQ_SIZE) {
2890 pr_err("val: %lu exceeds MAX_SRPT_SRQ_SIZE: %d\n", val,
2891 MAX_SRPT_SRQ_SIZE);
2892 return -EINVAL;
2893 }
2894 if (val < MIN_SRPT_SRQ_SIZE) {
2895 pr_err("val: %lu smaller than MIN_SRPT_SRQ_SIZE: %d\n", val,
2896 MIN_SRPT_SRQ_SIZE);
2897 return -EINVAL;
2898 }
2899 sport->port_attrib.srp_sq_size = val;
2900
2901 return count;
2902}
2903
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002904CONFIGFS_ATTR(srpt_tpg_attrib_, srp_max_rdma_size);
2905CONFIGFS_ATTR(srpt_tpg_attrib_, srp_max_rsp_size);
2906CONFIGFS_ATTR(srpt_tpg_attrib_, srp_sq_size);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002907
2908static struct configfs_attribute *srpt_tpg_attrib_attrs[] = {
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002909 &srpt_tpg_attrib_attr_srp_max_rdma_size,
2910 &srpt_tpg_attrib_attr_srp_max_rsp_size,
2911 &srpt_tpg_attrib_attr_srp_sq_size,
Bart Van Asschea42d9852011-10-14 01:30:46 +00002912 NULL,
2913};
2914
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002915static ssize_t srpt_tpg_enable_show(struct config_item *item, char *page)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002916{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002917 struct se_portal_group *se_tpg = to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002918 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2919
2920 return snprintf(page, PAGE_SIZE, "%d\n", (sport->enabled) ? 1: 0);
2921}
2922
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002923static ssize_t srpt_tpg_enable_store(struct config_item *item,
2924 const char *page, size_t count)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002925{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002926 struct se_portal_group *se_tpg = to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002927 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
Bart Van Assche043a6802016-02-11 11:09:28 -08002928 struct srpt_device *sdev = sport->sdev;
2929 struct srpt_rdma_ch *ch;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002930 unsigned long tmp;
2931 int ret;
2932
Jingoo Han9d8abf42014-02-05 11:22:05 +09002933 ret = kstrtoul(page, 0, &tmp);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002934 if (ret < 0) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002935 pr_err("Unable to extract srpt_tpg_store_enable\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +00002936 return -EINVAL;
2937 }
2938
2939 if ((tmp != 0) && (tmp != 1)) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002940 pr_err("Illegal value for srpt_tpg_store_enable: %lu\n", tmp);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002941 return -EINVAL;
2942 }
Bart Van Assche043a6802016-02-11 11:09:28 -08002943 if (sport->enabled == tmp)
2944 goto out;
2945 sport->enabled = tmp;
2946 if (sport->enabled)
2947 goto out;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002948
Bart Van Assche043a6802016-02-11 11:09:28 -08002949 mutex_lock(&sdev->mutex);
2950 list_for_each_entry(ch, &sdev->rch_list, list) {
2951 if (ch->sport == sport) {
2952 pr_debug("%s: ch %p %s-%d\n", __func__, ch,
2953 ch->sess_name, ch->qp->qp_num);
2954 srpt_disconnect_ch(ch);
2955 srpt_close_ch(ch);
2956 }
2957 }
2958 mutex_unlock(&sdev->mutex);
2959
2960out:
Bart Van Asschea42d9852011-10-14 01:30:46 +00002961 return count;
2962}
2963
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002964CONFIGFS_ATTR(srpt_tpg_, enable);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002965
2966static struct configfs_attribute *srpt_tpg_attrs[] = {
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002967 &srpt_tpg_attr_enable,
Bart Van Asschea42d9852011-10-14 01:30:46 +00002968 NULL,
2969};
2970
2971/**
2972 * configfs callback invoked for
2973 * mkdir /sys/kernel/config/target/$driver/$port/$tpg
2974 */
2975static struct se_portal_group *srpt_make_tpg(struct se_wwn *wwn,
2976 struct config_group *group,
2977 const char *name)
2978{
2979 struct srpt_port *sport = container_of(wwn, struct srpt_port, port_wwn);
2980 int res;
2981
2982 /* Initialize sport->port_wwn and sport->port_tpg_1 */
Nicholas Bellingerbc0c94b2015-05-20 21:48:03 -07002983 res = core_tpg_register(&sport->port_wwn, &sport->port_tpg_1, SCSI_PROTOCOL_SRP);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002984 if (res)
2985 return ERR_PTR(res);
2986
2987 return &sport->port_tpg_1;
2988}
2989
2990/**
2991 * configfs callback invoked for
2992 * rmdir /sys/kernel/config/target/$driver/$port/$tpg
2993 */
2994static void srpt_drop_tpg(struct se_portal_group *tpg)
2995{
2996 struct srpt_port *sport = container_of(tpg,
2997 struct srpt_port, port_tpg_1);
2998
2999 sport->enabled = false;
3000 core_tpg_deregister(&sport->port_tpg_1);
3001}
3002
3003/**
3004 * configfs callback invoked for
3005 * mkdir /sys/kernel/config/target/$driver/$port
3006 */
3007static struct se_wwn *srpt_make_tport(struct target_fabric_configfs *tf,
3008 struct config_group *group,
3009 const char *name)
3010{
3011 struct srpt_port *sport;
3012 int ret;
3013
3014 sport = srpt_lookup_port(name);
3015 pr_debug("make_tport(%s)\n", name);
3016 ret = -EINVAL;
3017 if (!sport)
3018 goto err;
3019
3020 return &sport->port_wwn;
3021
3022err:
3023 return ERR_PTR(ret);
3024}
3025
3026/**
3027 * configfs callback invoked for
3028 * rmdir /sys/kernel/config/target/$driver/$port
3029 */
3030static void srpt_drop_tport(struct se_wwn *wwn)
3031{
3032 struct srpt_port *sport = container_of(wwn, struct srpt_port, port_wwn);
3033
3034 pr_debug("drop_tport(%s\n", config_item_name(&sport->port_wwn.wwn_group.cg_item));
3035}
3036
Christoph Hellwig2eafd722015-10-03 15:32:55 +02003037static ssize_t srpt_wwn_version_show(struct config_item *item, char *buf)
Bart Van Asschea42d9852011-10-14 01:30:46 +00003038{
3039 return scnprintf(buf, PAGE_SIZE, "%s\n", DRV_VERSION);
3040}
3041
Christoph Hellwig2eafd722015-10-03 15:32:55 +02003042CONFIGFS_ATTR_RO(srpt_wwn_, version);
Bart Van Asschea42d9852011-10-14 01:30:46 +00003043
3044static struct configfs_attribute *srpt_wwn_attrs[] = {
Christoph Hellwig2eafd722015-10-03 15:32:55 +02003045 &srpt_wwn_attr_version,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003046 NULL,
3047};
3048
Christoph Hellwig9ac89282015-04-08 20:01:35 +02003049static const struct target_core_fabric_ops srpt_template = {
3050 .module = THIS_MODULE,
3051 .name = "srpt",
Bart Van Asschea42d9852011-10-14 01:30:46 +00003052 .get_fabric_name = srpt_get_fabric_name,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003053 .tpg_get_wwn = srpt_get_fabric_wwn,
3054 .tpg_get_tag = srpt_get_tag,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003055 .tpg_check_demo_mode = srpt_check_false,
3056 .tpg_check_demo_mode_cache = srpt_check_true,
3057 .tpg_check_demo_mode_write_protect = srpt_check_true,
3058 .tpg_check_prod_mode_write_protect = srpt_check_false,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003059 .tpg_get_inst_index = srpt_tpg_get_inst_index,
3060 .release_cmd = srpt_release_cmd,
3061 .check_stop_free = srpt_check_stop_free,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003062 .close_session = srpt_close_session,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003063 .sess_get_index = srpt_sess_get_index,
3064 .sess_get_initiator_sid = NULL,
3065 .write_pending = srpt_write_pending,
3066 .write_pending_status = srpt_write_pending_status,
3067 .set_default_node_attributes = srpt_set_default_node_attrs,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003068 .get_cmd_state = srpt_get_tcm_cmd_state,
Joern Engelb79fafa2013-07-03 11:22:17 -04003069 .queue_data_in = srpt_queue_data_in,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003070 .queue_status = srpt_queue_status,
Joern Engelb79fafa2013-07-03 11:22:17 -04003071 .queue_tm_rsp = srpt_queue_tm_rsp,
Nicholas Bellinger131e6ab2014-03-22 14:55:56 -07003072 .aborted_task = srpt_aborted_task,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003073 /*
3074 * Setup function pointers for generic logic in
3075 * target_core_fabric_configfs.c
3076 */
3077 .fabric_make_wwn = srpt_make_tport,
3078 .fabric_drop_wwn = srpt_drop_tport,
3079 .fabric_make_tpg = srpt_make_tpg,
3080 .fabric_drop_tpg = srpt_drop_tpg,
Christoph Hellwigc7d6a802015-04-13 19:51:14 +02003081 .fabric_init_nodeacl = srpt_init_nodeacl,
Christoph Hellwig9ac89282015-04-08 20:01:35 +02003082
3083 .tfc_wwn_attrs = srpt_wwn_attrs,
3084 .tfc_tpg_base_attrs = srpt_tpg_attrs,
3085 .tfc_tpg_attrib_attrs = srpt_tpg_attrib_attrs,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003086};
3087
3088/**
3089 * srpt_init_module() - Kernel module initialization.
3090 *
3091 * Note: Since ib_register_client() registers callback functions, and since at
3092 * least one of these callback functions (srpt_add_one()) calls target core
3093 * functions, this driver must be registered with the target core before
3094 * ib_register_client() is called.
3095 */
3096static int __init srpt_init_module(void)
3097{
3098 int ret;
3099
3100 ret = -EINVAL;
3101 if (srp_max_req_size < MIN_MAX_REQ_SIZE) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04003102 pr_err("invalid value %d for kernel module parameter"
Bart Van Asschea42d9852011-10-14 01:30:46 +00003103 " srp_max_req_size -- must be at least %d.\n",
3104 srp_max_req_size, MIN_MAX_REQ_SIZE);
3105 goto out;
3106 }
3107
3108 if (srpt_srq_size < MIN_SRPT_SRQ_SIZE
3109 || srpt_srq_size > MAX_SRPT_SRQ_SIZE) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04003110 pr_err("invalid value %d for kernel module parameter"
Bart Van Asschea42d9852011-10-14 01:30:46 +00003111 " srpt_srq_size -- must be in the range [%d..%d].\n",
3112 srpt_srq_size, MIN_SRPT_SRQ_SIZE, MAX_SRPT_SRQ_SIZE);
3113 goto out;
3114 }
3115
Christoph Hellwig9ac89282015-04-08 20:01:35 +02003116 ret = target_register_template(&srpt_template);
3117 if (ret)
Bart Van Asschea42d9852011-10-14 01:30:46 +00003118 goto out;
Bart Van Asschea42d9852011-10-14 01:30:46 +00003119
3120 ret = ib_register_client(&srpt_client);
3121 if (ret) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04003122 pr_err("couldn't register IB client\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +00003123 goto out_unregister_target;
3124 }
3125
3126 return 0;
3127
3128out_unregister_target:
Christoph Hellwig9ac89282015-04-08 20:01:35 +02003129 target_unregister_template(&srpt_template);
Bart Van Asschea42d9852011-10-14 01:30:46 +00003130out:
3131 return ret;
3132}
3133
3134static void __exit srpt_cleanup_module(void)
3135{
3136 ib_unregister_client(&srpt_client);
Christoph Hellwig9ac89282015-04-08 20:01:35 +02003137 target_unregister_template(&srpt_template);
Bart Van Asschea42d9852011-10-14 01:30:46 +00003138}
3139
3140module_init(srpt_init_module);
3141module_exit(srpt_cleanup_module);