blob: 876f438aa048995fec62db5a29a7af01be1302c7 [file] [log] [blame]
Bart Van Asschea42d9852011-10-14 01:30:46 +00001/*
2 * Copyright (c) 2006 - 2009 Mellanox Technology Inc. All rights reserved.
3 * Copyright (C) 2008 - 2011 Bart Van Assche <bvanassche@acm.org>.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 *
33 */
34
35#include <linux/module.h>
36#include <linux/init.h>
37#include <linux/slab.h>
38#include <linux/err.h>
39#include <linux/ctype.h>
40#include <linux/kthread.h>
41#include <linux/string.h>
42#include <linux/delay.h>
43#include <linux/atomic.h>
Bart Van Asscheba929992015-05-08 10:11:12 +020044#include <scsi/scsi_proto.h>
Bart Van Asschea42d9852011-10-14 01:30:46 +000045#include <scsi/scsi_tcq.h>
Bart Van Asschea42d9852011-10-14 01:30:46 +000046#include <target/target_core_base.h>
Bart Van Asschea42d9852011-10-14 01:30:46 +000047#include <target/target_core_fabric.h>
Bart Van Asschea42d9852011-10-14 01:30:46 +000048#include "ib_srpt.h"
49
50/* Name of this kernel module. */
51#define DRV_NAME "ib_srpt"
52#define DRV_VERSION "2.0.0"
53#define DRV_RELDATE "2011-02-14"
54
55#define SRPT_ID_STRING "Linux SRP target"
56
57#undef pr_fmt
58#define pr_fmt(fmt) DRV_NAME " " fmt
59
60MODULE_AUTHOR("Vu Pham and Bart Van Assche");
61MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
62 "v" DRV_VERSION " (" DRV_RELDATE ")");
63MODULE_LICENSE("Dual BSD/GPL");
64
65/*
66 * Global Variables
67 */
68
69static u64 srpt_service_guid;
Roland Dreier486d8b92012-02-02 12:55:58 -080070static DEFINE_SPINLOCK(srpt_dev_lock); /* Protects srpt_dev_list. */
71static LIST_HEAD(srpt_dev_list); /* List of srpt_device structures. */
Bart Van Asschea42d9852011-10-14 01:30:46 +000072
73static unsigned srp_max_req_size = DEFAULT_MAX_REQ_SIZE;
74module_param(srp_max_req_size, int, 0444);
75MODULE_PARM_DESC(srp_max_req_size,
76 "Maximum size of SRP request messages in bytes.");
77
78static int srpt_srq_size = DEFAULT_SRPT_SRQ_SIZE;
79module_param(srpt_srq_size, int, 0444);
80MODULE_PARM_DESC(srpt_srq_size,
81 "Shared receive queue (SRQ) size.");
82
83static int srpt_get_u64_x(char *buffer, struct kernel_param *kp)
84{
85 return sprintf(buffer, "0x%016llx", *(u64 *)kp->arg);
86}
87module_param_call(srpt_service_guid, NULL, srpt_get_u64_x, &srpt_service_guid,
88 0444);
89MODULE_PARM_DESC(srpt_service_guid,
90 "Using this value for ioc_guid, id_ext, and cm_listen_id"
91 " instead of using the node_guid of the first HCA.");
92
93static struct ib_client srpt_client;
Bart Van Assche2c7f37f2016-02-11 11:06:55 -080094static void srpt_release_cmd(struct se_cmd *se_cmd);
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -080095static void srpt_free_ch(struct kref *kref);
Bart Van Asschea42d9852011-10-14 01:30:46 +000096static int srpt_queue_status(struct se_cmd *cmd);
Christoph Hellwig59fae4d2015-09-29 13:00:44 +020097static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc);
98static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc);
Bart Van Assche387add42016-02-11 11:10:09 -080099static void srpt_process_wait_list(struct srpt_rdma_ch *ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000100
Bart Van Asschef130c222016-02-11 11:05:38 -0800101/*
102 * The only allowed channel state changes are those that change the channel
103 * state into a state with a higher numerical value. Hence the new > prev test.
Bart Van Asschea42d9852011-10-14 01:30:46 +0000104 */
Bart Van Asschef130c222016-02-11 11:05:38 -0800105static bool srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new)
Bart Van Asschea42d9852011-10-14 01:30:46 +0000106{
107 unsigned long flags;
108 enum rdma_ch_state prev;
Bart Van Asschef130c222016-02-11 11:05:38 -0800109 bool changed = false;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000110
111 spin_lock_irqsave(&ch->spinlock, flags);
112 prev = ch->state;
Bart Van Asschef130c222016-02-11 11:05:38 -0800113 if (new > prev) {
Bart Van Asschea42d9852011-10-14 01:30:46 +0000114 ch->state = new;
Bart Van Asschef130c222016-02-11 11:05:38 -0800115 changed = true;
116 }
Bart Van Asschea42d9852011-10-14 01:30:46 +0000117 spin_unlock_irqrestore(&ch->spinlock, flags);
Bart Van Asschef130c222016-02-11 11:05:38 -0800118
119 return changed;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000120}
121
122/**
123 * srpt_event_handler() - Asynchronous IB event callback function.
124 *
125 * Callback function called by the InfiniBand core when an asynchronous IB
126 * event occurs. This callback may occur in interrupt context. See also
127 * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
128 * Architecture Specification.
129 */
130static void srpt_event_handler(struct ib_event_handler *handler,
131 struct ib_event *event)
132{
133 struct srpt_device *sdev;
134 struct srpt_port *sport;
135
136 sdev = ib_get_client_data(event->device, &srpt_client);
137 if (!sdev || sdev->device != event->device)
138 return;
139
140 pr_debug("ASYNC event= %d on device= %s\n", event->event,
Bart Van Asschef68cba4e92016-02-11 11:04:20 -0800141 sdev->device->name);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000142
143 switch (event->event) {
144 case IB_EVENT_PORT_ERR:
145 if (event->element.port_num <= sdev->device->phys_port_cnt) {
146 sport = &sdev->port[event->element.port_num - 1];
147 sport->lid = 0;
148 sport->sm_lid = 0;
149 }
150 break;
151 case IB_EVENT_PORT_ACTIVE:
152 case IB_EVENT_LID_CHANGE:
153 case IB_EVENT_PKEY_CHANGE:
154 case IB_EVENT_SM_CHANGE:
155 case IB_EVENT_CLIENT_REREGISTER:
Doug Ledford2aa1cf62014-08-12 19:20:10 -0400156 case IB_EVENT_GID_CHANGE:
Bart Van Asschea42d9852011-10-14 01:30:46 +0000157 /* Refresh port data asynchronously. */
158 if (event->element.port_num <= sdev->device->phys_port_cnt) {
159 sport = &sdev->port[event->element.port_num - 1];
160 if (!sport->lid && !sport->sm_lid)
161 schedule_work(&sport->work);
162 }
163 break;
164 default:
Doug Ledford9f5d32a2014-10-20 18:25:15 -0400165 pr_err("received unrecognized IB event %d\n",
Bart Van Asschea42d9852011-10-14 01:30:46 +0000166 event->event);
167 break;
168 }
169}
170
171/**
172 * srpt_srq_event() - SRQ event callback function.
173 */
174static void srpt_srq_event(struct ib_event *event, void *ctx)
175{
Doug Ledford9f5d32a2014-10-20 18:25:15 -0400176 pr_info("SRQ event %d\n", event->event);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000177}
178
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -0800179static const char *get_ch_state_name(enum rdma_ch_state s)
180{
181 switch (s) {
182 case CH_CONNECTING:
183 return "connecting";
184 case CH_LIVE:
185 return "live";
186 case CH_DISCONNECTING:
187 return "disconnecting";
188 case CH_DRAINING:
189 return "draining";
190 case CH_DISCONNECTED:
191 return "disconnected";
192 }
193 return "???";
194}
195
Bart Van Asschea42d9852011-10-14 01:30:46 +0000196/**
197 * srpt_qp_event() - QP event callback function.
198 */
199static void srpt_qp_event(struct ib_event *event, struct srpt_rdma_ch *ch)
200{
201 pr_debug("QP event %d on cm_id=%p sess_name=%s state=%d\n",
Bart Van Assche33912d72016-02-11 11:04:43 -0800202 event->event, ch->cm_id, ch->sess_name, ch->state);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000203
204 switch (event->event) {
205 case IB_EVENT_COMM_EST:
206 ib_cm_notify(ch->cm_id, event->event);
207 break;
208 case IB_EVENT_QP_LAST_WQE_REACHED:
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -0800209 pr_debug("%s-%d, state %s: received Last WQE event.\n",
210 ch->sess_name, ch->qp->qp_num,
211 get_ch_state_name(ch->state));
Bart Van Asschea42d9852011-10-14 01:30:46 +0000212 break;
213 default:
Doug Ledford9f5d32a2014-10-20 18:25:15 -0400214 pr_err("received unrecognized IB QP event %d\n", event->event);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000215 break;
216 }
217}
218
219/**
220 * srpt_set_ioc() - Helper function for initializing an IOUnitInfo structure.
221 *
222 * @slot: one-based slot number.
223 * @value: four-bit value.
224 *
225 * Copies the lowest four bits of value in element slot of the array of four
226 * bit elements called c_list (controller list). The index slot is one-based.
227 */
228static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
229{
230 u16 id;
231 u8 tmp;
232
233 id = (slot - 1) / 2;
234 if (slot & 0x1) {
235 tmp = c_list[id] & 0xf;
236 c_list[id] = (value << 4) | tmp;
237 } else {
238 tmp = c_list[id] & 0xf0;
239 c_list[id] = (value & 0xf) | tmp;
240 }
241}
242
243/**
244 * srpt_get_class_port_info() - Copy ClassPortInfo to a management datagram.
245 *
246 * See also section 16.3.3.1 ClassPortInfo in the InfiniBand Architecture
247 * Specification.
248 */
249static void srpt_get_class_port_info(struct ib_dm_mad *mad)
250{
251 struct ib_class_port_info *cif;
252
253 cif = (struct ib_class_port_info *)mad->data;
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -0800254 memset(cif, 0, sizeof(*cif));
Bart Van Asschea42d9852011-10-14 01:30:46 +0000255 cif->base_version = 1;
256 cif->class_version = 1;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000257
Erez Shitrit507f6af2016-05-25 22:02:04 +0300258 ib_set_cpi_resp_time(cif, 20);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000259 mad->mad_hdr.status = 0;
260}
261
262/**
263 * srpt_get_iou() - Write IOUnitInfo to a management datagram.
264 *
265 * See also section 16.3.3.3 IOUnitInfo in the InfiniBand Architecture
266 * Specification. See also section B.7, table B.6 in the SRP r16a document.
267 */
268static void srpt_get_iou(struct ib_dm_mad *mad)
269{
270 struct ib_dm_iou_info *ioui;
271 u8 slot;
272 int i;
273
274 ioui = (struct ib_dm_iou_info *)mad->data;
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530275 ioui->change_id = cpu_to_be16(1);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000276 ioui->max_controllers = 16;
277
278 /* set present for slot 1 and empty for the rest */
279 srpt_set_ioc(ioui->controller_list, 1, 1);
280 for (i = 1, slot = 2; i < 16; i++, slot++)
281 srpt_set_ioc(ioui->controller_list, slot, 0);
282
283 mad->mad_hdr.status = 0;
284}
285
286/**
287 * srpt_get_ioc() - Write IOControllerprofile to a management datagram.
288 *
289 * See also section 16.3.3.4 IOControllerProfile in the InfiniBand
290 * Architecture Specification. See also section B.7, table B.7 in the SRP
291 * r16a document.
292 */
293static void srpt_get_ioc(struct srpt_port *sport, u32 slot,
294 struct ib_dm_mad *mad)
295{
296 struct srpt_device *sdev = sport->sdev;
297 struct ib_dm_ioc_profile *iocp;
298
299 iocp = (struct ib_dm_ioc_profile *)mad->data;
300
301 if (!slot || slot > 16) {
302 mad->mad_hdr.status
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530303 = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000304 return;
305 }
306
307 if (slot > 2) {
308 mad->mad_hdr.status
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530309 = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000310 return;
311 }
312
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -0800313 memset(iocp, 0, sizeof(*iocp));
Bart Van Asschea42d9852011-10-14 01:30:46 +0000314 strcpy(iocp->id_string, SRPT_ID_STRING);
315 iocp->guid = cpu_to_be64(srpt_service_guid);
Or Gerlitz4a061b22015-12-18 10:59:46 +0200316 iocp->vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
317 iocp->device_id = cpu_to_be32(sdev->device->attrs.vendor_part_id);
318 iocp->device_version = cpu_to_be16(sdev->device->attrs.hw_ver);
319 iocp->subsys_vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000320 iocp->subsys_device_id = 0x0;
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530321 iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
322 iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
323 iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
324 iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000325 iocp->send_queue_depth = cpu_to_be16(sdev->srq_size);
326 iocp->rdma_read_depth = 4;
327 iocp->send_size = cpu_to_be32(srp_max_req_size);
328 iocp->rdma_size = cpu_to_be32(min(sport->port_attrib.srp_max_rdma_size,
329 1U << 24));
330 iocp->num_svc_entries = 1;
331 iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
332 SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
333
334 mad->mad_hdr.status = 0;
335}
336
337/**
338 * srpt_get_svc_entries() - Write ServiceEntries to a management datagram.
339 *
340 * See also section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
341 * Specification. See also section B.7, table B.8 in the SRP r16a document.
342 */
343static void srpt_get_svc_entries(u64 ioc_guid,
344 u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
345{
346 struct ib_dm_svc_entries *svc_entries;
347
348 WARN_ON(!ioc_guid);
349
350 if (!slot || slot > 16) {
351 mad->mad_hdr.status
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530352 = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000353 return;
354 }
355
356 if (slot > 2 || lo > hi || hi > 1) {
357 mad->mad_hdr.status
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530358 = cpu_to_be16(DM_MAD_STATUS_NO_IOC);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000359 return;
360 }
361
362 svc_entries = (struct ib_dm_svc_entries *)mad->data;
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -0800363 memset(svc_entries, 0, sizeof(*svc_entries));
Bart Van Asschea42d9852011-10-14 01:30:46 +0000364 svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
365 snprintf(svc_entries->service_entries[0].name,
366 sizeof(svc_entries->service_entries[0].name),
367 "%s%016llx",
368 SRP_SERVICE_NAME_PREFIX,
369 ioc_guid);
370
371 mad->mad_hdr.status = 0;
372}
373
374/**
375 * srpt_mgmt_method_get() - Process a received management datagram.
376 * @sp: source port through which the MAD has been received.
377 * @rq_mad: received MAD.
378 * @rsp_mad: response MAD.
379 */
380static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
381 struct ib_dm_mad *rsp_mad)
382{
383 u16 attr_id;
384 u32 slot;
385 u8 hi, lo;
386
387 attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
388 switch (attr_id) {
389 case DM_ATTR_CLASS_PORT_INFO:
390 srpt_get_class_port_info(rsp_mad);
391 break;
392 case DM_ATTR_IOU_INFO:
393 srpt_get_iou(rsp_mad);
394 break;
395 case DM_ATTR_IOC_PROFILE:
396 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
397 srpt_get_ioc(sp, slot, rsp_mad);
398 break;
399 case DM_ATTR_SVC_ENTRIES:
400 slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
401 hi = (u8) ((slot >> 8) & 0xff);
402 lo = (u8) (slot & 0xff);
403 slot = (u16) ((slot >> 16) & 0xffff);
404 srpt_get_svc_entries(srpt_service_guid,
405 slot, hi, lo, rsp_mad);
406 break;
407 default:
408 rsp_mad->mad_hdr.status =
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530409 cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000410 break;
411 }
412}
413
414/**
415 * srpt_mad_send_handler() - Post MAD-send callback function.
416 */
417static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
418 struct ib_mad_send_wc *mad_wc)
419{
420 ib_destroy_ah(mad_wc->send_buf->ah);
421 ib_free_send_mad(mad_wc->send_buf);
422}
423
424/**
425 * srpt_mad_recv_handler() - MAD reception callback function.
426 */
427static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
Christoph Hellwigca281262016-01-04 14:15:58 +0100428 struct ib_mad_send_buf *send_buf,
Bart Van Asschea42d9852011-10-14 01:30:46 +0000429 struct ib_mad_recv_wc *mad_wc)
430{
431 struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
432 struct ib_ah *ah;
433 struct ib_mad_send_buf *rsp;
434 struct ib_dm_mad *dm_mad;
435
436 if (!mad_wc || !mad_wc->recv_buf.mad)
437 return;
438
439 ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
440 mad_wc->recv_buf.grh, mad_agent->port_num);
441 if (IS_ERR(ah))
442 goto err;
443
444 BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
445
446 rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
447 mad_wc->wc->pkey_index, 0,
448 IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
Ira Weinyda2dfaa2015-06-06 14:38:28 -0400449 GFP_KERNEL,
450 IB_MGMT_BASE_VERSION);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000451 if (IS_ERR(rsp))
452 goto err_rsp;
453
454 rsp->ah = ah;
455
456 dm_mad = rsp->mad;
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -0800457 memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof(*dm_mad));
Bart Van Asschea42d9852011-10-14 01:30:46 +0000458 dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
459 dm_mad->mad_hdr.status = 0;
460
461 switch (mad_wc->recv_buf.mad->mad_hdr.method) {
462 case IB_MGMT_METHOD_GET:
463 srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
464 break;
465 case IB_MGMT_METHOD_SET:
466 dm_mad->mad_hdr.status =
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530467 cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000468 break;
469 default:
470 dm_mad->mad_hdr.status =
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +0530471 cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000472 break;
473 }
474
475 if (!ib_post_send_mad(rsp, NULL)) {
476 ib_free_recv_mad(mad_wc);
477 /* will destroy_ah & free_send_mad in send completion */
478 return;
479 }
480
481 ib_free_send_mad(rsp);
482
483err_rsp:
484 ib_destroy_ah(ah);
485err:
486 ib_free_recv_mad(mad_wc);
487}
488
489/**
490 * srpt_refresh_port() - Configure a HCA port.
491 *
492 * Enable InfiniBand management datagram processing, update the cached sm_lid,
493 * lid and gid values, and register a callback function for processing MADs
494 * on the specified port.
495 *
496 * Note: It is safe to call this function more than once for the same port.
497 */
498static int srpt_refresh_port(struct srpt_port *sport)
499{
500 struct ib_mad_reg_req reg_req;
501 struct ib_port_modify port_modify;
502 struct ib_port_attr port_attr;
503 int ret;
504
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -0800505 memset(&port_modify, 0, sizeof(port_modify));
Bart Van Asschea42d9852011-10-14 01:30:46 +0000506 port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
507 port_modify.clr_port_cap_mask = 0;
508
509 ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
510 if (ret)
511 goto err_mod_port;
512
513 ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
514 if (ret)
515 goto err_query_port;
516
517 sport->sm_lid = port_attr.sm_lid;
518 sport->lid = port_attr.lid;
519
Matan Barak55ee3ab2015-10-15 18:38:45 +0300520 ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid,
521 NULL);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000522 if (ret)
523 goto err_query_port;
524
Doug Ledford716b076b2016-08-24 12:14:19 -0400525 snprintf(sport->port_guid, sizeof(sport->port_guid),
526 "0x%016llx%016llx",
527 be64_to_cpu(sport->gid.global.subnet_prefix),
528 be64_to_cpu(sport->gid.global.interface_id));
529
Bart Van Asschea42d9852011-10-14 01:30:46 +0000530 if (!sport->mad_agent) {
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -0800531 memset(&reg_req, 0, sizeof(reg_req));
Bart Van Asschea42d9852011-10-14 01:30:46 +0000532 reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
533 reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
534 set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
535 set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
536
537 sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
538 sport->port,
539 IB_QPT_GSI,
540 &reg_req, 0,
541 srpt_mad_send_handler,
542 srpt_mad_recv_handler,
Ira Weiny0f29b462014-08-08 19:00:55 -0400543 sport, 0);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000544 if (IS_ERR(sport->mad_agent)) {
545 ret = PTR_ERR(sport->mad_agent);
546 sport->mad_agent = NULL;
547 goto err_query_port;
548 }
549 }
550
551 return 0;
552
553err_query_port:
554
555 port_modify.set_port_cap_mask = 0;
556 port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
557 ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
558
559err_mod_port:
560
561 return ret;
562}
563
564/**
565 * srpt_unregister_mad_agent() - Unregister MAD callback functions.
566 *
567 * Note: It is safe to call this function more than once for the same device.
568 */
569static void srpt_unregister_mad_agent(struct srpt_device *sdev)
570{
571 struct ib_port_modify port_modify = {
572 .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
573 };
574 struct srpt_port *sport;
575 int i;
576
577 for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
578 sport = &sdev->port[i - 1];
579 WARN_ON(sport->port != i);
580 if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
Doug Ledford9f5d32a2014-10-20 18:25:15 -0400581 pr_err("disabling MAD processing failed.\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +0000582 if (sport->mad_agent) {
583 ib_unregister_mad_agent(sport->mad_agent);
584 sport->mad_agent = NULL;
585 }
586 }
587}
588
589/**
590 * srpt_alloc_ioctx() - Allocate an SRPT I/O context structure.
591 */
592static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev,
593 int ioctx_size, int dma_size,
594 enum dma_data_direction dir)
595{
596 struct srpt_ioctx *ioctx;
597
598 ioctx = kmalloc(ioctx_size, GFP_KERNEL);
599 if (!ioctx)
600 goto err;
601
602 ioctx->buf = kmalloc(dma_size, GFP_KERNEL);
603 if (!ioctx->buf)
604 goto err_free_ioctx;
605
606 ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf, dma_size, dir);
607 if (ib_dma_mapping_error(sdev->device, ioctx->dma))
608 goto err_free_buf;
609
610 return ioctx;
611
612err_free_buf:
613 kfree(ioctx->buf);
614err_free_ioctx:
615 kfree(ioctx);
616err:
617 return NULL;
618}
619
620/**
621 * srpt_free_ioctx() - Free an SRPT I/O context structure.
622 */
623static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx,
624 int dma_size, enum dma_data_direction dir)
625{
626 if (!ioctx)
627 return;
628
629 ib_dma_unmap_single(sdev->device, ioctx->dma, dma_size, dir);
630 kfree(ioctx->buf);
631 kfree(ioctx);
632}
633
634/**
635 * srpt_alloc_ioctx_ring() - Allocate a ring of SRPT I/O context structures.
636 * @sdev: Device to allocate the I/O context ring for.
637 * @ring_size: Number of elements in the I/O context ring.
638 * @ioctx_size: I/O context size.
639 * @dma_size: DMA buffer size.
640 * @dir: DMA data direction.
641 */
642static struct srpt_ioctx **srpt_alloc_ioctx_ring(struct srpt_device *sdev,
643 int ring_size, int ioctx_size,
644 int dma_size, enum dma_data_direction dir)
645{
646 struct srpt_ioctx **ring;
647 int i;
648
649 WARN_ON(ioctx_size != sizeof(struct srpt_recv_ioctx)
650 && ioctx_size != sizeof(struct srpt_send_ioctx));
651
652 ring = kmalloc(ring_size * sizeof(ring[0]), GFP_KERNEL);
653 if (!ring)
654 goto out;
655 for (i = 0; i < ring_size; ++i) {
656 ring[i] = srpt_alloc_ioctx(sdev, ioctx_size, dma_size, dir);
657 if (!ring[i])
658 goto err;
659 ring[i]->index = i;
660 }
661 goto out;
662
663err:
664 while (--i >= 0)
665 srpt_free_ioctx(sdev, ring[i], dma_size, dir);
666 kfree(ring);
Jesper Juhl715252d2012-02-04 23:49:40 +0100667 ring = NULL;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000668out:
669 return ring;
670}
671
672/**
673 * srpt_free_ioctx_ring() - Free the ring of SRPT I/O context structures.
674 */
675static void srpt_free_ioctx_ring(struct srpt_ioctx **ioctx_ring,
676 struct srpt_device *sdev, int ring_size,
677 int dma_size, enum dma_data_direction dir)
678{
679 int i;
680
681 for (i = 0; i < ring_size; ++i)
682 srpt_free_ioctx(sdev, ioctx_ring[i], dma_size, dir);
683 kfree(ioctx_ring);
684}
685
686/**
687 * srpt_get_cmd_state() - Get the state of a SCSI command.
688 */
689static enum srpt_command_state srpt_get_cmd_state(struct srpt_send_ioctx *ioctx)
690{
691 enum srpt_command_state state;
692 unsigned long flags;
693
694 BUG_ON(!ioctx);
695
696 spin_lock_irqsave(&ioctx->spinlock, flags);
697 state = ioctx->state;
698 spin_unlock_irqrestore(&ioctx->spinlock, flags);
699 return state;
700}
701
702/**
703 * srpt_set_cmd_state() - Set the state of a SCSI command.
704 *
705 * Does not modify the state of aborted commands. Returns the previous command
706 * state.
707 */
708static enum srpt_command_state srpt_set_cmd_state(struct srpt_send_ioctx *ioctx,
709 enum srpt_command_state new)
710{
711 enum srpt_command_state previous;
712 unsigned long flags;
713
714 BUG_ON(!ioctx);
715
716 spin_lock_irqsave(&ioctx->spinlock, flags);
717 previous = ioctx->state;
718 if (previous != SRPT_STATE_DONE)
719 ioctx->state = new;
720 spin_unlock_irqrestore(&ioctx->spinlock, flags);
721
722 return previous;
723}
724
725/**
726 * srpt_test_and_set_cmd_state() - Test and set the state of a command.
727 *
728 * Returns true if and only if the previous command state was equal to 'old'.
729 */
730static bool srpt_test_and_set_cmd_state(struct srpt_send_ioctx *ioctx,
731 enum srpt_command_state old,
732 enum srpt_command_state new)
733{
734 enum srpt_command_state previous;
735 unsigned long flags;
736
737 WARN_ON(!ioctx);
738 WARN_ON(old == SRPT_STATE_DONE);
739 WARN_ON(new == SRPT_STATE_NEW);
740
741 spin_lock_irqsave(&ioctx->spinlock, flags);
742 previous = ioctx->state;
743 if (previous == old)
744 ioctx->state = new;
745 spin_unlock_irqrestore(&ioctx->spinlock, flags);
746 return previous == old;
747}
748
749/**
750 * srpt_post_recv() - Post an IB receive request.
751 */
752static int srpt_post_recv(struct srpt_device *sdev,
753 struct srpt_recv_ioctx *ioctx)
754{
755 struct ib_sge list;
756 struct ib_recv_wr wr, *bad_wr;
757
758 BUG_ON(!sdev);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000759 list.addr = ioctx->ioctx.dma;
760 list.length = srp_max_req_size;
Jason Gunthorpe5a783952015-07-30 17:22:24 -0600761 list.lkey = sdev->pd->local_dma_lkey;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000762
Christoph Hellwig59fae4d2015-09-29 13:00:44 +0200763 ioctx->ioctx.cqe.done = srpt_recv_done;
764 wr.wr_cqe = &ioctx->ioctx.cqe;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000765 wr.next = NULL;
766 wr.sg_list = &list;
767 wr.num_sge = 1;
768
769 return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
770}
771
772/**
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -0800773 * srpt_zerolength_write() - Perform a zero-length RDMA write.
774 *
775 * A quote from the InfiniBand specification: C9-88: For an HCA responder
776 * using Reliable Connection service, for each zero-length RDMA READ or WRITE
777 * request, the R_Key shall not be validated, even if the request includes
778 * Immediate data.
779 */
780static int srpt_zerolength_write(struct srpt_rdma_ch *ch)
781{
782 struct ib_send_wr wr, *bad_wr;
783
784 memset(&wr, 0, sizeof(wr));
785 wr.opcode = IB_WR_RDMA_WRITE;
786 wr.wr_cqe = &ch->zw_cqe;
787 wr.send_flags = IB_SEND_SIGNALED;
788 return ib_post_send(ch->qp, &wr, &bad_wr);
789}
790
791static void srpt_zerolength_write_done(struct ib_cq *cq, struct ib_wc *wc)
792{
793 struct srpt_rdma_ch *ch = cq->cq_context;
794
Bart Van Assche387add42016-02-11 11:10:09 -0800795 if (wc->status == IB_WC_SUCCESS) {
796 srpt_process_wait_list(ch);
797 } else {
798 if (srpt_set_ch_state(ch, CH_DISCONNECTED))
799 schedule_work(&ch->release_work);
800 else
Dan Carpenter56586002016-03-18 08:41:59 +0300801 WARN_ONCE(1, "%s-%d\n", ch->sess_name, ch->qp->qp_num);
Bart Van Assche387add42016-02-11 11:10:09 -0800802 }
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -0800803}
804
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200805static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx,
806 struct srp_direct_buf *db, int nbufs, struct scatterlist **sg,
807 unsigned *sg_cnt)
808{
809 enum dma_data_direction dir = target_reverse_dma_direction(&ioctx->cmd);
810 struct srpt_rdma_ch *ch = ioctx->ch;
811 struct scatterlist *prev = NULL;
812 unsigned prev_nents;
813 int ret, i;
814
815 if (nbufs == 1) {
816 ioctx->rw_ctxs = &ioctx->s_rw_ctx;
817 } else {
818 ioctx->rw_ctxs = kmalloc_array(nbufs, sizeof(*ioctx->rw_ctxs),
819 GFP_KERNEL);
820 if (!ioctx->rw_ctxs)
821 return -ENOMEM;
822 }
823
824 for (i = ioctx->n_rw_ctx; i < nbufs; i++, db++) {
825 struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
826 u64 remote_addr = be64_to_cpu(db->va);
827 u32 size = be32_to_cpu(db->len);
828 u32 rkey = be32_to_cpu(db->key);
829
830 ret = target_alloc_sgl(&ctx->sg, &ctx->nents, size, false,
831 i < nbufs - 1);
832 if (ret)
833 goto unwind;
834
835 ret = rdma_rw_ctx_init(&ctx->rw, ch->qp, ch->sport->port,
836 ctx->sg, ctx->nents, 0, remote_addr, rkey, dir);
837 if (ret < 0) {
838 target_free_sgl(ctx->sg, ctx->nents);
839 goto unwind;
840 }
841
842 ioctx->n_rdma += ret;
843 ioctx->n_rw_ctx++;
844
845 if (prev) {
846 sg_unmark_end(&prev[prev_nents - 1]);
847 sg_chain(prev, prev_nents + 1, ctx->sg);
848 } else {
849 *sg = ctx->sg;
850 }
851
852 prev = ctx->sg;
853 prev_nents = ctx->nents;
854
855 *sg_cnt += ctx->nents;
856 }
857
858 return 0;
859
860unwind:
861 while (--i >= 0) {
862 struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
863
864 rdma_rw_ctx_destroy(&ctx->rw, ch->qp, ch->sport->port,
865 ctx->sg, ctx->nents, dir);
866 target_free_sgl(ctx->sg, ctx->nents);
867 }
868 if (ioctx->rw_ctxs != &ioctx->s_rw_ctx)
869 kfree(ioctx->rw_ctxs);
870 return ret;
871}
872
873static void srpt_free_rw_ctxs(struct srpt_rdma_ch *ch,
874 struct srpt_send_ioctx *ioctx)
875{
876 enum dma_data_direction dir = target_reverse_dma_direction(&ioctx->cmd);
877 int i;
878
879 for (i = 0; i < ioctx->n_rw_ctx; i++) {
880 struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
881
882 rdma_rw_ctx_destroy(&ctx->rw, ch->qp, ch->sport->port,
883 ctx->sg, ctx->nents, dir);
884 target_free_sgl(ctx->sg, ctx->nents);
885 }
886
887 if (ioctx->rw_ctxs != &ioctx->s_rw_ctx)
888 kfree(ioctx->rw_ctxs);
889}
890
891static inline void *srpt_get_desc_buf(struct srp_cmd *srp_cmd)
892{
893 /*
894 * The pointer computations below will only be compiled correctly
895 * if srp_cmd::add_data is declared as s8*, u8*, s8[] or u8[], so check
896 * whether srp_cmd::add_data has been declared as a byte pointer.
897 */
898 BUILD_BUG_ON(!__same_type(srp_cmd->add_data[0], (s8)0) &&
899 !__same_type(srp_cmd->add_data[0], (u8)0));
900
901 /*
902 * According to the SRP spec, the lower two bits of the 'ADDITIONAL
903 * CDB LENGTH' field are reserved and the size in bytes of this field
904 * is four times the value specified in bits 3..7. Hence the "& ~3".
905 */
906 return srp_cmd->add_data + (srp_cmd->add_cdb_len & ~3);
907}
908
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -0800909/**
Bart Van Asschea42d9852011-10-14 01:30:46 +0000910 * srpt_get_desc_tbl() - Parse the data descriptors of an SRP_CMD request.
911 * @ioctx: Pointer to the I/O context associated with the request.
912 * @srp_cmd: Pointer to the SRP_CMD request data.
913 * @dir: Pointer to the variable to which the transfer direction will be
914 * written.
915 * @data_len: Pointer to the variable to which the total data length of all
916 * descriptors in the SRP_CMD request will be written.
917 *
918 * This function initializes ioctx->nrbuf and ioctx->r_bufs.
919 *
920 * Returns -EINVAL when the SRP_CMD request contains inconsistent descriptors;
921 * -ENOMEM when memory allocation fails and zero upon success.
922 */
923static int srpt_get_desc_tbl(struct srpt_send_ioctx *ioctx,
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200924 struct srp_cmd *srp_cmd, enum dma_data_direction *dir,
925 struct scatterlist **sg, unsigned *sg_cnt, u64 *data_len)
Bart Van Asschea42d9852011-10-14 01:30:46 +0000926{
Bart Van Asschea42d9852011-10-14 01:30:46 +0000927 BUG_ON(!dir);
928 BUG_ON(!data_len);
929
Bart Van Asschea42d9852011-10-14 01:30:46 +0000930 /*
931 * The lower four bits of the buffer format field contain the DATA-IN
932 * buffer descriptor format, and the highest four bits contain the
933 * DATA-OUT buffer descriptor format.
934 */
Bart Van Asschea42d9852011-10-14 01:30:46 +0000935 if (srp_cmd->buf_fmt & 0xf)
936 /* DATA-IN: transfer data from target to initiator (read). */
937 *dir = DMA_FROM_DEVICE;
938 else if (srp_cmd->buf_fmt >> 4)
939 /* DATA-OUT: transfer data from initiator to target (write). */
940 *dir = DMA_TO_DEVICE;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200941 else
942 *dir = DMA_NONE;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000943
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200944 /* initialize data_direction early as srpt_alloc_rw_ctxs needs it */
945 ioctx->cmd.data_direction = *dir;
946
Bart Van Asschea42d9852011-10-14 01:30:46 +0000947 if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
948 ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200949 struct srp_direct_buf *db = srpt_get_desc_buf(srp_cmd);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000950
Bart Van Asschea42d9852011-10-14 01:30:46 +0000951 *data_len = be32_to_cpu(db->len);
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200952 return srpt_alloc_rw_ctxs(ioctx, db, 1, sg, sg_cnt);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000953 } else if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_INDIRECT) ||
954 ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_INDIRECT)) {
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200955 struct srp_indirect_buf *idb = srpt_get_desc_buf(srp_cmd);
956 int nbufs = be32_to_cpu(idb->table_desc.len) /
957 sizeof(struct srp_direct_buf);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000958
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200959 if (nbufs >
Bart Van Asschea42d9852011-10-14 01:30:46 +0000960 (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -0400961 pr_err("received unsupported SRP_CMD request"
Bart Van Asschea42d9852011-10-14 01:30:46 +0000962 " type (%u out + %u in != %u / %zu)\n",
963 srp_cmd->data_out_desc_cnt,
964 srp_cmd->data_in_desc_cnt,
965 be32_to_cpu(idb->table_desc.len),
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200966 sizeof(struct srp_direct_buf));
967 return -EINVAL;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000968 }
969
Bart Van Asschea42d9852011-10-14 01:30:46 +0000970 *data_len = be32_to_cpu(idb->len);
Christoph Hellwigb99f8e42016-05-03 18:01:11 +0200971 return srpt_alloc_rw_ctxs(ioctx, idb->desc_list, nbufs,
972 sg, sg_cnt);
973 } else {
974 *data_len = 0;
975 return 0;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000976 }
Bart Van Asschea42d9852011-10-14 01:30:46 +0000977}
978
979/**
980 * srpt_init_ch_qp() - Initialize queue pair attributes.
981 *
982 * Initialized the attributes of queue pair 'qp' by allowing local write,
983 * remote read and remote write. Also transitions 'qp' to state IB_QPS_INIT.
984 */
985static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
986{
987 struct ib_qp_attr *attr;
988 int ret;
989
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -0800990 attr = kzalloc(sizeof(*attr), GFP_KERNEL);
Bart Van Asschea42d9852011-10-14 01:30:46 +0000991 if (!attr)
992 return -ENOMEM;
993
994 attr->qp_state = IB_QPS_INIT;
Bart Van Assche30191712018-01-03 13:39:15 -0800995 attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE;
Bart Van Asschea42d9852011-10-14 01:30:46 +0000996 attr->port_num = ch->sport->port;
997 attr->pkey_index = 0;
998
999 ret = ib_modify_qp(qp, attr,
1000 IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
1001 IB_QP_PKEY_INDEX);
1002
1003 kfree(attr);
1004 return ret;
1005}
1006
1007/**
1008 * srpt_ch_qp_rtr() - Change the state of a channel to 'ready to receive' (RTR).
1009 * @ch: channel of the queue pair.
1010 * @qp: queue pair to change the state of.
1011 *
1012 * Returns zero upon success and a negative value upon failure.
1013 *
1014 * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
1015 * If this structure ever becomes larger, it might be necessary to allocate
1016 * it dynamically instead of on the stack.
1017 */
1018static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1019{
1020 struct ib_qp_attr qp_attr;
1021 int attr_mask;
1022 int ret;
1023
1024 qp_attr.qp_state = IB_QPS_RTR;
1025 ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
1026 if (ret)
1027 goto out;
1028
1029 qp_attr.max_dest_rd_atomic = 4;
1030
1031 ret = ib_modify_qp(qp, &qp_attr, attr_mask);
1032
1033out:
1034 return ret;
1035}
1036
1037/**
1038 * srpt_ch_qp_rts() - Change the state of a channel to 'ready to send' (RTS).
1039 * @ch: channel of the queue pair.
1040 * @qp: queue pair to change the state of.
1041 *
1042 * Returns zero upon success and a negative value upon failure.
1043 *
1044 * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
1045 * If this structure ever becomes larger, it might be necessary to allocate
1046 * it dynamically instead of on the stack.
1047 */
1048static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1049{
1050 struct ib_qp_attr qp_attr;
1051 int attr_mask;
1052 int ret;
1053
1054 qp_attr.qp_state = IB_QPS_RTS;
1055 ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
1056 if (ret)
1057 goto out;
1058
1059 qp_attr.max_rd_atomic = 4;
1060
1061 ret = ib_modify_qp(qp, &qp_attr, attr_mask);
1062
1063out:
1064 return ret;
1065}
1066
1067/**
1068 * srpt_ch_qp_err() - Set the channel queue pair state to 'error'.
1069 */
1070static int srpt_ch_qp_err(struct srpt_rdma_ch *ch)
1071{
1072 struct ib_qp_attr qp_attr;
1073
1074 qp_attr.qp_state = IB_QPS_ERR;
1075 return ib_modify_qp(ch->qp, &qp_attr, IB_QP_STATE);
1076}
1077
1078/**
Bart Van Asschea42d9852011-10-14 01:30:46 +00001079 * srpt_get_send_ioctx() - Obtain an I/O context for sending to the initiator.
1080 */
1081static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch)
1082{
1083 struct srpt_send_ioctx *ioctx;
Bart Van Assche3c968882016-04-07 15:55:04 -07001084 unsigned long flags;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001085
1086 BUG_ON(!ch);
1087
Bart Van Assche3c968882016-04-07 15:55:04 -07001088 ioctx = NULL;
1089 spin_lock_irqsave(&ch->spinlock, flags);
1090 if (!list_empty(&ch->free_list)) {
1091 ioctx = list_first_entry(&ch->free_list,
1092 struct srpt_send_ioctx, free_list);
1093 list_del(&ioctx->free_list);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001094 }
Bart Van Assche3c968882016-04-07 15:55:04 -07001095 spin_unlock_irqrestore(&ch->spinlock, flags);
1096
1097 if (!ioctx)
1098 return ioctx;
1099
1100 BUG_ON(ioctx->ch != ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001101 spin_lock_init(&ioctx->spinlock);
1102 ioctx->state = SRPT_STATE_NEW;
Bart Van Assche3c968882016-04-07 15:55:04 -07001103 ioctx->n_rdma = 0;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001104 ioctx->n_rw_ctx = 0;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001105 init_completion(&ioctx->tx_done);
Bart Van Assche3c968882016-04-07 15:55:04 -07001106 ioctx->queue_status_only = false;
1107 /*
1108 * transport_init_se_cmd() does not initialize all fields, so do it
1109 * here.
1110 */
1111 memset(&ioctx->cmd, 0, sizeof(ioctx->cmd));
1112 memset(&ioctx->sense_data, 0, sizeof(ioctx->sense_data));
Bart Van Asschea42d9852011-10-14 01:30:46 +00001113
1114 return ioctx;
1115}
1116
1117/**
Bart Van Asschea42d9852011-10-14 01:30:46 +00001118 * srpt_abort_cmd() - Abort a SCSI command.
1119 * @ioctx: I/O context associated with the SCSI command.
1120 * @context: Preferred execution context.
1121 */
1122static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx)
1123{
1124 enum srpt_command_state state;
1125 unsigned long flags;
1126
1127 BUG_ON(!ioctx);
1128
1129 /*
1130 * If the command is in a state where the target core is waiting for
Bart Van Assche49f40162016-02-11 11:07:11 -08001131 * the ib_srpt driver, change the state to the next state.
Bart Van Asschea42d9852011-10-14 01:30:46 +00001132 */
1133
1134 spin_lock_irqsave(&ioctx->spinlock, flags);
1135 state = ioctx->state;
1136 switch (state) {
1137 case SRPT_STATE_NEED_DATA:
1138 ioctx->state = SRPT_STATE_DATA_IN;
1139 break;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001140 case SRPT_STATE_CMD_RSP_SENT:
1141 case SRPT_STATE_MGMT_RSP_SENT:
1142 ioctx->state = SRPT_STATE_DONE;
1143 break;
1144 default:
Bart Van Assche49f40162016-02-11 11:07:11 -08001145 WARN_ONCE(true, "%s: unexpected I/O context state %d\n",
1146 __func__, state);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001147 break;
1148 }
1149 spin_unlock_irqrestore(&ioctx->spinlock, flags);
1150
Bart Van Asschea42d9852011-10-14 01:30:46 +00001151 pr_debug("Aborting cmd with state %d and tag %lld\n", state,
Bart Van Assche649ee052015-04-14 13:26:44 +02001152 ioctx->cmd.tag);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001153
1154 switch (state) {
1155 case SRPT_STATE_NEW:
1156 case SRPT_STATE_DATA_IN:
1157 case SRPT_STATE_MGMT:
Bart Van Assche49f40162016-02-11 11:07:11 -08001158 case SRPT_STATE_DONE:
Bart Van Asschea42d9852011-10-14 01:30:46 +00001159 /*
1160 * Do nothing - defer abort processing until
1161 * srpt_queue_response() is invoked.
1162 */
Bart Van Asschea42d9852011-10-14 01:30:46 +00001163 break;
1164 case SRPT_STATE_NEED_DATA:
Bart Van Assche49f40162016-02-11 11:07:11 -08001165 pr_debug("tag %#llx: RDMA read error\n", ioctx->cmd.tag);
1166 transport_generic_request_failure(&ioctx->cmd,
1167 TCM_CHECK_CONDITION_ABORT_CMD);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001168 break;
1169 case SRPT_STATE_CMD_RSP_SENT:
1170 /*
1171 * SRP_RSP sending failed or the SRP_RSP send completion has
1172 * not been received in time.
1173 */
Bart Van Assche49f40162016-02-11 11:07:11 -08001174 transport_generic_free_cmd(&ioctx->cmd, 0);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001175 break;
1176 case SRPT_STATE_MGMT_RSP_SENT:
Bart Van Assche49f40162016-02-11 11:07:11 -08001177 transport_generic_free_cmd(&ioctx->cmd, 0);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001178 break;
1179 default:
Grant Grundler532ec6f2013-03-26 21:48:28 +00001180 WARN(1, "Unexpected command state (%d)", state);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001181 break;
1182 }
1183
Bart Van Asschea42d9852011-10-14 01:30:46 +00001184 return state;
1185}
1186
1187/**
Christoph Hellwige672a472012-07-08 15:58:43 -04001188 * XXX: what is now target_execute_cmd used to be asynchronous, and unmapping
1189 * the data that has been transferred via IB RDMA had to be postponed until the
Masanari Iida142ad5d2012-08-10 00:07:58 +00001190 * check_stop_free() callback. None of this is necessary anymore and needs to
Christoph Hellwige672a472012-07-08 15:58:43 -04001191 * be cleaned up.
Bart Van Asschea42d9852011-10-14 01:30:46 +00001192 */
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001193static void srpt_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001194{
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001195 struct srpt_rdma_ch *ch = cq->cq_context;
1196 struct srpt_send_ioctx *ioctx =
Bart Van Assche19f57292015-12-31 09:56:43 +01001197 container_of(wc->wr_cqe, struct srpt_send_ioctx, rdma_cqe);
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001198
Bart Van Asschea42d9852011-10-14 01:30:46 +00001199 WARN_ON(ioctx->n_rdma <= 0);
1200 atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001201 ioctx->n_rdma = 0;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001202
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001203 if (unlikely(wc->status != IB_WC_SUCCESS)) {
1204 pr_info("RDMA_READ for ioctx 0x%p failed with status %d\n",
1205 ioctx, wc->status);
1206 srpt_abort_cmd(ioctx);
1207 return;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001208 }
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001209
1210 if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA,
1211 SRPT_STATE_DATA_IN))
1212 target_execute_cmd(&ioctx->cmd);
1213 else
1214 pr_err("%s[%d]: wrong state = %d\n", __func__,
1215 __LINE__, srpt_get_cmd_state(ioctx));
Bart Van Asschea42d9852011-10-14 01:30:46 +00001216}
1217
Bart Van Asschea42d9852011-10-14 01:30:46 +00001218/**
1219 * srpt_build_cmd_rsp() - Build an SRP_RSP response.
1220 * @ch: RDMA channel through which the request has been received.
1221 * @ioctx: I/O context associated with the SRP_CMD request. The response will
1222 * be built in the buffer ioctx->buf points at and hence this function will
1223 * overwrite the request data.
1224 * @tag: tag of the request for which this response is being generated.
1225 * @status: value for the STATUS field of the SRP_RSP information unit.
1226 *
1227 * Returns the size in bytes of the SRP_RSP response.
1228 *
1229 * An SRP_RSP response contains a SCSI status or service response. See also
1230 * section 6.9 in the SRP r16a document for the format of an SRP_RSP
1231 * response. See also SPC-2 for more information about sense data.
1232 */
1233static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1234 struct srpt_send_ioctx *ioctx, u64 tag,
1235 int status)
1236{
1237 struct srp_rsp *srp_rsp;
1238 const u8 *sense_data;
1239 int sense_data_len, max_sense_len;
1240
1241 /*
1242 * The lowest bit of all SAM-3 status codes is zero (see also
1243 * paragraph 5.3 in SAM-3).
1244 */
1245 WARN_ON(status & 1);
1246
1247 srp_rsp = ioctx->ioctx.buf;
1248 BUG_ON(!srp_rsp);
1249
1250 sense_data = ioctx->sense_data;
1251 sense_data_len = ioctx->cmd.scsi_sense_length;
1252 WARN_ON(sense_data_len > sizeof(ioctx->sense_data));
1253
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08001254 memset(srp_rsp, 0, sizeof(*srp_rsp));
Bart Van Asschea42d9852011-10-14 01:30:46 +00001255 srp_rsp->opcode = SRP_RSP;
1256 srp_rsp->req_lim_delta =
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301257 cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0));
Bart Van Asschea42d9852011-10-14 01:30:46 +00001258 srp_rsp->tag = tag;
1259 srp_rsp->status = status;
1260
1261 if (sense_data_len) {
1262 BUILD_BUG_ON(MIN_MAX_RSP_SIZE <= sizeof(*srp_rsp));
1263 max_sense_len = ch->max_ti_iu_len - sizeof(*srp_rsp);
1264 if (sense_data_len > max_sense_len) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001265 pr_warn("truncated sense data from %d to %d"
1266 " bytes\n", sense_data_len, max_sense_len);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001267 sense_data_len = max_sense_len;
1268 }
1269
1270 srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1271 srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1272 memcpy(srp_rsp + 1, sense_data, sense_data_len);
1273 }
1274
1275 return sizeof(*srp_rsp) + sense_data_len;
1276}
1277
1278/**
1279 * srpt_build_tskmgmt_rsp() - Build a task management response.
1280 * @ch: RDMA channel through which the request has been received.
1281 * @ioctx: I/O context in which the SRP_RSP response will be built.
1282 * @rsp_code: RSP_CODE that will be stored in the response.
1283 * @tag: Tag of the request for which this response is being generated.
1284 *
1285 * Returns the size in bytes of the SRP_RSP response.
1286 *
1287 * An SRP_RSP response contains a SCSI status or service response. See also
1288 * section 6.9 in the SRP r16a document for the format of an SRP_RSP
1289 * response.
1290 */
1291static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1292 struct srpt_send_ioctx *ioctx,
1293 u8 rsp_code, u64 tag)
1294{
1295 struct srp_rsp *srp_rsp;
1296 int resp_data_len;
1297 int resp_len;
1298
Jack Wangc807f642013-09-30 10:09:05 +02001299 resp_data_len = 4;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001300 resp_len = sizeof(*srp_rsp) + resp_data_len;
1301
1302 srp_rsp = ioctx->ioctx.buf;
1303 BUG_ON(!srp_rsp);
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08001304 memset(srp_rsp, 0, sizeof(*srp_rsp));
Bart Van Asschea42d9852011-10-14 01:30:46 +00001305
1306 srp_rsp->opcode = SRP_RSP;
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301307 srp_rsp->req_lim_delta =
1308 cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0));
Bart Van Asschea42d9852011-10-14 01:30:46 +00001309 srp_rsp->tag = tag;
1310
Jack Wangc807f642013-09-30 10:09:05 +02001311 srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1312 srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1313 srp_rsp->data[3] = rsp_code;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001314
1315 return resp_len;
1316}
1317
Bart Van Asschea42d9852011-10-14 01:30:46 +00001318static int srpt_check_stop_free(struct se_cmd *cmd)
1319{
Nicholas Bellinger9474b042012-11-27 23:55:57 -08001320 struct srpt_send_ioctx *ioctx = container_of(cmd,
1321 struct srpt_send_ioctx, cmd);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001322
Bart Van Asscheafc16602015-04-27 13:52:36 +02001323 return target_put_sess_cmd(&ioctx->cmd);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001324}
1325
1326/**
1327 * srpt_handle_cmd() - Process SRP_CMD.
1328 */
Bart Van Assche2c7f37f2016-02-11 11:06:55 -08001329static void srpt_handle_cmd(struct srpt_rdma_ch *ch,
1330 struct srpt_recv_ioctx *recv_ioctx,
1331 struct srpt_send_ioctx *send_ioctx)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001332{
1333 struct se_cmd *cmd;
1334 struct srp_cmd *srp_cmd;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001335 struct scatterlist *sg = NULL;
1336 unsigned sg_cnt = 0;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001337 u64 data_len;
1338 enum dma_data_direction dir;
Nicholas Bellinger9474b042012-11-27 23:55:57 -08001339 int rc;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001340
1341 BUG_ON(!send_ioctx);
1342
1343 srp_cmd = recv_ioctx->ioctx.buf;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001344 cmd = &send_ioctx->cmd;
Bart Van Assche649ee052015-04-14 13:26:44 +02001345 cmd->tag = srp_cmd->tag;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001346
1347 switch (srp_cmd->task_attr) {
1348 case SRP_CMD_SIMPLE_Q:
Christoph Hellwig68d81f42014-11-24 07:07:25 -08001349 cmd->sam_task_attr = TCM_SIMPLE_TAG;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001350 break;
1351 case SRP_CMD_ORDERED_Q:
1352 default:
Christoph Hellwig68d81f42014-11-24 07:07:25 -08001353 cmd->sam_task_attr = TCM_ORDERED_TAG;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001354 break;
1355 case SRP_CMD_HEAD_OF_Q:
Christoph Hellwig68d81f42014-11-24 07:07:25 -08001356 cmd->sam_task_attr = TCM_HEAD_TAG;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001357 break;
1358 case SRP_CMD_ACA:
Christoph Hellwig68d81f42014-11-24 07:07:25 -08001359 cmd->sam_task_attr = TCM_ACA_TAG;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001360 break;
1361 }
1362
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001363 rc = srpt_get_desc_tbl(send_ioctx, srp_cmd, &dir, &sg, &sg_cnt,
1364 &data_len);
1365 if (rc) {
1366 if (rc != -EAGAIN) {
1367 pr_err("0x%llx: parsing SRP descriptor table failed.\n",
1368 srp_cmd->tag);
1369 }
Bart Van Assche2c7f37f2016-02-11 11:06:55 -08001370 goto release_ioctx;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001371 }
1372
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001373 rc = target_submit_cmd_map_sgls(cmd, ch->sess, srp_cmd->cdb,
Bart Van Asschee1dd4132016-02-11 11:05:19 -08001374 &send_ioctx->sense_data[0],
1375 scsilun_to_int(&srp_cmd->lun), data_len,
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001376 TCM_SIMPLE_TAG, dir, TARGET_SCF_ACK_KREF,
1377 sg, sg_cnt, NULL, 0, NULL, 0);
Nicholas Bellinger9474b042012-11-27 23:55:57 -08001378 if (rc != 0) {
Bart Van Assche2c7f37f2016-02-11 11:06:55 -08001379 pr_debug("target_submit_cmd() returned %d for tag %#llx\n", rc,
1380 srp_cmd->tag);
1381 goto release_ioctx;
Nicholas Bellinger187e70a2012-03-17 20:12:36 -07001382 }
Bart Van Assche2c7f37f2016-02-11 11:06:55 -08001383 return;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001384
Bart Van Assche2c7f37f2016-02-11 11:06:55 -08001385release_ioctx:
1386 send_ioctx->state = SRPT_STATE_DONE;
1387 srpt_release_cmd(cmd);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001388}
1389
Bart Van Asschea42d9852011-10-14 01:30:46 +00001390static int srp_tmr_to_tcm(int fn)
1391{
1392 switch (fn) {
1393 case SRP_TSK_ABORT_TASK:
1394 return TMR_ABORT_TASK;
1395 case SRP_TSK_ABORT_TASK_SET:
1396 return TMR_ABORT_TASK_SET;
1397 case SRP_TSK_CLEAR_TASK_SET:
1398 return TMR_CLEAR_TASK_SET;
1399 case SRP_TSK_LUN_RESET:
1400 return TMR_LUN_RESET;
1401 case SRP_TSK_CLEAR_ACA:
1402 return TMR_CLEAR_ACA;
1403 default:
1404 return -1;
1405 }
1406}
1407
1408/**
1409 * srpt_handle_tsk_mgmt() - Process an SRP_TSK_MGMT information unit.
1410 *
1411 * Returns 0 if and only if the request will be processed by the target core.
1412 *
1413 * For more information about SRP_TSK_MGMT information units, see also section
1414 * 6.7 in the SRP r16a document.
1415 */
1416static void srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1417 struct srpt_recv_ioctx *recv_ioctx,
1418 struct srpt_send_ioctx *send_ioctx)
1419{
1420 struct srp_tsk_mgmt *srp_tsk;
1421 struct se_cmd *cmd;
Nicholas Bellinger3e4f5742012-11-28 01:38:04 -08001422 struct se_session *sess = ch->sess;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001423 int tcm_tmr;
Nicholas Bellinger3e4f5742012-11-28 01:38:04 -08001424 int rc;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001425
1426 BUG_ON(!send_ioctx);
1427
1428 srp_tsk = recv_ioctx->ioctx.buf;
1429 cmd = &send_ioctx->cmd;
1430
1431 pr_debug("recv tsk_mgmt fn %d for task_tag %lld and cmd tag %lld"
1432 " cm_id %p sess %p\n", srp_tsk->tsk_mgmt_func,
1433 srp_tsk->task_tag, srp_tsk->tag, ch->cm_id, ch->sess);
1434
1435 srpt_set_cmd_state(send_ioctx, SRPT_STATE_MGMT);
Bart Van Assche649ee052015-04-14 13:26:44 +02001436 send_ioctx->cmd.tag = srp_tsk->tag;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001437 tcm_tmr = srp_tmr_to_tcm(srp_tsk->tsk_mgmt_func);
Bart Van Asschee1dd4132016-02-11 11:05:19 -08001438 rc = target_submit_tmr(&send_ioctx->cmd, sess, NULL,
1439 scsilun_to_int(&srp_tsk->lun), srp_tsk, tcm_tmr,
1440 GFP_KERNEL, srp_tsk->task_tag,
1441 TARGET_SCF_ACK_KREF);
Nicholas Bellinger3e4f5742012-11-28 01:38:04 -08001442 if (rc != 0) {
Bart Van Asschea42d9852011-10-14 01:30:46 +00001443 send_ioctx->cmd.se_tmr_req->response = TMR_FUNCTION_REJECTED;
Christoph Hellwigde103c92012-11-06 12:24:09 -08001444 goto fail;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001445 }
Christoph Hellwigde103c92012-11-06 12:24:09 -08001446 return;
1447fail:
Christoph Hellwigde103c92012-11-06 12:24:09 -08001448 transport_send_check_condition_and_sense(cmd, 0, 0); // XXX:
Bart Van Asschea42d9852011-10-14 01:30:46 +00001449}
1450
1451/**
1452 * srpt_handle_new_iu() - Process a newly received information unit.
1453 * @ch: RDMA channel through which the information unit has been received.
1454 * @ioctx: SRPT I/O context associated with the information unit.
1455 */
1456static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1457 struct srpt_recv_ioctx *recv_ioctx,
1458 struct srpt_send_ioctx *send_ioctx)
1459{
1460 struct srp_cmd *srp_cmd;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001461
1462 BUG_ON(!ch);
1463 BUG_ON(!recv_ioctx);
1464
1465 ib_dma_sync_single_for_cpu(ch->sport->sdev->device,
1466 recv_ioctx->ioctx.dma, srp_max_req_size,
1467 DMA_FROM_DEVICE);
1468
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001469 if (unlikely(ch->state == CH_CONNECTING))
1470 goto out_wait;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001471
Bart Van Assche33912d72016-02-11 11:04:43 -08001472 if (unlikely(ch->state != CH_LIVE))
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001473 return;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001474
1475 srp_cmd = recv_ioctx->ioctx.buf;
1476 if (srp_cmd->opcode == SRP_CMD || srp_cmd->opcode == SRP_TSK_MGMT) {
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001477 if (!send_ioctx) {
1478 if (!list_empty(&ch->cmd_wait_list))
1479 goto out_wait;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001480 send_ioctx = srpt_get_send_ioctx(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001481 }
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001482 if (unlikely(!send_ioctx))
1483 goto out_wait;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001484 }
1485
Bart Van Asschea42d9852011-10-14 01:30:46 +00001486 switch (srp_cmd->opcode) {
1487 case SRP_CMD:
1488 srpt_handle_cmd(ch, recv_ioctx, send_ioctx);
1489 break;
1490 case SRP_TSK_MGMT:
1491 srpt_handle_tsk_mgmt(ch, recv_ioctx, send_ioctx);
1492 break;
1493 case SRP_I_LOGOUT:
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001494 pr_err("Not yet implemented: SRP_I_LOGOUT\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +00001495 break;
1496 case SRP_CRED_RSP:
1497 pr_debug("received SRP_CRED_RSP\n");
1498 break;
1499 case SRP_AER_RSP:
1500 pr_debug("received SRP_AER_RSP\n");
1501 break;
1502 case SRP_RSP:
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001503 pr_err("Received SRP_RSP\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +00001504 break;
1505 default:
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001506 pr_err("received IU with unknown opcode 0x%x\n",
Bart Van Asschea42d9852011-10-14 01:30:46 +00001507 srp_cmd->opcode);
1508 break;
1509 }
1510
1511 srpt_post_recv(ch->sport->sdev, recv_ioctx);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001512 return;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001513
1514out_wait:
1515 list_add_tail(&recv_ioctx->wait_list, &ch->cmd_wait_list);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001516}
1517
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001518static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001519{
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001520 struct srpt_rdma_ch *ch = cq->cq_context;
1521 struct srpt_recv_ioctx *ioctx =
1522 container_of(wc->wr_cqe, struct srpt_recv_ioctx, ioctx.cqe);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001523
Bart Van Asschea42d9852011-10-14 01:30:46 +00001524 if (wc->status == IB_WC_SUCCESS) {
1525 int req_lim;
1526
1527 req_lim = atomic_dec_return(&ch->req_lim);
1528 if (unlikely(req_lim < 0))
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001529 pr_err("req_lim = %d < 0\n", req_lim);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001530 srpt_handle_new_iu(ch, ioctx, NULL);
1531 } else {
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001532 pr_info("receiving failed for ioctx %p with status %d\n",
1533 ioctx, wc->status);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001534 }
1535}
1536
Bart Van Assche539b3242016-02-11 11:09:50 -08001537/*
1538 * This function must be called from the context in which RDMA completions are
1539 * processed because it accesses the wait list without protection against
1540 * access from other threads.
1541 */
1542static void srpt_process_wait_list(struct srpt_rdma_ch *ch)
1543{
1544 struct srpt_send_ioctx *ioctx;
1545
1546 while (!list_empty(&ch->cmd_wait_list) &&
1547 ch->state >= CH_LIVE &&
1548 (ioctx = srpt_get_send_ioctx(ch)) != NULL) {
1549 struct srpt_recv_ioctx *recv_ioctx;
1550
1551 recv_ioctx = list_first_entry(&ch->cmd_wait_list,
1552 struct srpt_recv_ioctx,
1553 wait_list);
1554 list_del(&recv_ioctx->wait_list);
1555 srpt_handle_new_iu(ch, recv_ioctx, ioctx);
1556 }
1557}
1558
Bart Van Asschea42d9852011-10-14 01:30:46 +00001559/**
Bart Van Asschea42d9852011-10-14 01:30:46 +00001560 * Note: Although this has not yet been observed during tests, at least in
1561 * theory it is possible that the srpt_get_send_ioctx() call invoked by
1562 * srpt_handle_new_iu() fails. This is possible because the req_lim_delta
1563 * value in each response is set to one, and it is possible that this response
1564 * makes the initiator send a new request before the send completion for that
1565 * response has been processed. This could e.g. happen if the call to
1566 * srpt_put_send_iotcx() is delayed because of a higher priority interrupt or
1567 * if IB retransmission causes generation of the send completion to be
1568 * delayed. Incoming information units for which srpt_get_send_ioctx() fails
1569 * are queued on cmd_wait_list. The code below processes these delayed
1570 * requests one at a time.
1571 */
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001572static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001573{
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001574 struct srpt_rdma_ch *ch = cq->cq_context;
1575 struct srpt_send_ioctx *ioctx =
1576 container_of(wc->wr_cqe, struct srpt_send_ioctx, ioctx.cqe);
1577 enum srpt_command_state state;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001578
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001579 state = srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
1580
1581 WARN_ON(state != SRPT_STATE_CMD_RSP_SENT &&
1582 state != SRPT_STATE_MGMT_RSP_SENT);
1583
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001584 atomic_add(1 + ioctx->n_rdma, &ch->sq_wr_avail);
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001585
Bart Van Assche49f40162016-02-11 11:07:11 -08001586 if (wc->status != IB_WC_SUCCESS)
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001587 pr_info("sending response for ioctx 0x%p failed"
1588 " with status %d\n", ioctx, wc->status);
1589
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001590 if (state != SRPT_STATE_DONE) {
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001591 transport_generic_free_cmd(&ioctx->cmd, 0);
1592 } else {
1593 pr_err("IB completion has been received too late for"
1594 " wr_id = %u.\n", ioctx->ioctx.index);
1595 }
1596
Bart Van Assche539b3242016-02-11 11:09:50 -08001597 srpt_process_wait_list(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001598}
1599
Bart Van Asschea42d9852011-10-14 01:30:46 +00001600/**
1601 * srpt_create_ch_ib() - Create receive and send completion queues.
1602 */
1603static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1604{
1605 struct ib_qp_init_attr *qp_init;
1606 struct srpt_port *sport = ch->sport;
1607 struct srpt_device *sdev = sport->sdev;
Bart Van Assche30c6d872016-07-21 13:03:47 -07001608 const struct ib_device_attr *attrs = &sdev->device->attrs;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001609 u32 srp_sq_size = sport->port_attrib.srp_sq_size;
1610 int ret;
1611
1612 WARN_ON(ch->rq_size < 1);
1613
1614 ret = -ENOMEM;
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08001615 qp_init = kzalloc(sizeof(*qp_init), GFP_KERNEL);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001616 if (!qp_init)
1617 goto out;
1618
Bart Van Asscheab477c12014-10-19 18:05:33 +03001619retry:
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001620 ch->cq = ib_alloc_cq(sdev->device, ch, ch->rq_size + srp_sq_size,
1621 0 /* XXX: spread CQs */, IB_POLL_WORKQUEUE);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001622 if (IS_ERR(ch->cq)) {
1623 ret = PTR_ERR(ch->cq);
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001624 pr_err("failed to create CQ cqe= %d ret= %d\n",
Bart Van Asschea42d9852011-10-14 01:30:46 +00001625 ch->rq_size + srp_sq_size, ret);
1626 goto out;
1627 }
1628
1629 qp_init->qp_context = (void *)ch;
1630 qp_init->event_handler
1631 = (void(*)(struct ib_event *, void*))srpt_qp_event;
1632 qp_init->send_cq = ch->cq;
1633 qp_init->recv_cq = ch->cq;
1634 qp_init->srq = sdev->srq;
1635 qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1636 qp_init->qp_type = IB_QPT_RC;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001637 /*
1638 * We divide up our send queue size into half SEND WRs to send the
1639 * completions, and half R/W contexts to actually do the RDMA
1640 * READ/WRITE transfers. Note that we need to allocate CQ slots for
1641 * both both, as RDMA contexts will also post completions for the
1642 * RDMA READ case.
1643 */
1644 qp_init->cap.max_send_wr = srp_sq_size / 2;
1645 qp_init->cap.max_rdma_ctxs = srp_sq_size / 2;
Bart Van Assche30c6d872016-07-21 13:03:47 -07001646 qp_init->cap.max_send_sge = min(attrs->max_sge, SRPT_MAX_SG_PER_WQE);
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02001647 qp_init->port_num = ch->sport->port;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001648
1649 ch->qp = ib_create_qp(sdev->pd, qp_init);
1650 if (IS_ERR(ch->qp)) {
1651 ret = PTR_ERR(ch->qp);
Bart Van Asscheab477c12014-10-19 18:05:33 +03001652 if (ret == -ENOMEM) {
1653 srp_sq_size /= 2;
1654 if (srp_sq_size >= MIN_SRPT_SQ_SIZE) {
1655 ib_destroy_cq(ch->cq);
1656 goto retry;
1657 }
1658 }
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001659 pr_err("failed to create_qp ret= %d\n", ret);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001660 goto err_destroy_cq;
1661 }
1662
1663 atomic_set(&ch->sq_wr_avail, qp_init->cap.max_send_wr);
1664
1665 pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n",
1666 __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1667 qp_init->cap.max_send_wr, ch->cm_id);
1668
1669 ret = srpt_init_ch_qp(ch, ch->qp);
1670 if (ret)
1671 goto err_destroy_qp;
1672
Bart Van Asschea42d9852011-10-14 01:30:46 +00001673out:
1674 kfree(qp_init);
1675 return ret;
1676
1677err_destroy_qp:
1678 ib_destroy_qp(ch->qp);
1679err_destroy_cq:
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001680 ib_free_cq(ch->cq);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001681 goto out;
1682}
1683
1684static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch)
1685{
Bart Van Asschea42d9852011-10-14 01:30:46 +00001686 ib_destroy_qp(ch->qp);
Christoph Hellwig59fae4d2015-09-29 13:00:44 +02001687 ib_free_cq(ch->cq);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001688}
1689
1690/**
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001691 * srpt_close_ch() - Close an RDMA channel.
Bart Van Asschea42d9852011-10-14 01:30:46 +00001692 *
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001693 * Make sure all resources associated with the channel will be deallocated at
1694 * an appropriate time.
Bart Van Asschea42d9852011-10-14 01:30:46 +00001695 *
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001696 * Returns true if and only if the channel state has been modified into
1697 * CH_DRAINING.
Bart Van Asschea42d9852011-10-14 01:30:46 +00001698 */
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001699static bool srpt_close_ch(struct srpt_rdma_ch *ch)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001700{
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001701 int ret;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001702
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001703 if (!srpt_set_ch_state(ch, CH_DRAINING)) {
1704 pr_debug("%s-%d: already closed\n", ch->sess_name,
1705 ch->qp->qp_num);
1706 return false;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001707 }
Bart Van Asschea42d9852011-10-14 01:30:46 +00001708
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001709 kref_get(&ch->kref);
1710
1711 ret = srpt_ch_qp_err(ch);
1712 if (ret < 0)
1713 pr_err("%s-%d: changing queue pair into error state failed: %d\n",
1714 ch->sess_name, ch->qp->qp_num, ret);
1715
1716 pr_debug("%s-%d: queued zerolength write\n", ch->sess_name,
1717 ch->qp->qp_num);
1718 ret = srpt_zerolength_write(ch);
1719 if (ret < 0) {
1720 pr_err("%s-%d: queuing zero-length write failed: %d\n",
1721 ch->sess_name, ch->qp->qp_num, ret);
1722 if (srpt_set_ch_state(ch, CH_DISCONNECTED))
1723 schedule_work(&ch->release_work);
1724 else
1725 WARN_ON_ONCE(true);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001726 }
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001727
1728 kref_put(&ch->kref, srpt_free_ch);
1729
1730 return true;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001731}
1732
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001733/*
1734 * Change the channel state into CH_DISCONNECTING. If a channel has not yet
1735 * reached the connected state, close it. If a channel is in the connected
1736 * state, send a DREQ. If a DREQ has been received, send a DREP. Note: it is
1737 * the responsibility of the caller to ensure that this function is not
1738 * invoked concurrently with the code that accepts a connection. This means
1739 * that this function must either be invoked from inside a CM callback
1740 * function or that it must be invoked with the srpt_port.mutex held.
Bart Van Asschea42d9852011-10-14 01:30:46 +00001741 */
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001742static int srpt_disconnect_ch(struct srpt_rdma_ch *ch)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001743{
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001744 int ret;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001745
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001746 if (!srpt_set_ch_state(ch, CH_DISCONNECTING))
1747 return -ENOTCONN;
1748
1749 ret = ib_send_cm_dreq(ch->cm_id, NULL, 0);
1750 if (ret < 0)
1751 ret = ib_send_cm_drep(ch->cm_id, NULL, 0);
1752
1753 if (ret < 0 && srpt_close_ch(ch))
1754 ret = 0;
1755
1756 return ret;
1757}
1758
1759static void __srpt_close_all_ch(struct srpt_device *sdev)
1760{
1761 struct srpt_rdma_ch *ch;
1762
1763 lockdep_assert_held(&sdev->mutex);
1764
1765 list_for_each_entry(ch, &sdev->rch_list, list) {
1766 if (srpt_disconnect_ch(ch) >= 0)
1767 pr_info("Closing channel %s-%d because target %s has been disabled\n",
1768 ch->sess_name, ch->qp->qp_num,
1769 sdev->device->name);
1770 srpt_close_ch(ch);
1771 }
Bart Van Asschea42d9852011-10-14 01:30:46 +00001772}
1773
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001774static void srpt_free_ch(struct kref *kref)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001775{
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001776 struct srpt_rdma_ch *ch = container_of(kref, struct srpt_rdma_ch, kref);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001777
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001778 kfree(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001779}
1780
1781static void srpt_release_channel_work(struct work_struct *w)
1782{
1783 struct srpt_rdma_ch *ch;
1784 struct srpt_device *sdev;
Nicholas Bellinger9474b042012-11-27 23:55:57 -08001785 struct se_session *se_sess;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001786
1787 ch = container_of(w, struct srpt_rdma_ch, release_work);
Bart Van Asschef108f0f2016-02-11 11:06:14 -08001788 pr_debug("%s: %s-%d; release_done = %p\n", __func__, ch->sess_name,
1789 ch->qp->qp_num, ch->release_done);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001790
1791 sdev = ch->sport->sdev;
1792 BUG_ON(!sdev);
1793
Nicholas Bellinger9474b042012-11-27 23:55:57 -08001794 se_sess = ch->sess;
1795 BUG_ON(!se_sess);
1796
Bart Van Assche88936252016-02-11 11:05:58 -08001797 target_sess_cmd_list_set_waiting(se_sess);
Joern Engelbe646c2d2013-05-15 00:44:07 -07001798 target_wait_for_sess_cmds(se_sess);
Nicholas Bellinger9474b042012-11-27 23:55:57 -08001799
1800 transport_deregister_session_configfs(se_sess);
1801 transport_deregister_session(se_sess);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001802 ch->sess = NULL;
1803
Nicholas Bellinger0b41d6c2013-09-18 12:48:27 -07001804 ib_destroy_cm_id(ch->cm_id);
1805
Bart Van Asschea42d9852011-10-14 01:30:46 +00001806 srpt_destroy_ch_ib(ch);
1807
1808 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
1809 ch->sport->sdev, ch->rq_size,
1810 ch->rsp_size, DMA_TO_DEVICE);
1811
Bart Van Assche86289912016-02-11 11:08:34 -08001812 mutex_lock(&sdev->mutex);
Bart Van Asschef108f0f2016-02-11 11:06:14 -08001813 list_del_init(&ch->list);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001814 if (ch->release_done)
1815 complete(ch->release_done);
Bart Van Assche86289912016-02-11 11:08:34 -08001816 mutex_unlock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001817
1818 wake_up(&sdev->ch_releaseQ);
1819
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001820 kref_put(&ch->kref, srpt_free_ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001821}
1822
Bart Van Asschea42d9852011-10-14 01:30:46 +00001823/**
1824 * srpt_cm_req_recv() - Process the event IB_CM_REQ_RECEIVED.
1825 *
1826 * Ownership of the cm_id is transferred to the target session if this
1827 * functions returns zero. Otherwise the caller remains the owner of cm_id.
1828 */
1829static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1830 struct ib_cm_req_event_param *param,
1831 void *private_data)
1832{
1833 struct srpt_device *sdev = cm_id->context;
1834 struct srpt_port *sport = &sdev->port[param->port - 1];
1835 struct srp_login_req *req;
1836 struct srp_login_rsp *rsp;
1837 struct srp_login_rej *rej;
1838 struct ib_cm_rep_param *rep_param;
1839 struct srpt_rdma_ch *ch, *tmp_ch;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001840 u32 it_iu_len;
Bart Van Assche3c968882016-04-07 15:55:04 -07001841 int i, ret = 0;
Nicholas Bellingerf246c942016-01-07 22:19:21 -08001842 unsigned char *p;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001843
1844 WARN_ON_ONCE(irqs_disabled());
1845
1846 if (WARN_ON(!sdev || !private_data))
1847 return -EINVAL;
1848
1849 req = (struct srp_login_req *)private_data;
1850
1851 it_iu_len = be32_to_cpu(req->req_it_iu_len);
1852
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001853 pr_info("Received SRP_LOGIN_REQ with i_port_id 0x%llx:0x%llx,"
1854 " t_port_id 0x%llx:0x%llx and it_iu_len %d on port %d"
1855 " (guid=0x%llx:0x%llx)\n",
1856 be64_to_cpu(*(__be64 *)&req->initiator_port_id[0]),
1857 be64_to_cpu(*(__be64 *)&req->initiator_port_id[8]),
1858 be64_to_cpu(*(__be64 *)&req->target_port_id[0]),
1859 be64_to_cpu(*(__be64 *)&req->target_port_id[8]),
1860 it_iu_len,
1861 param->port,
1862 be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[0]),
1863 be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[8]));
Bart Van Asschea42d9852011-10-14 01:30:46 +00001864
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08001865 rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
1866 rej = kzalloc(sizeof(*rej), GFP_KERNEL);
1867 rep_param = kzalloc(sizeof(*rep_param), GFP_KERNEL);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001868
1869 if (!rsp || !rej || !rep_param) {
1870 ret = -ENOMEM;
1871 goto out;
1872 }
1873
1874 if (it_iu_len > srp_max_req_size || it_iu_len < 64) {
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301875 rej->reason = cpu_to_be32(
1876 SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001877 ret = -EINVAL;
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001878 pr_err("rejected SRP_LOGIN_REQ because its"
Bart Van Asschea42d9852011-10-14 01:30:46 +00001879 " length (%d bytes) is out of range (%d .. %d)\n",
1880 it_iu_len, 64, srp_max_req_size);
1881 goto reject;
1882 }
1883
1884 if (!sport->enabled) {
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301885 rej->reason = cpu_to_be32(
1886 SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001887 ret = -EINVAL;
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001888 pr_err("rejected SRP_LOGIN_REQ because the target port"
Bart Van Asschea42d9852011-10-14 01:30:46 +00001889 " has not yet been enabled\n");
1890 goto reject;
1891 }
1892
1893 if ((req->req_flags & SRP_MTCH_ACTION) == SRP_MULTICHAN_SINGLE) {
1894 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1895
Bart Van Assche86289912016-02-11 11:08:34 -08001896 mutex_lock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001897
1898 list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1899 if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1900 && !memcmp(ch->t_port_id, req->target_port_id, 16)
1901 && param->port == ch->sport->port
1902 && param->listen_id == ch->sport->sdev->cm_id
1903 && ch->cm_id) {
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001904 if (srpt_disconnect_ch(ch) < 0)
Bart Van Asschea42d9852011-10-14 01:30:46 +00001905 continue;
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001906 pr_info("Relogin - closed existing channel %s\n",
1907 ch->sess_name);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001908 rsp->rsp_flags =
1909 SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1910 }
1911 }
1912
Bart Van Assche86289912016-02-11 11:08:34 -08001913 mutex_unlock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001914
1915 } else
1916 rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1917
1918 if (*(__be64 *)req->target_port_id != cpu_to_be64(srpt_service_guid)
1919 || *(__be64 *)(req->target_port_id + 8) !=
1920 cpu_to_be64(srpt_service_guid)) {
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301921 rej->reason = cpu_to_be32(
1922 SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001923 ret = -ENOMEM;
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001924 pr_err("rejected SRP_LOGIN_REQ because it"
Bart Van Asschea42d9852011-10-14 01:30:46 +00001925 " has an invalid target port identifier.\n");
1926 goto reject;
1927 }
1928
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08001929 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
Bart Van Asschea42d9852011-10-14 01:30:46 +00001930 if (!ch) {
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301931 rej->reason = cpu_to_be32(
1932 SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001933 pr_err("rejected SRP_LOGIN_REQ because no memory.\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +00001934 ret = -ENOMEM;
1935 goto reject;
1936 }
1937
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08001938 kref_init(&ch->kref);
1939 ch->zw_cqe.done = srpt_zerolength_write_done;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001940 INIT_WORK(&ch->release_work, srpt_release_channel_work);
1941 memcpy(ch->i_port_id, req->initiator_port_id, 16);
1942 memcpy(ch->t_port_id, req->target_port_id, 16);
1943 ch->sport = &sdev->port[param->port - 1];
1944 ch->cm_id = cm_id;
Bart Van Assche2739b592016-02-11 11:07:49 -08001945 cm_id->context = ch;
Bart Van Asschea42d9852011-10-14 01:30:46 +00001946 /*
1947 * Avoid QUEUE_FULL conditions by limiting the number of buffers used
1948 * for the SRP protocol to the command queue size.
1949 */
1950 ch->rq_size = SRPT_RQ_SIZE;
1951 spin_lock_init(&ch->spinlock);
1952 ch->state = CH_CONNECTING;
1953 INIT_LIST_HEAD(&ch->cmd_wait_list);
1954 ch->rsp_size = ch->sport->port_attrib.srp_max_rsp_size;
1955
1956 ch->ioctx_ring = (struct srpt_send_ioctx **)
1957 srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size,
1958 sizeof(*ch->ioctx_ring[0]),
1959 ch->rsp_size, DMA_TO_DEVICE);
1960 if (!ch->ioctx_ring)
1961 goto free_ch;
1962
Bart Van Assche3c968882016-04-07 15:55:04 -07001963 INIT_LIST_HEAD(&ch->free_list);
1964 for (i = 0; i < ch->rq_size; i++) {
1965 ch->ioctx_ring[i]->ch = ch;
1966 list_add_tail(&ch->ioctx_ring[i]->free_list, &ch->free_list);
1967 }
1968
Bart Van Asschea42d9852011-10-14 01:30:46 +00001969 ret = srpt_create_ch_ib(ch);
1970 if (ret) {
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301971 rej->reason = cpu_to_be32(
1972 SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001973 pr_err("rejected SRP_LOGIN_REQ because creating"
Bart Van Asschea42d9852011-10-14 01:30:46 +00001974 " a new RDMA channel failed.\n");
1975 goto free_ring;
1976 }
1977
1978 ret = srpt_ch_qp_rtr(ch, ch->qp);
1979 if (ret) {
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05301980 rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
Doug Ledford9f5d32a2014-10-20 18:25:15 -04001981 pr_err("rejected SRP_LOGIN_REQ because enabling"
Bart Van Asschea42d9852011-10-14 01:30:46 +00001982 " RTR failed (error code = %d)\n", ret);
1983 goto destroy_ib;
1984 }
Nicholas Bellingerf246c942016-01-07 22:19:21 -08001985
Bart Van Asschea42d9852011-10-14 01:30:46 +00001986 /*
Nicholas Bellingerf246c942016-01-07 22:19:21 -08001987 * Use the initator port identifier as the session name, when
1988 * checking against se_node_acl->initiatorname[] this can be
1989 * with or without preceeding '0x'.
Bart Van Asschea42d9852011-10-14 01:30:46 +00001990 */
1991 snprintf(ch->sess_name, sizeof(ch->sess_name), "0x%016llx%016llx",
1992 be64_to_cpu(*(__be64 *)ch->i_port_id),
1993 be64_to_cpu(*(__be64 *)(ch->i_port_id + 8)));
1994
1995 pr_debug("registering session %s\n", ch->sess_name);
Nicholas Bellingerf246c942016-01-07 22:19:21 -08001996 p = &ch->sess_name[0];
Bart Van Asschea42d9852011-10-14 01:30:46 +00001997
Nicholas Bellingerf246c942016-01-07 22:19:21 -08001998try_again:
Bart Van Assche3c968882016-04-07 15:55:04 -07001999 ch->sess = target_alloc_session(&sport->port_tpg_1, 0, 0,
Nicholas Bellingerb42057a2016-01-09 20:42:43 -08002000 TARGET_PROT_NORMAL, p, ch, NULL);
2001 if (IS_ERR(ch->sess)) {
Nicholas Bellingerf246c942016-01-07 22:19:21 -08002002 pr_info("Rejected login because no ACL has been"
Nicholas Bellingerb42057a2016-01-09 20:42:43 -08002003 " configured yet for initiator %s.\n", p);
Nicholas Bellingerf246c942016-01-07 22:19:21 -08002004 /*
2005 * XXX: Hack to retry of ch->i_port_id without leading '0x'
2006 */
2007 if (p == &ch->sess_name[0]) {
2008 p += 2;
2009 goto try_again;
2010 }
Nicholas Bellingerb42057a2016-01-09 20:42:43 -08002011 rej->reason = cpu_to_be32((PTR_ERR(ch->sess) == -ENOMEM) ?
2012 SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES :
Nicholas Bellingerf246c942016-01-07 22:19:21 -08002013 SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED);
Nicholas Bellingerf246c942016-01-07 22:19:21 -08002014 goto destroy_ib;
2015 }
Bart Van Asschea42d9852011-10-14 01:30:46 +00002016
2017 pr_debug("Establish connection sess=%p name=%s cm_id=%p\n", ch->sess,
2018 ch->sess_name, ch->cm_id);
2019
2020 /* create srp_login_response */
2021 rsp->opcode = SRP_LOGIN_RSP;
2022 rsp->tag = req->tag;
2023 rsp->max_it_iu_len = req->req_it_iu_len;
2024 rsp->max_ti_iu_len = req->req_it_iu_len;
2025 ch->max_ti_iu_len = it_iu_len;
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05302026 rsp->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT
2027 | SRP_BUF_FORMAT_INDIRECT);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002028 rsp->req_lim_delta = cpu_to_be32(ch->rq_size);
2029 atomic_set(&ch->req_lim, ch->rq_size);
2030 atomic_set(&ch->req_lim_delta, 0);
2031
2032 /* create cm reply */
2033 rep_param->qp_num = ch->qp->qp_num;
2034 rep_param->private_data = (void *)rsp;
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08002035 rep_param->private_data_len = sizeof(*rsp);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002036 rep_param->rnr_retry_count = 7;
2037 rep_param->flow_control = 1;
2038 rep_param->failover_accepted = 0;
2039 rep_param->srq = 1;
2040 rep_param->responder_resources = 4;
2041 rep_param->initiator_depth = 4;
2042
2043 ret = ib_send_cm_rep(cm_id, rep_param);
2044 if (ret) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002045 pr_err("sending SRP_LOGIN_REQ response failed"
Bart Van Asschea42d9852011-10-14 01:30:46 +00002046 " (error code = %d)\n", ret);
2047 goto release_channel;
2048 }
2049
Bart Van Assche86289912016-02-11 11:08:34 -08002050 mutex_lock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002051 list_add_tail(&ch->list, &sdev->rch_list);
Bart Van Assche86289912016-02-11 11:08:34 -08002052 mutex_unlock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002053
2054 goto out;
2055
2056release_channel:
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08002057 srpt_disconnect_ch(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002058 transport_deregister_session_configfs(ch->sess);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002059 transport_deregister_session(ch->sess);
2060 ch->sess = NULL;
2061
2062destroy_ib:
2063 srpt_destroy_ch_ib(ch);
2064
2065free_ring:
2066 srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
2067 ch->sport->sdev, ch->rq_size,
2068 ch->rsp_size, DMA_TO_DEVICE);
2069free_ch:
2070 kfree(ch);
2071
2072reject:
2073 rej->opcode = SRP_LOGIN_REJ;
2074 rej->tag = req->tag;
Vaishali Thakkarb356c1c2015-06-24 10:12:13 +05302075 rej->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT
2076 | SRP_BUF_FORMAT_INDIRECT);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002077
2078 ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08002079 (void *)rej, sizeof(*rej));
Bart Van Asschea42d9852011-10-14 01:30:46 +00002080
2081out:
2082 kfree(rep_param);
2083 kfree(rsp);
2084 kfree(rej);
2085
2086 return ret;
2087}
2088
Bart Van Assche2739b592016-02-11 11:07:49 -08002089static void srpt_cm_rej_recv(struct srpt_rdma_ch *ch,
2090 enum ib_cm_rej_reason reason,
2091 const u8 *private_data,
2092 u8 private_data_len)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002093{
Bart Van Asschec13c90e2016-02-11 11:08:12 -08002094 char *priv = NULL;
2095 int i;
2096
2097 if (private_data_len && (priv = kmalloc(private_data_len * 3 + 1,
2098 GFP_KERNEL))) {
2099 for (i = 0; i < private_data_len; i++)
2100 sprintf(priv + 3 * i, " %02x", private_data[i]);
2101 }
2102 pr_info("Received CM REJ for ch %s-%d; reason %d%s%s.\n",
2103 ch->sess_name, ch->qp->qp_num, reason, private_data_len ?
2104 "; private data" : "", priv ? priv : " (?)");
2105 kfree(priv);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002106}
2107
2108/**
2109 * srpt_cm_rtu_recv() - Process an IB_CM_RTU_RECEIVED or USER_ESTABLISHED event.
2110 *
2111 * An IB_CM_RTU_RECEIVED message indicates that the connection is established
2112 * and that the recipient may begin transmitting (RTU = ready to use).
2113 */
Bart Van Assche2739b592016-02-11 11:07:49 -08002114static void srpt_cm_rtu_recv(struct srpt_rdma_ch *ch)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002115{
Bart Van Asschea42d9852011-10-14 01:30:46 +00002116 int ret;
2117
Bart Van Asschef130c222016-02-11 11:05:38 -08002118 if (srpt_set_ch_state(ch, CH_LIVE)) {
Bart Van Asschea42d9852011-10-14 01:30:46 +00002119 ret = srpt_ch_qp_rts(ch, ch->qp);
2120
Bart Van Assche387add42016-02-11 11:10:09 -08002121 if (ret == 0) {
2122 /* Trigger wait list processing. */
2123 ret = srpt_zerolength_write(ch);
2124 WARN_ONCE(ret < 0, "%d\n", ret);
2125 } else {
Bart Van Asschea42d9852011-10-14 01:30:46 +00002126 srpt_close_ch(ch);
Bart Van Assche387add42016-02-11 11:10:09 -08002127 }
Bart Van Asschea42d9852011-10-14 01:30:46 +00002128 }
2129}
2130
Bart Van Asschea42d9852011-10-14 01:30:46 +00002131/**
Bart Van Asschea42d9852011-10-14 01:30:46 +00002132 * srpt_cm_handler() - IB connection manager callback function.
2133 *
2134 * A non-zero return value will cause the caller destroy the CM ID.
2135 *
2136 * Note: srpt_cm_handler() must only return a non-zero value when transferring
2137 * ownership of the cm_id to a channel by srpt_cm_req_recv() failed. Returning
2138 * a non-zero value in any other case will trigger a race with the
2139 * ib_destroy_cm_id() call in srpt_release_channel().
2140 */
2141static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2142{
Bart Van Assche2739b592016-02-11 11:07:49 -08002143 struct srpt_rdma_ch *ch = cm_id->context;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002144 int ret;
2145
2146 ret = 0;
2147 switch (event->event) {
2148 case IB_CM_REQ_RECEIVED:
2149 ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
2150 event->private_data);
2151 break;
2152 case IB_CM_REJ_RECEIVED:
Bart Van Assche2739b592016-02-11 11:07:49 -08002153 srpt_cm_rej_recv(ch, event->param.rej_rcvd.reason,
2154 event->private_data,
2155 IB_CM_REJ_PRIVATE_DATA_SIZE);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002156 break;
2157 case IB_CM_RTU_RECEIVED:
2158 case IB_CM_USER_ESTABLISHED:
Bart Van Assche2739b592016-02-11 11:07:49 -08002159 srpt_cm_rtu_recv(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002160 break;
2161 case IB_CM_DREQ_RECEIVED:
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08002162 srpt_disconnect_ch(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002163 break;
2164 case IB_CM_DREP_RECEIVED:
Bart Van Assche2739b592016-02-11 11:07:49 -08002165 pr_info("Received CM DREP message for ch %s-%d.\n",
2166 ch->sess_name, ch->qp->qp_num);
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08002167 srpt_close_ch(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002168 break;
2169 case IB_CM_TIMEWAIT_EXIT:
Bart Van Assche2739b592016-02-11 11:07:49 -08002170 pr_info("Received CM TimeWait exit for ch %s-%d.\n",
2171 ch->sess_name, ch->qp->qp_num);
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08002172 srpt_close_ch(ch);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002173 break;
2174 case IB_CM_REP_ERROR:
Bart Van Assche2739b592016-02-11 11:07:49 -08002175 pr_info("Received CM REP error for ch %s-%d.\n", ch->sess_name,
2176 ch->qp->qp_num);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002177 break;
2178 case IB_CM_DREQ_ERROR:
Bart Van Assche1e20a2a2016-02-11 11:07:29 -08002179 pr_info("Received CM DREQ ERROR event.\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +00002180 break;
2181 case IB_CM_MRA_RECEIVED:
Bart Van Assche1e20a2a2016-02-11 11:07:29 -08002182 pr_info("Received CM MRA event\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +00002183 break;
2184 default:
Bart Van Assche1e20a2a2016-02-11 11:07:29 -08002185 pr_err("received unrecognized CM event %d\n", event->event);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002186 break;
2187 }
2188
2189 return ret;
2190}
2191
Bart Van Asschea42d9852011-10-14 01:30:46 +00002192static int srpt_write_pending_status(struct se_cmd *se_cmd)
2193{
2194 struct srpt_send_ioctx *ioctx;
2195
2196 ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
2197 return srpt_get_cmd_state(ioctx) == SRPT_STATE_NEED_DATA;
2198}
2199
2200/*
2201 * srpt_write_pending() - Start data transfer from initiator to target (write).
2202 */
2203static int srpt_write_pending(struct se_cmd *se_cmd)
2204{
Bart Van Asschefc3af582016-02-11 11:09:10 -08002205 struct srpt_send_ioctx *ioctx =
2206 container_of(se_cmd, struct srpt_send_ioctx, cmd);
2207 struct srpt_rdma_ch *ch = ioctx->ch;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002208 struct ib_send_wr *first_wr = NULL, *bad_wr;
2209 struct ib_cqe *cqe = &ioctx->rdma_cqe;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002210 enum srpt_command_state new_state;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002211 int ret, i;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002212
2213 new_state = srpt_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA);
2214 WARN_ON(new_state == SRPT_STATE_DONE);
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002215
2216 if (atomic_sub_return(ioctx->n_rdma, &ch->sq_wr_avail) < 0) {
2217 pr_warn("%s: IB send queue full (needed %d)\n",
2218 __func__, ioctx->n_rdma);
2219 ret = -ENOMEM;
2220 goto out_undo;
2221 }
2222
2223 cqe->done = srpt_rdma_read_done;
2224 for (i = ioctx->n_rw_ctx - 1; i >= 0; i--) {
2225 struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
2226
2227 first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp, ch->sport->port,
2228 cqe, first_wr);
2229 cqe = NULL;
2230 }
2231
2232 ret = ib_post_send(ch->qp, first_wr, &bad_wr);
2233 if (ret) {
2234 pr_err("%s: ib_post_send() returned %d for %d (avail: %d)\n",
2235 __func__, ret, ioctx->n_rdma,
2236 atomic_read(&ch->sq_wr_avail));
2237 goto out_undo;
2238 }
2239
2240 return 0;
2241out_undo:
2242 atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);
2243 return ret;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002244}
2245
2246static u8 tcm_to_srp_tsk_mgmt_status(const int tcm_mgmt_status)
2247{
2248 switch (tcm_mgmt_status) {
2249 case TMR_FUNCTION_COMPLETE:
2250 return SRP_TSK_MGMT_SUCCESS;
2251 case TMR_FUNCTION_REJECTED:
2252 return SRP_TSK_MGMT_FUNC_NOT_SUPP;
2253 }
2254 return SRP_TSK_MGMT_FAILED;
2255}
2256
2257/**
2258 * srpt_queue_response() - Transmits the response to a SCSI command.
2259 *
2260 * Callback function called by the TCM core. Must not block since it can be
2261 * invoked on the context of the IB completion handler.
2262 */
Joern Engelb79fafa2013-07-03 11:22:17 -04002263static void srpt_queue_response(struct se_cmd *cmd)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002264{
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002265 struct srpt_send_ioctx *ioctx =
2266 container_of(cmd, struct srpt_send_ioctx, cmd);
2267 struct srpt_rdma_ch *ch = ioctx->ch;
2268 struct srpt_device *sdev = ch->sport->sdev;
Bart Van Assche10fce582016-07-21 13:04:06 -07002269 struct ib_send_wr send_wr, *first_wr = &send_wr, *bad_wr;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002270 struct ib_sge sge;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002271 enum srpt_command_state state;
2272 unsigned long flags;
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002273 int resp_len, ret, i;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002274 u8 srp_tm_status;
2275
Bart Van Asschea42d9852011-10-14 01:30:46 +00002276 BUG_ON(!ch);
2277
2278 spin_lock_irqsave(&ioctx->spinlock, flags);
2279 state = ioctx->state;
2280 switch (state) {
2281 case SRPT_STATE_NEW:
2282 case SRPT_STATE_DATA_IN:
2283 ioctx->state = SRPT_STATE_CMD_RSP_SENT;
2284 break;
2285 case SRPT_STATE_MGMT:
2286 ioctx->state = SRPT_STATE_MGMT_RSP_SENT;
2287 break;
2288 default:
2289 WARN(true, "ch %p; cmd %d: unexpected command state %d\n",
2290 ch, ioctx->ioctx.index, ioctx->state);
2291 break;
2292 }
2293 spin_unlock_irqrestore(&ioctx->spinlock, flags);
2294
Bart Van Asschef9800a12017-05-04 15:50:53 -07002295 if (unlikely(WARN_ON_ONCE(state == SRPT_STATE_CMD_RSP_SENT)))
Joern Engelb79fafa2013-07-03 11:22:17 -04002296 return;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002297
Bart Van Asschea42d9852011-10-14 01:30:46 +00002298 /* For read commands, transfer the data to the initiator. */
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002299 if (ioctx->cmd.data_direction == DMA_FROM_DEVICE &&
2300 ioctx->cmd.data_length &&
Bart Van Asschea42d9852011-10-14 01:30:46 +00002301 !ioctx->queue_status_only) {
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002302 for (i = ioctx->n_rw_ctx - 1; i >= 0; i--) {
2303 struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
2304
2305 first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp,
Bart Van Assche10fce582016-07-21 13:04:06 -07002306 ch->sport->port, NULL, first_wr);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002307 }
2308 }
2309
2310 if (state != SRPT_STATE_MGMT)
Bart Van Assche649ee052015-04-14 13:26:44 +02002311 resp_len = srpt_build_cmd_rsp(ch, ioctx, ioctx->cmd.tag,
Bart Van Asschea42d9852011-10-14 01:30:46 +00002312 cmd->scsi_status);
2313 else {
2314 srp_tm_status
2315 = tcm_to_srp_tsk_mgmt_status(cmd->se_tmr_req->response);
2316 resp_len = srpt_build_tskmgmt_rsp(ch, ioctx, srp_tm_status,
Bart Van Assche649ee052015-04-14 13:26:44 +02002317 ioctx->cmd.tag);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002318 }
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002319
2320 atomic_inc(&ch->req_lim);
2321
2322 if (unlikely(atomic_sub_return(1 + ioctx->n_rdma,
2323 &ch->sq_wr_avail) < 0)) {
2324 pr_warn("%s: IB send queue full (needed %d)\n",
2325 __func__, ioctx->n_rdma);
2326 ret = -ENOMEM;
2327 goto out;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002328 }
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002329
2330 ib_dma_sync_single_for_device(sdev->device, ioctx->ioctx.dma, resp_len,
2331 DMA_TO_DEVICE);
2332
2333 sge.addr = ioctx->ioctx.dma;
2334 sge.length = resp_len;
2335 sge.lkey = sdev->pd->local_dma_lkey;
2336
2337 ioctx->ioctx.cqe.done = srpt_send_done;
2338 send_wr.next = NULL;
2339 send_wr.wr_cqe = &ioctx->ioctx.cqe;
2340 send_wr.sg_list = &sge;
2341 send_wr.num_sge = 1;
2342 send_wr.opcode = IB_WR_SEND;
2343 send_wr.send_flags = IB_SEND_SIGNALED;
2344
2345 ret = ib_post_send(ch->qp, first_wr, &bad_wr);
2346 if (ret < 0) {
2347 pr_err("%s: sending cmd response failed for tag %llu (%d)\n",
2348 __func__, ioctx->cmd.tag, ret);
2349 goto out;
2350 }
2351
2352 return;
2353
2354out:
2355 atomic_add(1 + ioctx->n_rdma, &ch->sq_wr_avail);
2356 atomic_dec(&ch->req_lim);
2357 srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
2358 target_put_sess_cmd(&ioctx->cmd);
Joern Engelb79fafa2013-07-03 11:22:17 -04002359}
Bart Van Asschea42d9852011-10-14 01:30:46 +00002360
Joern Engelb79fafa2013-07-03 11:22:17 -04002361static int srpt_queue_data_in(struct se_cmd *cmd)
2362{
2363 srpt_queue_response(cmd);
2364 return 0;
2365}
2366
2367static void srpt_queue_tm_rsp(struct se_cmd *cmd)
2368{
2369 srpt_queue_response(cmd);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002370}
2371
Nicholas Bellinger131e6ab2014-03-22 14:55:56 -07002372static void srpt_aborted_task(struct se_cmd *cmd)
2373{
Nicholas Bellinger131e6ab2014-03-22 14:55:56 -07002374}
2375
Bart Van Asschea42d9852011-10-14 01:30:46 +00002376static int srpt_queue_status(struct se_cmd *cmd)
2377{
2378 struct srpt_send_ioctx *ioctx;
2379
2380 ioctx = container_of(cmd, struct srpt_send_ioctx, cmd);
2381 BUG_ON(ioctx->sense_data != cmd->sense_buffer);
2382 if (cmd->se_cmd_flags &
2383 (SCF_TRANSPORT_TASK_SENSE | SCF_EMULATED_TASK_SENSE))
2384 WARN_ON(cmd->scsi_status != SAM_STAT_CHECK_CONDITION);
2385 ioctx->queue_status_only = true;
Joern Engelb79fafa2013-07-03 11:22:17 -04002386 srpt_queue_response(cmd);
2387 return 0;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002388}
2389
2390static void srpt_refresh_port_work(struct work_struct *work)
2391{
2392 struct srpt_port *sport = container_of(work, struct srpt_port, work);
2393
2394 srpt_refresh_port(sport);
2395}
2396
Bart Van Asschea42d9852011-10-14 01:30:46 +00002397/**
2398 * srpt_release_sdev() - Free the channel resources associated with a target.
2399 */
2400static int srpt_release_sdev(struct srpt_device *sdev)
2401{
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08002402 int i, res;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002403
2404 WARN_ON_ONCE(irqs_disabled());
2405
2406 BUG_ON(!sdev);
2407
Bart Van Assche86289912016-02-11 11:08:34 -08002408 mutex_lock(&sdev->mutex);
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08002409 for (i = 0; i < ARRAY_SIZE(sdev->port); i++)
2410 sdev->port[i].enabled = false;
2411 __srpt_close_all_ch(sdev);
Bart Van Assche86289912016-02-11 11:08:34 -08002412 mutex_unlock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002413
2414 res = wait_event_interruptible(sdev->ch_releaseQ,
Bart Van Assche86289912016-02-11 11:08:34 -08002415 list_empty_careful(&sdev->rch_list));
Bart Van Asschea42d9852011-10-14 01:30:46 +00002416 if (res)
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002417 pr_err("%s: interrupted.\n", __func__);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002418
2419 return 0;
2420}
2421
2422static struct srpt_port *__srpt_lookup_port(const char *name)
2423{
2424 struct ib_device *dev;
2425 struct srpt_device *sdev;
2426 struct srpt_port *sport;
2427 int i;
2428
2429 list_for_each_entry(sdev, &srpt_dev_list, list) {
2430 dev = sdev->device;
2431 if (!dev)
2432 continue;
2433
2434 for (i = 0; i < dev->phys_port_cnt; i++) {
2435 sport = &sdev->port[i];
2436
2437 if (!strcmp(sport->port_guid, name))
2438 return sport;
2439 }
2440 }
2441
2442 return NULL;
2443}
2444
2445static struct srpt_port *srpt_lookup_port(const char *name)
2446{
2447 struct srpt_port *sport;
2448
2449 spin_lock(&srpt_dev_lock);
2450 sport = __srpt_lookup_port(name);
2451 spin_unlock(&srpt_dev_lock);
2452
2453 return sport;
2454}
2455
2456/**
2457 * srpt_add_one() - Infiniband device addition callback function.
2458 */
2459static void srpt_add_one(struct ib_device *device)
2460{
2461 struct srpt_device *sdev;
2462 struct srpt_port *sport;
2463 struct ib_srq_init_attr srq_attr;
2464 int i;
2465
2466 pr_debug("device = %p, device->dma_ops = %p\n", device,
2467 device->dma_ops);
2468
Bart Van Assche9d2aa2b2016-02-11 11:03:31 -08002469 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002470 if (!sdev)
2471 goto err;
2472
2473 sdev->device = device;
2474 INIT_LIST_HEAD(&sdev->rch_list);
2475 init_waitqueue_head(&sdev->ch_releaseQ);
Bart Van Assche86289912016-02-11 11:08:34 -08002476 mutex_init(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002477
Christoph Hellwiged082d32016-09-05 12:56:17 +02002478 sdev->pd = ib_alloc_pd(device, 0);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002479 if (IS_ERR(sdev->pd))
2480 goto free_dev;
2481
Or Gerlitz4a061b22015-12-18 10:59:46 +02002482 sdev->srq_size = min(srpt_srq_size, sdev->device->attrs.max_srq_wr);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002483
2484 srq_attr.event_handler = srpt_srq_event;
2485 srq_attr.srq_context = (void *)sdev;
2486 srq_attr.attr.max_wr = sdev->srq_size;
2487 srq_attr.attr.max_sge = 1;
2488 srq_attr.attr.srq_limit = 0;
Roland Dreier6f360332012-04-12 07:51:08 -07002489 srq_attr.srq_type = IB_SRQT_BASIC;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002490
2491 sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
2492 if (IS_ERR(sdev->srq))
Jason Gunthorpe5a783952015-07-30 17:22:24 -06002493 goto err_pd;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002494
2495 pr_debug("%s: create SRQ #wr= %d max_allow=%d dev= %s\n",
Or Gerlitz4a061b22015-12-18 10:59:46 +02002496 __func__, sdev->srq_size, sdev->device->attrs.max_srq_wr,
Bart Van Asschea42d9852011-10-14 01:30:46 +00002497 device->name);
2498
2499 if (!srpt_service_guid)
2500 srpt_service_guid = be64_to_cpu(device->node_guid);
2501
2502 sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
2503 if (IS_ERR(sdev->cm_id))
2504 goto err_srq;
2505
2506 /* print out target login information */
2507 pr_debug("Target login info: id_ext=%016llx,ioc_guid=%016llx,"
2508 "pkey=ffff,service_id=%016llx\n", srpt_service_guid,
2509 srpt_service_guid, srpt_service_guid);
2510
2511 /*
2512 * We do not have a consistent service_id (ie. also id_ext of target_id)
2513 * to identify this target. We currently use the guid of the first HCA
2514 * in the system as service_id; therefore, the target_id will change
2515 * if this HCA is gone bad and replaced by different HCA
2516 */
Haggai Eran73fec7f2015-07-30 17:50:26 +03002517 if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0))
Bart Van Asschea42d9852011-10-14 01:30:46 +00002518 goto err_cm;
2519
2520 INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
2521 srpt_event_handler);
2522 if (ib_register_event_handler(&sdev->event_handler))
2523 goto err_cm;
2524
2525 sdev->ioctx_ring = (struct srpt_recv_ioctx **)
2526 srpt_alloc_ioctx_ring(sdev, sdev->srq_size,
2527 sizeof(*sdev->ioctx_ring[0]),
2528 srp_max_req_size, DMA_FROM_DEVICE);
2529 if (!sdev->ioctx_ring)
2530 goto err_event;
2531
2532 for (i = 0; i < sdev->srq_size; ++i)
2533 srpt_post_recv(sdev, sdev->ioctx_ring[i]);
2534
Roland Dreierf2250662012-02-02 12:55:58 -08002535 WARN_ON(sdev->device->phys_port_cnt > ARRAY_SIZE(sdev->port));
Bart Van Asschea42d9852011-10-14 01:30:46 +00002536
2537 for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
2538 sport = &sdev->port[i - 1];
2539 sport->sdev = sdev;
2540 sport->port = i;
2541 sport->port_attrib.srp_max_rdma_size = DEFAULT_MAX_RDMA_SIZE;
2542 sport->port_attrib.srp_max_rsp_size = DEFAULT_MAX_RSP_SIZE;
2543 sport->port_attrib.srp_sq_size = DEF_SRPT_SQ_SIZE;
2544 INIT_WORK(&sport->work, srpt_refresh_port_work);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002545
2546 if (srpt_refresh_port(sport)) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002547 pr_err("MAD registration failed for %s-%d.\n",
Bart Van Asschef68cba4e92016-02-11 11:04:20 -08002548 sdev->device->name, i);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002549 goto err_ring;
2550 }
Bart Van Asschea42d9852011-10-14 01:30:46 +00002551 }
2552
2553 spin_lock(&srpt_dev_lock);
2554 list_add_tail(&sdev->list, &srpt_dev_list);
2555 spin_unlock(&srpt_dev_lock);
2556
2557out:
2558 ib_set_client_data(device, &srpt_client, sdev);
2559 pr_debug("added %s.\n", device->name);
2560 return;
2561
2562err_ring:
2563 srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,
2564 sdev->srq_size, srp_max_req_size,
2565 DMA_FROM_DEVICE);
2566err_event:
2567 ib_unregister_event_handler(&sdev->event_handler);
2568err_cm:
2569 ib_destroy_cm_id(sdev->cm_id);
2570err_srq:
2571 ib_destroy_srq(sdev->srq);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002572err_pd:
2573 ib_dealloc_pd(sdev->pd);
2574free_dev:
2575 kfree(sdev);
2576err:
2577 sdev = NULL;
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002578 pr_info("%s(%s) failed.\n", __func__, device->name);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002579 goto out;
2580}
2581
2582/**
2583 * srpt_remove_one() - InfiniBand device removal callback function.
2584 */
Haggai Eran7c1eb452015-07-30 17:50:14 +03002585static void srpt_remove_one(struct ib_device *device, void *client_data)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002586{
Haggai Eran7c1eb452015-07-30 17:50:14 +03002587 struct srpt_device *sdev = client_data;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002588 int i;
2589
Bart Van Asschea42d9852011-10-14 01:30:46 +00002590 if (!sdev) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002591 pr_info("%s(%s): nothing to do.\n", __func__, device->name);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002592 return;
2593 }
2594
2595 srpt_unregister_mad_agent(sdev);
2596
2597 ib_unregister_event_handler(&sdev->event_handler);
2598
2599 /* Cancel any work queued by the just unregistered IB event handler. */
2600 for (i = 0; i < sdev->device->phys_port_cnt; i++)
2601 cancel_work_sync(&sdev->port[i].work);
2602
2603 ib_destroy_cm_id(sdev->cm_id);
2604
2605 /*
2606 * Unregistering a target must happen after destroying sdev->cm_id
2607 * such that no new SRP_LOGIN_REQ information units can arrive while
2608 * destroying the target.
2609 */
2610 spin_lock(&srpt_dev_lock);
2611 list_del(&sdev->list);
2612 spin_unlock(&srpt_dev_lock);
2613 srpt_release_sdev(sdev);
2614
2615 ib_destroy_srq(sdev->srq);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002616 ib_dealloc_pd(sdev->pd);
2617
2618 srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,
2619 sdev->srq_size, srp_max_req_size, DMA_FROM_DEVICE);
2620 sdev->ioctx_ring = NULL;
2621 kfree(sdev);
2622}
2623
2624static struct ib_client srpt_client = {
2625 .name = DRV_NAME,
2626 .add = srpt_add_one,
2627 .remove = srpt_remove_one
2628};
2629
2630static int srpt_check_true(struct se_portal_group *se_tpg)
2631{
2632 return 1;
2633}
2634
2635static int srpt_check_false(struct se_portal_group *se_tpg)
2636{
2637 return 0;
2638}
2639
2640static char *srpt_get_fabric_name(void)
2641{
2642 return "srpt";
2643}
2644
Bart Van Asschea42d9852011-10-14 01:30:46 +00002645static char *srpt_get_fabric_wwn(struct se_portal_group *tpg)
2646{
2647 struct srpt_port *sport = container_of(tpg, struct srpt_port, port_tpg_1);
2648
2649 return sport->port_guid;
2650}
2651
2652static u16 srpt_get_tag(struct se_portal_group *tpg)
2653{
2654 return 1;
2655}
2656
Bart Van Asschea42d9852011-10-14 01:30:46 +00002657static u32 srpt_tpg_get_inst_index(struct se_portal_group *se_tpg)
2658{
2659 return 1;
2660}
2661
2662static void srpt_release_cmd(struct se_cmd *se_cmd)
2663{
Nicholas Bellinger9474b042012-11-27 23:55:57 -08002664 struct srpt_send_ioctx *ioctx = container_of(se_cmd,
2665 struct srpt_send_ioctx, cmd);
2666 struct srpt_rdma_ch *ch = ioctx->ch;
Bart Van Assche3c968882016-04-07 15:55:04 -07002667 unsigned long flags;
Nicholas Bellinger9474b042012-11-27 23:55:57 -08002668
Bart Van Assche84284962017-05-04 15:50:54 -07002669 WARN_ON_ONCE(ioctx->state != SRPT_STATE_DONE &&
2670 !(ioctx->cmd.transport_state & CMD_T_ABORTED));
Nicholas Bellinger9474b042012-11-27 23:55:57 -08002671
Christoph Hellwigb99f8e42016-05-03 18:01:11 +02002672 if (ioctx->n_rw_ctx) {
2673 srpt_free_rw_ctxs(ch, ioctx);
2674 ioctx->n_rw_ctx = 0;
Nicholas Bellinger9474b042012-11-27 23:55:57 -08002675 }
2676
Bart Van Assche3c968882016-04-07 15:55:04 -07002677 spin_lock_irqsave(&ch->spinlock, flags);
2678 list_add(&ioctx->free_list, &ch->free_list);
2679 spin_unlock_irqrestore(&ch->spinlock, flags);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002680}
2681
2682/**
Bart Van Asschea42d9852011-10-14 01:30:46 +00002683 * srpt_close_session() - Forcibly close a session.
2684 *
2685 * Callback function invoked by the TCM core to clean up sessions associated
2686 * with a node ACL when the user invokes
2687 * rmdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id
2688 */
2689static void srpt_close_session(struct se_session *se_sess)
2690{
2691 DECLARE_COMPLETION_ONSTACK(release_done);
Bart Van Asschef108f0f2016-02-11 11:06:14 -08002692 struct srpt_rdma_ch *ch = se_sess->fabric_sess_ptr;
2693 struct srpt_device *sdev = ch->sport->sdev;
2694 bool wait;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002695
Bart Van Asschef108f0f2016-02-11 11:06:14 -08002696 pr_debug("ch %s-%d state %d\n", ch->sess_name, ch->qp->qp_num,
2697 ch->state);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002698
Bart Van Assche86289912016-02-11 11:08:34 -08002699 mutex_lock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002700 BUG_ON(ch->release_done);
2701 ch->release_done = &release_done;
Bart Van Asschef108f0f2016-02-11 11:06:14 -08002702 wait = !list_empty(&ch->list);
Bart Van Asscheaaf45bd2016-02-11 11:08:53 -08002703 srpt_disconnect_ch(ch);
Bart Van Assche86289912016-02-11 11:08:34 -08002704 mutex_unlock(&sdev->mutex);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002705
Bart Van Asschef108f0f2016-02-11 11:06:14 -08002706 if (!wait)
2707 return;
2708
2709 while (wait_for_completion_timeout(&release_done, 180 * HZ) == 0)
2710 pr_info("%s(%s-%d state %d): still waiting ...\n", __func__,
2711 ch->sess_name, ch->qp->qp_num, ch->state);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002712}
2713
2714/**
Bart Van Asschea42d9852011-10-14 01:30:46 +00002715 * srpt_sess_get_index() - Return the value of scsiAttIntrPortIndex (SCSI-MIB).
2716 *
2717 * A quote from RFC 4455 (SCSI-MIB) about this MIB object:
2718 * This object represents an arbitrary integer used to uniquely identify a
2719 * particular attached remote initiator port to a particular SCSI target port
2720 * within a particular SCSI target device within a particular SCSI instance.
2721 */
2722static u32 srpt_sess_get_index(struct se_session *se_sess)
2723{
2724 return 0;
2725}
2726
2727static void srpt_set_default_node_attrs(struct se_node_acl *nacl)
2728{
2729}
2730
Bart Van Asschea42d9852011-10-14 01:30:46 +00002731/* Note: only used from inside debug printk's by the TCM core. */
2732static int srpt_get_tcm_cmd_state(struct se_cmd *se_cmd)
2733{
2734 struct srpt_send_ioctx *ioctx;
2735
2736 ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
2737 return srpt_get_cmd_state(ioctx);
2738}
2739
Bart Van Asschea42d9852011-10-14 01:30:46 +00002740/**
2741 * srpt_parse_i_port_id() - Parse an initiator port ID.
2742 * @name: ASCII representation of a 128-bit initiator port ID.
2743 * @i_port_id: Binary 128-bit port ID.
2744 */
2745static int srpt_parse_i_port_id(u8 i_port_id[16], const char *name)
2746{
2747 const char *p;
2748 unsigned len, count, leading_zero_bytes;
Bart Van Assche8105ea32017-10-11 10:27:22 -07002749 int ret;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002750
2751 p = name;
Rasmus Villemoesb60459f2014-10-13 15:54:46 -07002752 if (strncasecmp(p, "0x", 2) == 0)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002753 p += 2;
2754 ret = -EINVAL;
2755 len = strlen(p);
2756 if (len % 2)
2757 goto out;
2758 count = min(len / 2, 16U);
2759 leading_zero_bytes = 16 - count;
2760 memset(i_port_id, 0, leading_zero_bytes);
Bart Van Assche8105ea32017-10-11 10:27:22 -07002761 ret = hex2bin(i_port_id + leading_zero_bytes, p, count);
2762 if (ret < 0)
2763 pr_debug("hex2bin failed for srpt_parse_i_port_id: %d\n", ret);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002764out:
2765 return ret;
2766}
2767
2768/*
2769 * configfs callback function invoked for
2770 * mkdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id
2771 */
Christoph Hellwigc7d6a802015-04-13 19:51:14 +02002772static int srpt_init_nodeacl(struct se_node_acl *se_nacl, const char *name)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002773{
Bart Van Asschea42d9852011-10-14 01:30:46 +00002774 u8 i_port_id[16];
2775
2776 if (srpt_parse_i_port_id(i_port_id, name) < 0) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002777 pr_err("invalid initiator port ID %s\n", name);
Christoph Hellwigc7d6a802015-04-13 19:51:14 +02002778 return -EINVAL;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002779 }
Christoph Hellwigc7d6a802015-04-13 19:51:14 +02002780 return 0;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002781}
2782
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002783static ssize_t srpt_tpg_attrib_srp_max_rdma_size_show(struct config_item *item,
2784 char *page)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002785{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002786 struct se_portal_group *se_tpg = attrib_to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002787 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2788
2789 return sprintf(page, "%u\n", sport->port_attrib.srp_max_rdma_size);
2790}
2791
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002792static ssize_t srpt_tpg_attrib_srp_max_rdma_size_store(struct config_item *item,
2793 const char *page, size_t count)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002794{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002795 struct se_portal_group *se_tpg = attrib_to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002796 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2797 unsigned long val;
2798 int ret;
2799
Jingoo Han9d8abf42014-02-05 11:22:05 +09002800 ret = kstrtoul(page, 0, &val);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002801 if (ret < 0) {
Jingoo Han9d8abf42014-02-05 11:22:05 +09002802 pr_err("kstrtoul() failed with ret: %d\n", ret);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002803 return -EINVAL;
2804 }
2805 if (val > MAX_SRPT_RDMA_SIZE) {
2806 pr_err("val: %lu exceeds MAX_SRPT_RDMA_SIZE: %d\n", val,
2807 MAX_SRPT_RDMA_SIZE);
2808 return -EINVAL;
2809 }
2810 if (val < DEFAULT_MAX_RDMA_SIZE) {
2811 pr_err("val: %lu smaller than DEFAULT_MAX_RDMA_SIZE: %d\n",
2812 val, DEFAULT_MAX_RDMA_SIZE);
2813 return -EINVAL;
2814 }
2815 sport->port_attrib.srp_max_rdma_size = val;
2816
2817 return count;
2818}
2819
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002820static ssize_t srpt_tpg_attrib_srp_max_rsp_size_show(struct config_item *item,
2821 char *page)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002822{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002823 struct se_portal_group *se_tpg = attrib_to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002824 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2825
2826 return sprintf(page, "%u\n", sport->port_attrib.srp_max_rsp_size);
2827}
2828
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002829static ssize_t srpt_tpg_attrib_srp_max_rsp_size_store(struct config_item *item,
2830 const char *page, size_t count)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002831{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002832 struct se_portal_group *se_tpg = attrib_to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002833 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2834 unsigned long val;
2835 int ret;
2836
Jingoo Han9d8abf42014-02-05 11:22:05 +09002837 ret = kstrtoul(page, 0, &val);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002838 if (ret < 0) {
Jingoo Han9d8abf42014-02-05 11:22:05 +09002839 pr_err("kstrtoul() failed with ret: %d\n", ret);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002840 return -EINVAL;
2841 }
2842 if (val > MAX_SRPT_RSP_SIZE) {
2843 pr_err("val: %lu exceeds MAX_SRPT_RSP_SIZE: %d\n", val,
2844 MAX_SRPT_RSP_SIZE);
2845 return -EINVAL;
2846 }
2847 if (val < MIN_MAX_RSP_SIZE) {
2848 pr_err("val: %lu smaller than MIN_MAX_RSP_SIZE: %d\n", val,
2849 MIN_MAX_RSP_SIZE);
2850 return -EINVAL;
2851 }
2852 sport->port_attrib.srp_max_rsp_size = val;
2853
2854 return count;
2855}
2856
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002857static ssize_t srpt_tpg_attrib_srp_sq_size_show(struct config_item *item,
2858 char *page)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002859{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002860 struct se_portal_group *se_tpg = attrib_to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002861 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2862
2863 return sprintf(page, "%u\n", sport->port_attrib.srp_sq_size);
2864}
2865
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002866static ssize_t srpt_tpg_attrib_srp_sq_size_store(struct config_item *item,
2867 const char *page, size_t count)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002868{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002869 struct se_portal_group *se_tpg = attrib_to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002870 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2871 unsigned long val;
2872 int ret;
2873
Jingoo Han9d8abf42014-02-05 11:22:05 +09002874 ret = kstrtoul(page, 0, &val);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002875 if (ret < 0) {
Jingoo Han9d8abf42014-02-05 11:22:05 +09002876 pr_err("kstrtoul() failed with ret: %d\n", ret);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002877 return -EINVAL;
2878 }
2879 if (val > MAX_SRPT_SRQ_SIZE) {
2880 pr_err("val: %lu exceeds MAX_SRPT_SRQ_SIZE: %d\n", val,
2881 MAX_SRPT_SRQ_SIZE);
2882 return -EINVAL;
2883 }
2884 if (val < MIN_SRPT_SRQ_SIZE) {
2885 pr_err("val: %lu smaller than MIN_SRPT_SRQ_SIZE: %d\n", val,
2886 MIN_SRPT_SRQ_SIZE);
2887 return -EINVAL;
2888 }
2889 sport->port_attrib.srp_sq_size = val;
2890
2891 return count;
2892}
2893
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002894CONFIGFS_ATTR(srpt_tpg_attrib_, srp_max_rdma_size);
2895CONFIGFS_ATTR(srpt_tpg_attrib_, srp_max_rsp_size);
2896CONFIGFS_ATTR(srpt_tpg_attrib_, srp_sq_size);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002897
2898static struct configfs_attribute *srpt_tpg_attrib_attrs[] = {
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002899 &srpt_tpg_attrib_attr_srp_max_rdma_size,
2900 &srpt_tpg_attrib_attr_srp_max_rsp_size,
2901 &srpt_tpg_attrib_attr_srp_sq_size,
Bart Van Asschea42d9852011-10-14 01:30:46 +00002902 NULL,
2903};
2904
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002905static ssize_t srpt_tpg_enable_show(struct config_item *item, char *page)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002906{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002907 struct se_portal_group *se_tpg = to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002908 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2909
2910 return snprintf(page, PAGE_SIZE, "%d\n", (sport->enabled) ? 1: 0);
2911}
2912
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002913static ssize_t srpt_tpg_enable_store(struct config_item *item,
2914 const char *page, size_t count)
Bart Van Asschea42d9852011-10-14 01:30:46 +00002915{
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002916 struct se_portal_group *se_tpg = to_tpg(item);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002917 struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
Bart Van Assche043a6802016-02-11 11:09:28 -08002918 struct srpt_device *sdev = sport->sdev;
2919 struct srpt_rdma_ch *ch;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002920 unsigned long tmp;
2921 int ret;
2922
Jingoo Han9d8abf42014-02-05 11:22:05 +09002923 ret = kstrtoul(page, 0, &tmp);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002924 if (ret < 0) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002925 pr_err("Unable to extract srpt_tpg_store_enable\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +00002926 return -EINVAL;
2927 }
2928
2929 if ((tmp != 0) && (tmp != 1)) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04002930 pr_err("Illegal value for srpt_tpg_store_enable: %lu\n", tmp);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002931 return -EINVAL;
2932 }
Bart Van Assche043a6802016-02-11 11:09:28 -08002933 if (sport->enabled == tmp)
2934 goto out;
2935 sport->enabled = tmp;
2936 if (sport->enabled)
2937 goto out;
Bart Van Asschea42d9852011-10-14 01:30:46 +00002938
Bart Van Assche043a6802016-02-11 11:09:28 -08002939 mutex_lock(&sdev->mutex);
2940 list_for_each_entry(ch, &sdev->rch_list, list) {
2941 if (ch->sport == sport) {
2942 pr_debug("%s: ch %p %s-%d\n", __func__, ch,
2943 ch->sess_name, ch->qp->qp_num);
2944 srpt_disconnect_ch(ch);
2945 srpt_close_ch(ch);
2946 }
2947 }
2948 mutex_unlock(&sdev->mutex);
2949
2950out:
Bart Van Asschea42d9852011-10-14 01:30:46 +00002951 return count;
2952}
2953
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002954CONFIGFS_ATTR(srpt_tpg_, enable);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002955
2956static struct configfs_attribute *srpt_tpg_attrs[] = {
Christoph Hellwig2eafd722015-10-03 15:32:55 +02002957 &srpt_tpg_attr_enable,
Bart Van Asschea42d9852011-10-14 01:30:46 +00002958 NULL,
2959};
2960
2961/**
2962 * configfs callback invoked for
2963 * mkdir /sys/kernel/config/target/$driver/$port/$tpg
2964 */
2965static struct se_portal_group *srpt_make_tpg(struct se_wwn *wwn,
2966 struct config_group *group,
2967 const char *name)
2968{
2969 struct srpt_port *sport = container_of(wwn, struct srpt_port, port_wwn);
2970 int res;
2971
2972 /* Initialize sport->port_wwn and sport->port_tpg_1 */
Nicholas Bellingerbc0c94b2015-05-20 21:48:03 -07002973 res = core_tpg_register(&sport->port_wwn, &sport->port_tpg_1, SCSI_PROTOCOL_SRP);
Bart Van Asschea42d9852011-10-14 01:30:46 +00002974 if (res)
2975 return ERR_PTR(res);
2976
2977 return &sport->port_tpg_1;
2978}
2979
2980/**
2981 * configfs callback invoked for
2982 * rmdir /sys/kernel/config/target/$driver/$port/$tpg
2983 */
2984static void srpt_drop_tpg(struct se_portal_group *tpg)
2985{
2986 struct srpt_port *sport = container_of(tpg,
2987 struct srpt_port, port_tpg_1);
2988
2989 sport->enabled = false;
2990 core_tpg_deregister(&sport->port_tpg_1);
2991}
2992
2993/**
2994 * configfs callback invoked for
2995 * mkdir /sys/kernel/config/target/$driver/$port
2996 */
2997static struct se_wwn *srpt_make_tport(struct target_fabric_configfs *tf,
2998 struct config_group *group,
2999 const char *name)
3000{
3001 struct srpt_port *sport;
3002 int ret;
3003
3004 sport = srpt_lookup_port(name);
3005 pr_debug("make_tport(%s)\n", name);
3006 ret = -EINVAL;
3007 if (!sport)
3008 goto err;
3009
3010 return &sport->port_wwn;
3011
3012err:
3013 return ERR_PTR(ret);
3014}
3015
3016/**
3017 * configfs callback invoked for
3018 * rmdir /sys/kernel/config/target/$driver/$port
3019 */
3020static void srpt_drop_tport(struct se_wwn *wwn)
3021{
3022 struct srpt_port *sport = container_of(wwn, struct srpt_port, port_wwn);
3023
3024 pr_debug("drop_tport(%s\n", config_item_name(&sport->port_wwn.wwn_group.cg_item));
3025}
3026
Christoph Hellwig2eafd722015-10-03 15:32:55 +02003027static ssize_t srpt_wwn_version_show(struct config_item *item, char *buf)
Bart Van Asschea42d9852011-10-14 01:30:46 +00003028{
3029 return scnprintf(buf, PAGE_SIZE, "%s\n", DRV_VERSION);
3030}
3031
Christoph Hellwig2eafd722015-10-03 15:32:55 +02003032CONFIGFS_ATTR_RO(srpt_wwn_, version);
Bart Van Asschea42d9852011-10-14 01:30:46 +00003033
3034static struct configfs_attribute *srpt_wwn_attrs[] = {
Christoph Hellwig2eafd722015-10-03 15:32:55 +02003035 &srpt_wwn_attr_version,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003036 NULL,
3037};
3038
Christoph Hellwig9ac89282015-04-08 20:01:35 +02003039static const struct target_core_fabric_ops srpt_template = {
3040 .module = THIS_MODULE,
3041 .name = "srpt",
Bart Van Asschea42d9852011-10-14 01:30:46 +00003042 .get_fabric_name = srpt_get_fabric_name,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003043 .tpg_get_wwn = srpt_get_fabric_wwn,
3044 .tpg_get_tag = srpt_get_tag,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003045 .tpg_check_demo_mode = srpt_check_false,
3046 .tpg_check_demo_mode_cache = srpt_check_true,
3047 .tpg_check_demo_mode_write_protect = srpt_check_true,
3048 .tpg_check_prod_mode_write_protect = srpt_check_false,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003049 .tpg_get_inst_index = srpt_tpg_get_inst_index,
3050 .release_cmd = srpt_release_cmd,
3051 .check_stop_free = srpt_check_stop_free,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003052 .close_session = srpt_close_session,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003053 .sess_get_index = srpt_sess_get_index,
3054 .sess_get_initiator_sid = NULL,
3055 .write_pending = srpt_write_pending,
3056 .write_pending_status = srpt_write_pending_status,
3057 .set_default_node_attributes = srpt_set_default_node_attrs,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003058 .get_cmd_state = srpt_get_tcm_cmd_state,
Joern Engelb79fafa2013-07-03 11:22:17 -04003059 .queue_data_in = srpt_queue_data_in,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003060 .queue_status = srpt_queue_status,
Joern Engelb79fafa2013-07-03 11:22:17 -04003061 .queue_tm_rsp = srpt_queue_tm_rsp,
Nicholas Bellinger131e6ab2014-03-22 14:55:56 -07003062 .aborted_task = srpt_aborted_task,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003063 /*
3064 * Setup function pointers for generic logic in
3065 * target_core_fabric_configfs.c
3066 */
3067 .fabric_make_wwn = srpt_make_tport,
3068 .fabric_drop_wwn = srpt_drop_tport,
3069 .fabric_make_tpg = srpt_make_tpg,
3070 .fabric_drop_tpg = srpt_drop_tpg,
Christoph Hellwigc7d6a802015-04-13 19:51:14 +02003071 .fabric_init_nodeacl = srpt_init_nodeacl,
Christoph Hellwig9ac89282015-04-08 20:01:35 +02003072
3073 .tfc_wwn_attrs = srpt_wwn_attrs,
3074 .tfc_tpg_base_attrs = srpt_tpg_attrs,
3075 .tfc_tpg_attrib_attrs = srpt_tpg_attrib_attrs,
Bart Van Asschea42d9852011-10-14 01:30:46 +00003076};
3077
3078/**
3079 * srpt_init_module() - Kernel module initialization.
3080 *
3081 * Note: Since ib_register_client() registers callback functions, and since at
3082 * least one of these callback functions (srpt_add_one()) calls target core
3083 * functions, this driver must be registered with the target core before
3084 * ib_register_client() is called.
3085 */
3086static int __init srpt_init_module(void)
3087{
3088 int ret;
3089
3090 ret = -EINVAL;
3091 if (srp_max_req_size < MIN_MAX_REQ_SIZE) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04003092 pr_err("invalid value %d for kernel module parameter"
Bart Van Asschea42d9852011-10-14 01:30:46 +00003093 " srp_max_req_size -- must be at least %d.\n",
3094 srp_max_req_size, MIN_MAX_REQ_SIZE);
3095 goto out;
3096 }
3097
3098 if (srpt_srq_size < MIN_SRPT_SRQ_SIZE
3099 || srpt_srq_size > MAX_SRPT_SRQ_SIZE) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04003100 pr_err("invalid value %d for kernel module parameter"
Bart Van Asschea42d9852011-10-14 01:30:46 +00003101 " srpt_srq_size -- must be in the range [%d..%d].\n",
3102 srpt_srq_size, MIN_SRPT_SRQ_SIZE, MAX_SRPT_SRQ_SIZE);
3103 goto out;
3104 }
3105
Christoph Hellwig9ac89282015-04-08 20:01:35 +02003106 ret = target_register_template(&srpt_template);
3107 if (ret)
Bart Van Asschea42d9852011-10-14 01:30:46 +00003108 goto out;
Bart Van Asschea42d9852011-10-14 01:30:46 +00003109
3110 ret = ib_register_client(&srpt_client);
3111 if (ret) {
Doug Ledford9f5d32a2014-10-20 18:25:15 -04003112 pr_err("couldn't register IB client\n");
Bart Van Asschea42d9852011-10-14 01:30:46 +00003113 goto out_unregister_target;
3114 }
3115
3116 return 0;
3117
3118out_unregister_target:
Christoph Hellwig9ac89282015-04-08 20:01:35 +02003119 target_unregister_template(&srpt_template);
Bart Van Asschea42d9852011-10-14 01:30:46 +00003120out:
3121 return ret;
3122}
3123
3124static void __exit srpt_cleanup_module(void)
3125{
3126 ib_unregister_client(&srpt_client);
Christoph Hellwig9ac89282015-04-08 20:01:35 +02003127 target_unregister_template(&srpt_template);
Bart Van Asschea42d9852011-10-14 01:30:46 +00003128}
3129
3130module_init(srpt_init_module);
3131module_exit(srpt_cleanup_module);