Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2006 - 2009 Mellanox Technology Inc. All rights reserved. |
| 3 | * Copyright (C) 2008 - 2011 Bart Van Assche <bvanassche@acm.org>. |
| 4 | * |
| 5 | * This software is available to you under a choice of one of two |
| 6 | * licenses. You may choose to be licensed under the terms of the GNU |
| 7 | * General Public License (GPL) Version 2, available from the file |
| 8 | * COPYING in the main directory of this source tree, or the |
| 9 | * OpenIB.org BSD license below: |
| 10 | * |
| 11 | * Redistribution and use in source and binary forms, with or |
| 12 | * without modification, are permitted provided that the following |
| 13 | * conditions are met: |
| 14 | * |
| 15 | * - Redistributions of source code must retain the above |
| 16 | * copyright notice, this list of conditions and the following |
| 17 | * disclaimer. |
| 18 | * |
| 19 | * - Redistributions in binary form must reproduce the above |
| 20 | * copyright notice, this list of conditions and the following |
| 21 | * disclaimer in the documentation and/or other materials |
| 22 | * provided with the distribution. |
| 23 | * |
| 24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
| 28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
| 29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 31 | * SOFTWARE. |
| 32 | * |
| 33 | */ |
| 34 | |
| 35 | #include <linux/module.h> |
| 36 | #include <linux/init.h> |
| 37 | #include <linux/slab.h> |
| 38 | #include <linux/err.h> |
| 39 | #include <linux/ctype.h> |
| 40 | #include <linux/kthread.h> |
| 41 | #include <linux/string.h> |
| 42 | #include <linux/delay.h> |
| 43 | #include <linux/atomic.h> |
Bart Van Assche | ba92999 | 2015-05-08 10:11:12 +0200 | [diff] [blame] | 44 | #include <scsi/scsi_proto.h> |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 45 | #include <scsi/scsi_tcq.h> |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 46 | #include <target/target_core_base.h> |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 47 | #include <target/target_core_fabric.h> |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 48 | #include "ib_srpt.h" |
| 49 | |
| 50 | /* Name of this kernel module. */ |
| 51 | #define DRV_NAME "ib_srpt" |
| 52 | #define DRV_VERSION "2.0.0" |
| 53 | #define DRV_RELDATE "2011-02-14" |
| 54 | |
| 55 | #define SRPT_ID_STRING "Linux SRP target" |
| 56 | |
| 57 | #undef pr_fmt |
| 58 | #define pr_fmt(fmt) DRV_NAME " " fmt |
| 59 | |
| 60 | MODULE_AUTHOR("Vu Pham and Bart Van Assche"); |
| 61 | MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target " |
| 62 | "v" DRV_VERSION " (" DRV_RELDATE ")"); |
| 63 | MODULE_LICENSE("Dual BSD/GPL"); |
| 64 | |
| 65 | /* |
| 66 | * Global Variables |
| 67 | */ |
| 68 | |
| 69 | static u64 srpt_service_guid; |
Roland Dreier | 486d8b9 | 2012-02-02 12:55:58 -0800 | [diff] [blame] | 70 | static DEFINE_SPINLOCK(srpt_dev_lock); /* Protects srpt_dev_list. */ |
| 71 | static LIST_HEAD(srpt_dev_list); /* List of srpt_device structures. */ |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 72 | |
| 73 | static unsigned srp_max_req_size = DEFAULT_MAX_REQ_SIZE; |
| 74 | module_param(srp_max_req_size, int, 0444); |
| 75 | MODULE_PARM_DESC(srp_max_req_size, |
| 76 | "Maximum size of SRP request messages in bytes."); |
| 77 | |
| 78 | static int srpt_srq_size = DEFAULT_SRPT_SRQ_SIZE; |
| 79 | module_param(srpt_srq_size, int, 0444); |
| 80 | MODULE_PARM_DESC(srpt_srq_size, |
| 81 | "Shared receive queue (SRQ) size."); |
| 82 | |
| 83 | static int srpt_get_u64_x(char *buffer, struct kernel_param *kp) |
| 84 | { |
| 85 | return sprintf(buffer, "0x%016llx", *(u64 *)kp->arg); |
| 86 | } |
| 87 | module_param_call(srpt_service_guid, NULL, srpt_get_u64_x, &srpt_service_guid, |
| 88 | 0444); |
| 89 | MODULE_PARM_DESC(srpt_service_guid, |
| 90 | "Using this value for ioc_guid, id_ext, and cm_listen_id" |
| 91 | " instead of using the node_guid of the first HCA."); |
| 92 | |
| 93 | static struct ib_client srpt_client; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 94 | static void srpt_release_channel(struct srpt_rdma_ch *ch); |
| 95 | static int srpt_queue_status(struct se_cmd *cmd); |
| 96 | |
| 97 | /** |
| 98 | * opposite_dma_dir() - Swap DMA_TO_DEVICE and DMA_FROM_DEVICE. |
| 99 | */ |
| 100 | static inline |
| 101 | enum dma_data_direction opposite_dma_dir(enum dma_data_direction dir) |
| 102 | { |
| 103 | switch (dir) { |
| 104 | case DMA_TO_DEVICE: return DMA_FROM_DEVICE; |
| 105 | case DMA_FROM_DEVICE: return DMA_TO_DEVICE; |
| 106 | default: return dir; |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | /** |
| 111 | * srpt_sdev_name() - Return the name associated with the HCA. |
| 112 | * |
| 113 | * Examples are ib0, ib1, ... |
| 114 | */ |
| 115 | static inline const char *srpt_sdev_name(struct srpt_device *sdev) |
| 116 | { |
| 117 | return sdev->device->name; |
| 118 | } |
| 119 | |
| 120 | static enum rdma_ch_state srpt_get_ch_state(struct srpt_rdma_ch *ch) |
| 121 | { |
| 122 | unsigned long flags; |
| 123 | enum rdma_ch_state state; |
| 124 | |
| 125 | spin_lock_irqsave(&ch->spinlock, flags); |
| 126 | state = ch->state; |
| 127 | spin_unlock_irqrestore(&ch->spinlock, flags); |
| 128 | return state; |
| 129 | } |
| 130 | |
| 131 | static enum rdma_ch_state |
| 132 | srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new_state) |
| 133 | { |
| 134 | unsigned long flags; |
| 135 | enum rdma_ch_state prev; |
| 136 | |
| 137 | spin_lock_irqsave(&ch->spinlock, flags); |
| 138 | prev = ch->state; |
| 139 | ch->state = new_state; |
| 140 | spin_unlock_irqrestore(&ch->spinlock, flags); |
| 141 | return prev; |
| 142 | } |
| 143 | |
| 144 | /** |
| 145 | * srpt_test_and_set_ch_state() - Test and set the channel state. |
| 146 | * |
| 147 | * Returns true if and only if the channel state has been set to the new state. |
| 148 | */ |
| 149 | static bool |
| 150 | srpt_test_and_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state old, |
| 151 | enum rdma_ch_state new) |
| 152 | { |
| 153 | unsigned long flags; |
| 154 | enum rdma_ch_state prev; |
| 155 | |
| 156 | spin_lock_irqsave(&ch->spinlock, flags); |
| 157 | prev = ch->state; |
| 158 | if (prev == old) |
| 159 | ch->state = new; |
| 160 | spin_unlock_irqrestore(&ch->spinlock, flags); |
| 161 | return prev == old; |
| 162 | } |
| 163 | |
| 164 | /** |
| 165 | * srpt_event_handler() - Asynchronous IB event callback function. |
| 166 | * |
| 167 | * Callback function called by the InfiniBand core when an asynchronous IB |
| 168 | * event occurs. This callback may occur in interrupt context. See also |
| 169 | * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand |
| 170 | * Architecture Specification. |
| 171 | */ |
| 172 | static void srpt_event_handler(struct ib_event_handler *handler, |
| 173 | struct ib_event *event) |
| 174 | { |
| 175 | struct srpt_device *sdev; |
| 176 | struct srpt_port *sport; |
| 177 | |
| 178 | sdev = ib_get_client_data(event->device, &srpt_client); |
| 179 | if (!sdev || sdev->device != event->device) |
| 180 | return; |
| 181 | |
| 182 | pr_debug("ASYNC event= %d on device= %s\n", event->event, |
| 183 | srpt_sdev_name(sdev)); |
| 184 | |
| 185 | switch (event->event) { |
| 186 | case IB_EVENT_PORT_ERR: |
| 187 | if (event->element.port_num <= sdev->device->phys_port_cnt) { |
| 188 | sport = &sdev->port[event->element.port_num - 1]; |
| 189 | sport->lid = 0; |
| 190 | sport->sm_lid = 0; |
| 191 | } |
| 192 | break; |
| 193 | case IB_EVENT_PORT_ACTIVE: |
| 194 | case IB_EVENT_LID_CHANGE: |
| 195 | case IB_EVENT_PKEY_CHANGE: |
| 196 | case IB_EVENT_SM_CHANGE: |
| 197 | case IB_EVENT_CLIENT_REREGISTER: |
Doug Ledford | 2aa1cf6 | 2014-08-12 19:20:10 -0400 | [diff] [blame] | 198 | case IB_EVENT_GID_CHANGE: |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 199 | /* Refresh port data asynchronously. */ |
| 200 | if (event->element.port_num <= sdev->device->phys_port_cnt) { |
| 201 | sport = &sdev->port[event->element.port_num - 1]; |
| 202 | if (!sport->lid && !sport->sm_lid) |
| 203 | schedule_work(&sport->work); |
| 204 | } |
| 205 | break; |
| 206 | default: |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 207 | pr_err("received unrecognized IB event %d\n", |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 208 | event->event); |
| 209 | break; |
| 210 | } |
| 211 | } |
| 212 | |
| 213 | /** |
| 214 | * srpt_srq_event() - SRQ event callback function. |
| 215 | */ |
| 216 | static void srpt_srq_event(struct ib_event *event, void *ctx) |
| 217 | { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 218 | pr_info("SRQ event %d\n", event->event); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 219 | } |
| 220 | |
| 221 | /** |
| 222 | * srpt_qp_event() - QP event callback function. |
| 223 | */ |
| 224 | static void srpt_qp_event(struct ib_event *event, struct srpt_rdma_ch *ch) |
| 225 | { |
| 226 | pr_debug("QP event %d on cm_id=%p sess_name=%s state=%d\n", |
| 227 | event->event, ch->cm_id, ch->sess_name, srpt_get_ch_state(ch)); |
| 228 | |
| 229 | switch (event->event) { |
| 230 | case IB_EVENT_COMM_EST: |
| 231 | ib_cm_notify(ch->cm_id, event->event); |
| 232 | break; |
| 233 | case IB_EVENT_QP_LAST_WQE_REACHED: |
| 234 | if (srpt_test_and_set_ch_state(ch, CH_DRAINING, |
| 235 | CH_RELEASING)) |
| 236 | srpt_release_channel(ch); |
| 237 | else |
| 238 | pr_debug("%s: state %d - ignored LAST_WQE.\n", |
| 239 | ch->sess_name, srpt_get_ch_state(ch)); |
| 240 | break; |
| 241 | default: |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 242 | pr_err("received unrecognized IB QP event %d\n", event->event); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 243 | break; |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | /** |
| 248 | * srpt_set_ioc() - Helper function for initializing an IOUnitInfo structure. |
| 249 | * |
| 250 | * @slot: one-based slot number. |
| 251 | * @value: four-bit value. |
| 252 | * |
| 253 | * Copies the lowest four bits of value in element slot of the array of four |
| 254 | * bit elements called c_list (controller list). The index slot is one-based. |
| 255 | */ |
| 256 | static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value) |
| 257 | { |
| 258 | u16 id; |
| 259 | u8 tmp; |
| 260 | |
| 261 | id = (slot - 1) / 2; |
| 262 | if (slot & 0x1) { |
| 263 | tmp = c_list[id] & 0xf; |
| 264 | c_list[id] = (value << 4) | tmp; |
| 265 | } else { |
| 266 | tmp = c_list[id] & 0xf0; |
| 267 | c_list[id] = (value & 0xf) | tmp; |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | /** |
| 272 | * srpt_get_class_port_info() - Copy ClassPortInfo to a management datagram. |
| 273 | * |
| 274 | * See also section 16.3.3.1 ClassPortInfo in the InfiniBand Architecture |
| 275 | * Specification. |
| 276 | */ |
| 277 | static void srpt_get_class_port_info(struct ib_dm_mad *mad) |
| 278 | { |
| 279 | struct ib_class_port_info *cif; |
| 280 | |
| 281 | cif = (struct ib_class_port_info *)mad->data; |
| 282 | memset(cif, 0, sizeof *cif); |
| 283 | cif->base_version = 1; |
| 284 | cif->class_version = 1; |
| 285 | cif->resp_time_value = 20; |
| 286 | |
| 287 | mad->mad_hdr.status = 0; |
| 288 | } |
| 289 | |
| 290 | /** |
| 291 | * srpt_get_iou() - Write IOUnitInfo to a management datagram. |
| 292 | * |
| 293 | * See also section 16.3.3.3 IOUnitInfo in the InfiniBand Architecture |
| 294 | * Specification. See also section B.7, table B.6 in the SRP r16a document. |
| 295 | */ |
| 296 | static void srpt_get_iou(struct ib_dm_mad *mad) |
| 297 | { |
| 298 | struct ib_dm_iou_info *ioui; |
| 299 | u8 slot; |
| 300 | int i; |
| 301 | |
| 302 | ioui = (struct ib_dm_iou_info *)mad->data; |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 303 | ioui->change_id = cpu_to_be16(1); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 304 | ioui->max_controllers = 16; |
| 305 | |
| 306 | /* set present for slot 1 and empty for the rest */ |
| 307 | srpt_set_ioc(ioui->controller_list, 1, 1); |
| 308 | for (i = 1, slot = 2; i < 16; i++, slot++) |
| 309 | srpt_set_ioc(ioui->controller_list, slot, 0); |
| 310 | |
| 311 | mad->mad_hdr.status = 0; |
| 312 | } |
| 313 | |
| 314 | /** |
| 315 | * srpt_get_ioc() - Write IOControllerprofile to a management datagram. |
| 316 | * |
| 317 | * See also section 16.3.3.4 IOControllerProfile in the InfiniBand |
| 318 | * Architecture Specification. See also section B.7, table B.7 in the SRP |
| 319 | * r16a document. |
| 320 | */ |
| 321 | static void srpt_get_ioc(struct srpt_port *sport, u32 slot, |
| 322 | struct ib_dm_mad *mad) |
| 323 | { |
| 324 | struct srpt_device *sdev = sport->sdev; |
| 325 | struct ib_dm_ioc_profile *iocp; |
| 326 | |
| 327 | iocp = (struct ib_dm_ioc_profile *)mad->data; |
| 328 | |
| 329 | if (!slot || slot > 16) { |
| 330 | mad->mad_hdr.status |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 331 | = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 332 | return; |
| 333 | } |
| 334 | |
| 335 | if (slot > 2) { |
| 336 | mad->mad_hdr.status |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 337 | = cpu_to_be16(DM_MAD_STATUS_NO_IOC); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 338 | return; |
| 339 | } |
| 340 | |
| 341 | memset(iocp, 0, sizeof *iocp); |
| 342 | strcpy(iocp->id_string, SRPT_ID_STRING); |
| 343 | iocp->guid = cpu_to_be64(srpt_service_guid); |
| 344 | iocp->vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id); |
| 345 | iocp->device_id = cpu_to_be32(sdev->dev_attr.vendor_part_id); |
| 346 | iocp->device_version = cpu_to_be16(sdev->dev_attr.hw_ver); |
| 347 | iocp->subsys_vendor_id = cpu_to_be32(sdev->dev_attr.vendor_id); |
| 348 | iocp->subsys_device_id = 0x0; |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 349 | iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS); |
| 350 | iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS); |
| 351 | iocp->protocol = cpu_to_be16(SRP_PROTOCOL); |
| 352 | iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 353 | iocp->send_queue_depth = cpu_to_be16(sdev->srq_size); |
| 354 | iocp->rdma_read_depth = 4; |
| 355 | iocp->send_size = cpu_to_be32(srp_max_req_size); |
| 356 | iocp->rdma_size = cpu_to_be32(min(sport->port_attrib.srp_max_rdma_size, |
| 357 | 1U << 24)); |
| 358 | iocp->num_svc_entries = 1; |
| 359 | iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC | |
| 360 | SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC; |
| 361 | |
| 362 | mad->mad_hdr.status = 0; |
| 363 | } |
| 364 | |
| 365 | /** |
| 366 | * srpt_get_svc_entries() - Write ServiceEntries to a management datagram. |
| 367 | * |
| 368 | * See also section 16.3.3.5 ServiceEntries in the InfiniBand Architecture |
| 369 | * Specification. See also section B.7, table B.8 in the SRP r16a document. |
| 370 | */ |
| 371 | static void srpt_get_svc_entries(u64 ioc_guid, |
| 372 | u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad) |
| 373 | { |
| 374 | struct ib_dm_svc_entries *svc_entries; |
| 375 | |
| 376 | WARN_ON(!ioc_guid); |
| 377 | |
| 378 | if (!slot || slot > 16) { |
| 379 | mad->mad_hdr.status |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 380 | = cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 381 | return; |
| 382 | } |
| 383 | |
| 384 | if (slot > 2 || lo > hi || hi > 1) { |
| 385 | mad->mad_hdr.status |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 386 | = cpu_to_be16(DM_MAD_STATUS_NO_IOC); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 387 | return; |
| 388 | } |
| 389 | |
| 390 | svc_entries = (struct ib_dm_svc_entries *)mad->data; |
| 391 | memset(svc_entries, 0, sizeof *svc_entries); |
| 392 | svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid); |
| 393 | snprintf(svc_entries->service_entries[0].name, |
| 394 | sizeof(svc_entries->service_entries[0].name), |
| 395 | "%s%016llx", |
| 396 | SRP_SERVICE_NAME_PREFIX, |
| 397 | ioc_guid); |
| 398 | |
| 399 | mad->mad_hdr.status = 0; |
| 400 | } |
| 401 | |
| 402 | /** |
| 403 | * srpt_mgmt_method_get() - Process a received management datagram. |
| 404 | * @sp: source port through which the MAD has been received. |
| 405 | * @rq_mad: received MAD. |
| 406 | * @rsp_mad: response MAD. |
| 407 | */ |
| 408 | static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad, |
| 409 | struct ib_dm_mad *rsp_mad) |
| 410 | { |
| 411 | u16 attr_id; |
| 412 | u32 slot; |
| 413 | u8 hi, lo; |
| 414 | |
| 415 | attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id); |
| 416 | switch (attr_id) { |
| 417 | case DM_ATTR_CLASS_PORT_INFO: |
| 418 | srpt_get_class_port_info(rsp_mad); |
| 419 | break; |
| 420 | case DM_ATTR_IOU_INFO: |
| 421 | srpt_get_iou(rsp_mad); |
| 422 | break; |
| 423 | case DM_ATTR_IOC_PROFILE: |
| 424 | slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod); |
| 425 | srpt_get_ioc(sp, slot, rsp_mad); |
| 426 | break; |
| 427 | case DM_ATTR_SVC_ENTRIES: |
| 428 | slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod); |
| 429 | hi = (u8) ((slot >> 8) & 0xff); |
| 430 | lo = (u8) (slot & 0xff); |
| 431 | slot = (u16) ((slot >> 16) & 0xffff); |
| 432 | srpt_get_svc_entries(srpt_service_guid, |
| 433 | slot, hi, lo, rsp_mad); |
| 434 | break; |
| 435 | default: |
| 436 | rsp_mad->mad_hdr.status = |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 437 | cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 438 | break; |
| 439 | } |
| 440 | } |
| 441 | |
| 442 | /** |
| 443 | * srpt_mad_send_handler() - Post MAD-send callback function. |
| 444 | */ |
| 445 | static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent, |
| 446 | struct ib_mad_send_wc *mad_wc) |
| 447 | { |
| 448 | ib_destroy_ah(mad_wc->send_buf->ah); |
| 449 | ib_free_send_mad(mad_wc->send_buf); |
| 450 | } |
| 451 | |
| 452 | /** |
| 453 | * srpt_mad_recv_handler() - MAD reception callback function. |
| 454 | */ |
| 455 | static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent, |
| 456 | struct ib_mad_recv_wc *mad_wc) |
| 457 | { |
| 458 | struct srpt_port *sport = (struct srpt_port *)mad_agent->context; |
| 459 | struct ib_ah *ah; |
| 460 | struct ib_mad_send_buf *rsp; |
| 461 | struct ib_dm_mad *dm_mad; |
| 462 | |
| 463 | if (!mad_wc || !mad_wc->recv_buf.mad) |
| 464 | return; |
| 465 | |
| 466 | ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc, |
| 467 | mad_wc->recv_buf.grh, mad_agent->port_num); |
| 468 | if (IS_ERR(ah)) |
| 469 | goto err; |
| 470 | |
| 471 | BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR); |
| 472 | |
| 473 | rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp, |
| 474 | mad_wc->wc->pkey_index, 0, |
| 475 | IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA, |
Ira Weiny | da2dfaa | 2015-06-06 14:38:28 -0400 | [diff] [blame] | 476 | GFP_KERNEL, |
| 477 | IB_MGMT_BASE_VERSION); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 478 | if (IS_ERR(rsp)) |
| 479 | goto err_rsp; |
| 480 | |
| 481 | rsp->ah = ah; |
| 482 | |
| 483 | dm_mad = rsp->mad; |
| 484 | memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof *dm_mad); |
| 485 | dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP; |
| 486 | dm_mad->mad_hdr.status = 0; |
| 487 | |
| 488 | switch (mad_wc->recv_buf.mad->mad_hdr.method) { |
| 489 | case IB_MGMT_METHOD_GET: |
| 490 | srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad); |
| 491 | break; |
| 492 | case IB_MGMT_METHOD_SET: |
| 493 | dm_mad->mad_hdr.status = |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 494 | cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 495 | break; |
| 496 | default: |
| 497 | dm_mad->mad_hdr.status = |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 498 | cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 499 | break; |
| 500 | } |
| 501 | |
| 502 | if (!ib_post_send_mad(rsp, NULL)) { |
| 503 | ib_free_recv_mad(mad_wc); |
| 504 | /* will destroy_ah & free_send_mad in send completion */ |
| 505 | return; |
| 506 | } |
| 507 | |
| 508 | ib_free_send_mad(rsp); |
| 509 | |
| 510 | err_rsp: |
| 511 | ib_destroy_ah(ah); |
| 512 | err: |
| 513 | ib_free_recv_mad(mad_wc); |
| 514 | } |
| 515 | |
| 516 | /** |
| 517 | * srpt_refresh_port() - Configure a HCA port. |
| 518 | * |
| 519 | * Enable InfiniBand management datagram processing, update the cached sm_lid, |
| 520 | * lid and gid values, and register a callback function for processing MADs |
| 521 | * on the specified port. |
| 522 | * |
| 523 | * Note: It is safe to call this function more than once for the same port. |
| 524 | */ |
| 525 | static int srpt_refresh_port(struct srpt_port *sport) |
| 526 | { |
| 527 | struct ib_mad_reg_req reg_req; |
| 528 | struct ib_port_modify port_modify; |
| 529 | struct ib_port_attr port_attr; |
| 530 | int ret; |
| 531 | |
| 532 | memset(&port_modify, 0, sizeof port_modify); |
| 533 | port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP; |
| 534 | port_modify.clr_port_cap_mask = 0; |
| 535 | |
| 536 | ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify); |
| 537 | if (ret) |
| 538 | goto err_mod_port; |
| 539 | |
| 540 | ret = ib_query_port(sport->sdev->device, sport->port, &port_attr); |
| 541 | if (ret) |
| 542 | goto err_query_port; |
| 543 | |
| 544 | sport->sm_lid = port_attr.sm_lid; |
| 545 | sport->lid = port_attr.lid; |
| 546 | |
| 547 | ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid); |
| 548 | if (ret) |
| 549 | goto err_query_port; |
| 550 | |
| 551 | if (!sport->mad_agent) { |
| 552 | memset(®_req, 0, sizeof reg_req); |
| 553 | reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT; |
| 554 | reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION; |
| 555 | set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask); |
| 556 | set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask); |
| 557 | |
| 558 | sport->mad_agent = ib_register_mad_agent(sport->sdev->device, |
| 559 | sport->port, |
| 560 | IB_QPT_GSI, |
| 561 | ®_req, 0, |
| 562 | srpt_mad_send_handler, |
| 563 | srpt_mad_recv_handler, |
Ira Weiny | 0f29b46 | 2014-08-08 19:00:55 -0400 | [diff] [blame] | 564 | sport, 0); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 565 | if (IS_ERR(sport->mad_agent)) { |
| 566 | ret = PTR_ERR(sport->mad_agent); |
| 567 | sport->mad_agent = NULL; |
| 568 | goto err_query_port; |
| 569 | } |
| 570 | } |
| 571 | |
| 572 | return 0; |
| 573 | |
| 574 | err_query_port: |
| 575 | |
| 576 | port_modify.set_port_cap_mask = 0; |
| 577 | port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP; |
| 578 | ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify); |
| 579 | |
| 580 | err_mod_port: |
| 581 | |
| 582 | return ret; |
| 583 | } |
| 584 | |
| 585 | /** |
| 586 | * srpt_unregister_mad_agent() - Unregister MAD callback functions. |
| 587 | * |
| 588 | * Note: It is safe to call this function more than once for the same device. |
| 589 | */ |
| 590 | static void srpt_unregister_mad_agent(struct srpt_device *sdev) |
| 591 | { |
| 592 | struct ib_port_modify port_modify = { |
| 593 | .clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP, |
| 594 | }; |
| 595 | struct srpt_port *sport; |
| 596 | int i; |
| 597 | |
| 598 | for (i = 1; i <= sdev->device->phys_port_cnt; i++) { |
| 599 | sport = &sdev->port[i - 1]; |
| 600 | WARN_ON(sport->port != i); |
| 601 | if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0) |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 602 | pr_err("disabling MAD processing failed.\n"); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 603 | if (sport->mad_agent) { |
| 604 | ib_unregister_mad_agent(sport->mad_agent); |
| 605 | sport->mad_agent = NULL; |
| 606 | } |
| 607 | } |
| 608 | } |
| 609 | |
| 610 | /** |
| 611 | * srpt_alloc_ioctx() - Allocate an SRPT I/O context structure. |
| 612 | */ |
| 613 | static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev, |
| 614 | int ioctx_size, int dma_size, |
| 615 | enum dma_data_direction dir) |
| 616 | { |
| 617 | struct srpt_ioctx *ioctx; |
| 618 | |
| 619 | ioctx = kmalloc(ioctx_size, GFP_KERNEL); |
| 620 | if (!ioctx) |
| 621 | goto err; |
| 622 | |
| 623 | ioctx->buf = kmalloc(dma_size, GFP_KERNEL); |
| 624 | if (!ioctx->buf) |
| 625 | goto err_free_ioctx; |
| 626 | |
| 627 | ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf, dma_size, dir); |
| 628 | if (ib_dma_mapping_error(sdev->device, ioctx->dma)) |
| 629 | goto err_free_buf; |
| 630 | |
| 631 | return ioctx; |
| 632 | |
| 633 | err_free_buf: |
| 634 | kfree(ioctx->buf); |
| 635 | err_free_ioctx: |
| 636 | kfree(ioctx); |
| 637 | err: |
| 638 | return NULL; |
| 639 | } |
| 640 | |
| 641 | /** |
| 642 | * srpt_free_ioctx() - Free an SRPT I/O context structure. |
| 643 | */ |
| 644 | static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx, |
| 645 | int dma_size, enum dma_data_direction dir) |
| 646 | { |
| 647 | if (!ioctx) |
| 648 | return; |
| 649 | |
| 650 | ib_dma_unmap_single(sdev->device, ioctx->dma, dma_size, dir); |
| 651 | kfree(ioctx->buf); |
| 652 | kfree(ioctx); |
| 653 | } |
| 654 | |
| 655 | /** |
| 656 | * srpt_alloc_ioctx_ring() - Allocate a ring of SRPT I/O context structures. |
| 657 | * @sdev: Device to allocate the I/O context ring for. |
| 658 | * @ring_size: Number of elements in the I/O context ring. |
| 659 | * @ioctx_size: I/O context size. |
| 660 | * @dma_size: DMA buffer size. |
| 661 | * @dir: DMA data direction. |
| 662 | */ |
| 663 | static struct srpt_ioctx **srpt_alloc_ioctx_ring(struct srpt_device *sdev, |
| 664 | int ring_size, int ioctx_size, |
| 665 | int dma_size, enum dma_data_direction dir) |
| 666 | { |
| 667 | struct srpt_ioctx **ring; |
| 668 | int i; |
| 669 | |
| 670 | WARN_ON(ioctx_size != sizeof(struct srpt_recv_ioctx) |
| 671 | && ioctx_size != sizeof(struct srpt_send_ioctx)); |
| 672 | |
| 673 | ring = kmalloc(ring_size * sizeof(ring[0]), GFP_KERNEL); |
| 674 | if (!ring) |
| 675 | goto out; |
| 676 | for (i = 0; i < ring_size; ++i) { |
| 677 | ring[i] = srpt_alloc_ioctx(sdev, ioctx_size, dma_size, dir); |
| 678 | if (!ring[i]) |
| 679 | goto err; |
| 680 | ring[i]->index = i; |
| 681 | } |
| 682 | goto out; |
| 683 | |
| 684 | err: |
| 685 | while (--i >= 0) |
| 686 | srpt_free_ioctx(sdev, ring[i], dma_size, dir); |
| 687 | kfree(ring); |
Jesper Juhl | 715252d | 2012-02-04 23:49:40 +0100 | [diff] [blame] | 688 | ring = NULL; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 689 | out: |
| 690 | return ring; |
| 691 | } |
| 692 | |
| 693 | /** |
| 694 | * srpt_free_ioctx_ring() - Free the ring of SRPT I/O context structures. |
| 695 | */ |
| 696 | static void srpt_free_ioctx_ring(struct srpt_ioctx **ioctx_ring, |
| 697 | struct srpt_device *sdev, int ring_size, |
| 698 | int dma_size, enum dma_data_direction dir) |
| 699 | { |
| 700 | int i; |
| 701 | |
| 702 | for (i = 0; i < ring_size; ++i) |
| 703 | srpt_free_ioctx(sdev, ioctx_ring[i], dma_size, dir); |
| 704 | kfree(ioctx_ring); |
| 705 | } |
| 706 | |
| 707 | /** |
| 708 | * srpt_get_cmd_state() - Get the state of a SCSI command. |
| 709 | */ |
| 710 | static enum srpt_command_state srpt_get_cmd_state(struct srpt_send_ioctx *ioctx) |
| 711 | { |
| 712 | enum srpt_command_state state; |
| 713 | unsigned long flags; |
| 714 | |
| 715 | BUG_ON(!ioctx); |
| 716 | |
| 717 | spin_lock_irqsave(&ioctx->spinlock, flags); |
| 718 | state = ioctx->state; |
| 719 | spin_unlock_irqrestore(&ioctx->spinlock, flags); |
| 720 | return state; |
| 721 | } |
| 722 | |
| 723 | /** |
| 724 | * srpt_set_cmd_state() - Set the state of a SCSI command. |
| 725 | * |
| 726 | * Does not modify the state of aborted commands. Returns the previous command |
| 727 | * state. |
| 728 | */ |
| 729 | static enum srpt_command_state srpt_set_cmd_state(struct srpt_send_ioctx *ioctx, |
| 730 | enum srpt_command_state new) |
| 731 | { |
| 732 | enum srpt_command_state previous; |
| 733 | unsigned long flags; |
| 734 | |
| 735 | BUG_ON(!ioctx); |
| 736 | |
| 737 | spin_lock_irqsave(&ioctx->spinlock, flags); |
| 738 | previous = ioctx->state; |
| 739 | if (previous != SRPT_STATE_DONE) |
| 740 | ioctx->state = new; |
| 741 | spin_unlock_irqrestore(&ioctx->spinlock, flags); |
| 742 | |
| 743 | return previous; |
| 744 | } |
| 745 | |
| 746 | /** |
| 747 | * srpt_test_and_set_cmd_state() - Test and set the state of a command. |
| 748 | * |
| 749 | * Returns true if and only if the previous command state was equal to 'old'. |
| 750 | */ |
| 751 | static bool srpt_test_and_set_cmd_state(struct srpt_send_ioctx *ioctx, |
| 752 | enum srpt_command_state old, |
| 753 | enum srpt_command_state new) |
| 754 | { |
| 755 | enum srpt_command_state previous; |
| 756 | unsigned long flags; |
| 757 | |
| 758 | WARN_ON(!ioctx); |
| 759 | WARN_ON(old == SRPT_STATE_DONE); |
| 760 | WARN_ON(new == SRPT_STATE_NEW); |
| 761 | |
| 762 | spin_lock_irqsave(&ioctx->spinlock, flags); |
| 763 | previous = ioctx->state; |
| 764 | if (previous == old) |
| 765 | ioctx->state = new; |
| 766 | spin_unlock_irqrestore(&ioctx->spinlock, flags); |
| 767 | return previous == old; |
| 768 | } |
| 769 | |
| 770 | /** |
| 771 | * srpt_post_recv() - Post an IB receive request. |
| 772 | */ |
| 773 | static int srpt_post_recv(struct srpt_device *sdev, |
| 774 | struct srpt_recv_ioctx *ioctx) |
| 775 | { |
| 776 | struct ib_sge list; |
| 777 | struct ib_recv_wr wr, *bad_wr; |
| 778 | |
| 779 | BUG_ON(!sdev); |
| 780 | wr.wr_id = encode_wr_id(SRPT_RECV, ioctx->ioctx.index); |
| 781 | |
| 782 | list.addr = ioctx->ioctx.dma; |
| 783 | list.length = srp_max_req_size; |
Jason Gunthorpe | 5a78395 | 2015-07-30 17:22:24 -0600 | [diff] [blame] | 784 | list.lkey = sdev->pd->local_dma_lkey; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 785 | |
| 786 | wr.next = NULL; |
| 787 | wr.sg_list = &list; |
| 788 | wr.num_sge = 1; |
| 789 | |
| 790 | return ib_post_srq_recv(sdev->srq, &wr, &bad_wr); |
| 791 | } |
| 792 | |
| 793 | /** |
| 794 | * srpt_post_send() - Post an IB send request. |
| 795 | * |
| 796 | * Returns zero upon success and a non-zero value upon failure. |
| 797 | */ |
| 798 | static int srpt_post_send(struct srpt_rdma_ch *ch, |
| 799 | struct srpt_send_ioctx *ioctx, int len) |
| 800 | { |
| 801 | struct ib_sge list; |
| 802 | struct ib_send_wr wr, *bad_wr; |
| 803 | struct srpt_device *sdev = ch->sport->sdev; |
| 804 | int ret; |
| 805 | |
| 806 | atomic_inc(&ch->req_lim); |
| 807 | |
| 808 | ret = -ENOMEM; |
| 809 | if (unlikely(atomic_dec_return(&ch->sq_wr_avail) < 0)) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 810 | pr_warn("IB send queue full (needed 1)\n"); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 811 | goto out; |
| 812 | } |
| 813 | |
| 814 | ib_dma_sync_single_for_device(sdev->device, ioctx->ioctx.dma, len, |
| 815 | DMA_TO_DEVICE); |
| 816 | |
| 817 | list.addr = ioctx->ioctx.dma; |
| 818 | list.length = len; |
Jason Gunthorpe | 5a78395 | 2015-07-30 17:22:24 -0600 | [diff] [blame] | 819 | list.lkey = sdev->pd->local_dma_lkey; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 820 | |
| 821 | wr.next = NULL; |
| 822 | wr.wr_id = encode_wr_id(SRPT_SEND, ioctx->ioctx.index); |
| 823 | wr.sg_list = &list; |
| 824 | wr.num_sge = 1; |
| 825 | wr.opcode = IB_WR_SEND; |
| 826 | wr.send_flags = IB_SEND_SIGNALED; |
| 827 | |
| 828 | ret = ib_post_send(ch->qp, &wr, &bad_wr); |
| 829 | |
| 830 | out: |
| 831 | if (ret < 0) { |
| 832 | atomic_inc(&ch->sq_wr_avail); |
| 833 | atomic_dec(&ch->req_lim); |
| 834 | } |
| 835 | return ret; |
| 836 | } |
| 837 | |
| 838 | /** |
| 839 | * srpt_get_desc_tbl() - Parse the data descriptors of an SRP_CMD request. |
| 840 | * @ioctx: Pointer to the I/O context associated with the request. |
| 841 | * @srp_cmd: Pointer to the SRP_CMD request data. |
| 842 | * @dir: Pointer to the variable to which the transfer direction will be |
| 843 | * written. |
| 844 | * @data_len: Pointer to the variable to which the total data length of all |
| 845 | * descriptors in the SRP_CMD request will be written. |
| 846 | * |
| 847 | * This function initializes ioctx->nrbuf and ioctx->r_bufs. |
| 848 | * |
| 849 | * Returns -EINVAL when the SRP_CMD request contains inconsistent descriptors; |
| 850 | * -ENOMEM when memory allocation fails and zero upon success. |
| 851 | */ |
| 852 | static int srpt_get_desc_tbl(struct srpt_send_ioctx *ioctx, |
| 853 | struct srp_cmd *srp_cmd, |
| 854 | enum dma_data_direction *dir, u64 *data_len) |
| 855 | { |
| 856 | struct srp_indirect_buf *idb; |
| 857 | struct srp_direct_buf *db; |
| 858 | unsigned add_cdb_offset; |
| 859 | int ret; |
| 860 | |
| 861 | /* |
| 862 | * The pointer computations below will only be compiled correctly |
| 863 | * if srp_cmd::add_data is declared as s8*, u8*, s8[] or u8[], so check |
| 864 | * whether srp_cmd::add_data has been declared as a byte pointer. |
| 865 | */ |
| 866 | BUILD_BUG_ON(!__same_type(srp_cmd->add_data[0], (s8)0) |
| 867 | && !__same_type(srp_cmd->add_data[0], (u8)0)); |
| 868 | |
| 869 | BUG_ON(!dir); |
| 870 | BUG_ON(!data_len); |
| 871 | |
| 872 | ret = 0; |
| 873 | *data_len = 0; |
| 874 | |
| 875 | /* |
| 876 | * The lower four bits of the buffer format field contain the DATA-IN |
| 877 | * buffer descriptor format, and the highest four bits contain the |
| 878 | * DATA-OUT buffer descriptor format. |
| 879 | */ |
| 880 | *dir = DMA_NONE; |
| 881 | if (srp_cmd->buf_fmt & 0xf) |
| 882 | /* DATA-IN: transfer data from target to initiator (read). */ |
| 883 | *dir = DMA_FROM_DEVICE; |
| 884 | else if (srp_cmd->buf_fmt >> 4) |
| 885 | /* DATA-OUT: transfer data from initiator to target (write). */ |
| 886 | *dir = DMA_TO_DEVICE; |
| 887 | |
| 888 | /* |
| 889 | * According to the SRP spec, the lower two bits of the 'ADDITIONAL |
| 890 | * CDB LENGTH' field are reserved and the size in bytes of this field |
| 891 | * is four times the value specified in bits 3..7. Hence the "& ~3". |
| 892 | */ |
| 893 | add_cdb_offset = srp_cmd->add_cdb_len & ~3; |
| 894 | if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) || |
| 895 | ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) { |
| 896 | ioctx->n_rbuf = 1; |
| 897 | ioctx->rbufs = &ioctx->single_rbuf; |
| 898 | |
| 899 | db = (struct srp_direct_buf *)(srp_cmd->add_data |
| 900 | + add_cdb_offset); |
| 901 | memcpy(ioctx->rbufs, db, sizeof *db); |
| 902 | *data_len = be32_to_cpu(db->len); |
| 903 | } else if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_INDIRECT) || |
| 904 | ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_INDIRECT)) { |
| 905 | idb = (struct srp_indirect_buf *)(srp_cmd->add_data |
| 906 | + add_cdb_offset); |
| 907 | |
| 908 | ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db; |
| 909 | |
| 910 | if (ioctx->n_rbuf > |
| 911 | (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 912 | pr_err("received unsupported SRP_CMD request" |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 913 | " type (%u out + %u in != %u / %zu)\n", |
| 914 | srp_cmd->data_out_desc_cnt, |
| 915 | srp_cmd->data_in_desc_cnt, |
| 916 | be32_to_cpu(idb->table_desc.len), |
| 917 | sizeof(*db)); |
| 918 | ioctx->n_rbuf = 0; |
| 919 | ret = -EINVAL; |
| 920 | goto out; |
| 921 | } |
| 922 | |
| 923 | if (ioctx->n_rbuf == 1) |
| 924 | ioctx->rbufs = &ioctx->single_rbuf; |
| 925 | else { |
| 926 | ioctx->rbufs = |
| 927 | kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC); |
| 928 | if (!ioctx->rbufs) { |
| 929 | ioctx->n_rbuf = 0; |
| 930 | ret = -ENOMEM; |
| 931 | goto out; |
| 932 | } |
| 933 | } |
| 934 | |
| 935 | db = idb->desc_list; |
| 936 | memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db); |
| 937 | *data_len = be32_to_cpu(idb->len); |
| 938 | } |
| 939 | out: |
| 940 | return ret; |
| 941 | } |
| 942 | |
| 943 | /** |
| 944 | * srpt_init_ch_qp() - Initialize queue pair attributes. |
| 945 | * |
| 946 | * Initialized the attributes of queue pair 'qp' by allowing local write, |
| 947 | * remote read and remote write. Also transitions 'qp' to state IB_QPS_INIT. |
| 948 | */ |
| 949 | static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp) |
| 950 | { |
| 951 | struct ib_qp_attr *attr; |
| 952 | int ret; |
| 953 | |
| 954 | attr = kzalloc(sizeof *attr, GFP_KERNEL); |
| 955 | if (!attr) |
| 956 | return -ENOMEM; |
| 957 | |
| 958 | attr->qp_state = IB_QPS_INIT; |
| 959 | attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ | |
| 960 | IB_ACCESS_REMOTE_WRITE; |
| 961 | attr->port_num = ch->sport->port; |
| 962 | attr->pkey_index = 0; |
| 963 | |
| 964 | ret = ib_modify_qp(qp, attr, |
| 965 | IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT | |
| 966 | IB_QP_PKEY_INDEX); |
| 967 | |
| 968 | kfree(attr); |
| 969 | return ret; |
| 970 | } |
| 971 | |
| 972 | /** |
| 973 | * srpt_ch_qp_rtr() - Change the state of a channel to 'ready to receive' (RTR). |
| 974 | * @ch: channel of the queue pair. |
| 975 | * @qp: queue pair to change the state of. |
| 976 | * |
| 977 | * Returns zero upon success and a negative value upon failure. |
| 978 | * |
| 979 | * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system. |
| 980 | * If this structure ever becomes larger, it might be necessary to allocate |
| 981 | * it dynamically instead of on the stack. |
| 982 | */ |
| 983 | static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp) |
| 984 | { |
| 985 | struct ib_qp_attr qp_attr; |
| 986 | int attr_mask; |
| 987 | int ret; |
| 988 | |
| 989 | qp_attr.qp_state = IB_QPS_RTR; |
| 990 | ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask); |
| 991 | if (ret) |
| 992 | goto out; |
| 993 | |
| 994 | qp_attr.max_dest_rd_atomic = 4; |
| 995 | |
| 996 | ret = ib_modify_qp(qp, &qp_attr, attr_mask); |
| 997 | |
| 998 | out: |
| 999 | return ret; |
| 1000 | } |
| 1001 | |
| 1002 | /** |
| 1003 | * srpt_ch_qp_rts() - Change the state of a channel to 'ready to send' (RTS). |
| 1004 | * @ch: channel of the queue pair. |
| 1005 | * @qp: queue pair to change the state of. |
| 1006 | * |
| 1007 | * Returns zero upon success and a negative value upon failure. |
| 1008 | * |
| 1009 | * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system. |
| 1010 | * If this structure ever becomes larger, it might be necessary to allocate |
| 1011 | * it dynamically instead of on the stack. |
| 1012 | */ |
| 1013 | static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp) |
| 1014 | { |
| 1015 | struct ib_qp_attr qp_attr; |
| 1016 | int attr_mask; |
| 1017 | int ret; |
| 1018 | |
| 1019 | qp_attr.qp_state = IB_QPS_RTS; |
| 1020 | ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask); |
| 1021 | if (ret) |
| 1022 | goto out; |
| 1023 | |
| 1024 | qp_attr.max_rd_atomic = 4; |
| 1025 | |
| 1026 | ret = ib_modify_qp(qp, &qp_attr, attr_mask); |
| 1027 | |
| 1028 | out: |
| 1029 | return ret; |
| 1030 | } |
| 1031 | |
| 1032 | /** |
| 1033 | * srpt_ch_qp_err() - Set the channel queue pair state to 'error'. |
| 1034 | */ |
| 1035 | static int srpt_ch_qp_err(struct srpt_rdma_ch *ch) |
| 1036 | { |
| 1037 | struct ib_qp_attr qp_attr; |
| 1038 | |
| 1039 | qp_attr.qp_state = IB_QPS_ERR; |
| 1040 | return ib_modify_qp(ch->qp, &qp_attr, IB_QP_STATE); |
| 1041 | } |
| 1042 | |
| 1043 | /** |
| 1044 | * srpt_unmap_sg_to_ib_sge() - Unmap an IB SGE list. |
| 1045 | */ |
| 1046 | static void srpt_unmap_sg_to_ib_sge(struct srpt_rdma_ch *ch, |
| 1047 | struct srpt_send_ioctx *ioctx) |
| 1048 | { |
| 1049 | struct scatterlist *sg; |
| 1050 | enum dma_data_direction dir; |
| 1051 | |
| 1052 | BUG_ON(!ch); |
| 1053 | BUG_ON(!ioctx); |
| 1054 | BUG_ON(ioctx->n_rdma && !ioctx->rdma_ius); |
| 1055 | |
| 1056 | while (ioctx->n_rdma) |
| 1057 | kfree(ioctx->rdma_ius[--ioctx->n_rdma].sge); |
| 1058 | |
| 1059 | kfree(ioctx->rdma_ius); |
| 1060 | ioctx->rdma_ius = NULL; |
| 1061 | |
| 1062 | if (ioctx->mapped_sg_count) { |
| 1063 | sg = ioctx->sg; |
| 1064 | WARN_ON(!sg); |
| 1065 | dir = ioctx->cmd.data_direction; |
| 1066 | BUG_ON(dir == DMA_NONE); |
| 1067 | ib_dma_unmap_sg(ch->sport->sdev->device, sg, ioctx->sg_cnt, |
| 1068 | opposite_dma_dir(dir)); |
| 1069 | ioctx->mapped_sg_count = 0; |
| 1070 | } |
| 1071 | } |
| 1072 | |
| 1073 | /** |
| 1074 | * srpt_map_sg_to_ib_sge() - Map an SG list to an IB SGE list. |
| 1075 | */ |
| 1076 | static int srpt_map_sg_to_ib_sge(struct srpt_rdma_ch *ch, |
| 1077 | struct srpt_send_ioctx *ioctx) |
| 1078 | { |
Mike Marciniszyn | b076808 | 2014-04-07 13:58:35 -0400 | [diff] [blame] | 1079 | struct ib_device *dev = ch->sport->sdev->device; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1080 | struct se_cmd *cmd; |
| 1081 | struct scatterlist *sg, *sg_orig; |
| 1082 | int sg_cnt; |
| 1083 | enum dma_data_direction dir; |
| 1084 | struct rdma_iu *riu; |
| 1085 | struct srp_direct_buf *db; |
| 1086 | dma_addr_t dma_addr; |
| 1087 | struct ib_sge *sge; |
| 1088 | u64 raddr; |
| 1089 | u32 rsize; |
| 1090 | u32 tsize; |
| 1091 | u32 dma_len; |
| 1092 | int count, nrdma; |
| 1093 | int i, j, k; |
| 1094 | |
| 1095 | BUG_ON(!ch); |
| 1096 | BUG_ON(!ioctx); |
| 1097 | cmd = &ioctx->cmd; |
| 1098 | dir = cmd->data_direction; |
| 1099 | BUG_ON(dir == DMA_NONE); |
| 1100 | |
Roland Dreier | 6f9e7f0 | 2012-03-30 11:29:12 -0700 | [diff] [blame] | 1101 | ioctx->sg = sg = sg_orig = cmd->t_data_sg; |
| 1102 | ioctx->sg_cnt = sg_cnt = cmd->t_data_nents; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1103 | |
| 1104 | count = ib_dma_map_sg(ch->sport->sdev->device, sg, sg_cnt, |
| 1105 | opposite_dma_dir(dir)); |
| 1106 | if (unlikely(!count)) |
| 1107 | return -EAGAIN; |
| 1108 | |
| 1109 | ioctx->mapped_sg_count = count; |
| 1110 | |
| 1111 | if (ioctx->rdma_ius && ioctx->n_rdma_ius) |
| 1112 | nrdma = ioctx->n_rdma_ius; |
| 1113 | else { |
| 1114 | nrdma = (count + SRPT_DEF_SG_PER_WQE - 1) / SRPT_DEF_SG_PER_WQE |
| 1115 | + ioctx->n_rbuf; |
| 1116 | |
| 1117 | ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu, GFP_KERNEL); |
| 1118 | if (!ioctx->rdma_ius) |
| 1119 | goto free_mem; |
| 1120 | |
| 1121 | ioctx->n_rdma_ius = nrdma; |
| 1122 | } |
| 1123 | |
| 1124 | db = ioctx->rbufs; |
| 1125 | tsize = cmd->data_length; |
Mike Marciniszyn | b076808 | 2014-04-07 13:58:35 -0400 | [diff] [blame] | 1126 | dma_len = ib_sg_dma_len(dev, &sg[0]); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1127 | riu = ioctx->rdma_ius; |
| 1128 | |
| 1129 | /* |
| 1130 | * For each remote desc - calculate the #ib_sge. |
| 1131 | * If #ib_sge < SRPT_DEF_SG_PER_WQE per rdma operation then |
| 1132 | * each remote desc rdma_iu is required a rdma wr; |
| 1133 | * else |
| 1134 | * we need to allocate extra rdma_iu to carry extra #ib_sge in |
| 1135 | * another rdma wr |
| 1136 | */ |
| 1137 | for (i = 0, j = 0; |
| 1138 | j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) { |
| 1139 | rsize = be32_to_cpu(db->len); |
| 1140 | raddr = be64_to_cpu(db->va); |
| 1141 | riu->raddr = raddr; |
| 1142 | riu->rkey = be32_to_cpu(db->key); |
| 1143 | riu->sge_cnt = 0; |
| 1144 | |
| 1145 | /* calculate how many sge required for this remote_buf */ |
| 1146 | while (rsize > 0 && tsize > 0) { |
| 1147 | |
| 1148 | if (rsize >= dma_len) { |
| 1149 | tsize -= dma_len; |
| 1150 | rsize -= dma_len; |
| 1151 | raddr += dma_len; |
| 1152 | |
| 1153 | if (tsize > 0) { |
| 1154 | ++j; |
| 1155 | if (j < count) { |
| 1156 | sg = sg_next(sg); |
Mike Marciniszyn | b076808 | 2014-04-07 13:58:35 -0400 | [diff] [blame] | 1157 | dma_len = ib_sg_dma_len( |
| 1158 | dev, sg); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1159 | } |
| 1160 | } |
| 1161 | } else { |
| 1162 | tsize -= rsize; |
| 1163 | dma_len -= rsize; |
| 1164 | rsize = 0; |
| 1165 | } |
| 1166 | |
| 1167 | ++riu->sge_cnt; |
| 1168 | |
| 1169 | if (rsize > 0 && riu->sge_cnt == SRPT_DEF_SG_PER_WQE) { |
| 1170 | ++ioctx->n_rdma; |
| 1171 | riu->sge = |
| 1172 | kmalloc(riu->sge_cnt * sizeof *riu->sge, |
| 1173 | GFP_KERNEL); |
| 1174 | if (!riu->sge) |
| 1175 | goto free_mem; |
| 1176 | |
| 1177 | ++riu; |
| 1178 | riu->sge_cnt = 0; |
| 1179 | riu->raddr = raddr; |
| 1180 | riu->rkey = be32_to_cpu(db->key); |
| 1181 | } |
| 1182 | } |
| 1183 | |
| 1184 | ++ioctx->n_rdma; |
| 1185 | riu->sge = kmalloc(riu->sge_cnt * sizeof *riu->sge, |
| 1186 | GFP_KERNEL); |
| 1187 | if (!riu->sge) |
| 1188 | goto free_mem; |
| 1189 | } |
| 1190 | |
| 1191 | db = ioctx->rbufs; |
| 1192 | tsize = cmd->data_length; |
| 1193 | riu = ioctx->rdma_ius; |
| 1194 | sg = sg_orig; |
Mike Marciniszyn | b076808 | 2014-04-07 13:58:35 -0400 | [diff] [blame] | 1195 | dma_len = ib_sg_dma_len(dev, &sg[0]); |
| 1196 | dma_addr = ib_sg_dma_address(dev, &sg[0]); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1197 | |
| 1198 | /* this second loop is really mapped sg_addres to rdma_iu->ib_sge */ |
| 1199 | for (i = 0, j = 0; |
| 1200 | j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) { |
| 1201 | rsize = be32_to_cpu(db->len); |
| 1202 | sge = riu->sge; |
| 1203 | k = 0; |
| 1204 | |
| 1205 | while (rsize > 0 && tsize > 0) { |
| 1206 | sge->addr = dma_addr; |
Jason Gunthorpe | 5a78395 | 2015-07-30 17:22:24 -0600 | [diff] [blame] | 1207 | sge->lkey = ch->sport->sdev->pd->local_dma_lkey; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1208 | |
| 1209 | if (rsize >= dma_len) { |
| 1210 | sge->length = |
| 1211 | (tsize < dma_len) ? tsize : dma_len; |
| 1212 | tsize -= dma_len; |
| 1213 | rsize -= dma_len; |
| 1214 | |
| 1215 | if (tsize > 0) { |
| 1216 | ++j; |
| 1217 | if (j < count) { |
| 1218 | sg = sg_next(sg); |
Mike Marciniszyn | b076808 | 2014-04-07 13:58:35 -0400 | [diff] [blame] | 1219 | dma_len = ib_sg_dma_len( |
| 1220 | dev, sg); |
| 1221 | dma_addr = ib_sg_dma_address( |
| 1222 | dev, sg); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1223 | } |
| 1224 | } |
| 1225 | } else { |
| 1226 | sge->length = (tsize < rsize) ? tsize : rsize; |
| 1227 | tsize -= rsize; |
| 1228 | dma_len -= rsize; |
| 1229 | dma_addr += rsize; |
| 1230 | rsize = 0; |
| 1231 | } |
| 1232 | |
| 1233 | ++k; |
| 1234 | if (k == riu->sge_cnt && rsize > 0 && tsize > 0) { |
| 1235 | ++riu; |
| 1236 | sge = riu->sge; |
| 1237 | k = 0; |
| 1238 | } else if (rsize > 0 && tsize > 0) |
| 1239 | ++sge; |
| 1240 | } |
| 1241 | } |
| 1242 | |
| 1243 | return 0; |
| 1244 | |
| 1245 | free_mem: |
| 1246 | srpt_unmap_sg_to_ib_sge(ch, ioctx); |
| 1247 | |
| 1248 | return -ENOMEM; |
| 1249 | } |
| 1250 | |
| 1251 | /** |
| 1252 | * srpt_get_send_ioctx() - Obtain an I/O context for sending to the initiator. |
| 1253 | */ |
| 1254 | static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch) |
| 1255 | { |
| 1256 | struct srpt_send_ioctx *ioctx; |
| 1257 | unsigned long flags; |
| 1258 | |
| 1259 | BUG_ON(!ch); |
| 1260 | |
| 1261 | ioctx = NULL; |
| 1262 | spin_lock_irqsave(&ch->spinlock, flags); |
| 1263 | if (!list_empty(&ch->free_list)) { |
| 1264 | ioctx = list_first_entry(&ch->free_list, |
| 1265 | struct srpt_send_ioctx, free_list); |
| 1266 | list_del(&ioctx->free_list); |
| 1267 | } |
| 1268 | spin_unlock_irqrestore(&ch->spinlock, flags); |
| 1269 | |
| 1270 | if (!ioctx) |
| 1271 | return ioctx; |
| 1272 | |
| 1273 | BUG_ON(ioctx->ch != ch); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1274 | spin_lock_init(&ioctx->spinlock); |
| 1275 | ioctx->state = SRPT_STATE_NEW; |
| 1276 | ioctx->n_rbuf = 0; |
| 1277 | ioctx->rbufs = NULL; |
| 1278 | ioctx->n_rdma = 0; |
| 1279 | ioctx->n_rdma_ius = 0; |
| 1280 | ioctx->rdma_ius = NULL; |
| 1281 | ioctx->mapped_sg_count = 0; |
| 1282 | init_completion(&ioctx->tx_done); |
| 1283 | ioctx->queue_status_only = false; |
| 1284 | /* |
| 1285 | * transport_init_se_cmd() does not initialize all fields, so do it |
| 1286 | * here. |
| 1287 | */ |
| 1288 | memset(&ioctx->cmd, 0, sizeof(ioctx->cmd)); |
| 1289 | memset(&ioctx->sense_data, 0, sizeof(ioctx->sense_data)); |
| 1290 | |
| 1291 | return ioctx; |
| 1292 | } |
| 1293 | |
| 1294 | /** |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1295 | * srpt_abort_cmd() - Abort a SCSI command. |
| 1296 | * @ioctx: I/O context associated with the SCSI command. |
| 1297 | * @context: Preferred execution context. |
| 1298 | */ |
| 1299 | static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx) |
| 1300 | { |
| 1301 | enum srpt_command_state state; |
| 1302 | unsigned long flags; |
| 1303 | |
| 1304 | BUG_ON(!ioctx); |
| 1305 | |
| 1306 | /* |
| 1307 | * If the command is in a state where the target core is waiting for |
| 1308 | * the ib_srpt driver, change the state to the next state. Changing |
| 1309 | * the state of the command from SRPT_STATE_NEED_DATA to |
| 1310 | * SRPT_STATE_DATA_IN ensures that srpt_xmit_response() will call this |
| 1311 | * function a second time. |
| 1312 | */ |
| 1313 | |
| 1314 | spin_lock_irqsave(&ioctx->spinlock, flags); |
| 1315 | state = ioctx->state; |
| 1316 | switch (state) { |
| 1317 | case SRPT_STATE_NEED_DATA: |
| 1318 | ioctx->state = SRPT_STATE_DATA_IN; |
| 1319 | break; |
| 1320 | case SRPT_STATE_DATA_IN: |
| 1321 | case SRPT_STATE_CMD_RSP_SENT: |
| 1322 | case SRPT_STATE_MGMT_RSP_SENT: |
| 1323 | ioctx->state = SRPT_STATE_DONE; |
| 1324 | break; |
| 1325 | default: |
| 1326 | break; |
| 1327 | } |
| 1328 | spin_unlock_irqrestore(&ioctx->spinlock, flags); |
| 1329 | |
Nicholas Bellinger | 9474b04 | 2012-11-27 23:55:57 -0800 | [diff] [blame] | 1330 | if (state == SRPT_STATE_DONE) { |
| 1331 | struct srpt_rdma_ch *ch = ioctx->ch; |
| 1332 | |
| 1333 | BUG_ON(ch->sess == NULL); |
| 1334 | |
Bart Van Assche | afc1660 | 2015-04-27 13:52:36 +0200 | [diff] [blame] | 1335 | target_put_sess_cmd(&ioctx->cmd); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1336 | goto out; |
Nicholas Bellinger | 9474b04 | 2012-11-27 23:55:57 -0800 | [diff] [blame] | 1337 | } |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1338 | |
| 1339 | pr_debug("Aborting cmd with state %d and tag %lld\n", state, |
Bart Van Assche | 649ee05 | 2015-04-14 13:26:44 +0200 | [diff] [blame] | 1340 | ioctx->cmd.tag); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1341 | |
| 1342 | switch (state) { |
| 1343 | case SRPT_STATE_NEW: |
| 1344 | case SRPT_STATE_DATA_IN: |
| 1345 | case SRPT_STATE_MGMT: |
| 1346 | /* |
| 1347 | * Do nothing - defer abort processing until |
| 1348 | * srpt_queue_response() is invoked. |
| 1349 | */ |
| 1350 | WARN_ON(!transport_check_aborted_status(&ioctx->cmd, false)); |
| 1351 | break; |
| 1352 | case SRPT_STATE_NEED_DATA: |
| 1353 | /* DMA_TO_DEVICE (write) - RDMA read error. */ |
Christoph Hellwig | e672a47 | 2012-07-08 15:58:43 -0400 | [diff] [blame] | 1354 | |
| 1355 | /* XXX(hch): this is a horrible layering violation.. */ |
Christoph Hellwig | 7d680f3 | 2011-12-21 14:13:47 -0500 | [diff] [blame] | 1356 | spin_lock_irqsave(&ioctx->cmd.t_state_lock, flags); |
Christoph Hellwig | e672a47 | 2012-07-08 15:58:43 -0400 | [diff] [blame] | 1357 | ioctx->cmd.transport_state &= ~CMD_T_ACTIVE; |
Christoph Hellwig | 7d680f3 | 2011-12-21 14:13:47 -0500 | [diff] [blame] | 1358 | spin_unlock_irqrestore(&ioctx->cmd.t_state_lock, flags); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1359 | break; |
| 1360 | case SRPT_STATE_CMD_RSP_SENT: |
| 1361 | /* |
| 1362 | * SRP_RSP sending failed or the SRP_RSP send completion has |
| 1363 | * not been received in time. |
| 1364 | */ |
| 1365 | srpt_unmap_sg_to_ib_sge(ioctx->ch, ioctx); |
Bart Van Assche | afc1660 | 2015-04-27 13:52:36 +0200 | [diff] [blame] | 1366 | target_put_sess_cmd(&ioctx->cmd); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1367 | break; |
| 1368 | case SRPT_STATE_MGMT_RSP_SENT: |
| 1369 | srpt_set_cmd_state(ioctx, SRPT_STATE_DONE); |
Bart Van Assche | afc1660 | 2015-04-27 13:52:36 +0200 | [diff] [blame] | 1370 | target_put_sess_cmd(&ioctx->cmd); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1371 | break; |
| 1372 | default: |
Grant Grundler | 532ec6f | 2013-03-26 21:48:28 +0000 | [diff] [blame] | 1373 | WARN(1, "Unexpected command state (%d)", state); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1374 | break; |
| 1375 | } |
| 1376 | |
| 1377 | out: |
| 1378 | return state; |
| 1379 | } |
| 1380 | |
| 1381 | /** |
| 1382 | * srpt_handle_send_err_comp() - Process an IB_WC_SEND error completion. |
| 1383 | */ |
| 1384 | static void srpt_handle_send_err_comp(struct srpt_rdma_ch *ch, u64 wr_id) |
| 1385 | { |
| 1386 | struct srpt_send_ioctx *ioctx; |
| 1387 | enum srpt_command_state state; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1388 | u32 index; |
| 1389 | |
| 1390 | atomic_inc(&ch->sq_wr_avail); |
| 1391 | |
| 1392 | index = idx_from_wr_id(wr_id); |
| 1393 | ioctx = ch->ioctx_ring[index]; |
| 1394 | state = srpt_get_cmd_state(ioctx); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1395 | |
| 1396 | WARN_ON(state != SRPT_STATE_CMD_RSP_SENT |
| 1397 | && state != SRPT_STATE_MGMT_RSP_SENT |
| 1398 | && state != SRPT_STATE_NEED_DATA |
| 1399 | && state != SRPT_STATE_DONE); |
| 1400 | |
| 1401 | /* If SRP_RSP sending failed, undo the ch->req_lim change. */ |
| 1402 | if (state == SRPT_STATE_CMD_RSP_SENT |
| 1403 | || state == SRPT_STATE_MGMT_RSP_SENT) |
| 1404 | atomic_dec(&ch->req_lim); |
| 1405 | |
| 1406 | srpt_abort_cmd(ioctx); |
| 1407 | } |
| 1408 | |
| 1409 | /** |
| 1410 | * srpt_handle_send_comp() - Process an IB send completion notification. |
| 1411 | */ |
| 1412 | static void srpt_handle_send_comp(struct srpt_rdma_ch *ch, |
| 1413 | struct srpt_send_ioctx *ioctx) |
| 1414 | { |
| 1415 | enum srpt_command_state state; |
| 1416 | |
| 1417 | atomic_inc(&ch->sq_wr_avail); |
| 1418 | |
| 1419 | state = srpt_set_cmd_state(ioctx, SRPT_STATE_DONE); |
| 1420 | |
| 1421 | if (WARN_ON(state != SRPT_STATE_CMD_RSP_SENT |
| 1422 | && state != SRPT_STATE_MGMT_RSP_SENT |
| 1423 | && state != SRPT_STATE_DONE)) |
| 1424 | pr_debug("state = %d\n", state); |
| 1425 | |
Nicholas Bellinger | 9474b04 | 2012-11-27 23:55:57 -0800 | [diff] [blame] | 1426 | if (state != SRPT_STATE_DONE) { |
| 1427 | srpt_unmap_sg_to_ib_sge(ch, ioctx); |
| 1428 | transport_generic_free_cmd(&ioctx->cmd, 0); |
| 1429 | } else { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1430 | pr_err("IB completion has been received too late for" |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1431 | " wr_id = %u.\n", ioctx->ioctx.index); |
Nicholas Bellinger | 9474b04 | 2012-11-27 23:55:57 -0800 | [diff] [blame] | 1432 | } |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1433 | } |
| 1434 | |
| 1435 | /** |
| 1436 | * srpt_handle_rdma_comp() - Process an IB RDMA completion notification. |
| 1437 | * |
Christoph Hellwig | e672a47 | 2012-07-08 15:58:43 -0400 | [diff] [blame] | 1438 | * XXX: what is now target_execute_cmd used to be asynchronous, and unmapping |
| 1439 | * the data that has been transferred via IB RDMA had to be postponed until the |
Masanari Iida | 142ad5d | 2012-08-10 00:07:58 +0000 | [diff] [blame] | 1440 | * check_stop_free() callback. None of this is necessary anymore and needs to |
Christoph Hellwig | e672a47 | 2012-07-08 15:58:43 -0400 | [diff] [blame] | 1441 | * be cleaned up. |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1442 | */ |
| 1443 | static void srpt_handle_rdma_comp(struct srpt_rdma_ch *ch, |
| 1444 | struct srpt_send_ioctx *ioctx, |
| 1445 | enum srpt_opcode opcode) |
| 1446 | { |
| 1447 | WARN_ON(ioctx->n_rdma <= 0); |
| 1448 | atomic_add(ioctx->n_rdma, &ch->sq_wr_avail); |
| 1449 | |
| 1450 | if (opcode == SRPT_RDMA_READ_LAST) { |
| 1451 | if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA, |
| 1452 | SRPT_STATE_DATA_IN)) |
Christoph Hellwig | e672a47 | 2012-07-08 15:58:43 -0400 | [diff] [blame] | 1453 | target_execute_cmd(&ioctx->cmd); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1454 | else |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1455 | pr_err("%s[%d]: wrong state = %d\n", __func__, |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1456 | __LINE__, srpt_get_cmd_state(ioctx)); |
| 1457 | } else if (opcode == SRPT_RDMA_ABORT) { |
| 1458 | ioctx->rdma_aborted = true; |
| 1459 | } else { |
| 1460 | WARN(true, "unexpected opcode %d\n", opcode); |
| 1461 | } |
| 1462 | } |
| 1463 | |
| 1464 | /** |
| 1465 | * srpt_handle_rdma_err_comp() - Process an IB RDMA error completion. |
| 1466 | */ |
| 1467 | static void srpt_handle_rdma_err_comp(struct srpt_rdma_ch *ch, |
| 1468 | struct srpt_send_ioctx *ioctx, |
| 1469 | enum srpt_opcode opcode) |
| 1470 | { |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1471 | enum srpt_command_state state; |
| 1472 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1473 | state = srpt_get_cmd_state(ioctx); |
| 1474 | switch (opcode) { |
| 1475 | case SRPT_RDMA_READ_LAST: |
| 1476 | if (ioctx->n_rdma <= 0) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1477 | pr_err("Received invalid RDMA read" |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1478 | " error completion with idx %d\n", |
| 1479 | ioctx->ioctx.index); |
| 1480 | break; |
| 1481 | } |
| 1482 | atomic_add(ioctx->n_rdma, &ch->sq_wr_avail); |
| 1483 | if (state == SRPT_STATE_NEED_DATA) |
| 1484 | srpt_abort_cmd(ioctx); |
| 1485 | else |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1486 | pr_err("%s[%d]: wrong state = %d\n", |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1487 | __func__, __LINE__, state); |
| 1488 | break; |
| 1489 | case SRPT_RDMA_WRITE_LAST: |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1490 | break; |
| 1491 | default: |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1492 | pr_err("%s[%d]: opcode = %u\n", __func__, __LINE__, opcode); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1493 | break; |
| 1494 | } |
| 1495 | } |
| 1496 | |
| 1497 | /** |
| 1498 | * srpt_build_cmd_rsp() - Build an SRP_RSP response. |
| 1499 | * @ch: RDMA channel through which the request has been received. |
| 1500 | * @ioctx: I/O context associated with the SRP_CMD request. The response will |
| 1501 | * be built in the buffer ioctx->buf points at and hence this function will |
| 1502 | * overwrite the request data. |
| 1503 | * @tag: tag of the request for which this response is being generated. |
| 1504 | * @status: value for the STATUS field of the SRP_RSP information unit. |
| 1505 | * |
| 1506 | * Returns the size in bytes of the SRP_RSP response. |
| 1507 | * |
| 1508 | * An SRP_RSP response contains a SCSI status or service response. See also |
| 1509 | * section 6.9 in the SRP r16a document for the format of an SRP_RSP |
| 1510 | * response. See also SPC-2 for more information about sense data. |
| 1511 | */ |
| 1512 | static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch, |
| 1513 | struct srpt_send_ioctx *ioctx, u64 tag, |
| 1514 | int status) |
| 1515 | { |
| 1516 | struct srp_rsp *srp_rsp; |
| 1517 | const u8 *sense_data; |
| 1518 | int sense_data_len, max_sense_len; |
| 1519 | |
| 1520 | /* |
| 1521 | * The lowest bit of all SAM-3 status codes is zero (see also |
| 1522 | * paragraph 5.3 in SAM-3). |
| 1523 | */ |
| 1524 | WARN_ON(status & 1); |
| 1525 | |
| 1526 | srp_rsp = ioctx->ioctx.buf; |
| 1527 | BUG_ON(!srp_rsp); |
| 1528 | |
| 1529 | sense_data = ioctx->sense_data; |
| 1530 | sense_data_len = ioctx->cmd.scsi_sense_length; |
| 1531 | WARN_ON(sense_data_len > sizeof(ioctx->sense_data)); |
| 1532 | |
| 1533 | memset(srp_rsp, 0, sizeof *srp_rsp); |
| 1534 | srp_rsp->opcode = SRP_RSP; |
| 1535 | srp_rsp->req_lim_delta = |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 1536 | cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0)); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1537 | srp_rsp->tag = tag; |
| 1538 | srp_rsp->status = status; |
| 1539 | |
| 1540 | if (sense_data_len) { |
| 1541 | BUILD_BUG_ON(MIN_MAX_RSP_SIZE <= sizeof(*srp_rsp)); |
| 1542 | max_sense_len = ch->max_ti_iu_len - sizeof(*srp_rsp); |
| 1543 | if (sense_data_len > max_sense_len) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1544 | pr_warn("truncated sense data from %d to %d" |
| 1545 | " bytes\n", sense_data_len, max_sense_len); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1546 | sense_data_len = max_sense_len; |
| 1547 | } |
| 1548 | |
| 1549 | srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID; |
| 1550 | srp_rsp->sense_data_len = cpu_to_be32(sense_data_len); |
| 1551 | memcpy(srp_rsp + 1, sense_data, sense_data_len); |
| 1552 | } |
| 1553 | |
| 1554 | return sizeof(*srp_rsp) + sense_data_len; |
| 1555 | } |
| 1556 | |
| 1557 | /** |
| 1558 | * srpt_build_tskmgmt_rsp() - Build a task management response. |
| 1559 | * @ch: RDMA channel through which the request has been received. |
| 1560 | * @ioctx: I/O context in which the SRP_RSP response will be built. |
| 1561 | * @rsp_code: RSP_CODE that will be stored in the response. |
| 1562 | * @tag: Tag of the request for which this response is being generated. |
| 1563 | * |
| 1564 | * Returns the size in bytes of the SRP_RSP response. |
| 1565 | * |
| 1566 | * An SRP_RSP response contains a SCSI status or service response. See also |
| 1567 | * section 6.9 in the SRP r16a document for the format of an SRP_RSP |
| 1568 | * response. |
| 1569 | */ |
| 1570 | static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch, |
| 1571 | struct srpt_send_ioctx *ioctx, |
| 1572 | u8 rsp_code, u64 tag) |
| 1573 | { |
| 1574 | struct srp_rsp *srp_rsp; |
| 1575 | int resp_data_len; |
| 1576 | int resp_len; |
| 1577 | |
Jack Wang | c807f64 | 2013-09-30 10:09:05 +0200 | [diff] [blame] | 1578 | resp_data_len = 4; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1579 | resp_len = sizeof(*srp_rsp) + resp_data_len; |
| 1580 | |
| 1581 | srp_rsp = ioctx->ioctx.buf; |
| 1582 | BUG_ON(!srp_rsp); |
| 1583 | memset(srp_rsp, 0, sizeof *srp_rsp); |
| 1584 | |
| 1585 | srp_rsp->opcode = SRP_RSP; |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 1586 | srp_rsp->req_lim_delta = |
| 1587 | cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0)); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1588 | srp_rsp->tag = tag; |
| 1589 | |
Jack Wang | c807f64 | 2013-09-30 10:09:05 +0200 | [diff] [blame] | 1590 | srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID; |
| 1591 | srp_rsp->resp_data_len = cpu_to_be32(resp_data_len); |
| 1592 | srp_rsp->data[3] = rsp_code; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1593 | |
| 1594 | return resp_len; |
| 1595 | } |
| 1596 | |
| 1597 | #define NO_SUCH_LUN ((uint64_t)-1LL) |
| 1598 | |
| 1599 | /* |
| 1600 | * SCSI LUN addressing method. See also SAM-2 and the section about |
| 1601 | * eight byte LUNs. |
| 1602 | */ |
| 1603 | enum scsi_lun_addr_method { |
| 1604 | SCSI_LUN_ADDR_METHOD_PERIPHERAL = 0, |
| 1605 | SCSI_LUN_ADDR_METHOD_FLAT = 1, |
| 1606 | SCSI_LUN_ADDR_METHOD_LUN = 2, |
| 1607 | SCSI_LUN_ADDR_METHOD_EXTENDED_LUN = 3, |
| 1608 | }; |
| 1609 | |
| 1610 | /* |
| 1611 | * srpt_unpack_lun() - Convert from network LUN to linear LUN. |
| 1612 | * |
| 1613 | * Convert an 2-byte, 4-byte, 6-byte or 8-byte LUN structure in network byte |
| 1614 | * order (big endian) to a linear LUN. Supports three LUN addressing methods: |
| 1615 | * peripheral, flat and logical unit. See also SAM-2, section 4.9.4 (page 40). |
| 1616 | */ |
| 1617 | static uint64_t srpt_unpack_lun(const uint8_t *lun, int len) |
| 1618 | { |
| 1619 | uint64_t res = NO_SUCH_LUN; |
| 1620 | int addressing_method; |
| 1621 | |
| 1622 | if (unlikely(len < 2)) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1623 | pr_err("Illegal LUN length %d, expected 2 bytes or more\n", |
| 1624 | len); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1625 | goto out; |
| 1626 | } |
| 1627 | |
| 1628 | switch (len) { |
| 1629 | case 8: |
| 1630 | if ((*((__be64 *)lun) & |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 1631 | cpu_to_be64(0x0000FFFFFFFFFFFFLL)) != 0) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1632 | goto out_err; |
| 1633 | break; |
| 1634 | case 4: |
| 1635 | if (*((__be16 *)&lun[2]) != 0) |
| 1636 | goto out_err; |
| 1637 | break; |
| 1638 | case 6: |
| 1639 | if (*((__be32 *)&lun[2]) != 0) |
| 1640 | goto out_err; |
| 1641 | break; |
| 1642 | case 2: |
| 1643 | break; |
| 1644 | default: |
| 1645 | goto out_err; |
| 1646 | } |
| 1647 | |
| 1648 | addressing_method = (*lun) >> 6; /* highest two bits of byte 0 */ |
| 1649 | switch (addressing_method) { |
| 1650 | case SCSI_LUN_ADDR_METHOD_PERIPHERAL: |
| 1651 | case SCSI_LUN_ADDR_METHOD_FLAT: |
| 1652 | case SCSI_LUN_ADDR_METHOD_LUN: |
| 1653 | res = *(lun + 1) | (((*lun) & 0x3f) << 8); |
| 1654 | break; |
| 1655 | |
| 1656 | case SCSI_LUN_ADDR_METHOD_EXTENDED_LUN: |
| 1657 | default: |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1658 | pr_err("Unimplemented LUN addressing method %u\n", |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1659 | addressing_method); |
| 1660 | break; |
| 1661 | } |
| 1662 | |
| 1663 | out: |
| 1664 | return res; |
| 1665 | |
| 1666 | out_err: |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1667 | pr_err("Support for multi-level LUNs has not yet been implemented\n"); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1668 | goto out; |
| 1669 | } |
| 1670 | |
| 1671 | static int srpt_check_stop_free(struct se_cmd *cmd) |
| 1672 | { |
Nicholas Bellinger | 9474b04 | 2012-11-27 23:55:57 -0800 | [diff] [blame] | 1673 | struct srpt_send_ioctx *ioctx = container_of(cmd, |
| 1674 | struct srpt_send_ioctx, cmd); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1675 | |
Bart Van Assche | afc1660 | 2015-04-27 13:52:36 +0200 | [diff] [blame] | 1676 | return target_put_sess_cmd(&ioctx->cmd); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1677 | } |
| 1678 | |
| 1679 | /** |
| 1680 | * srpt_handle_cmd() - Process SRP_CMD. |
| 1681 | */ |
| 1682 | static int srpt_handle_cmd(struct srpt_rdma_ch *ch, |
| 1683 | struct srpt_recv_ioctx *recv_ioctx, |
| 1684 | struct srpt_send_ioctx *send_ioctx) |
| 1685 | { |
| 1686 | struct se_cmd *cmd; |
| 1687 | struct srp_cmd *srp_cmd; |
| 1688 | uint64_t unpacked_lun; |
| 1689 | u64 data_len; |
| 1690 | enum dma_data_direction dir; |
Christoph Hellwig | de103c9 | 2012-11-06 12:24:09 -0800 | [diff] [blame] | 1691 | sense_reason_t ret; |
Nicholas Bellinger | 9474b04 | 2012-11-27 23:55:57 -0800 | [diff] [blame] | 1692 | int rc; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1693 | |
| 1694 | BUG_ON(!send_ioctx); |
| 1695 | |
| 1696 | srp_cmd = recv_ioctx->ioctx.buf; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1697 | cmd = &send_ioctx->cmd; |
Bart Van Assche | 649ee05 | 2015-04-14 13:26:44 +0200 | [diff] [blame] | 1698 | cmd->tag = srp_cmd->tag; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1699 | |
| 1700 | switch (srp_cmd->task_attr) { |
| 1701 | case SRP_CMD_SIMPLE_Q: |
Christoph Hellwig | 68d81f4 | 2014-11-24 07:07:25 -0800 | [diff] [blame] | 1702 | cmd->sam_task_attr = TCM_SIMPLE_TAG; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1703 | break; |
| 1704 | case SRP_CMD_ORDERED_Q: |
| 1705 | default: |
Christoph Hellwig | 68d81f4 | 2014-11-24 07:07:25 -0800 | [diff] [blame] | 1706 | cmd->sam_task_attr = TCM_ORDERED_TAG; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1707 | break; |
| 1708 | case SRP_CMD_HEAD_OF_Q: |
Christoph Hellwig | 68d81f4 | 2014-11-24 07:07:25 -0800 | [diff] [blame] | 1709 | cmd->sam_task_attr = TCM_HEAD_TAG; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1710 | break; |
| 1711 | case SRP_CMD_ACA: |
Christoph Hellwig | 68d81f4 | 2014-11-24 07:07:25 -0800 | [diff] [blame] | 1712 | cmd->sam_task_attr = TCM_ACA_TAG; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1713 | break; |
| 1714 | } |
| 1715 | |
Christoph Hellwig | de103c9 | 2012-11-06 12:24:09 -0800 | [diff] [blame] | 1716 | if (srpt_get_desc_tbl(send_ioctx, srp_cmd, &dir, &data_len)) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1717 | pr_err("0x%llx: parsing SRP descriptor table failed.\n", |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1718 | srp_cmd->tag); |
Christoph Hellwig | de103c9 | 2012-11-06 12:24:09 -0800 | [diff] [blame] | 1719 | ret = TCM_INVALID_CDB_FIELD; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1720 | goto send_sense; |
| 1721 | } |
| 1722 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1723 | unpacked_lun = srpt_unpack_lun((uint8_t *)&srp_cmd->lun, |
| 1724 | sizeof(srp_cmd->lun)); |
Nicholas Bellinger | 9474b04 | 2012-11-27 23:55:57 -0800 | [diff] [blame] | 1725 | rc = target_submit_cmd(cmd, ch->sess, srp_cmd->cdb, |
| 1726 | &send_ioctx->sense_data[0], unpacked_lun, data_len, |
Christoph Hellwig | 68d81f4 | 2014-11-24 07:07:25 -0800 | [diff] [blame] | 1727 | TCM_SIMPLE_TAG, dir, TARGET_SCF_ACK_KREF); |
Nicholas Bellinger | 9474b04 | 2012-11-27 23:55:57 -0800 | [diff] [blame] | 1728 | if (rc != 0) { |
| 1729 | ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1730 | goto send_sense; |
Nicholas Bellinger | 187e70a | 2012-03-17 20:12:36 -0700 | [diff] [blame] | 1731 | } |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1732 | return 0; |
| 1733 | |
| 1734 | send_sense: |
Christoph Hellwig | de103c9 | 2012-11-06 12:24:09 -0800 | [diff] [blame] | 1735 | transport_send_check_condition_and_sense(cmd, ret, 0); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1736 | return -1; |
| 1737 | } |
| 1738 | |
| 1739 | /** |
| 1740 | * srpt_rx_mgmt_fn_tag() - Process a task management function by tag. |
| 1741 | * @ch: RDMA channel of the task management request. |
| 1742 | * @fn: Task management function to perform. |
| 1743 | * @req_tag: Tag of the SRP task management request. |
| 1744 | * @mgmt_ioctx: I/O context of the task management request. |
| 1745 | * |
| 1746 | * Returns zero if the target core will process the task management |
| 1747 | * request asynchronously. |
| 1748 | * |
| 1749 | * Note: It is assumed that the initiator serializes tag-based task management |
| 1750 | * requests. |
| 1751 | */ |
| 1752 | static int srpt_rx_mgmt_fn_tag(struct srpt_send_ioctx *ioctx, u64 tag) |
| 1753 | { |
| 1754 | struct srpt_device *sdev; |
| 1755 | struct srpt_rdma_ch *ch; |
| 1756 | struct srpt_send_ioctx *target; |
| 1757 | int ret, i; |
| 1758 | |
| 1759 | ret = -EINVAL; |
| 1760 | ch = ioctx->ch; |
| 1761 | BUG_ON(!ch); |
| 1762 | BUG_ON(!ch->sport); |
| 1763 | sdev = ch->sport->sdev; |
| 1764 | BUG_ON(!sdev); |
| 1765 | spin_lock_irq(&sdev->spinlock); |
| 1766 | for (i = 0; i < ch->rq_size; ++i) { |
| 1767 | target = ch->ioctx_ring[i]; |
| 1768 | if (target->cmd.se_lun == ioctx->cmd.se_lun && |
Bart Van Assche | 649ee05 | 2015-04-14 13:26:44 +0200 | [diff] [blame] | 1769 | target->cmd.tag == tag && |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1770 | srpt_get_cmd_state(target) != SRPT_STATE_DONE) { |
| 1771 | ret = 0; |
| 1772 | /* now let the target core abort &target->cmd; */ |
| 1773 | break; |
| 1774 | } |
| 1775 | } |
| 1776 | spin_unlock_irq(&sdev->spinlock); |
| 1777 | return ret; |
| 1778 | } |
| 1779 | |
| 1780 | static int srp_tmr_to_tcm(int fn) |
| 1781 | { |
| 1782 | switch (fn) { |
| 1783 | case SRP_TSK_ABORT_TASK: |
| 1784 | return TMR_ABORT_TASK; |
| 1785 | case SRP_TSK_ABORT_TASK_SET: |
| 1786 | return TMR_ABORT_TASK_SET; |
| 1787 | case SRP_TSK_CLEAR_TASK_SET: |
| 1788 | return TMR_CLEAR_TASK_SET; |
| 1789 | case SRP_TSK_LUN_RESET: |
| 1790 | return TMR_LUN_RESET; |
| 1791 | case SRP_TSK_CLEAR_ACA: |
| 1792 | return TMR_CLEAR_ACA; |
| 1793 | default: |
| 1794 | return -1; |
| 1795 | } |
| 1796 | } |
| 1797 | |
| 1798 | /** |
| 1799 | * srpt_handle_tsk_mgmt() - Process an SRP_TSK_MGMT information unit. |
| 1800 | * |
| 1801 | * Returns 0 if and only if the request will be processed by the target core. |
| 1802 | * |
| 1803 | * For more information about SRP_TSK_MGMT information units, see also section |
| 1804 | * 6.7 in the SRP r16a document. |
| 1805 | */ |
| 1806 | static void srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch, |
| 1807 | struct srpt_recv_ioctx *recv_ioctx, |
| 1808 | struct srpt_send_ioctx *send_ioctx) |
| 1809 | { |
| 1810 | struct srp_tsk_mgmt *srp_tsk; |
| 1811 | struct se_cmd *cmd; |
Nicholas Bellinger | 3e4f574 | 2012-11-28 01:38:04 -0800 | [diff] [blame] | 1812 | struct se_session *sess = ch->sess; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1813 | uint64_t unpacked_lun; |
Nicholas Bellinger | 3e4f574 | 2012-11-28 01:38:04 -0800 | [diff] [blame] | 1814 | uint32_t tag = 0; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1815 | int tcm_tmr; |
Nicholas Bellinger | 3e4f574 | 2012-11-28 01:38:04 -0800 | [diff] [blame] | 1816 | int rc; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1817 | |
| 1818 | BUG_ON(!send_ioctx); |
| 1819 | |
| 1820 | srp_tsk = recv_ioctx->ioctx.buf; |
| 1821 | cmd = &send_ioctx->cmd; |
| 1822 | |
| 1823 | pr_debug("recv tsk_mgmt fn %d for task_tag %lld and cmd tag %lld" |
| 1824 | " cm_id %p sess %p\n", srp_tsk->tsk_mgmt_func, |
| 1825 | srp_tsk->task_tag, srp_tsk->tag, ch->cm_id, ch->sess); |
| 1826 | |
| 1827 | srpt_set_cmd_state(send_ioctx, SRPT_STATE_MGMT); |
Bart Van Assche | 649ee05 | 2015-04-14 13:26:44 +0200 | [diff] [blame] | 1828 | send_ioctx->cmd.tag = srp_tsk->tag; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1829 | tcm_tmr = srp_tmr_to_tcm(srp_tsk->tsk_mgmt_func); |
| 1830 | if (tcm_tmr < 0) { |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1831 | send_ioctx->cmd.se_tmr_req->response = |
| 1832 | TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED; |
Christoph Hellwig | de103c9 | 2012-11-06 12:24:09 -0800 | [diff] [blame] | 1833 | goto fail; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1834 | } |
Nicholas Bellinger | 3e4f574 | 2012-11-28 01:38:04 -0800 | [diff] [blame] | 1835 | unpacked_lun = srpt_unpack_lun((uint8_t *)&srp_tsk->lun, |
| 1836 | sizeof(srp_tsk->lun)); |
Nicholas Bellinger | 9474b04 | 2012-11-27 23:55:57 -0800 | [diff] [blame] | 1837 | |
Nicholas Bellinger | 3e4f574 | 2012-11-28 01:38:04 -0800 | [diff] [blame] | 1838 | if (srp_tsk->tsk_mgmt_func == SRP_TSK_ABORT_TASK) { |
| 1839 | rc = srpt_rx_mgmt_fn_tag(send_ioctx, srp_tsk->task_tag); |
| 1840 | if (rc < 0) { |
| 1841 | send_ioctx->cmd.se_tmr_req->response = |
| 1842 | TMR_TASK_DOES_NOT_EXIST; |
| 1843 | goto fail; |
| 1844 | } |
| 1845 | tag = srp_tsk->task_tag; |
| 1846 | } |
| 1847 | rc = target_submit_tmr(&send_ioctx->cmd, sess, NULL, unpacked_lun, |
| 1848 | srp_tsk, tcm_tmr, GFP_KERNEL, tag, |
| 1849 | TARGET_SCF_ACK_KREF); |
| 1850 | if (rc != 0) { |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1851 | send_ioctx->cmd.se_tmr_req->response = TMR_FUNCTION_REJECTED; |
Christoph Hellwig | de103c9 | 2012-11-06 12:24:09 -0800 | [diff] [blame] | 1852 | goto fail; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1853 | } |
Christoph Hellwig | de103c9 | 2012-11-06 12:24:09 -0800 | [diff] [blame] | 1854 | return; |
| 1855 | fail: |
Christoph Hellwig | de103c9 | 2012-11-06 12:24:09 -0800 | [diff] [blame] | 1856 | transport_send_check_condition_and_sense(cmd, 0, 0); // XXX: |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1857 | } |
| 1858 | |
| 1859 | /** |
| 1860 | * srpt_handle_new_iu() - Process a newly received information unit. |
| 1861 | * @ch: RDMA channel through which the information unit has been received. |
| 1862 | * @ioctx: SRPT I/O context associated with the information unit. |
| 1863 | */ |
| 1864 | static void srpt_handle_new_iu(struct srpt_rdma_ch *ch, |
| 1865 | struct srpt_recv_ioctx *recv_ioctx, |
| 1866 | struct srpt_send_ioctx *send_ioctx) |
| 1867 | { |
| 1868 | struct srp_cmd *srp_cmd; |
| 1869 | enum rdma_ch_state ch_state; |
| 1870 | |
| 1871 | BUG_ON(!ch); |
| 1872 | BUG_ON(!recv_ioctx); |
| 1873 | |
| 1874 | ib_dma_sync_single_for_cpu(ch->sport->sdev->device, |
| 1875 | recv_ioctx->ioctx.dma, srp_max_req_size, |
| 1876 | DMA_FROM_DEVICE); |
| 1877 | |
| 1878 | ch_state = srpt_get_ch_state(ch); |
| 1879 | if (unlikely(ch_state == CH_CONNECTING)) { |
| 1880 | list_add_tail(&recv_ioctx->wait_list, &ch->cmd_wait_list); |
| 1881 | goto out; |
| 1882 | } |
| 1883 | |
| 1884 | if (unlikely(ch_state != CH_LIVE)) |
| 1885 | goto out; |
| 1886 | |
| 1887 | srp_cmd = recv_ioctx->ioctx.buf; |
| 1888 | if (srp_cmd->opcode == SRP_CMD || srp_cmd->opcode == SRP_TSK_MGMT) { |
| 1889 | if (!send_ioctx) |
| 1890 | send_ioctx = srpt_get_send_ioctx(ch); |
| 1891 | if (unlikely(!send_ioctx)) { |
| 1892 | list_add_tail(&recv_ioctx->wait_list, |
| 1893 | &ch->cmd_wait_list); |
| 1894 | goto out; |
| 1895 | } |
| 1896 | } |
| 1897 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1898 | switch (srp_cmd->opcode) { |
| 1899 | case SRP_CMD: |
| 1900 | srpt_handle_cmd(ch, recv_ioctx, send_ioctx); |
| 1901 | break; |
| 1902 | case SRP_TSK_MGMT: |
| 1903 | srpt_handle_tsk_mgmt(ch, recv_ioctx, send_ioctx); |
| 1904 | break; |
| 1905 | case SRP_I_LOGOUT: |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1906 | pr_err("Not yet implemented: SRP_I_LOGOUT\n"); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1907 | break; |
| 1908 | case SRP_CRED_RSP: |
| 1909 | pr_debug("received SRP_CRED_RSP\n"); |
| 1910 | break; |
| 1911 | case SRP_AER_RSP: |
| 1912 | pr_debug("received SRP_AER_RSP\n"); |
| 1913 | break; |
| 1914 | case SRP_RSP: |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1915 | pr_err("Received SRP_RSP\n"); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1916 | break; |
| 1917 | default: |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1918 | pr_err("received IU with unknown opcode 0x%x\n", |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1919 | srp_cmd->opcode); |
| 1920 | break; |
| 1921 | } |
| 1922 | |
| 1923 | srpt_post_recv(ch->sport->sdev, recv_ioctx); |
| 1924 | out: |
| 1925 | return; |
| 1926 | } |
| 1927 | |
| 1928 | static void srpt_process_rcv_completion(struct ib_cq *cq, |
| 1929 | struct srpt_rdma_ch *ch, |
| 1930 | struct ib_wc *wc) |
| 1931 | { |
| 1932 | struct srpt_device *sdev = ch->sport->sdev; |
| 1933 | struct srpt_recv_ioctx *ioctx; |
| 1934 | u32 index; |
| 1935 | |
| 1936 | index = idx_from_wr_id(wc->wr_id); |
| 1937 | if (wc->status == IB_WC_SUCCESS) { |
| 1938 | int req_lim; |
| 1939 | |
| 1940 | req_lim = atomic_dec_return(&ch->req_lim); |
| 1941 | if (unlikely(req_lim < 0)) |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1942 | pr_err("req_lim = %d < 0\n", req_lim); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1943 | ioctx = sdev->ioctx_ring[index]; |
| 1944 | srpt_handle_new_iu(ch, ioctx, NULL); |
| 1945 | } else { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1946 | pr_info("receiving failed for idx %u with status %d\n", |
| 1947 | index, wc->status); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1948 | } |
| 1949 | } |
| 1950 | |
| 1951 | /** |
| 1952 | * srpt_process_send_completion() - Process an IB send completion. |
| 1953 | * |
| 1954 | * Note: Although this has not yet been observed during tests, at least in |
| 1955 | * theory it is possible that the srpt_get_send_ioctx() call invoked by |
| 1956 | * srpt_handle_new_iu() fails. This is possible because the req_lim_delta |
| 1957 | * value in each response is set to one, and it is possible that this response |
| 1958 | * makes the initiator send a new request before the send completion for that |
| 1959 | * response has been processed. This could e.g. happen if the call to |
| 1960 | * srpt_put_send_iotcx() is delayed because of a higher priority interrupt or |
| 1961 | * if IB retransmission causes generation of the send completion to be |
| 1962 | * delayed. Incoming information units for which srpt_get_send_ioctx() fails |
| 1963 | * are queued on cmd_wait_list. The code below processes these delayed |
| 1964 | * requests one at a time. |
| 1965 | */ |
| 1966 | static void srpt_process_send_completion(struct ib_cq *cq, |
| 1967 | struct srpt_rdma_ch *ch, |
| 1968 | struct ib_wc *wc) |
| 1969 | { |
| 1970 | struct srpt_send_ioctx *send_ioctx; |
| 1971 | uint32_t index; |
| 1972 | enum srpt_opcode opcode; |
| 1973 | |
| 1974 | index = idx_from_wr_id(wc->wr_id); |
| 1975 | opcode = opcode_from_wr_id(wc->wr_id); |
| 1976 | send_ioctx = ch->ioctx_ring[index]; |
| 1977 | if (wc->status == IB_WC_SUCCESS) { |
| 1978 | if (opcode == SRPT_SEND) |
| 1979 | srpt_handle_send_comp(ch, send_ioctx); |
| 1980 | else { |
| 1981 | WARN_ON(opcode != SRPT_RDMA_ABORT && |
| 1982 | wc->opcode != IB_WC_RDMA_READ); |
| 1983 | srpt_handle_rdma_comp(ch, send_ioctx, opcode); |
| 1984 | } |
| 1985 | } else { |
| 1986 | if (opcode == SRPT_SEND) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1987 | pr_info("sending response for idx %u failed" |
| 1988 | " with status %d\n", index, wc->status); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1989 | srpt_handle_send_err_comp(ch, wc->wr_id); |
| 1990 | } else if (opcode != SRPT_RDMA_MID) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 1991 | pr_info("RDMA t %d for idx %u failed with" |
| 1992 | " status %d\n", opcode, index, wc->status); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 1993 | srpt_handle_rdma_err_comp(ch, send_ioctx, opcode); |
| 1994 | } |
| 1995 | } |
| 1996 | |
| 1997 | while (unlikely(opcode == SRPT_SEND |
| 1998 | && !list_empty(&ch->cmd_wait_list) |
| 1999 | && srpt_get_ch_state(ch) == CH_LIVE |
| 2000 | && (send_ioctx = srpt_get_send_ioctx(ch)) != NULL)) { |
| 2001 | struct srpt_recv_ioctx *recv_ioctx; |
| 2002 | |
| 2003 | recv_ioctx = list_first_entry(&ch->cmd_wait_list, |
| 2004 | struct srpt_recv_ioctx, |
| 2005 | wait_list); |
| 2006 | list_del(&recv_ioctx->wait_list); |
| 2007 | srpt_handle_new_iu(ch, recv_ioctx, send_ioctx); |
| 2008 | } |
| 2009 | } |
| 2010 | |
| 2011 | static void srpt_process_completion(struct ib_cq *cq, struct srpt_rdma_ch *ch) |
| 2012 | { |
| 2013 | struct ib_wc *const wc = ch->wc; |
| 2014 | int i, n; |
| 2015 | |
| 2016 | WARN_ON(cq != ch->cq); |
| 2017 | |
| 2018 | ib_req_notify_cq(cq, IB_CQ_NEXT_COMP); |
| 2019 | while ((n = ib_poll_cq(cq, ARRAY_SIZE(ch->wc), wc)) > 0) { |
| 2020 | for (i = 0; i < n; i++) { |
| 2021 | if (opcode_from_wr_id(wc[i].wr_id) == SRPT_RECV) |
| 2022 | srpt_process_rcv_completion(cq, ch, &wc[i]); |
| 2023 | else |
| 2024 | srpt_process_send_completion(cq, ch, &wc[i]); |
| 2025 | } |
| 2026 | } |
| 2027 | } |
| 2028 | |
| 2029 | /** |
| 2030 | * srpt_completion() - IB completion queue callback function. |
| 2031 | * |
| 2032 | * Notes: |
| 2033 | * - It is guaranteed that a completion handler will never be invoked |
| 2034 | * concurrently on two different CPUs for the same completion queue. See also |
| 2035 | * Documentation/infiniband/core_locking.txt and the implementation of |
| 2036 | * handle_edge_irq() in kernel/irq/chip.c. |
| 2037 | * - When threaded IRQs are enabled, completion handlers are invoked in thread |
| 2038 | * context instead of interrupt context. |
| 2039 | */ |
| 2040 | static void srpt_completion(struct ib_cq *cq, void *ctx) |
| 2041 | { |
| 2042 | struct srpt_rdma_ch *ch = ctx; |
| 2043 | |
| 2044 | wake_up_interruptible(&ch->wait_queue); |
| 2045 | } |
| 2046 | |
| 2047 | static int srpt_compl_thread(void *arg) |
| 2048 | { |
| 2049 | struct srpt_rdma_ch *ch; |
| 2050 | |
| 2051 | /* Hibernation / freezing of the SRPT kernel thread is not supported. */ |
| 2052 | current->flags |= PF_NOFREEZE; |
| 2053 | |
| 2054 | ch = arg; |
| 2055 | BUG_ON(!ch); |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2056 | pr_info("Session %s: kernel thread %s (PID %d) started\n", |
| 2057 | ch->sess_name, ch->thread->comm, current->pid); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2058 | while (!kthread_should_stop()) { |
| 2059 | wait_event_interruptible(ch->wait_queue, |
| 2060 | (srpt_process_completion(ch->cq, ch), |
| 2061 | kthread_should_stop())); |
| 2062 | } |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2063 | pr_info("Session %s: kernel thread %s (PID %d) stopped\n", |
| 2064 | ch->sess_name, ch->thread->comm, current->pid); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2065 | return 0; |
| 2066 | } |
| 2067 | |
| 2068 | /** |
| 2069 | * srpt_create_ch_ib() - Create receive and send completion queues. |
| 2070 | */ |
| 2071 | static int srpt_create_ch_ib(struct srpt_rdma_ch *ch) |
| 2072 | { |
| 2073 | struct ib_qp_init_attr *qp_init; |
| 2074 | struct srpt_port *sport = ch->sport; |
| 2075 | struct srpt_device *sdev = sport->sdev; |
| 2076 | u32 srp_sq_size = sport->port_attrib.srp_sq_size; |
Matan Barak | 8e37210 | 2015-06-11 16:35:21 +0300 | [diff] [blame] | 2077 | struct ib_cq_init_attr cq_attr = {}; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2078 | int ret; |
| 2079 | |
| 2080 | WARN_ON(ch->rq_size < 1); |
| 2081 | |
| 2082 | ret = -ENOMEM; |
| 2083 | qp_init = kzalloc(sizeof *qp_init, GFP_KERNEL); |
| 2084 | if (!qp_init) |
| 2085 | goto out; |
| 2086 | |
Bart Van Assche | ab477c1 | 2014-10-19 18:05:33 +0300 | [diff] [blame] | 2087 | retry: |
Matan Barak | 8e37210 | 2015-06-11 16:35:21 +0300 | [diff] [blame] | 2088 | cq_attr.cqe = ch->rq_size + srp_sq_size; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2089 | ch->cq = ib_create_cq(sdev->device, srpt_completion, NULL, ch, |
Matan Barak | 8e37210 | 2015-06-11 16:35:21 +0300 | [diff] [blame] | 2090 | &cq_attr); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2091 | if (IS_ERR(ch->cq)) { |
| 2092 | ret = PTR_ERR(ch->cq); |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2093 | pr_err("failed to create CQ cqe= %d ret= %d\n", |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2094 | ch->rq_size + srp_sq_size, ret); |
| 2095 | goto out; |
| 2096 | } |
| 2097 | |
| 2098 | qp_init->qp_context = (void *)ch; |
| 2099 | qp_init->event_handler |
| 2100 | = (void(*)(struct ib_event *, void*))srpt_qp_event; |
| 2101 | qp_init->send_cq = ch->cq; |
| 2102 | qp_init->recv_cq = ch->cq; |
| 2103 | qp_init->srq = sdev->srq; |
| 2104 | qp_init->sq_sig_type = IB_SIGNAL_REQ_WR; |
| 2105 | qp_init->qp_type = IB_QPT_RC; |
| 2106 | qp_init->cap.max_send_wr = srp_sq_size; |
| 2107 | qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE; |
| 2108 | |
| 2109 | ch->qp = ib_create_qp(sdev->pd, qp_init); |
| 2110 | if (IS_ERR(ch->qp)) { |
| 2111 | ret = PTR_ERR(ch->qp); |
Bart Van Assche | ab477c1 | 2014-10-19 18:05:33 +0300 | [diff] [blame] | 2112 | if (ret == -ENOMEM) { |
| 2113 | srp_sq_size /= 2; |
| 2114 | if (srp_sq_size >= MIN_SRPT_SQ_SIZE) { |
| 2115 | ib_destroy_cq(ch->cq); |
| 2116 | goto retry; |
| 2117 | } |
| 2118 | } |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2119 | pr_err("failed to create_qp ret= %d\n", ret); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2120 | goto err_destroy_cq; |
| 2121 | } |
| 2122 | |
| 2123 | atomic_set(&ch->sq_wr_avail, qp_init->cap.max_send_wr); |
| 2124 | |
| 2125 | pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n", |
| 2126 | __func__, ch->cq->cqe, qp_init->cap.max_send_sge, |
| 2127 | qp_init->cap.max_send_wr, ch->cm_id); |
| 2128 | |
| 2129 | ret = srpt_init_ch_qp(ch, ch->qp); |
| 2130 | if (ret) |
| 2131 | goto err_destroy_qp; |
| 2132 | |
| 2133 | init_waitqueue_head(&ch->wait_queue); |
| 2134 | |
| 2135 | pr_debug("creating thread for session %s\n", ch->sess_name); |
| 2136 | |
| 2137 | ch->thread = kthread_run(srpt_compl_thread, ch, "ib_srpt_compl"); |
| 2138 | if (IS_ERR(ch->thread)) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2139 | pr_err("failed to create kernel thread %ld\n", |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2140 | PTR_ERR(ch->thread)); |
| 2141 | ch->thread = NULL; |
| 2142 | goto err_destroy_qp; |
| 2143 | } |
| 2144 | |
| 2145 | out: |
| 2146 | kfree(qp_init); |
| 2147 | return ret; |
| 2148 | |
| 2149 | err_destroy_qp: |
| 2150 | ib_destroy_qp(ch->qp); |
| 2151 | err_destroy_cq: |
| 2152 | ib_destroy_cq(ch->cq); |
| 2153 | goto out; |
| 2154 | } |
| 2155 | |
| 2156 | static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch) |
| 2157 | { |
| 2158 | if (ch->thread) |
| 2159 | kthread_stop(ch->thread); |
| 2160 | |
| 2161 | ib_destroy_qp(ch->qp); |
| 2162 | ib_destroy_cq(ch->cq); |
| 2163 | } |
| 2164 | |
| 2165 | /** |
| 2166 | * __srpt_close_ch() - Close an RDMA channel by setting the QP error state. |
| 2167 | * |
| 2168 | * Reset the QP and make sure all resources associated with the channel will |
| 2169 | * be deallocated at an appropriate time. |
| 2170 | * |
| 2171 | * Note: The caller must hold ch->sport->sdev->spinlock. |
| 2172 | */ |
| 2173 | static void __srpt_close_ch(struct srpt_rdma_ch *ch) |
| 2174 | { |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2175 | enum rdma_ch_state prev_state; |
| 2176 | unsigned long flags; |
| 2177 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2178 | spin_lock_irqsave(&ch->spinlock, flags); |
| 2179 | prev_state = ch->state; |
| 2180 | switch (prev_state) { |
| 2181 | case CH_CONNECTING: |
| 2182 | case CH_LIVE: |
| 2183 | ch->state = CH_DISCONNECTING; |
| 2184 | break; |
| 2185 | default: |
| 2186 | break; |
| 2187 | } |
| 2188 | spin_unlock_irqrestore(&ch->spinlock, flags); |
| 2189 | |
| 2190 | switch (prev_state) { |
| 2191 | case CH_CONNECTING: |
| 2192 | ib_send_cm_rej(ch->cm_id, IB_CM_REJ_NO_RESOURCES, NULL, 0, |
| 2193 | NULL, 0); |
| 2194 | /* fall through */ |
| 2195 | case CH_LIVE: |
| 2196 | if (ib_send_cm_dreq(ch->cm_id, NULL, 0) < 0) |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2197 | pr_err("sending CM DREQ failed.\n"); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2198 | break; |
| 2199 | case CH_DISCONNECTING: |
| 2200 | break; |
| 2201 | case CH_DRAINING: |
| 2202 | case CH_RELEASING: |
| 2203 | break; |
| 2204 | } |
| 2205 | } |
| 2206 | |
| 2207 | /** |
| 2208 | * srpt_close_ch() - Close an RDMA channel. |
| 2209 | */ |
| 2210 | static void srpt_close_ch(struct srpt_rdma_ch *ch) |
| 2211 | { |
| 2212 | struct srpt_device *sdev; |
| 2213 | |
| 2214 | sdev = ch->sport->sdev; |
| 2215 | spin_lock_irq(&sdev->spinlock); |
| 2216 | __srpt_close_ch(ch); |
| 2217 | spin_unlock_irq(&sdev->spinlock); |
| 2218 | } |
| 2219 | |
| 2220 | /** |
Nicholas Bellinger | 1d19f78 | 2013-05-15 01:30:01 -0700 | [diff] [blame] | 2221 | * srpt_shutdown_session() - Whether or not a session may be shut down. |
| 2222 | */ |
| 2223 | static int srpt_shutdown_session(struct se_session *se_sess) |
| 2224 | { |
| 2225 | struct srpt_rdma_ch *ch = se_sess->fabric_sess_ptr; |
| 2226 | unsigned long flags; |
| 2227 | |
| 2228 | spin_lock_irqsave(&ch->spinlock, flags); |
| 2229 | if (ch->in_shutdown) { |
| 2230 | spin_unlock_irqrestore(&ch->spinlock, flags); |
| 2231 | return true; |
| 2232 | } |
| 2233 | |
| 2234 | ch->in_shutdown = true; |
| 2235 | target_sess_cmd_list_set_waiting(se_sess); |
| 2236 | spin_unlock_irqrestore(&ch->spinlock, flags); |
| 2237 | |
| 2238 | return true; |
| 2239 | } |
| 2240 | |
| 2241 | /** |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2242 | * srpt_drain_channel() - Drain a channel by resetting the IB queue pair. |
| 2243 | * @cm_id: Pointer to the CM ID of the channel to be drained. |
| 2244 | * |
| 2245 | * Note: Must be called from inside srpt_cm_handler to avoid a race between |
| 2246 | * accessing sdev->spinlock and the call to kfree(sdev) in srpt_remove_one() |
| 2247 | * (the caller of srpt_cm_handler holds the cm_id spinlock; srpt_remove_one() |
| 2248 | * waits until all target sessions for the associated IB device have been |
| 2249 | * unregistered and target session registration involves a call to |
| 2250 | * ib_destroy_cm_id(), which locks the cm_id spinlock and hence waits until |
| 2251 | * this function has finished). |
| 2252 | */ |
| 2253 | static void srpt_drain_channel(struct ib_cm_id *cm_id) |
| 2254 | { |
| 2255 | struct srpt_device *sdev; |
| 2256 | struct srpt_rdma_ch *ch; |
| 2257 | int ret; |
| 2258 | bool do_reset = false; |
| 2259 | |
| 2260 | WARN_ON_ONCE(irqs_disabled()); |
| 2261 | |
| 2262 | sdev = cm_id->context; |
| 2263 | BUG_ON(!sdev); |
| 2264 | spin_lock_irq(&sdev->spinlock); |
| 2265 | list_for_each_entry(ch, &sdev->rch_list, list) { |
| 2266 | if (ch->cm_id == cm_id) { |
| 2267 | do_reset = srpt_test_and_set_ch_state(ch, |
| 2268 | CH_CONNECTING, CH_DRAINING) || |
| 2269 | srpt_test_and_set_ch_state(ch, |
| 2270 | CH_LIVE, CH_DRAINING) || |
| 2271 | srpt_test_and_set_ch_state(ch, |
| 2272 | CH_DISCONNECTING, CH_DRAINING); |
| 2273 | break; |
| 2274 | } |
| 2275 | } |
| 2276 | spin_unlock_irq(&sdev->spinlock); |
| 2277 | |
| 2278 | if (do_reset) { |
Nicholas Bellinger | 1d19f78 | 2013-05-15 01:30:01 -0700 | [diff] [blame] | 2279 | if (ch->sess) |
| 2280 | srpt_shutdown_session(ch->sess); |
| 2281 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2282 | ret = srpt_ch_qp_err(ch); |
| 2283 | if (ret < 0) |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2284 | pr_err("Setting queue pair in error state" |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2285 | " failed: %d\n", ret); |
| 2286 | } |
| 2287 | } |
| 2288 | |
| 2289 | /** |
| 2290 | * srpt_find_channel() - Look up an RDMA channel. |
| 2291 | * @cm_id: Pointer to the CM ID of the channel to be looked up. |
| 2292 | * |
| 2293 | * Return NULL if no matching RDMA channel has been found. |
| 2294 | */ |
| 2295 | static struct srpt_rdma_ch *srpt_find_channel(struct srpt_device *sdev, |
| 2296 | struct ib_cm_id *cm_id) |
| 2297 | { |
| 2298 | struct srpt_rdma_ch *ch; |
| 2299 | bool found; |
| 2300 | |
| 2301 | WARN_ON_ONCE(irqs_disabled()); |
| 2302 | BUG_ON(!sdev); |
| 2303 | |
| 2304 | found = false; |
| 2305 | spin_lock_irq(&sdev->spinlock); |
| 2306 | list_for_each_entry(ch, &sdev->rch_list, list) { |
| 2307 | if (ch->cm_id == cm_id) { |
| 2308 | found = true; |
| 2309 | break; |
| 2310 | } |
| 2311 | } |
| 2312 | spin_unlock_irq(&sdev->spinlock); |
| 2313 | |
| 2314 | return found ? ch : NULL; |
| 2315 | } |
| 2316 | |
| 2317 | /** |
| 2318 | * srpt_release_channel() - Release channel resources. |
| 2319 | * |
| 2320 | * Schedules the actual release because: |
| 2321 | * - Calling the ib_destroy_cm_id() call from inside an IB CM callback would |
| 2322 | * trigger a deadlock. |
| 2323 | * - It is not safe to call TCM transport_* functions from interrupt context. |
| 2324 | */ |
| 2325 | static void srpt_release_channel(struct srpt_rdma_ch *ch) |
| 2326 | { |
| 2327 | schedule_work(&ch->release_work); |
| 2328 | } |
| 2329 | |
| 2330 | static void srpt_release_channel_work(struct work_struct *w) |
| 2331 | { |
| 2332 | struct srpt_rdma_ch *ch; |
| 2333 | struct srpt_device *sdev; |
Nicholas Bellinger | 9474b04 | 2012-11-27 23:55:57 -0800 | [diff] [blame] | 2334 | struct se_session *se_sess; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2335 | |
| 2336 | ch = container_of(w, struct srpt_rdma_ch, release_work); |
| 2337 | pr_debug("ch = %p; ch->sess = %p; release_done = %p\n", ch, ch->sess, |
| 2338 | ch->release_done); |
| 2339 | |
| 2340 | sdev = ch->sport->sdev; |
| 2341 | BUG_ON(!sdev); |
| 2342 | |
Nicholas Bellinger | 9474b04 | 2012-11-27 23:55:57 -0800 | [diff] [blame] | 2343 | se_sess = ch->sess; |
| 2344 | BUG_ON(!se_sess); |
| 2345 | |
Joern Engel | be646c2d | 2013-05-15 00:44:07 -0700 | [diff] [blame] | 2346 | target_wait_for_sess_cmds(se_sess); |
Nicholas Bellinger | 9474b04 | 2012-11-27 23:55:57 -0800 | [diff] [blame] | 2347 | |
| 2348 | transport_deregister_session_configfs(se_sess); |
| 2349 | transport_deregister_session(se_sess); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2350 | ch->sess = NULL; |
| 2351 | |
Nicholas Bellinger | 0b41d6c | 2013-09-18 12:48:27 -0700 | [diff] [blame] | 2352 | ib_destroy_cm_id(ch->cm_id); |
| 2353 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2354 | srpt_destroy_ch_ib(ch); |
| 2355 | |
| 2356 | srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring, |
| 2357 | ch->sport->sdev, ch->rq_size, |
| 2358 | ch->rsp_size, DMA_TO_DEVICE); |
| 2359 | |
| 2360 | spin_lock_irq(&sdev->spinlock); |
| 2361 | list_del(&ch->list); |
| 2362 | spin_unlock_irq(&sdev->spinlock); |
| 2363 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2364 | if (ch->release_done) |
| 2365 | complete(ch->release_done); |
| 2366 | |
| 2367 | wake_up(&sdev->ch_releaseQ); |
| 2368 | |
| 2369 | kfree(ch); |
| 2370 | } |
| 2371 | |
| 2372 | static struct srpt_node_acl *__srpt_lookup_acl(struct srpt_port *sport, |
| 2373 | u8 i_port_id[16]) |
| 2374 | { |
| 2375 | struct srpt_node_acl *nacl; |
| 2376 | |
| 2377 | list_for_each_entry(nacl, &sport->port_acl_list, list) |
| 2378 | if (memcmp(nacl->i_port_id, i_port_id, |
| 2379 | sizeof(nacl->i_port_id)) == 0) |
| 2380 | return nacl; |
| 2381 | |
| 2382 | return NULL; |
| 2383 | } |
| 2384 | |
| 2385 | static struct srpt_node_acl *srpt_lookup_acl(struct srpt_port *sport, |
| 2386 | u8 i_port_id[16]) |
| 2387 | { |
| 2388 | struct srpt_node_acl *nacl; |
| 2389 | |
| 2390 | spin_lock_irq(&sport->port_acl_lock); |
| 2391 | nacl = __srpt_lookup_acl(sport, i_port_id); |
| 2392 | spin_unlock_irq(&sport->port_acl_lock); |
| 2393 | |
| 2394 | return nacl; |
| 2395 | } |
| 2396 | |
| 2397 | /** |
| 2398 | * srpt_cm_req_recv() - Process the event IB_CM_REQ_RECEIVED. |
| 2399 | * |
| 2400 | * Ownership of the cm_id is transferred to the target session if this |
| 2401 | * functions returns zero. Otherwise the caller remains the owner of cm_id. |
| 2402 | */ |
| 2403 | static int srpt_cm_req_recv(struct ib_cm_id *cm_id, |
| 2404 | struct ib_cm_req_event_param *param, |
| 2405 | void *private_data) |
| 2406 | { |
| 2407 | struct srpt_device *sdev = cm_id->context; |
| 2408 | struct srpt_port *sport = &sdev->port[param->port - 1]; |
| 2409 | struct srp_login_req *req; |
| 2410 | struct srp_login_rsp *rsp; |
| 2411 | struct srp_login_rej *rej; |
| 2412 | struct ib_cm_rep_param *rep_param; |
| 2413 | struct srpt_rdma_ch *ch, *tmp_ch; |
| 2414 | struct srpt_node_acl *nacl; |
| 2415 | u32 it_iu_len; |
| 2416 | int i; |
| 2417 | int ret = 0; |
| 2418 | |
| 2419 | WARN_ON_ONCE(irqs_disabled()); |
| 2420 | |
| 2421 | if (WARN_ON(!sdev || !private_data)) |
| 2422 | return -EINVAL; |
| 2423 | |
| 2424 | req = (struct srp_login_req *)private_data; |
| 2425 | |
| 2426 | it_iu_len = be32_to_cpu(req->req_it_iu_len); |
| 2427 | |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2428 | pr_info("Received SRP_LOGIN_REQ with i_port_id 0x%llx:0x%llx," |
| 2429 | " t_port_id 0x%llx:0x%llx and it_iu_len %d on port %d" |
| 2430 | " (guid=0x%llx:0x%llx)\n", |
| 2431 | be64_to_cpu(*(__be64 *)&req->initiator_port_id[0]), |
| 2432 | be64_to_cpu(*(__be64 *)&req->initiator_port_id[8]), |
| 2433 | be64_to_cpu(*(__be64 *)&req->target_port_id[0]), |
| 2434 | be64_to_cpu(*(__be64 *)&req->target_port_id[8]), |
| 2435 | it_iu_len, |
| 2436 | param->port, |
| 2437 | be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[0]), |
| 2438 | be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[8])); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2439 | |
| 2440 | rsp = kzalloc(sizeof *rsp, GFP_KERNEL); |
| 2441 | rej = kzalloc(sizeof *rej, GFP_KERNEL); |
| 2442 | rep_param = kzalloc(sizeof *rep_param, GFP_KERNEL); |
| 2443 | |
| 2444 | if (!rsp || !rej || !rep_param) { |
| 2445 | ret = -ENOMEM; |
| 2446 | goto out; |
| 2447 | } |
| 2448 | |
| 2449 | if (it_iu_len > srp_max_req_size || it_iu_len < 64) { |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 2450 | rej->reason = cpu_to_be32( |
| 2451 | SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2452 | ret = -EINVAL; |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2453 | pr_err("rejected SRP_LOGIN_REQ because its" |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2454 | " length (%d bytes) is out of range (%d .. %d)\n", |
| 2455 | it_iu_len, 64, srp_max_req_size); |
| 2456 | goto reject; |
| 2457 | } |
| 2458 | |
| 2459 | if (!sport->enabled) { |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 2460 | rej->reason = cpu_to_be32( |
| 2461 | SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2462 | ret = -EINVAL; |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2463 | pr_err("rejected SRP_LOGIN_REQ because the target port" |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2464 | " has not yet been enabled\n"); |
| 2465 | goto reject; |
| 2466 | } |
| 2467 | |
| 2468 | if ((req->req_flags & SRP_MTCH_ACTION) == SRP_MULTICHAN_SINGLE) { |
| 2469 | rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN; |
| 2470 | |
| 2471 | spin_lock_irq(&sdev->spinlock); |
| 2472 | |
| 2473 | list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) { |
| 2474 | if (!memcmp(ch->i_port_id, req->initiator_port_id, 16) |
| 2475 | && !memcmp(ch->t_port_id, req->target_port_id, 16) |
| 2476 | && param->port == ch->sport->port |
| 2477 | && param->listen_id == ch->sport->sdev->cm_id |
| 2478 | && ch->cm_id) { |
| 2479 | enum rdma_ch_state ch_state; |
| 2480 | |
| 2481 | ch_state = srpt_get_ch_state(ch); |
| 2482 | if (ch_state != CH_CONNECTING |
| 2483 | && ch_state != CH_LIVE) |
| 2484 | continue; |
| 2485 | |
| 2486 | /* found an existing channel */ |
| 2487 | pr_debug("Found existing channel %s" |
| 2488 | " cm_id= %p state= %d\n", |
| 2489 | ch->sess_name, ch->cm_id, ch_state); |
| 2490 | |
| 2491 | __srpt_close_ch(ch); |
| 2492 | |
| 2493 | rsp->rsp_flags = |
| 2494 | SRP_LOGIN_RSP_MULTICHAN_TERMINATED; |
| 2495 | } |
| 2496 | } |
| 2497 | |
| 2498 | spin_unlock_irq(&sdev->spinlock); |
| 2499 | |
| 2500 | } else |
| 2501 | rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED; |
| 2502 | |
| 2503 | if (*(__be64 *)req->target_port_id != cpu_to_be64(srpt_service_guid) |
| 2504 | || *(__be64 *)(req->target_port_id + 8) != |
| 2505 | cpu_to_be64(srpt_service_guid)) { |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 2506 | rej->reason = cpu_to_be32( |
| 2507 | SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2508 | ret = -ENOMEM; |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2509 | pr_err("rejected SRP_LOGIN_REQ because it" |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2510 | " has an invalid target port identifier.\n"); |
| 2511 | goto reject; |
| 2512 | } |
| 2513 | |
| 2514 | ch = kzalloc(sizeof *ch, GFP_KERNEL); |
| 2515 | if (!ch) { |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 2516 | rej->reason = cpu_to_be32( |
| 2517 | SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES); |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2518 | pr_err("rejected SRP_LOGIN_REQ because no memory.\n"); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2519 | ret = -ENOMEM; |
| 2520 | goto reject; |
| 2521 | } |
| 2522 | |
| 2523 | INIT_WORK(&ch->release_work, srpt_release_channel_work); |
| 2524 | memcpy(ch->i_port_id, req->initiator_port_id, 16); |
| 2525 | memcpy(ch->t_port_id, req->target_port_id, 16); |
| 2526 | ch->sport = &sdev->port[param->port - 1]; |
| 2527 | ch->cm_id = cm_id; |
| 2528 | /* |
| 2529 | * Avoid QUEUE_FULL conditions by limiting the number of buffers used |
| 2530 | * for the SRP protocol to the command queue size. |
| 2531 | */ |
| 2532 | ch->rq_size = SRPT_RQ_SIZE; |
| 2533 | spin_lock_init(&ch->spinlock); |
| 2534 | ch->state = CH_CONNECTING; |
| 2535 | INIT_LIST_HEAD(&ch->cmd_wait_list); |
| 2536 | ch->rsp_size = ch->sport->port_attrib.srp_max_rsp_size; |
| 2537 | |
| 2538 | ch->ioctx_ring = (struct srpt_send_ioctx **) |
| 2539 | srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size, |
| 2540 | sizeof(*ch->ioctx_ring[0]), |
| 2541 | ch->rsp_size, DMA_TO_DEVICE); |
| 2542 | if (!ch->ioctx_ring) |
| 2543 | goto free_ch; |
| 2544 | |
| 2545 | INIT_LIST_HEAD(&ch->free_list); |
| 2546 | for (i = 0; i < ch->rq_size; i++) { |
| 2547 | ch->ioctx_ring[i]->ch = ch; |
| 2548 | list_add_tail(&ch->ioctx_ring[i]->free_list, &ch->free_list); |
| 2549 | } |
| 2550 | |
| 2551 | ret = srpt_create_ch_ib(ch); |
| 2552 | if (ret) { |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 2553 | rej->reason = cpu_to_be32( |
| 2554 | SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES); |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2555 | pr_err("rejected SRP_LOGIN_REQ because creating" |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2556 | " a new RDMA channel failed.\n"); |
| 2557 | goto free_ring; |
| 2558 | } |
| 2559 | |
| 2560 | ret = srpt_ch_qp_rtr(ch, ch->qp); |
| 2561 | if (ret) { |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 2562 | rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES); |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2563 | pr_err("rejected SRP_LOGIN_REQ because enabling" |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2564 | " RTR failed (error code = %d)\n", ret); |
| 2565 | goto destroy_ib; |
| 2566 | } |
| 2567 | /* |
| 2568 | * Use the initator port identifier as the session name. |
| 2569 | */ |
| 2570 | snprintf(ch->sess_name, sizeof(ch->sess_name), "0x%016llx%016llx", |
| 2571 | be64_to_cpu(*(__be64 *)ch->i_port_id), |
| 2572 | be64_to_cpu(*(__be64 *)(ch->i_port_id + 8))); |
| 2573 | |
| 2574 | pr_debug("registering session %s\n", ch->sess_name); |
| 2575 | |
| 2576 | nacl = srpt_lookup_acl(sport, ch->i_port_id); |
| 2577 | if (!nacl) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2578 | pr_info("Rejected login because no ACL has been" |
| 2579 | " configured yet for initiator %s.\n", ch->sess_name); |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 2580 | rej->reason = cpu_to_be32( |
| 2581 | SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2582 | goto destroy_ib; |
| 2583 | } |
| 2584 | |
Nicholas Bellinger | e70beee | 2014-04-02 12:52:38 -0700 | [diff] [blame] | 2585 | ch->sess = transport_init_session(TARGET_PROT_NORMAL); |
Dan Carpenter | 3af3363 | 2011-11-04 21:27:32 +0300 | [diff] [blame] | 2586 | if (IS_ERR(ch->sess)) { |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 2587 | rej->reason = cpu_to_be32( |
| 2588 | SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2589 | pr_debug("Failed to create session\n"); |
| 2590 | goto deregister_session; |
| 2591 | } |
| 2592 | ch->sess->se_node_acl = &nacl->nacl; |
| 2593 | transport_register_session(&sport->port_tpg_1, &nacl->nacl, ch->sess, ch); |
| 2594 | |
| 2595 | pr_debug("Establish connection sess=%p name=%s cm_id=%p\n", ch->sess, |
| 2596 | ch->sess_name, ch->cm_id); |
| 2597 | |
| 2598 | /* create srp_login_response */ |
| 2599 | rsp->opcode = SRP_LOGIN_RSP; |
| 2600 | rsp->tag = req->tag; |
| 2601 | rsp->max_it_iu_len = req->req_it_iu_len; |
| 2602 | rsp->max_ti_iu_len = req->req_it_iu_len; |
| 2603 | ch->max_ti_iu_len = it_iu_len; |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 2604 | rsp->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
| 2605 | | SRP_BUF_FORMAT_INDIRECT); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2606 | rsp->req_lim_delta = cpu_to_be32(ch->rq_size); |
| 2607 | atomic_set(&ch->req_lim, ch->rq_size); |
| 2608 | atomic_set(&ch->req_lim_delta, 0); |
| 2609 | |
| 2610 | /* create cm reply */ |
| 2611 | rep_param->qp_num = ch->qp->qp_num; |
| 2612 | rep_param->private_data = (void *)rsp; |
| 2613 | rep_param->private_data_len = sizeof *rsp; |
| 2614 | rep_param->rnr_retry_count = 7; |
| 2615 | rep_param->flow_control = 1; |
| 2616 | rep_param->failover_accepted = 0; |
| 2617 | rep_param->srq = 1; |
| 2618 | rep_param->responder_resources = 4; |
| 2619 | rep_param->initiator_depth = 4; |
| 2620 | |
| 2621 | ret = ib_send_cm_rep(cm_id, rep_param); |
| 2622 | if (ret) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2623 | pr_err("sending SRP_LOGIN_REQ response failed" |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2624 | " (error code = %d)\n", ret); |
| 2625 | goto release_channel; |
| 2626 | } |
| 2627 | |
| 2628 | spin_lock_irq(&sdev->spinlock); |
| 2629 | list_add_tail(&ch->list, &sdev->rch_list); |
| 2630 | spin_unlock_irq(&sdev->spinlock); |
| 2631 | |
| 2632 | goto out; |
| 2633 | |
| 2634 | release_channel: |
| 2635 | srpt_set_ch_state(ch, CH_RELEASING); |
| 2636 | transport_deregister_session_configfs(ch->sess); |
| 2637 | |
| 2638 | deregister_session: |
| 2639 | transport_deregister_session(ch->sess); |
| 2640 | ch->sess = NULL; |
| 2641 | |
| 2642 | destroy_ib: |
| 2643 | srpt_destroy_ch_ib(ch); |
| 2644 | |
| 2645 | free_ring: |
| 2646 | srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring, |
| 2647 | ch->sport->sdev, ch->rq_size, |
| 2648 | ch->rsp_size, DMA_TO_DEVICE); |
| 2649 | free_ch: |
| 2650 | kfree(ch); |
| 2651 | |
| 2652 | reject: |
| 2653 | rej->opcode = SRP_LOGIN_REJ; |
| 2654 | rej->tag = req->tag; |
Vaishali Thakkar | b356c1c | 2015-06-24 10:12:13 +0530 | [diff] [blame] | 2655 | rej->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
| 2656 | | SRP_BUF_FORMAT_INDIRECT); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2657 | |
| 2658 | ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0, |
| 2659 | (void *)rej, sizeof *rej); |
| 2660 | |
| 2661 | out: |
| 2662 | kfree(rep_param); |
| 2663 | kfree(rsp); |
| 2664 | kfree(rej); |
| 2665 | |
| 2666 | return ret; |
| 2667 | } |
| 2668 | |
| 2669 | static void srpt_cm_rej_recv(struct ib_cm_id *cm_id) |
| 2670 | { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2671 | pr_info("Received IB REJ for cm_id %p.\n", cm_id); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2672 | srpt_drain_channel(cm_id); |
| 2673 | } |
| 2674 | |
| 2675 | /** |
| 2676 | * srpt_cm_rtu_recv() - Process an IB_CM_RTU_RECEIVED or USER_ESTABLISHED event. |
| 2677 | * |
| 2678 | * An IB_CM_RTU_RECEIVED message indicates that the connection is established |
| 2679 | * and that the recipient may begin transmitting (RTU = ready to use). |
| 2680 | */ |
| 2681 | static void srpt_cm_rtu_recv(struct ib_cm_id *cm_id) |
| 2682 | { |
| 2683 | struct srpt_rdma_ch *ch; |
| 2684 | int ret; |
| 2685 | |
| 2686 | ch = srpt_find_channel(cm_id->context, cm_id); |
| 2687 | BUG_ON(!ch); |
| 2688 | |
| 2689 | if (srpt_test_and_set_ch_state(ch, CH_CONNECTING, CH_LIVE)) { |
| 2690 | struct srpt_recv_ioctx *ioctx, *ioctx_tmp; |
| 2691 | |
| 2692 | ret = srpt_ch_qp_rts(ch, ch->qp); |
| 2693 | |
| 2694 | list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list, |
| 2695 | wait_list) { |
| 2696 | list_del(&ioctx->wait_list); |
| 2697 | srpt_handle_new_iu(ch, ioctx, NULL); |
| 2698 | } |
| 2699 | if (ret) |
| 2700 | srpt_close_ch(ch); |
| 2701 | } |
| 2702 | } |
| 2703 | |
| 2704 | static void srpt_cm_timewait_exit(struct ib_cm_id *cm_id) |
| 2705 | { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2706 | pr_info("Received IB TimeWait exit for cm_id %p.\n", cm_id); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2707 | srpt_drain_channel(cm_id); |
| 2708 | } |
| 2709 | |
| 2710 | static void srpt_cm_rep_error(struct ib_cm_id *cm_id) |
| 2711 | { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2712 | pr_info("Received IB REP error for cm_id %p.\n", cm_id); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2713 | srpt_drain_channel(cm_id); |
| 2714 | } |
| 2715 | |
| 2716 | /** |
| 2717 | * srpt_cm_dreq_recv() - Process reception of a DREQ message. |
| 2718 | */ |
| 2719 | static void srpt_cm_dreq_recv(struct ib_cm_id *cm_id) |
| 2720 | { |
| 2721 | struct srpt_rdma_ch *ch; |
| 2722 | unsigned long flags; |
| 2723 | bool send_drep = false; |
| 2724 | |
| 2725 | ch = srpt_find_channel(cm_id->context, cm_id); |
| 2726 | BUG_ON(!ch); |
| 2727 | |
| 2728 | pr_debug("cm_id= %p ch->state= %d\n", cm_id, srpt_get_ch_state(ch)); |
| 2729 | |
| 2730 | spin_lock_irqsave(&ch->spinlock, flags); |
| 2731 | switch (ch->state) { |
| 2732 | case CH_CONNECTING: |
| 2733 | case CH_LIVE: |
| 2734 | send_drep = true; |
| 2735 | ch->state = CH_DISCONNECTING; |
| 2736 | break; |
| 2737 | case CH_DISCONNECTING: |
| 2738 | case CH_DRAINING: |
| 2739 | case CH_RELEASING: |
| 2740 | WARN(true, "unexpected channel state %d\n", ch->state); |
| 2741 | break; |
| 2742 | } |
| 2743 | spin_unlock_irqrestore(&ch->spinlock, flags); |
| 2744 | |
| 2745 | if (send_drep) { |
| 2746 | if (ib_send_cm_drep(ch->cm_id, NULL, 0) < 0) |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2747 | pr_err("Sending IB DREP failed.\n"); |
| 2748 | pr_info("Received DREQ and sent DREP for session %s.\n", |
| 2749 | ch->sess_name); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2750 | } |
| 2751 | } |
| 2752 | |
| 2753 | /** |
| 2754 | * srpt_cm_drep_recv() - Process reception of a DREP message. |
| 2755 | */ |
| 2756 | static void srpt_cm_drep_recv(struct ib_cm_id *cm_id) |
| 2757 | { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2758 | pr_info("Received InfiniBand DREP message for cm_id %p.\n", cm_id); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2759 | srpt_drain_channel(cm_id); |
| 2760 | } |
| 2761 | |
| 2762 | /** |
| 2763 | * srpt_cm_handler() - IB connection manager callback function. |
| 2764 | * |
| 2765 | * A non-zero return value will cause the caller destroy the CM ID. |
| 2766 | * |
| 2767 | * Note: srpt_cm_handler() must only return a non-zero value when transferring |
| 2768 | * ownership of the cm_id to a channel by srpt_cm_req_recv() failed. Returning |
| 2769 | * a non-zero value in any other case will trigger a race with the |
| 2770 | * ib_destroy_cm_id() call in srpt_release_channel(). |
| 2771 | */ |
| 2772 | static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event) |
| 2773 | { |
| 2774 | int ret; |
| 2775 | |
| 2776 | ret = 0; |
| 2777 | switch (event->event) { |
| 2778 | case IB_CM_REQ_RECEIVED: |
| 2779 | ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd, |
| 2780 | event->private_data); |
| 2781 | break; |
| 2782 | case IB_CM_REJ_RECEIVED: |
| 2783 | srpt_cm_rej_recv(cm_id); |
| 2784 | break; |
| 2785 | case IB_CM_RTU_RECEIVED: |
| 2786 | case IB_CM_USER_ESTABLISHED: |
| 2787 | srpt_cm_rtu_recv(cm_id); |
| 2788 | break; |
| 2789 | case IB_CM_DREQ_RECEIVED: |
| 2790 | srpt_cm_dreq_recv(cm_id); |
| 2791 | break; |
| 2792 | case IB_CM_DREP_RECEIVED: |
| 2793 | srpt_cm_drep_recv(cm_id); |
| 2794 | break; |
| 2795 | case IB_CM_TIMEWAIT_EXIT: |
| 2796 | srpt_cm_timewait_exit(cm_id); |
| 2797 | break; |
| 2798 | case IB_CM_REP_ERROR: |
| 2799 | srpt_cm_rep_error(cm_id); |
| 2800 | break; |
| 2801 | case IB_CM_DREQ_ERROR: |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2802 | pr_info("Received IB DREQ ERROR event.\n"); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2803 | break; |
| 2804 | case IB_CM_MRA_RECEIVED: |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2805 | pr_info("Received IB MRA event\n"); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2806 | break; |
| 2807 | default: |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2808 | pr_err("received unrecognized IB CM event %d\n", event->event); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2809 | break; |
| 2810 | } |
| 2811 | |
| 2812 | return ret; |
| 2813 | } |
| 2814 | |
| 2815 | /** |
| 2816 | * srpt_perform_rdmas() - Perform IB RDMA. |
| 2817 | * |
| 2818 | * Returns zero upon success or a negative number upon failure. |
| 2819 | */ |
| 2820 | static int srpt_perform_rdmas(struct srpt_rdma_ch *ch, |
| 2821 | struct srpt_send_ioctx *ioctx) |
| 2822 | { |
| 2823 | struct ib_send_wr wr; |
| 2824 | struct ib_send_wr *bad_wr; |
| 2825 | struct rdma_iu *riu; |
| 2826 | int i; |
| 2827 | int ret; |
| 2828 | int sq_wr_avail; |
| 2829 | enum dma_data_direction dir; |
| 2830 | const int n_rdma = ioctx->n_rdma; |
| 2831 | |
| 2832 | dir = ioctx->cmd.data_direction; |
| 2833 | if (dir == DMA_TO_DEVICE) { |
| 2834 | /* write */ |
| 2835 | ret = -ENOMEM; |
| 2836 | sq_wr_avail = atomic_sub_return(n_rdma, &ch->sq_wr_avail); |
| 2837 | if (sq_wr_avail < 0) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2838 | pr_warn("IB send queue full (needed %d)\n", |
| 2839 | n_rdma); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2840 | goto out; |
| 2841 | } |
| 2842 | } |
| 2843 | |
| 2844 | ioctx->rdma_aborted = false; |
| 2845 | ret = 0; |
| 2846 | riu = ioctx->rdma_ius; |
| 2847 | memset(&wr, 0, sizeof wr); |
| 2848 | |
| 2849 | for (i = 0; i < n_rdma; ++i, ++riu) { |
| 2850 | if (dir == DMA_FROM_DEVICE) { |
| 2851 | wr.opcode = IB_WR_RDMA_WRITE; |
| 2852 | wr.wr_id = encode_wr_id(i == n_rdma - 1 ? |
| 2853 | SRPT_RDMA_WRITE_LAST : |
| 2854 | SRPT_RDMA_MID, |
| 2855 | ioctx->ioctx.index); |
| 2856 | } else { |
| 2857 | wr.opcode = IB_WR_RDMA_READ; |
| 2858 | wr.wr_id = encode_wr_id(i == n_rdma - 1 ? |
| 2859 | SRPT_RDMA_READ_LAST : |
| 2860 | SRPT_RDMA_MID, |
| 2861 | ioctx->ioctx.index); |
| 2862 | } |
| 2863 | wr.next = NULL; |
| 2864 | wr.wr.rdma.remote_addr = riu->raddr; |
| 2865 | wr.wr.rdma.rkey = riu->rkey; |
| 2866 | wr.num_sge = riu->sge_cnt; |
| 2867 | wr.sg_list = riu->sge; |
| 2868 | |
| 2869 | /* only get completion event for the last rdma write */ |
| 2870 | if (i == (n_rdma - 1) && dir == DMA_TO_DEVICE) |
| 2871 | wr.send_flags = IB_SEND_SIGNALED; |
| 2872 | |
| 2873 | ret = ib_post_send(ch->qp, &wr, &bad_wr); |
| 2874 | if (ret) |
| 2875 | break; |
| 2876 | } |
| 2877 | |
| 2878 | if (ret) |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2879 | pr_err("%s[%d]: ib_post_send() returned %d for %d/%d\n", |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2880 | __func__, __LINE__, ret, i, n_rdma); |
| 2881 | if (ret && i > 0) { |
| 2882 | wr.num_sge = 0; |
| 2883 | wr.wr_id = encode_wr_id(SRPT_RDMA_ABORT, ioctx->ioctx.index); |
| 2884 | wr.send_flags = IB_SEND_SIGNALED; |
| 2885 | while (ch->state == CH_LIVE && |
| 2886 | ib_post_send(ch->qp, &wr, &bad_wr) != 0) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2887 | pr_info("Trying to abort failed RDMA transfer [%d]\n", |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2888 | ioctx->ioctx.index); |
| 2889 | msleep(1000); |
| 2890 | } |
| 2891 | while (ch->state != CH_RELEASING && !ioctx->rdma_aborted) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2892 | pr_info("Waiting until RDMA abort finished [%d]\n", |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2893 | ioctx->ioctx.index); |
| 2894 | msleep(1000); |
| 2895 | } |
| 2896 | } |
| 2897 | out: |
| 2898 | if (unlikely(dir == DMA_TO_DEVICE && ret < 0)) |
| 2899 | atomic_add(n_rdma, &ch->sq_wr_avail); |
| 2900 | return ret; |
| 2901 | } |
| 2902 | |
| 2903 | /** |
| 2904 | * srpt_xfer_data() - Start data transfer from initiator to target. |
| 2905 | */ |
| 2906 | static int srpt_xfer_data(struct srpt_rdma_ch *ch, |
| 2907 | struct srpt_send_ioctx *ioctx) |
| 2908 | { |
| 2909 | int ret; |
| 2910 | |
| 2911 | ret = srpt_map_sg_to_ib_sge(ch, ioctx); |
| 2912 | if (ret) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2913 | pr_err("%s[%d] ret=%d\n", __func__, __LINE__, ret); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2914 | goto out; |
| 2915 | } |
| 2916 | |
| 2917 | ret = srpt_perform_rdmas(ch, ioctx); |
| 2918 | if (ret) { |
| 2919 | if (ret == -EAGAIN || ret == -ENOMEM) |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2920 | pr_info("%s[%d] queue full -- ret=%d\n", |
| 2921 | __func__, __LINE__, ret); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2922 | else |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 2923 | pr_err("%s[%d] fatal error -- ret=%d\n", |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2924 | __func__, __LINE__, ret); |
| 2925 | goto out_unmap; |
| 2926 | } |
| 2927 | |
| 2928 | out: |
| 2929 | return ret; |
| 2930 | out_unmap: |
| 2931 | srpt_unmap_sg_to_ib_sge(ch, ioctx); |
| 2932 | goto out; |
| 2933 | } |
| 2934 | |
| 2935 | static int srpt_write_pending_status(struct se_cmd *se_cmd) |
| 2936 | { |
| 2937 | struct srpt_send_ioctx *ioctx; |
| 2938 | |
| 2939 | ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd); |
| 2940 | return srpt_get_cmd_state(ioctx) == SRPT_STATE_NEED_DATA; |
| 2941 | } |
| 2942 | |
| 2943 | /* |
| 2944 | * srpt_write_pending() - Start data transfer from initiator to target (write). |
| 2945 | */ |
| 2946 | static int srpt_write_pending(struct se_cmd *se_cmd) |
| 2947 | { |
| 2948 | struct srpt_rdma_ch *ch; |
| 2949 | struct srpt_send_ioctx *ioctx; |
| 2950 | enum srpt_command_state new_state; |
| 2951 | enum rdma_ch_state ch_state; |
| 2952 | int ret; |
| 2953 | |
| 2954 | ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd); |
| 2955 | |
| 2956 | new_state = srpt_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA); |
| 2957 | WARN_ON(new_state == SRPT_STATE_DONE); |
| 2958 | |
| 2959 | ch = ioctx->ch; |
| 2960 | BUG_ON(!ch); |
| 2961 | |
| 2962 | ch_state = srpt_get_ch_state(ch); |
| 2963 | switch (ch_state) { |
| 2964 | case CH_CONNECTING: |
| 2965 | WARN(true, "unexpected channel state %d\n", ch_state); |
| 2966 | ret = -EINVAL; |
| 2967 | goto out; |
| 2968 | case CH_LIVE: |
| 2969 | break; |
| 2970 | case CH_DISCONNECTING: |
| 2971 | case CH_DRAINING: |
| 2972 | case CH_RELEASING: |
| 2973 | pr_debug("cmd with tag %lld: channel disconnecting\n", |
Bart Van Assche | 649ee05 | 2015-04-14 13:26:44 +0200 | [diff] [blame] | 2974 | ioctx->cmd.tag); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 2975 | srpt_set_cmd_state(ioctx, SRPT_STATE_DATA_IN); |
| 2976 | ret = -EINVAL; |
| 2977 | goto out; |
| 2978 | } |
| 2979 | ret = srpt_xfer_data(ch, ioctx); |
| 2980 | |
| 2981 | out: |
| 2982 | return ret; |
| 2983 | } |
| 2984 | |
| 2985 | static u8 tcm_to_srp_tsk_mgmt_status(const int tcm_mgmt_status) |
| 2986 | { |
| 2987 | switch (tcm_mgmt_status) { |
| 2988 | case TMR_FUNCTION_COMPLETE: |
| 2989 | return SRP_TSK_MGMT_SUCCESS; |
| 2990 | case TMR_FUNCTION_REJECTED: |
| 2991 | return SRP_TSK_MGMT_FUNC_NOT_SUPP; |
| 2992 | } |
| 2993 | return SRP_TSK_MGMT_FAILED; |
| 2994 | } |
| 2995 | |
| 2996 | /** |
| 2997 | * srpt_queue_response() - Transmits the response to a SCSI command. |
| 2998 | * |
| 2999 | * Callback function called by the TCM core. Must not block since it can be |
| 3000 | * invoked on the context of the IB completion handler. |
| 3001 | */ |
Joern Engel | b79fafa | 2013-07-03 11:22:17 -0400 | [diff] [blame] | 3002 | static void srpt_queue_response(struct se_cmd *cmd) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3003 | { |
| 3004 | struct srpt_rdma_ch *ch; |
| 3005 | struct srpt_send_ioctx *ioctx; |
| 3006 | enum srpt_command_state state; |
| 3007 | unsigned long flags; |
| 3008 | int ret; |
| 3009 | enum dma_data_direction dir; |
| 3010 | int resp_len; |
| 3011 | u8 srp_tm_status; |
| 3012 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3013 | ioctx = container_of(cmd, struct srpt_send_ioctx, cmd); |
| 3014 | ch = ioctx->ch; |
| 3015 | BUG_ON(!ch); |
| 3016 | |
| 3017 | spin_lock_irqsave(&ioctx->spinlock, flags); |
| 3018 | state = ioctx->state; |
| 3019 | switch (state) { |
| 3020 | case SRPT_STATE_NEW: |
| 3021 | case SRPT_STATE_DATA_IN: |
| 3022 | ioctx->state = SRPT_STATE_CMD_RSP_SENT; |
| 3023 | break; |
| 3024 | case SRPT_STATE_MGMT: |
| 3025 | ioctx->state = SRPT_STATE_MGMT_RSP_SENT; |
| 3026 | break; |
| 3027 | default: |
| 3028 | WARN(true, "ch %p; cmd %d: unexpected command state %d\n", |
| 3029 | ch, ioctx->ioctx.index, ioctx->state); |
| 3030 | break; |
| 3031 | } |
| 3032 | spin_unlock_irqrestore(&ioctx->spinlock, flags); |
| 3033 | |
| 3034 | if (unlikely(transport_check_aborted_status(&ioctx->cmd, false) |
| 3035 | || WARN_ON_ONCE(state == SRPT_STATE_CMD_RSP_SENT))) { |
| 3036 | atomic_inc(&ch->req_lim_delta); |
| 3037 | srpt_abort_cmd(ioctx); |
Joern Engel | b79fafa | 2013-07-03 11:22:17 -0400 | [diff] [blame] | 3038 | return; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3039 | } |
| 3040 | |
| 3041 | dir = ioctx->cmd.data_direction; |
| 3042 | |
| 3043 | /* For read commands, transfer the data to the initiator. */ |
| 3044 | if (dir == DMA_FROM_DEVICE && ioctx->cmd.data_length && |
| 3045 | !ioctx->queue_status_only) { |
| 3046 | ret = srpt_xfer_data(ch, ioctx); |
| 3047 | if (ret) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 3048 | pr_err("xfer_data failed for tag %llu\n", |
Bart Van Assche | 649ee05 | 2015-04-14 13:26:44 +0200 | [diff] [blame] | 3049 | ioctx->cmd.tag); |
Joern Engel | b79fafa | 2013-07-03 11:22:17 -0400 | [diff] [blame] | 3050 | return; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3051 | } |
| 3052 | } |
| 3053 | |
| 3054 | if (state != SRPT_STATE_MGMT) |
Bart Van Assche | 649ee05 | 2015-04-14 13:26:44 +0200 | [diff] [blame] | 3055 | resp_len = srpt_build_cmd_rsp(ch, ioctx, ioctx->cmd.tag, |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3056 | cmd->scsi_status); |
| 3057 | else { |
| 3058 | srp_tm_status |
| 3059 | = tcm_to_srp_tsk_mgmt_status(cmd->se_tmr_req->response); |
| 3060 | resp_len = srpt_build_tskmgmt_rsp(ch, ioctx, srp_tm_status, |
Bart Van Assche | 649ee05 | 2015-04-14 13:26:44 +0200 | [diff] [blame] | 3061 | ioctx->cmd.tag); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3062 | } |
| 3063 | ret = srpt_post_send(ch, ioctx, resp_len); |
| 3064 | if (ret) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 3065 | pr_err("sending cmd response failed for tag %llu\n", |
Bart Van Assche | 649ee05 | 2015-04-14 13:26:44 +0200 | [diff] [blame] | 3066 | ioctx->cmd.tag); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3067 | srpt_unmap_sg_to_ib_sge(ch, ioctx); |
| 3068 | srpt_set_cmd_state(ioctx, SRPT_STATE_DONE); |
Bart Van Assche | afc1660 | 2015-04-27 13:52:36 +0200 | [diff] [blame] | 3069 | target_put_sess_cmd(&ioctx->cmd); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3070 | } |
Joern Engel | b79fafa | 2013-07-03 11:22:17 -0400 | [diff] [blame] | 3071 | } |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3072 | |
Joern Engel | b79fafa | 2013-07-03 11:22:17 -0400 | [diff] [blame] | 3073 | static int srpt_queue_data_in(struct se_cmd *cmd) |
| 3074 | { |
| 3075 | srpt_queue_response(cmd); |
| 3076 | return 0; |
| 3077 | } |
| 3078 | |
| 3079 | static void srpt_queue_tm_rsp(struct se_cmd *cmd) |
| 3080 | { |
| 3081 | srpt_queue_response(cmd); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3082 | } |
| 3083 | |
Nicholas Bellinger | 131e6ab | 2014-03-22 14:55:56 -0700 | [diff] [blame] | 3084 | static void srpt_aborted_task(struct se_cmd *cmd) |
| 3085 | { |
| 3086 | struct srpt_send_ioctx *ioctx = container_of(cmd, |
| 3087 | struct srpt_send_ioctx, cmd); |
| 3088 | |
| 3089 | srpt_unmap_sg_to_ib_sge(ioctx->ch, ioctx); |
| 3090 | } |
| 3091 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3092 | static int srpt_queue_status(struct se_cmd *cmd) |
| 3093 | { |
| 3094 | struct srpt_send_ioctx *ioctx; |
| 3095 | |
| 3096 | ioctx = container_of(cmd, struct srpt_send_ioctx, cmd); |
| 3097 | BUG_ON(ioctx->sense_data != cmd->sense_buffer); |
| 3098 | if (cmd->se_cmd_flags & |
| 3099 | (SCF_TRANSPORT_TASK_SENSE | SCF_EMULATED_TASK_SENSE)) |
| 3100 | WARN_ON(cmd->scsi_status != SAM_STAT_CHECK_CONDITION); |
| 3101 | ioctx->queue_status_only = true; |
Joern Engel | b79fafa | 2013-07-03 11:22:17 -0400 | [diff] [blame] | 3102 | srpt_queue_response(cmd); |
| 3103 | return 0; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3104 | } |
| 3105 | |
| 3106 | static void srpt_refresh_port_work(struct work_struct *work) |
| 3107 | { |
| 3108 | struct srpt_port *sport = container_of(work, struct srpt_port, work); |
| 3109 | |
| 3110 | srpt_refresh_port(sport); |
| 3111 | } |
| 3112 | |
| 3113 | static int srpt_ch_list_empty(struct srpt_device *sdev) |
| 3114 | { |
| 3115 | int res; |
| 3116 | |
| 3117 | spin_lock_irq(&sdev->spinlock); |
| 3118 | res = list_empty(&sdev->rch_list); |
| 3119 | spin_unlock_irq(&sdev->spinlock); |
| 3120 | |
| 3121 | return res; |
| 3122 | } |
| 3123 | |
| 3124 | /** |
| 3125 | * srpt_release_sdev() - Free the channel resources associated with a target. |
| 3126 | */ |
| 3127 | static int srpt_release_sdev(struct srpt_device *sdev) |
| 3128 | { |
| 3129 | struct srpt_rdma_ch *ch, *tmp_ch; |
| 3130 | int res; |
| 3131 | |
| 3132 | WARN_ON_ONCE(irqs_disabled()); |
| 3133 | |
| 3134 | BUG_ON(!sdev); |
| 3135 | |
| 3136 | spin_lock_irq(&sdev->spinlock); |
| 3137 | list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) |
| 3138 | __srpt_close_ch(ch); |
| 3139 | spin_unlock_irq(&sdev->spinlock); |
| 3140 | |
| 3141 | res = wait_event_interruptible(sdev->ch_releaseQ, |
| 3142 | srpt_ch_list_empty(sdev)); |
| 3143 | if (res) |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 3144 | pr_err("%s: interrupted.\n", __func__); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3145 | |
| 3146 | return 0; |
| 3147 | } |
| 3148 | |
| 3149 | static struct srpt_port *__srpt_lookup_port(const char *name) |
| 3150 | { |
| 3151 | struct ib_device *dev; |
| 3152 | struct srpt_device *sdev; |
| 3153 | struct srpt_port *sport; |
| 3154 | int i; |
| 3155 | |
| 3156 | list_for_each_entry(sdev, &srpt_dev_list, list) { |
| 3157 | dev = sdev->device; |
| 3158 | if (!dev) |
| 3159 | continue; |
| 3160 | |
| 3161 | for (i = 0; i < dev->phys_port_cnt; i++) { |
| 3162 | sport = &sdev->port[i]; |
| 3163 | |
| 3164 | if (!strcmp(sport->port_guid, name)) |
| 3165 | return sport; |
| 3166 | } |
| 3167 | } |
| 3168 | |
| 3169 | return NULL; |
| 3170 | } |
| 3171 | |
| 3172 | static struct srpt_port *srpt_lookup_port(const char *name) |
| 3173 | { |
| 3174 | struct srpt_port *sport; |
| 3175 | |
| 3176 | spin_lock(&srpt_dev_lock); |
| 3177 | sport = __srpt_lookup_port(name); |
| 3178 | spin_unlock(&srpt_dev_lock); |
| 3179 | |
| 3180 | return sport; |
| 3181 | } |
| 3182 | |
| 3183 | /** |
| 3184 | * srpt_add_one() - Infiniband device addition callback function. |
| 3185 | */ |
| 3186 | static void srpt_add_one(struct ib_device *device) |
| 3187 | { |
| 3188 | struct srpt_device *sdev; |
| 3189 | struct srpt_port *sport; |
| 3190 | struct ib_srq_init_attr srq_attr; |
| 3191 | int i; |
| 3192 | |
| 3193 | pr_debug("device = %p, device->dma_ops = %p\n", device, |
| 3194 | device->dma_ops); |
| 3195 | |
| 3196 | sdev = kzalloc(sizeof *sdev, GFP_KERNEL); |
| 3197 | if (!sdev) |
| 3198 | goto err; |
| 3199 | |
| 3200 | sdev->device = device; |
| 3201 | INIT_LIST_HEAD(&sdev->rch_list); |
| 3202 | init_waitqueue_head(&sdev->ch_releaseQ); |
| 3203 | spin_lock_init(&sdev->spinlock); |
| 3204 | |
| 3205 | if (ib_query_device(device, &sdev->dev_attr)) |
| 3206 | goto free_dev; |
| 3207 | |
| 3208 | sdev->pd = ib_alloc_pd(device); |
| 3209 | if (IS_ERR(sdev->pd)) |
| 3210 | goto free_dev; |
| 3211 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3212 | sdev->srq_size = min(srpt_srq_size, sdev->dev_attr.max_srq_wr); |
| 3213 | |
| 3214 | srq_attr.event_handler = srpt_srq_event; |
| 3215 | srq_attr.srq_context = (void *)sdev; |
| 3216 | srq_attr.attr.max_wr = sdev->srq_size; |
| 3217 | srq_attr.attr.max_sge = 1; |
| 3218 | srq_attr.attr.srq_limit = 0; |
Roland Dreier | 6f36033 | 2012-04-12 07:51:08 -0700 | [diff] [blame] | 3219 | srq_attr.srq_type = IB_SRQT_BASIC; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3220 | |
| 3221 | sdev->srq = ib_create_srq(sdev->pd, &srq_attr); |
| 3222 | if (IS_ERR(sdev->srq)) |
Jason Gunthorpe | 5a78395 | 2015-07-30 17:22:24 -0600 | [diff] [blame] | 3223 | goto err_pd; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3224 | |
| 3225 | pr_debug("%s: create SRQ #wr= %d max_allow=%d dev= %s\n", |
| 3226 | __func__, sdev->srq_size, sdev->dev_attr.max_srq_wr, |
| 3227 | device->name); |
| 3228 | |
| 3229 | if (!srpt_service_guid) |
| 3230 | srpt_service_guid = be64_to_cpu(device->node_guid); |
| 3231 | |
| 3232 | sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev); |
| 3233 | if (IS_ERR(sdev->cm_id)) |
| 3234 | goto err_srq; |
| 3235 | |
| 3236 | /* print out target login information */ |
| 3237 | pr_debug("Target login info: id_ext=%016llx,ioc_guid=%016llx," |
| 3238 | "pkey=ffff,service_id=%016llx\n", srpt_service_guid, |
| 3239 | srpt_service_guid, srpt_service_guid); |
| 3240 | |
| 3241 | /* |
| 3242 | * We do not have a consistent service_id (ie. also id_ext of target_id) |
| 3243 | * to identify this target. We currently use the guid of the first HCA |
| 3244 | * in the system as service_id; therefore, the target_id will change |
| 3245 | * if this HCA is gone bad and replaced by different HCA |
| 3246 | */ |
Haggai Eran | 73fec7f | 2015-07-30 17:50:26 +0300 | [diff] [blame] | 3247 | if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0)) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3248 | goto err_cm; |
| 3249 | |
| 3250 | INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device, |
| 3251 | srpt_event_handler); |
| 3252 | if (ib_register_event_handler(&sdev->event_handler)) |
| 3253 | goto err_cm; |
| 3254 | |
| 3255 | sdev->ioctx_ring = (struct srpt_recv_ioctx **) |
| 3256 | srpt_alloc_ioctx_ring(sdev, sdev->srq_size, |
| 3257 | sizeof(*sdev->ioctx_ring[0]), |
| 3258 | srp_max_req_size, DMA_FROM_DEVICE); |
| 3259 | if (!sdev->ioctx_ring) |
| 3260 | goto err_event; |
| 3261 | |
| 3262 | for (i = 0; i < sdev->srq_size; ++i) |
| 3263 | srpt_post_recv(sdev, sdev->ioctx_ring[i]); |
| 3264 | |
Roland Dreier | f225066 | 2012-02-02 12:55:58 -0800 | [diff] [blame] | 3265 | WARN_ON(sdev->device->phys_port_cnt > ARRAY_SIZE(sdev->port)); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3266 | |
| 3267 | for (i = 1; i <= sdev->device->phys_port_cnt; i++) { |
| 3268 | sport = &sdev->port[i - 1]; |
| 3269 | sport->sdev = sdev; |
| 3270 | sport->port = i; |
| 3271 | sport->port_attrib.srp_max_rdma_size = DEFAULT_MAX_RDMA_SIZE; |
| 3272 | sport->port_attrib.srp_max_rsp_size = DEFAULT_MAX_RSP_SIZE; |
| 3273 | sport->port_attrib.srp_sq_size = DEF_SRPT_SQ_SIZE; |
| 3274 | INIT_WORK(&sport->work, srpt_refresh_port_work); |
| 3275 | INIT_LIST_HEAD(&sport->port_acl_list); |
| 3276 | spin_lock_init(&sport->port_acl_lock); |
| 3277 | |
| 3278 | if (srpt_refresh_port(sport)) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 3279 | pr_err("MAD registration failed for %s-%d.\n", |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3280 | srpt_sdev_name(sdev), i); |
| 3281 | goto err_ring; |
| 3282 | } |
| 3283 | snprintf(sport->port_guid, sizeof(sport->port_guid), |
| 3284 | "0x%016llx%016llx", |
| 3285 | be64_to_cpu(sport->gid.global.subnet_prefix), |
| 3286 | be64_to_cpu(sport->gid.global.interface_id)); |
| 3287 | } |
| 3288 | |
| 3289 | spin_lock(&srpt_dev_lock); |
| 3290 | list_add_tail(&sdev->list, &srpt_dev_list); |
| 3291 | spin_unlock(&srpt_dev_lock); |
| 3292 | |
| 3293 | out: |
| 3294 | ib_set_client_data(device, &srpt_client, sdev); |
| 3295 | pr_debug("added %s.\n", device->name); |
| 3296 | return; |
| 3297 | |
| 3298 | err_ring: |
| 3299 | srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev, |
| 3300 | sdev->srq_size, srp_max_req_size, |
| 3301 | DMA_FROM_DEVICE); |
| 3302 | err_event: |
| 3303 | ib_unregister_event_handler(&sdev->event_handler); |
| 3304 | err_cm: |
| 3305 | ib_destroy_cm_id(sdev->cm_id); |
| 3306 | err_srq: |
| 3307 | ib_destroy_srq(sdev->srq); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3308 | err_pd: |
| 3309 | ib_dealloc_pd(sdev->pd); |
| 3310 | free_dev: |
| 3311 | kfree(sdev); |
| 3312 | err: |
| 3313 | sdev = NULL; |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 3314 | pr_info("%s(%s) failed.\n", __func__, device->name); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3315 | goto out; |
| 3316 | } |
| 3317 | |
| 3318 | /** |
| 3319 | * srpt_remove_one() - InfiniBand device removal callback function. |
| 3320 | */ |
Haggai Eran | 7c1eb45 | 2015-07-30 17:50:14 +0300 | [diff] [blame] | 3321 | static void srpt_remove_one(struct ib_device *device, void *client_data) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3322 | { |
Haggai Eran | 7c1eb45 | 2015-07-30 17:50:14 +0300 | [diff] [blame] | 3323 | struct srpt_device *sdev = client_data; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3324 | int i; |
| 3325 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3326 | if (!sdev) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 3327 | pr_info("%s(%s): nothing to do.\n", __func__, device->name); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3328 | return; |
| 3329 | } |
| 3330 | |
| 3331 | srpt_unregister_mad_agent(sdev); |
| 3332 | |
| 3333 | ib_unregister_event_handler(&sdev->event_handler); |
| 3334 | |
| 3335 | /* Cancel any work queued by the just unregistered IB event handler. */ |
| 3336 | for (i = 0; i < sdev->device->phys_port_cnt; i++) |
| 3337 | cancel_work_sync(&sdev->port[i].work); |
| 3338 | |
| 3339 | ib_destroy_cm_id(sdev->cm_id); |
| 3340 | |
| 3341 | /* |
| 3342 | * Unregistering a target must happen after destroying sdev->cm_id |
| 3343 | * such that no new SRP_LOGIN_REQ information units can arrive while |
| 3344 | * destroying the target. |
| 3345 | */ |
| 3346 | spin_lock(&srpt_dev_lock); |
| 3347 | list_del(&sdev->list); |
| 3348 | spin_unlock(&srpt_dev_lock); |
| 3349 | srpt_release_sdev(sdev); |
| 3350 | |
| 3351 | ib_destroy_srq(sdev->srq); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3352 | ib_dealloc_pd(sdev->pd); |
| 3353 | |
| 3354 | srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev, |
| 3355 | sdev->srq_size, srp_max_req_size, DMA_FROM_DEVICE); |
| 3356 | sdev->ioctx_ring = NULL; |
| 3357 | kfree(sdev); |
| 3358 | } |
| 3359 | |
| 3360 | static struct ib_client srpt_client = { |
| 3361 | .name = DRV_NAME, |
| 3362 | .add = srpt_add_one, |
| 3363 | .remove = srpt_remove_one |
| 3364 | }; |
| 3365 | |
| 3366 | static int srpt_check_true(struct se_portal_group *se_tpg) |
| 3367 | { |
| 3368 | return 1; |
| 3369 | } |
| 3370 | |
| 3371 | static int srpt_check_false(struct se_portal_group *se_tpg) |
| 3372 | { |
| 3373 | return 0; |
| 3374 | } |
| 3375 | |
| 3376 | static char *srpt_get_fabric_name(void) |
| 3377 | { |
| 3378 | return "srpt"; |
| 3379 | } |
| 3380 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3381 | static char *srpt_get_fabric_wwn(struct se_portal_group *tpg) |
| 3382 | { |
| 3383 | struct srpt_port *sport = container_of(tpg, struct srpt_port, port_tpg_1); |
| 3384 | |
| 3385 | return sport->port_guid; |
| 3386 | } |
| 3387 | |
| 3388 | static u16 srpt_get_tag(struct se_portal_group *tpg) |
| 3389 | { |
| 3390 | return 1; |
| 3391 | } |
| 3392 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3393 | static u32 srpt_tpg_get_inst_index(struct se_portal_group *se_tpg) |
| 3394 | { |
| 3395 | return 1; |
| 3396 | } |
| 3397 | |
| 3398 | static void srpt_release_cmd(struct se_cmd *se_cmd) |
| 3399 | { |
Nicholas Bellinger | 9474b04 | 2012-11-27 23:55:57 -0800 | [diff] [blame] | 3400 | struct srpt_send_ioctx *ioctx = container_of(se_cmd, |
| 3401 | struct srpt_send_ioctx, cmd); |
| 3402 | struct srpt_rdma_ch *ch = ioctx->ch; |
| 3403 | unsigned long flags; |
| 3404 | |
| 3405 | WARN_ON(ioctx->state != SRPT_STATE_DONE); |
| 3406 | WARN_ON(ioctx->mapped_sg_count != 0); |
| 3407 | |
| 3408 | if (ioctx->n_rbuf > 1) { |
| 3409 | kfree(ioctx->rbufs); |
| 3410 | ioctx->rbufs = NULL; |
| 3411 | ioctx->n_rbuf = 0; |
| 3412 | } |
| 3413 | |
| 3414 | spin_lock_irqsave(&ch->spinlock, flags); |
| 3415 | list_add(&ioctx->free_list, &ch->free_list); |
| 3416 | spin_unlock_irqrestore(&ch->spinlock, flags); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3417 | } |
| 3418 | |
| 3419 | /** |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3420 | * srpt_close_session() - Forcibly close a session. |
| 3421 | * |
| 3422 | * Callback function invoked by the TCM core to clean up sessions associated |
| 3423 | * with a node ACL when the user invokes |
| 3424 | * rmdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id |
| 3425 | */ |
| 3426 | static void srpt_close_session(struct se_session *se_sess) |
| 3427 | { |
| 3428 | DECLARE_COMPLETION_ONSTACK(release_done); |
| 3429 | struct srpt_rdma_ch *ch; |
| 3430 | struct srpt_device *sdev; |
Nicholas Mc Guire | ecc3f3e | 2015-01-16 12:20:17 +0100 | [diff] [blame] | 3431 | unsigned long res; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3432 | |
| 3433 | ch = se_sess->fabric_sess_ptr; |
| 3434 | WARN_ON(ch->sess != se_sess); |
| 3435 | |
| 3436 | pr_debug("ch %p state %d\n", ch, srpt_get_ch_state(ch)); |
| 3437 | |
| 3438 | sdev = ch->sport->sdev; |
| 3439 | spin_lock_irq(&sdev->spinlock); |
| 3440 | BUG_ON(ch->release_done); |
| 3441 | ch->release_done = &release_done; |
| 3442 | __srpt_close_ch(ch); |
| 3443 | spin_unlock_irq(&sdev->spinlock); |
| 3444 | |
| 3445 | res = wait_for_completion_timeout(&release_done, 60 * HZ); |
Nicholas Mc Guire | ecc3f3e | 2015-01-16 12:20:17 +0100 | [diff] [blame] | 3446 | WARN_ON(res == 0); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3447 | } |
| 3448 | |
| 3449 | /** |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3450 | * srpt_sess_get_index() - Return the value of scsiAttIntrPortIndex (SCSI-MIB). |
| 3451 | * |
| 3452 | * A quote from RFC 4455 (SCSI-MIB) about this MIB object: |
| 3453 | * This object represents an arbitrary integer used to uniquely identify a |
| 3454 | * particular attached remote initiator port to a particular SCSI target port |
| 3455 | * within a particular SCSI target device within a particular SCSI instance. |
| 3456 | */ |
| 3457 | static u32 srpt_sess_get_index(struct se_session *se_sess) |
| 3458 | { |
| 3459 | return 0; |
| 3460 | } |
| 3461 | |
| 3462 | static void srpt_set_default_node_attrs(struct se_node_acl *nacl) |
| 3463 | { |
| 3464 | } |
| 3465 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3466 | /* Note: only used from inside debug printk's by the TCM core. */ |
| 3467 | static int srpt_get_tcm_cmd_state(struct se_cmd *se_cmd) |
| 3468 | { |
| 3469 | struct srpt_send_ioctx *ioctx; |
| 3470 | |
| 3471 | ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd); |
| 3472 | return srpt_get_cmd_state(ioctx); |
| 3473 | } |
| 3474 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3475 | /** |
| 3476 | * srpt_parse_i_port_id() - Parse an initiator port ID. |
| 3477 | * @name: ASCII representation of a 128-bit initiator port ID. |
| 3478 | * @i_port_id: Binary 128-bit port ID. |
| 3479 | */ |
| 3480 | static int srpt_parse_i_port_id(u8 i_port_id[16], const char *name) |
| 3481 | { |
| 3482 | const char *p; |
| 3483 | unsigned len, count, leading_zero_bytes; |
| 3484 | int ret, rc; |
| 3485 | |
| 3486 | p = name; |
Rasmus Villemoes | b60459f | 2014-10-13 15:54:46 -0700 | [diff] [blame] | 3487 | if (strncasecmp(p, "0x", 2) == 0) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3488 | p += 2; |
| 3489 | ret = -EINVAL; |
| 3490 | len = strlen(p); |
| 3491 | if (len % 2) |
| 3492 | goto out; |
| 3493 | count = min(len / 2, 16U); |
| 3494 | leading_zero_bytes = 16 - count; |
| 3495 | memset(i_port_id, 0, leading_zero_bytes); |
| 3496 | rc = hex2bin(i_port_id + leading_zero_bytes, p, count); |
| 3497 | if (rc < 0) |
| 3498 | pr_debug("hex2bin failed for srpt_parse_i_port_id: %d\n", rc); |
| 3499 | ret = 0; |
| 3500 | out: |
| 3501 | return ret; |
| 3502 | } |
| 3503 | |
| 3504 | /* |
| 3505 | * configfs callback function invoked for |
| 3506 | * mkdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id |
| 3507 | */ |
Christoph Hellwig | c7d6a80 | 2015-04-13 19:51:14 +0200 | [diff] [blame] | 3508 | static int srpt_init_nodeacl(struct se_node_acl *se_nacl, const char *name) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3509 | { |
Christoph Hellwig | c7d6a80 | 2015-04-13 19:51:14 +0200 | [diff] [blame] | 3510 | struct srpt_port *sport = |
| 3511 | container_of(se_nacl->se_tpg, struct srpt_port, port_tpg_1); |
| 3512 | struct srpt_node_acl *nacl = |
| 3513 | container_of(se_nacl, struct srpt_node_acl, nacl); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3514 | u8 i_port_id[16]; |
| 3515 | |
| 3516 | if (srpt_parse_i_port_id(i_port_id, name) < 0) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 3517 | pr_err("invalid initiator port ID %s\n", name); |
Christoph Hellwig | c7d6a80 | 2015-04-13 19:51:14 +0200 | [diff] [blame] | 3518 | return -EINVAL; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3519 | } |
| 3520 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3521 | memcpy(&nacl->i_port_id[0], &i_port_id[0], 16); |
| 3522 | nacl->sport = sport; |
| 3523 | |
| 3524 | spin_lock_irq(&sport->port_acl_lock); |
| 3525 | list_add_tail(&nacl->list, &sport->port_acl_list); |
| 3526 | spin_unlock_irq(&sport->port_acl_lock); |
| 3527 | |
Christoph Hellwig | c7d6a80 | 2015-04-13 19:51:14 +0200 | [diff] [blame] | 3528 | return 0; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3529 | } |
| 3530 | |
| 3531 | /* |
| 3532 | * configfs callback function invoked for |
| 3533 | * rmdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id |
| 3534 | */ |
Christoph Hellwig | c7d6a80 | 2015-04-13 19:51:14 +0200 | [diff] [blame] | 3535 | static void srpt_cleanup_nodeacl(struct se_node_acl *se_nacl) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3536 | { |
Christoph Hellwig | c7d6a80 | 2015-04-13 19:51:14 +0200 | [diff] [blame] | 3537 | struct srpt_node_acl *nacl = |
| 3538 | container_of(se_nacl, struct srpt_node_acl, nacl); |
| 3539 | struct srpt_port *sport = nacl->sport; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3540 | |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3541 | spin_lock_irq(&sport->port_acl_lock); |
| 3542 | list_del(&nacl->list); |
| 3543 | spin_unlock_irq(&sport->port_acl_lock); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3544 | } |
| 3545 | |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3546 | static ssize_t srpt_tpg_attrib_srp_max_rdma_size_show(struct config_item *item, |
| 3547 | char *page) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3548 | { |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3549 | struct se_portal_group *se_tpg = attrib_to_tpg(item); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3550 | struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1); |
| 3551 | |
| 3552 | return sprintf(page, "%u\n", sport->port_attrib.srp_max_rdma_size); |
| 3553 | } |
| 3554 | |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3555 | static ssize_t srpt_tpg_attrib_srp_max_rdma_size_store(struct config_item *item, |
| 3556 | const char *page, size_t count) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3557 | { |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3558 | struct se_portal_group *se_tpg = attrib_to_tpg(item); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3559 | struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1); |
| 3560 | unsigned long val; |
| 3561 | int ret; |
| 3562 | |
Jingoo Han | 9d8abf4 | 2014-02-05 11:22:05 +0900 | [diff] [blame] | 3563 | ret = kstrtoul(page, 0, &val); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3564 | if (ret < 0) { |
Jingoo Han | 9d8abf4 | 2014-02-05 11:22:05 +0900 | [diff] [blame] | 3565 | pr_err("kstrtoul() failed with ret: %d\n", ret); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3566 | return -EINVAL; |
| 3567 | } |
| 3568 | if (val > MAX_SRPT_RDMA_SIZE) { |
| 3569 | pr_err("val: %lu exceeds MAX_SRPT_RDMA_SIZE: %d\n", val, |
| 3570 | MAX_SRPT_RDMA_SIZE); |
| 3571 | return -EINVAL; |
| 3572 | } |
| 3573 | if (val < DEFAULT_MAX_RDMA_SIZE) { |
| 3574 | pr_err("val: %lu smaller than DEFAULT_MAX_RDMA_SIZE: %d\n", |
| 3575 | val, DEFAULT_MAX_RDMA_SIZE); |
| 3576 | return -EINVAL; |
| 3577 | } |
| 3578 | sport->port_attrib.srp_max_rdma_size = val; |
| 3579 | |
| 3580 | return count; |
| 3581 | } |
| 3582 | |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3583 | static ssize_t srpt_tpg_attrib_srp_max_rsp_size_show(struct config_item *item, |
| 3584 | char *page) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3585 | { |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3586 | struct se_portal_group *se_tpg = attrib_to_tpg(item); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3587 | struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1); |
| 3588 | |
| 3589 | return sprintf(page, "%u\n", sport->port_attrib.srp_max_rsp_size); |
| 3590 | } |
| 3591 | |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3592 | static ssize_t srpt_tpg_attrib_srp_max_rsp_size_store(struct config_item *item, |
| 3593 | const char *page, size_t count) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3594 | { |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3595 | struct se_portal_group *se_tpg = attrib_to_tpg(item); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3596 | struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1); |
| 3597 | unsigned long val; |
| 3598 | int ret; |
| 3599 | |
Jingoo Han | 9d8abf4 | 2014-02-05 11:22:05 +0900 | [diff] [blame] | 3600 | ret = kstrtoul(page, 0, &val); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3601 | if (ret < 0) { |
Jingoo Han | 9d8abf4 | 2014-02-05 11:22:05 +0900 | [diff] [blame] | 3602 | pr_err("kstrtoul() failed with ret: %d\n", ret); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3603 | return -EINVAL; |
| 3604 | } |
| 3605 | if (val > MAX_SRPT_RSP_SIZE) { |
| 3606 | pr_err("val: %lu exceeds MAX_SRPT_RSP_SIZE: %d\n", val, |
| 3607 | MAX_SRPT_RSP_SIZE); |
| 3608 | return -EINVAL; |
| 3609 | } |
| 3610 | if (val < MIN_MAX_RSP_SIZE) { |
| 3611 | pr_err("val: %lu smaller than MIN_MAX_RSP_SIZE: %d\n", val, |
| 3612 | MIN_MAX_RSP_SIZE); |
| 3613 | return -EINVAL; |
| 3614 | } |
| 3615 | sport->port_attrib.srp_max_rsp_size = val; |
| 3616 | |
| 3617 | return count; |
| 3618 | } |
| 3619 | |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3620 | static ssize_t srpt_tpg_attrib_srp_sq_size_show(struct config_item *item, |
| 3621 | char *page) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3622 | { |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3623 | struct se_portal_group *se_tpg = attrib_to_tpg(item); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3624 | struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1); |
| 3625 | |
| 3626 | return sprintf(page, "%u\n", sport->port_attrib.srp_sq_size); |
| 3627 | } |
| 3628 | |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3629 | static ssize_t srpt_tpg_attrib_srp_sq_size_store(struct config_item *item, |
| 3630 | const char *page, size_t count) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3631 | { |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3632 | struct se_portal_group *se_tpg = attrib_to_tpg(item); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3633 | struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1); |
| 3634 | unsigned long val; |
| 3635 | int ret; |
| 3636 | |
Jingoo Han | 9d8abf4 | 2014-02-05 11:22:05 +0900 | [diff] [blame] | 3637 | ret = kstrtoul(page, 0, &val); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3638 | if (ret < 0) { |
Jingoo Han | 9d8abf4 | 2014-02-05 11:22:05 +0900 | [diff] [blame] | 3639 | pr_err("kstrtoul() failed with ret: %d\n", ret); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3640 | return -EINVAL; |
| 3641 | } |
| 3642 | if (val > MAX_SRPT_SRQ_SIZE) { |
| 3643 | pr_err("val: %lu exceeds MAX_SRPT_SRQ_SIZE: %d\n", val, |
| 3644 | MAX_SRPT_SRQ_SIZE); |
| 3645 | return -EINVAL; |
| 3646 | } |
| 3647 | if (val < MIN_SRPT_SRQ_SIZE) { |
| 3648 | pr_err("val: %lu smaller than MIN_SRPT_SRQ_SIZE: %d\n", val, |
| 3649 | MIN_SRPT_SRQ_SIZE); |
| 3650 | return -EINVAL; |
| 3651 | } |
| 3652 | sport->port_attrib.srp_sq_size = val; |
| 3653 | |
| 3654 | return count; |
| 3655 | } |
| 3656 | |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3657 | CONFIGFS_ATTR(srpt_tpg_attrib_, srp_max_rdma_size); |
| 3658 | CONFIGFS_ATTR(srpt_tpg_attrib_, srp_max_rsp_size); |
| 3659 | CONFIGFS_ATTR(srpt_tpg_attrib_, srp_sq_size); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3660 | |
| 3661 | static struct configfs_attribute *srpt_tpg_attrib_attrs[] = { |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3662 | &srpt_tpg_attrib_attr_srp_max_rdma_size, |
| 3663 | &srpt_tpg_attrib_attr_srp_max_rsp_size, |
| 3664 | &srpt_tpg_attrib_attr_srp_sq_size, |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3665 | NULL, |
| 3666 | }; |
| 3667 | |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3668 | static ssize_t srpt_tpg_enable_show(struct config_item *item, char *page) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3669 | { |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3670 | struct se_portal_group *se_tpg = to_tpg(item); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3671 | struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1); |
| 3672 | |
| 3673 | return snprintf(page, PAGE_SIZE, "%d\n", (sport->enabled) ? 1: 0); |
| 3674 | } |
| 3675 | |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3676 | static ssize_t srpt_tpg_enable_store(struct config_item *item, |
| 3677 | const char *page, size_t count) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3678 | { |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3679 | struct se_portal_group *se_tpg = to_tpg(item); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3680 | struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1); |
| 3681 | unsigned long tmp; |
| 3682 | int ret; |
| 3683 | |
Jingoo Han | 9d8abf4 | 2014-02-05 11:22:05 +0900 | [diff] [blame] | 3684 | ret = kstrtoul(page, 0, &tmp); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3685 | if (ret < 0) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 3686 | pr_err("Unable to extract srpt_tpg_store_enable\n"); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3687 | return -EINVAL; |
| 3688 | } |
| 3689 | |
| 3690 | if ((tmp != 0) && (tmp != 1)) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 3691 | pr_err("Illegal value for srpt_tpg_store_enable: %lu\n", tmp); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3692 | return -EINVAL; |
| 3693 | } |
| 3694 | if (tmp == 1) |
| 3695 | sport->enabled = true; |
| 3696 | else |
| 3697 | sport->enabled = false; |
| 3698 | |
| 3699 | return count; |
| 3700 | } |
| 3701 | |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3702 | CONFIGFS_ATTR(srpt_tpg_, enable); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3703 | |
| 3704 | static struct configfs_attribute *srpt_tpg_attrs[] = { |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3705 | &srpt_tpg_attr_enable, |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3706 | NULL, |
| 3707 | }; |
| 3708 | |
| 3709 | /** |
| 3710 | * configfs callback invoked for |
| 3711 | * mkdir /sys/kernel/config/target/$driver/$port/$tpg |
| 3712 | */ |
| 3713 | static struct se_portal_group *srpt_make_tpg(struct se_wwn *wwn, |
| 3714 | struct config_group *group, |
| 3715 | const char *name) |
| 3716 | { |
| 3717 | struct srpt_port *sport = container_of(wwn, struct srpt_port, port_wwn); |
| 3718 | int res; |
| 3719 | |
| 3720 | /* Initialize sport->port_wwn and sport->port_tpg_1 */ |
Nicholas Bellinger | bc0c94b | 2015-05-20 21:48:03 -0700 | [diff] [blame] | 3721 | res = core_tpg_register(&sport->port_wwn, &sport->port_tpg_1, SCSI_PROTOCOL_SRP); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3722 | if (res) |
| 3723 | return ERR_PTR(res); |
| 3724 | |
| 3725 | return &sport->port_tpg_1; |
| 3726 | } |
| 3727 | |
| 3728 | /** |
| 3729 | * configfs callback invoked for |
| 3730 | * rmdir /sys/kernel/config/target/$driver/$port/$tpg |
| 3731 | */ |
| 3732 | static void srpt_drop_tpg(struct se_portal_group *tpg) |
| 3733 | { |
| 3734 | struct srpt_port *sport = container_of(tpg, |
| 3735 | struct srpt_port, port_tpg_1); |
| 3736 | |
| 3737 | sport->enabled = false; |
| 3738 | core_tpg_deregister(&sport->port_tpg_1); |
| 3739 | } |
| 3740 | |
| 3741 | /** |
| 3742 | * configfs callback invoked for |
| 3743 | * mkdir /sys/kernel/config/target/$driver/$port |
| 3744 | */ |
| 3745 | static struct se_wwn *srpt_make_tport(struct target_fabric_configfs *tf, |
| 3746 | struct config_group *group, |
| 3747 | const char *name) |
| 3748 | { |
| 3749 | struct srpt_port *sport; |
| 3750 | int ret; |
| 3751 | |
| 3752 | sport = srpt_lookup_port(name); |
| 3753 | pr_debug("make_tport(%s)\n", name); |
| 3754 | ret = -EINVAL; |
| 3755 | if (!sport) |
| 3756 | goto err; |
| 3757 | |
| 3758 | return &sport->port_wwn; |
| 3759 | |
| 3760 | err: |
| 3761 | return ERR_PTR(ret); |
| 3762 | } |
| 3763 | |
| 3764 | /** |
| 3765 | * configfs callback invoked for |
| 3766 | * rmdir /sys/kernel/config/target/$driver/$port |
| 3767 | */ |
| 3768 | static void srpt_drop_tport(struct se_wwn *wwn) |
| 3769 | { |
| 3770 | struct srpt_port *sport = container_of(wwn, struct srpt_port, port_wwn); |
| 3771 | |
| 3772 | pr_debug("drop_tport(%s\n", config_item_name(&sport->port_wwn.wwn_group.cg_item)); |
| 3773 | } |
| 3774 | |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3775 | static ssize_t srpt_wwn_version_show(struct config_item *item, char *buf) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3776 | { |
| 3777 | return scnprintf(buf, PAGE_SIZE, "%s\n", DRV_VERSION); |
| 3778 | } |
| 3779 | |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3780 | CONFIGFS_ATTR_RO(srpt_wwn_, version); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3781 | |
| 3782 | static struct configfs_attribute *srpt_wwn_attrs[] = { |
Christoph Hellwig | 2eafd72 | 2015-10-03 15:32:55 +0200 | [diff] [blame^] | 3783 | &srpt_wwn_attr_version, |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3784 | NULL, |
| 3785 | }; |
| 3786 | |
Christoph Hellwig | 9ac8928 | 2015-04-08 20:01:35 +0200 | [diff] [blame] | 3787 | static const struct target_core_fabric_ops srpt_template = { |
| 3788 | .module = THIS_MODULE, |
| 3789 | .name = "srpt", |
Christoph Hellwig | 144bc4c | 2015-04-13 19:51:16 +0200 | [diff] [blame] | 3790 | .node_acl_size = sizeof(struct srpt_node_acl), |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3791 | .get_fabric_name = srpt_get_fabric_name, |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3792 | .tpg_get_wwn = srpt_get_fabric_wwn, |
| 3793 | .tpg_get_tag = srpt_get_tag, |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3794 | .tpg_check_demo_mode = srpt_check_false, |
| 3795 | .tpg_check_demo_mode_cache = srpt_check_true, |
| 3796 | .tpg_check_demo_mode_write_protect = srpt_check_true, |
| 3797 | .tpg_check_prod_mode_write_protect = srpt_check_false, |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3798 | .tpg_get_inst_index = srpt_tpg_get_inst_index, |
| 3799 | .release_cmd = srpt_release_cmd, |
| 3800 | .check_stop_free = srpt_check_stop_free, |
| 3801 | .shutdown_session = srpt_shutdown_session, |
| 3802 | .close_session = srpt_close_session, |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3803 | .sess_get_index = srpt_sess_get_index, |
| 3804 | .sess_get_initiator_sid = NULL, |
| 3805 | .write_pending = srpt_write_pending, |
| 3806 | .write_pending_status = srpt_write_pending_status, |
| 3807 | .set_default_node_attributes = srpt_set_default_node_attrs, |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3808 | .get_cmd_state = srpt_get_tcm_cmd_state, |
Joern Engel | b79fafa | 2013-07-03 11:22:17 -0400 | [diff] [blame] | 3809 | .queue_data_in = srpt_queue_data_in, |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3810 | .queue_status = srpt_queue_status, |
Joern Engel | b79fafa | 2013-07-03 11:22:17 -0400 | [diff] [blame] | 3811 | .queue_tm_rsp = srpt_queue_tm_rsp, |
Nicholas Bellinger | 131e6ab | 2014-03-22 14:55:56 -0700 | [diff] [blame] | 3812 | .aborted_task = srpt_aborted_task, |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3813 | /* |
| 3814 | * Setup function pointers for generic logic in |
| 3815 | * target_core_fabric_configfs.c |
| 3816 | */ |
| 3817 | .fabric_make_wwn = srpt_make_tport, |
| 3818 | .fabric_drop_wwn = srpt_drop_tport, |
| 3819 | .fabric_make_tpg = srpt_make_tpg, |
| 3820 | .fabric_drop_tpg = srpt_drop_tpg, |
Christoph Hellwig | c7d6a80 | 2015-04-13 19:51:14 +0200 | [diff] [blame] | 3821 | .fabric_init_nodeacl = srpt_init_nodeacl, |
| 3822 | .fabric_cleanup_nodeacl = srpt_cleanup_nodeacl, |
Christoph Hellwig | 9ac8928 | 2015-04-08 20:01:35 +0200 | [diff] [blame] | 3823 | |
| 3824 | .tfc_wwn_attrs = srpt_wwn_attrs, |
| 3825 | .tfc_tpg_base_attrs = srpt_tpg_attrs, |
| 3826 | .tfc_tpg_attrib_attrs = srpt_tpg_attrib_attrs, |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3827 | }; |
| 3828 | |
| 3829 | /** |
| 3830 | * srpt_init_module() - Kernel module initialization. |
| 3831 | * |
| 3832 | * Note: Since ib_register_client() registers callback functions, and since at |
| 3833 | * least one of these callback functions (srpt_add_one()) calls target core |
| 3834 | * functions, this driver must be registered with the target core before |
| 3835 | * ib_register_client() is called. |
| 3836 | */ |
| 3837 | static int __init srpt_init_module(void) |
| 3838 | { |
| 3839 | int ret; |
| 3840 | |
| 3841 | ret = -EINVAL; |
| 3842 | if (srp_max_req_size < MIN_MAX_REQ_SIZE) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 3843 | pr_err("invalid value %d for kernel module parameter" |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3844 | " srp_max_req_size -- must be at least %d.\n", |
| 3845 | srp_max_req_size, MIN_MAX_REQ_SIZE); |
| 3846 | goto out; |
| 3847 | } |
| 3848 | |
| 3849 | if (srpt_srq_size < MIN_SRPT_SRQ_SIZE |
| 3850 | || srpt_srq_size > MAX_SRPT_SRQ_SIZE) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 3851 | pr_err("invalid value %d for kernel module parameter" |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3852 | " srpt_srq_size -- must be in the range [%d..%d].\n", |
| 3853 | srpt_srq_size, MIN_SRPT_SRQ_SIZE, MAX_SRPT_SRQ_SIZE); |
| 3854 | goto out; |
| 3855 | } |
| 3856 | |
Christoph Hellwig | 9ac8928 | 2015-04-08 20:01:35 +0200 | [diff] [blame] | 3857 | ret = target_register_template(&srpt_template); |
| 3858 | if (ret) |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3859 | goto out; |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3860 | |
| 3861 | ret = ib_register_client(&srpt_client); |
| 3862 | if (ret) { |
Doug Ledford | 9f5d32a | 2014-10-20 18:25:15 -0400 | [diff] [blame] | 3863 | pr_err("couldn't register IB client\n"); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3864 | goto out_unregister_target; |
| 3865 | } |
| 3866 | |
| 3867 | return 0; |
| 3868 | |
| 3869 | out_unregister_target: |
Christoph Hellwig | 9ac8928 | 2015-04-08 20:01:35 +0200 | [diff] [blame] | 3870 | target_unregister_template(&srpt_template); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3871 | out: |
| 3872 | return ret; |
| 3873 | } |
| 3874 | |
| 3875 | static void __exit srpt_cleanup_module(void) |
| 3876 | { |
| 3877 | ib_unregister_client(&srpt_client); |
Christoph Hellwig | 9ac8928 | 2015-04-08 20:01:35 +0200 | [diff] [blame] | 3878 | target_unregister_template(&srpt_template); |
Bart Van Assche | a42d985 | 2011-10-14 01:30:46 +0000 | [diff] [blame] | 3879 | } |
| 3880 | |
| 3881 | module_init(srpt_init_module); |
| 3882 | module_exit(srpt_cleanup_module); |