James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1 | /******************************************************************* |
| 2 | * This file is part of the Emulex Linux Device Driver for * |
| 3 | * Fibre Channel Host Bus Adapters. * |
James Smart | d080abe | 2017-02-12 13:52:39 -0800 | [diff] [blame] | 4 | * Copyright (C) 2017 Broadcom. All Rights Reserved. The term * |
| 5 | * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. * |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 6 | * Copyright (C) 2004-2016 Emulex. All rights reserved. * |
| 7 | * EMULEX and SLI are trademarks of Emulex. * |
James Smart | d080abe | 2017-02-12 13:52:39 -0800 | [diff] [blame] | 8 | * www.broadcom.com * |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 9 | * Portions Copyright (C) 2004-2005 Christoph Hellwig * |
| 10 | * * |
| 11 | * This program is free software; you can redistribute it and/or * |
| 12 | * modify it under the terms of version 2 of the GNU General * |
| 13 | * Public License as published by the Free Software Foundation. * |
| 14 | * This program is distributed in the hope that it will be useful. * |
| 15 | * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND * |
| 16 | * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, * |
| 17 | * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE * |
| 18 | * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD * |
| 19 | * TO BE LEGALLY INVALID. See the GNU General Public License for * |
| 20 | * more details, a copy of which can be found in the file COPYING * |
| 21 | * included with this package. * |
| 22 | ********************************************************************/ |
| 23 | #include <linux/pci.h> |
| 24 | #include <linux/slab.h> |
| 25 | #include <linux/interrupt.h> |
| 26 | #include <linux/delay.h> |
| 27 | #include <asm/unaligned.h> |
| 28 | #include <linux/crc-t10dif.h> |
| 29 | #include <net/checksum.h> |
| 30 | |
| 31 | #include <scsi/scsi.h> |
| 32 | #include <scsi/scsi_device.h> |
| 33 | #include <scsi/scsi_eh.h> |
| 34 | #include <scsi/scsi_host.h> |
| 35 | #include <scsi/scsi_tcq.h> |
| 36 | #include <scsi/scsi_transport_fc.h> |
| 37 | #include <scsi/fc/fc_fs.h> |
| 38 | |
| 39 | #include <linux/nvme.h> |
| 40 | #include <linux/nvme-fc-driver.h> |
| 41 | #include <linux/nvme-fc.h> |
| 42 | #include "lpfc_version.h" |
| 43 | #include "lpfc_hw4.h" |
| 44 | #include "lpfc_hw.h" |
| 45 | #include "lpfc_sli.h" |
| 46 | #include "lpfc_sli4.h" |
| 47 | #include "lpfc_nl.h" |
| 48 | #include "lpfc_disc.h" |
| 49 | #include "lpfc.h" |
| 50 | #include "lpfc_nvme.h" |
| 51 | #include "lpfc_scsi.h" |
| 52 | #include "lpfc_logmsg.h" |
| 53 | #include "lpfc_crtn.h" |
| 54 | #include "lpfc_vport.h" |
James Smart | bd2cdd5 | 2017-02-12 13:52:33 -0800 | [diff] [blame] | 55 | #include "lpfc_debugfs.h" |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 56 | |
| 57 | /* NVME initiator-based functions */ |
| 58 | |
| 59 | static struct lpfc_nvme_buf * |
| 60 | lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp); |
| 61 | |
| 62 | static void |
| 63 | lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_nvme_buf *); |
| 64 | |
| 65 | |
| 66 | /** |
| 67 | * lpfc_nvme_create_queue - |
| 68 | * @lpfc_pnvme: Pointer to the driver's nvme instance data |
| 69 | * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. |
| 70 | * @handle: An opaque driver handle used in follow-up calls. |
| 71 | * |
| 72 | * Driver registers this routine to preallocate and initialize any |
| 73 | * internal data structures to bind the @qidx to its internal IO queues. |
| 74 | * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ. |
| 75 | * |
| 76 | * Return value : |
| 77 | * 0 - Success |
| 78 | * -EINVAL - Unsupported input value. |
| 79 | * -ENOMEM - Could not alloc necessary memory |
| 80 | **/ |
| 81 | static int |
| 82 | lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport, |
| 83 | unsigned int qidx, u16 qsize, |
| 84 | void **handle) |
| 85 | { |
| 86 | struct lpfc_nvme_lport *lport; |
| 87 | struct lpfc_vport *vport; |
| 88 | struct lpfc_nvme_qhandle *qhandle; |
| 89 | char *str; |
| 90 | |
| 91 | lport = (struct lpfc_nvme_lport *)pnvme_lport->private; |
| 92 | vport = lport->vport; |
| 93 | qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL); |
| 94 | if (qhandle == NULL) |
| 95 | return -ENOMEM; |
| 96 | |
| 97 | qhandle->cpu_id = smp_processor_id(); |
| 98 | qhandle->qidx = qidx; |
| 99 | /* |
| 100 | * NVME qidx == 0 is the admin queue, so both admin queue |
| 101 | * and first IO queue will use MSI-X vector and associated |
| 102 | * EQ/CQ/WQ at index 0. After that they are sequentially assigned. |
| 103 | */ |
| 104 | if (qidx) { |
| 105 | str = "IO "; /* IO queue */ |
| 106 | qhandle->index = ((qidx - 1) % |
| 107 | vport->phba->cfg_nvme_io_channel); |
| 108 | } else { |
| 109 | str = "ADM"; /* Admin queue */ |
| 110 | qhandle->index = qidx; |
| 111 | } |
| 112 | |
James Smart | d58734f | 2017-08-23 16:55:46 -0700 | [diff] [blame] | 113 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 114 | "6073 Binding %s HdwQueue %d (cpu %d) to " |
| 115 | "io_channel %d qhandle %p\n", str, |
| 116 | qidx, qhandle->cpu_id, qhandle->index, qhandle); |
| 117 | *handle = (void *)qhandle; |
| 118 | return 0; |
| 119 | } |
| 120 | |
| 121 | /** |
| 122 | * lpfc_nvme_delete_queue - |
| 123 | * @lpfc_pnvme: Pointer to the driver's nvme instance data |
| 124 | * @qidx: An cpu index used to affinitize IO queues and MSIX vectors. |
| 125 | * @handle: An opaque driver handle from lpfc_nvme_create_queue |
| 126 | * |
| 127 | * Driver registers this routine to free |
| 128 | * any internal data structures to bind the @qidx to its internal |
| 129 | * IO queues. |
| 130 | * |
| 131 | * Return value : |
| 132 | * 0 - Success |
| 133 | * TODO: What are the failure codes. |
| 134 | **/ |
| 135 | static void |
| 136 | lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport, |
| 137 | unsigned int qidx, |
| 138 | void *handle) |
| 139 | { |
| 140 | struct lpfc_nvme_lport *lport; |
| 141 | struct lpfc_vport *vport; |
| 142 | |
| 143 | lport = (struct lpfc_nvme_lport *)pnvme_lport->private; |
| 144 | vport = lport->vport; |
| 145 | |
| 146 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, |
| 147 | "6001 ENTER. lpfc_pnvme %p, qidx x%xi qhandle %p\n", |
| 148 | lport, qidx, handle); |
| 149 | kfree(handle); |
| 150 | } |
| 151 | |
| 152 | static void |
| 153 | lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport) |
| 154 | { |
| 155 | struct lpfc_nvme_lport *lport = localport->private; |
| 156 | |
| 157 | /* release any threads waiting for the unreg to complete */ |
| 158 | complete(&lport->lport_unreg_done); |
| 159 | } |
| 160 | |
| 161 | /* lpfc_nvme_remoteport_delete |
| 162 | * |
| 163 | * @remoteport: Pointer to an nvme transport remoteport instance. |
| 164 | * |
| 165 | * This is a template downcall. NVME transport calls this function |
| 166 | * when it has completed the unregistration of a previously |
| 167 | * registered remoteport. |
| 168 | * |
| 169 | * Return value : |
| 170 | * None |
| 171 | */ |
| 172 | void |
| 173 | lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport) |
| 174 | { |
| 175 | struct lpfc_nvme_rport *rport = remoteport->private; |
| 176 | struct lpfc_vport *vport; |
| 177 | struct lpfc_nodelist *ndlp; |
| 178 | |
| 179 | ndlp = rport->ndlp; |
| 180 | if (!ndlp) |
| 181 | goto rport_err; |
| 182 | |
| 183 | vport = ndlp->vport; |
| 184 | if (!vport) |
| 185 | goto rport_err; |
| 186 | |
| 187 | /* Remove this rport from the lport's list - memory is owned by the |
| 188 | * transport. Remove the ndlp reference for the NVME transport before |
James Smart | 7a06dcd | 2017-06-01 21:06:55 -0700 | [diff] [blame] | 189 | * calling state machine to remove the node. |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 190 | */ |
| 191 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, |
| 192 | "6146 remoteport delete complete %p\n", |
| 193 | remoteport); |
James Smart | 7a06dcd | 2017-06-01 21:06:55 -0700 | [diff] [blame] | 194 | ndlp->nrport = NULL; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 195 | lpfc_nlp_put(ndlp); |
| 196 | |
| 197 | rport_err: |
| 198 | /* This call has to execute as long as the rport is valid. |
| 199 | * Release any threads waiting for the unreg to complete. |
| 200 | */ |
| 201 | complete(&rport->rport_unreg_done); |
| 202 | } |
| 203 | |
| 204 | static void |
| 205 | lpfc_nvme_cmpl_gen_req(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe, |
| 206 | struct lpfc_wcqe_complete *wcqe) |
| 207 | { |
| 208 | struct lpfc_vport *vport = cmdwqe->vport; |
| 209 | uint32_t status; |
| 210 | struct nvmefc_ls_req *pnvme_lsreq; |
| 211 | struct lpfc_dmabuf *buf_ptr; |
| 212 | struct lpfc_nodelist *ndlp; |
| 213 | |
James Smart | 2cee780 | 2017-06-01 21:07:02 -0700 | [diff] [blame] | 214 | atomic_inc(&vport->phba->fc4NvmeLsCmpls); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 215 | |
| 216 | pnvme_lsreq = (struct nvmefc_ls_req *)cmdwqe->context2; |
| 217 | status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK; |
| 218 | ndlp = (struct lpfc_nodelist *)cmdwqe->context1; |
| 219 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, |
| 220 | "6047 nvme cmpl Enter " |
| 221 | "Data %p DID %x Xri: %x status %x cmd:%p lsreg:%p " |
| 222 | "bmp:%p ndlp:%p\n", |
| 223 | pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, |
| 224 | cmdwqe->sli4_xritag, status, |
| 225 | cmdwqe, pnvme_lsreq, cmdwqe->context3, ndlp); |
| 226 | |
James Smart | bd2cdd5 | 2017-02-12 13:52:33 -0800 | [diff] [blame] | 227 | lpfc_nvmeio_data(phba, "NVME LS CMPL: xri x%x stat x%x parm x%x\n", |
| 228 | cmdwqe->sli4_xritag, status, wcqe->parameter); |
| 229 | |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 230 | if (cmdwqe->context3) { |
| 231 | buf_ptr = (struct lpfc_dmabuf *)cmdwqe->context3; |
| 232 | lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys); |
| 233 | kfree(buf_ptr); |
| 234 | cmdwqe->context3 = NULL; |
| 235 | } |
| 236 | if (pnvme_lsreq->done) |
| 237 | pnvme_lsreq->done(pnvme_lsreq, status); |
| 238 | else |
| 239 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, |
| 240 | "6046 nvme cmpl without done call back? " |
| 241 | "Data %p DID %x Xri: %x status %x\n", |
| 242 | pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0, |
| 243 | cmdwqe->sli4_xritag, status); |
| 244 | if (ndlp) { |
| 245 | lpfc_nlp_put(ndlp); |
| 246 | cmdwqe->context1 = NULL; |
| 247 | } |
| 248 | lpfc_sli_release_iocbq(phba, cmdwqe); |
| 249 | } |
| 250 | |
| 251 | static int |
| 252 | lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp, |
| 253 | struct lpfc_dmabuf *inp, |
| 254 | struct nvmefc_ls_req *pnvme_lsreq, |
| 255 | void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *, |
| 256 | struct lpfc_wcqe_complete *), |
| 257 | struct lpfc_nodelist *ndlp, uint32_t num_entry, |
| 258 | uint32_t tmo, uint8_t retry) |
| 259 | { |
| 260 | struct lpfc_hba *phba = vport->phba; |
| 261 | union lpfc_wqe *wqe; |
| 262 | struct lpfc_iocbq *genwqe; |
| 263 | struct ulp_bde64 *bpl; |
| 264 | struct ulp_bde64 bde; |
| 265 | int i, rc, xmit_len, first_len; |
| 266 | |
| 267 | /* Allocate buffer for command WQE */ |
| 268 | genwqe = lpfc_sli_get_iocbq(phba); |
| 269 | if (genwqe == NULL) |
| 270 | return 1; |
| 271 | |
| 272 | wqe = &genwqe->wqe; |
| 273 | memset(wqe, 0, sizeof(union lpfc_wqe)); |
| 274 | |
| 275 | genwqe->context3 = (uint8_t *)bmp; |
| 276 | genwqe->iocb_flag |= LPFC_IO_NVME_LS; |
| 277 | |
| 278 | /* Save for completion so we can release these resources */ |
| 279 | genwqe->context1 = lpfc_nlp_get(ndlp); |
| 280 | genwqe->context2 = (uint8_t *)pnvme_lsreq; |
| 281 | /* Fill in payload, bp points to frame payload */ |
| 282 | |
| 283 | if (!tmo) |
| 284 | /* FC spec states we need 3 * ratov for CT requests */ |
| 285 | tmo = (3 * phba->fc_ratov); |
| 286 | |
| 287 | /* For this command calculate the xmit length of the request bde. */ |
| 288 | xmit_len = 0; |
| 289 | first_len = 0; |
| 290 | bpl = (struct ulp_bde64 *)bmp->virt; |
| 291 | for (i = 0; i < num_entry; i++) { |
| 292 | bde.tus.w = bpl[i].tus.w; |
| 293 | if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64) |
| 294 | break; |
| 295 | xmit_len += bde.tus.f.bdeSize; |
| 296 | if (i == 0) |
| 297 | first_len = xmit_len; |
| 298 | } |
| 299 | |
| 300 | genwqe->rsvd2 = num_entry; |
| 301 | genwqe->hba_wqidx = 0; |
| 302 | |
| 303 | /* Words 0 - 2 */ |
| 304 | wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64; |
| 305 | wqe->generic.bde.tus.f.bdeSize = first_len; |
| 306 | wqe->generic.bde.addrLow = bpl[0].addrLow; |
| 307 | wqe->generic.bde.addrHigh = bpl[0].addrHigh; |
| 308 | |
| 309 | /* Word 3 */ |
| 310 | wqe->gen_req.request_payload_len = first_len; |
| 311 | |
| 312 | /* Word 4 */ |
| 313 | |
| 314 | /* Word 5 */ |
| 315 | bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0); |
| 316 | bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1); |
| 317 | bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1); |
James Smart | 8b36163 | 2017-03-04 09:30:26 -0800 | [diff] [blame] | 318 | bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 319 | bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME); |
| 320 | |
| 321 | /* Word 6 */ |
| 322 | bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com, |
| 323 | phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]); |
| 324 | bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag); |
| 325 | |
| 326 | /* Word 7 */ |
| 327 | bf_set(wqe_tmo, &wqe->gen_req.wqe_com, (vport->phba->fc_ratov-1)); |
| 328 | bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3); |
| 329 | bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE); |
| 330 | bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI); |
| 331 | |
| 332 | /* Word 8 */ |
| 333 | wqe->gen_req.wqe_com.abort_tag = genwqe->iotag; |
| 334 | |
| 335 | /* Word 9 */ |
| 336 | bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag); |
| 337 | |
| 338 | /* Word 10 */ |
| 339 | bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1); |
| 340 | bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ); |
| 341 | bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1); |
| 342 | bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE); |
| 343 | bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0); |
| 344 | |
| 345 | /* Word 11 */ |
| 346 | bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); |
| 347 | bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND); |
| 348 | |
| 349 | |
| 350 | /* Issue GEN REQ WQE for NPORT <did> */ |
| 351 | lpfc_printf_vlog(vport, KERN_INFO, LOG_ELS, |
| 352 | "6050 Issue GEN REQ WQE to NPORT x%x " |
| 353 | "Data: x%x x%x wq:%p lsreq:%p bmp:%p xmit:%d 1st:%d\n", |
| 354 | ndlp->nlp_DID, genwqe->iotag, |
| 355 | vport->port_state, |
| 356 | genwqe, pnvme_lsreq, bmp, xmit_len, first_len); |
| 357 | genwqe->wqe_cmpl = cmpl; |
| 358 | genwqe->iocb_cmpl = NULL; |
| 359 | genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT; |
| 360 | genwqe->vport = vport; |
| 361 | genwqe->retry = retry; |
| 362 | |
James Smart | bd2cdd5 | 2017-02-12 13:52:33 -0800 | [diff] [blame] | 363 | lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n", |
| 364 | genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID); |
| 365 | |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 366 | rc = lpfc_sli4_issue_wqe(phba, LPFC_ELS_RING, genwqe); |
Dick Kennedy | cd22d60 | 2017-08-23 16:55:35 -0700 | [diff] [blame] | 367 | if (rc) { |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 368 | lpfc_printf_vlog(vport, KERN_ERR, LOG_ELS, |
| 369 | "6045 Issue GEN REQ WQE to NPORT x%x " |
| 370 | "Data: x%x x%x\n", |
| 371 | ndlp->nlp_DID, genwqe->iotag, |
| 372 | vport->port_state); |
| 373 | lpfc_sli_release_iocbq(phba, genwqe); |
| 374 | return 1; |
| 375 | } |
| 376 | return 0; |
| 377 | } |
| 378 | |
| 379 | /** |
| 380 | * lpfc_nvme_ls_req - Issue an Link Service request |
| 381 | * @lpfc_pnvme: Pointer to the driver's nvme instance data |
| 382 | * @lpfc_nvme_lport: Pointer to the driver's local port data |
| 383 | * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq |
| 384 | * |
| 385 | * Driver registers this routine to handle any link service request |
| 386 | * from the nvme_fc transport to a remote nvme-aware port. |
| 387 | * |
| 388 | * Return value : |
| 389 | * 0 - Success |
| 390 | * TODO: What are the failure codes. |
| 391 | **/ |
| 392 | static int |
| 393 | lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport, |
| 394 | struct nvme_fc_remote_port *pnvme_rport, |
| 395 | struct nvmefc_ls_req *pnvme_lsreq) |
| 396 | { |
| 397 | int ret = 0; |
| 398 | struct lpfc_nvme_lport *lport; |
| 399 | struct lpfc_vport *vport; |
| 400 | struct lpfc_nodelist *ndlp; |
| 401 | struct ulp_bde64 *bpl; |
| 402 | struct lpfc_dmabuf *bmp; |
James Smart | ba43c4d | 2017-04-21 16:04:47 -0700 | [diff] [blame] | 403 | uint16_t ntype, nstate; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 404 | |
| 405 | /* there are two dma buf in the request, actually there is one and |
| 406 | * the second one is just the start address + cmd size. |
| 407 | * Before calling lpfc_nvme_gen_req these buffers need to be wrapped |
| 408 | * in a lpfc_dmabuf struct. When freeing we just free the wrapper |
| 409 | * because the nvem layer owns the data bufs. |
| 410 | * We do not have to break these packets open, we don't care what is in |
| 411 | * them. And we do not have to look at the resonse data, we only care |
| 412 | * that we got a response. All of the caring is going to happen in the |
| 413 | * nvme-fc layer. |
| 414 | */ |
| 415 | |
| 416 | lport = (struct lpfc_nvme_lport *)pnvme_lport->private; |
| 417 | vport = lport->vport; |
| 418 | |
| 419 | ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); |
James Smart | ba43c4d | 2017-04-21 16:04:47 -0700 | [diff] [blame] | 420 | if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { |
| 421 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, |
| 422 | "6051 DID x%06x not an active rport.\n", |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 423 | pnvme_rport->port_id); |
James Smart | ba43c4d | 2017-04-21 16:04:47 -0700 | [diff] [blame] | 424 | return -ENODEV; |
| 425 | } |
| 426 | |
| 427 | /* The remote node has to be a mapped nvme target or an |
| 428 | * unmapped nvme initiator or it's an error. |
| 429 | */ |
| 430 | ntype = ndlp->nlp_type; |
| 431 | nstate = ndlp->nlp_state; |
| 432 | if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) || |
| 433 | (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) { |
| 434 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, |
| 435 | "6088 DID x%06x not ready for " |
| 436 | "IO. State x%x, Type x%x\n", |
| 437 | pnvme_rport->port_id, |
| 438 | ndlp->nlp_state, ndlp->nlp_type); |
| 439 | return -ENODEV; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 440 | } |
| 441 | bmp = kmalloc(sizeof(struct lpfc_dmabuf), GFP_KERNEL); |
| 442 | if (!bmp) { |
| 443 | |
| 444 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, |
| 445 | "6044 Could not find node for DID %x\n", |
| 446 | pnvme_rport->port_id); |
| 447 | return 2; |
| 448 | } |
| 449 | INIT_LIST_HEAD(&bmp->list); |
| 450 | bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys)); |
| 451 | if (!bmp->virt) { |
| 452 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, |
| 453 | "6042 Could not find node for DID %x\n", |
| 454 | pnvme_rport->port_id); |
| 455 | kfree(bmp); |
| 456 | return 3; |
| 457 | } |
| 458 | bpl = (struct ulp_bde64 *)bmp->virt; |
| 459 | bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma)); |
| 460 | bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma)); |
| 461 | bpl->tus.f.bdeFlags = 0; |
| 462 | bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen; |
| 463 | bpl->tus.w = le32_to_cpu(bpl->tus.w); |
| 464 | bpl++; |
| 465 | |
| 466 | bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma)); |
| 467 | bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma)); |
| 468 | bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I; |
| 469 | bpl->tus.f.bdeSize = pnvme_lsreq->rsplen; |
| 470 | bpl->tus.w = le32_to_cpu(bpl->tus.w); |
| 471 | |
| 472 | /* Expand print to include key fields. */ |
| 473 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, |
James Smart | ba43c4d | 2017-04-21 16:04:47 -0700 | [diff] [blame] | 474 | "6149 ENTER. lport %p, rport %p lsreq%p rqstlen:%d " |
Arnd Bergmann | 825c6ab | 2017-02-27 21:37:12 +0100 | [diff] [blame] | 475 | "rsplen:%d %pad %pad\n", |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 476 | pnvme_lport, pnvme_rport, |
| 477 | pnvme_lsreq, pnvme_lsreq->rqstlen, |
Arnd Bergmann | 825c6ab | 2017-02-27 21:37:12 +0100 | [diff] [blame] | 478 | pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, |
| 479 | &pnvme_lsreq->rspdma); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 480 | |
James Smart | 2cee780 | 2017-06-01 21:07:02 -0700 | [diff] [blame] | 481 | atomic_inc(&vport->phba->fc4NvmeLsRequests); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 482 | |
| 483 | /* Hardcode the wait to 30 seconds. Connections are failing otherwise. |
| 484 | * This code allows it all to work. |
| 485 | */ |
| 486 | ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr, |
| 487 | pnvme_lsreq, lpfc_nvme_cmpl_gen_req, |
| 488 | ndlp, 2, 30, 0); |
| 489 | if (ret != WQE_SUCCESS) { |
| 490 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, |
| 491 | "6052 EXIT. issue ls wqe failed lport %p, " |
| 492 | "rport %p lsreq%p Status %x DID %x\n", |
| 493 | pnvme_lport, pnvme_rport, pnvme_lsreq, |
| 494 | ret, ndlp->nlp_DID); |
| 495 | lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys); |
| 496 | kfree(bmp); |
| 497 | return ret; |
| 498 | } |
| 499 | |
| 500 | /* Stub in routine and return 0 for now. */ |
| 501 | return ret; |
| 502 | } |
| 503 | |
| 504 | /** |
| 505 | * lpfc_nvme_ls_abort - Issue an Link Service request |
| 506 | * @lpfc_pnvme: Pointer to the driver's nvme instance data |
| 507 | * @lpfc_nvme_lport: Pointer to the driver's local port data |
| 508 | * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq |
| 509 | * |
| 510 | * Driver registers this routine to handle any link service request |
| 511 | * from the nvme_fc transport to a remote nvme-aware port. |
| 512 | * |
| 513 | * Return value : |
| 514 | * 0 - Success |
| 515 | * TODO: What are the failure codes. |
| 516 | **/ |
| 517 | static void |
| 518 | lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport, |
| 519 | struct nvme_fc_remote_port *pnvme_rport, |
| 520 | struct nvmefc_ls_req *pnvme_lsreq) |
| 521 | { |
| 522 | struct lpfc_nvme_lport *lport; |
| 523 | struct lpfc_vport *vport; |
| 524 | struct lpfc_hba *phba; |
| 525 | struct lpfc_nodelist *ndlp; |
| 526 | LIST_HEAD(abort_list); |
| 527 | struct lpfc_sli_ring *pring; |
| 528 | struct lpfc_iocbq *wqe, *next_wqe; |
| 529 | |
| 530 | lport = (struct lpfc_nvme_lport *)pnvme_lport->private; |
| 531 | vport = lport->vport; |
| 532 | phba = vport->phba; |
| 533 | |
| 534 | ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); |
| 535 | if (!ndlp) { |
| 536 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, |
| 537 | "6049 Could not find node for DID %x\n", |
| 538 | pnvme_rport->port_id); |
| 539 | return; |
| 540 | } |
| 541 | |
| 542 | /* Expand print to include key fields. */ |
| 543 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, |
| 544 | "6040 ENTER. lport %p, rport %p lsreq %p rqstlen:%d " |
Arnd Bergmann | 825c6ab | 2017-02-27 21:37:12 +0100 | [diff] [blame] | 545 | "rsplen:%d %pad %pad\n", |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 546 | pnvme_lport, pnvme_rport, |
| 547 | pnvme_lsreq, pnvme_lsreq->rqstlen, |
Arnd Bergmann | 825c6ab | 2017-02-27 21:37:12 +0100 | [diff] [blame] | 548 | pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma, |
| 549 | &pnvme_lsreq->rspdma); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 550 | |
| 551 | /* |
| 552 | * Lock the ELS ring txcmplq and build a local list of all ELS IOs |
| 553 | * that need an ABTS. The IOs need to stay on the txcmplq so that |
| 554 | * the abort operation completes them successfully. |
| 555 | */ |
| 556 | pring = phba->sli4_hba.nvmels_wq->pring; |
| 557 | spin_lock_irq(&phba->hbalock); |
| 558 | spin_lock(&pring->ring_lock); |
| 559 | list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) { |
| 560 | /* Add to abort_list on on NDLP match. */ |
| 561 | if (lpfc_check_sli_ndlp(phba, pring, wqe, ndlp)) { |
| 562 | wqe->iocb_flag |= LPFC_DRIVER_ABORTED; |
| 563 | list_add_tail(&wqe->dlist, &abort_list); |
| 564 | } |
| 565 | } |
| 566 | spin_unlock(&pring->ring_lock); |
| 567 | spin_unlock_irq(&phba->hbalock); |
| 568 | |
| 569 | /* Abort the targeted IOs and remove them from the abort list. */ |
| 570 | list_for_each_entry_safe(wqe, next_wqe, &abort_list, dlist) { |
| 571 | spin_lock_irq(&phba->hbalock); |
| 572 | list_del_init(&wqe->dlist); |
| 573 | lpfc_sli_issue_abort_iotag(phba, pring, wqe); |
| 574 | spin_unlock_irq(&phba->hbalock); |
| 575 | } |
| 576 | } |
| 577 | |
| 578 | /* Fix up the existing sgls for NVME IO. */ |
| 579 | static void |
| 580 | lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport, |
| 581 | struct lpfc_nvme_buf *lpfc_ncmd, |
| 582 | struct nvmefc_fcp_req *nCmd) |
| 583 | { |
| 584 | struct sli4_sge *sgl; |
| 585 | union lpfc_wqe128 *wqe; |
| 586 | uint32_t *wptr, *dptr; |
| 587 | |
| 588 | /* |
| 589 | * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to |
| 590 | * match NVME. NVME sends 96 bytes. Also, use the |
| 591 | * nvme commands command and response dma addresses |
| 592 | * rather than the virtual memory to ease the restore |
| 593 | * operation. |
| 594 | */ |
| 595 | sgl = lpfc_ncmd->nvme_sgl; |
| 596 | sgl->sge_len = cpu_to_le32(nCmd->cmdlen); |
| 597 | |
| 598 | sgl++; |
| 599 | |
| 600 | /* Setup the physical region for the FCP RSP */ |
| 601 | sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma)); |
| 602 | sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma)); |
| 603 | sgl->word2 = le32_to_cpu(sgl->word2); |
| 604 | if (nCmd->sg_cnt) |
| 605 | bf_set(lpfc_sli4_sge_last, sgl, 0); |
| 606 | else |
| 607 | bf_set(lpfc_sli4_sge_last, sgl, 1); |
| 608 | sgl->word2 = cpu_to_le32(sgl->word2); |
| 609 | sgl->sge_len = cpu_to_le32(nCmd->rsplen); |
| 610 | |
| 611 | /* |
| 612 | * Get a local pointer to the built-in wqe and correct |
| 613 | * the cmd size to match NVME's 96 bytes and fix |
| 614 | * the dma address. |
| 615 | */ |
| 616 | |
| 617 | /* 128 byte wqe support here */ |
| 618 | wqe = (union lpfc_wqe128 *)&lpfc_ncmd->cur_iocbq.wqe; |
| 619 | |
| 620 | /* Word 0-2 - NVME CMND IU (embedded payload) */ |
| 621 | wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED; |
| 622 | wqe->generic.bde.tus.f.bdeSize = 60; |
| 623 | wqe->generic.bde.addrHigh = 0; |
| 624 | wqe->generic.bde.addrLow = 64; /* Word 16 */ |
| 625 | |
| 626 | /* Word 3 */ |
| 627 | bf_set(payload_offset_len, &wqe->fcp_icmd, |
| 628 | (nCmd->rsplen + nCmd->cmdlen)); |
| 629 | |
| 630 | /* Word 10 */ |
| 631 | bf_set(wqe_nvme, &wqe->fcp_icmd.wqe_com, 1); |
| 632 | bf_set(wqe_wqes, &wqe->fcp_icmd.wqe_com, 1); |
| 633 | |
| 634 | /* |
| 635 | * Embed the payload in the last half of the WQE |
| 636 | * WQE words 16-30 get the NVME CMD IU payload |
| 637 | * |
James Smart | b06a622 | 2017-03-04 09:30:27 -0800 | [diff] [blame] | 638 | * WQE words 16-19 get payload Words 1-4 |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 639 | * WQE words 20-21 get payload Words 6-7 |
| 640 | * WQE words 22-29 get payload Words 16-23 |
| 641 | */ |
James Smart | b06a622 | 2017-03-04 09:30:27 -0800 | [diff] [blame] | 642 | wptr = &wqe->words[16]; /* WQE ptr */ |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 643 | dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */ |
James Smart | b06a622 | 2017-03-04 09:30:27 -0800 | [diff] [blame] | 644 | dptr++; /* Skip Word 0 in payload */ |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 645 | |
James Smart | b06a622 | 2017-03-04 09:30:27 -0800 | [diff] [blame] | 646 | *wptr++ = *dptr++; /* Word 1 */ |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 647 | *wptr++ = *dptr++; /* Word 2 */ |
| 648 | *wptr++ = *dptr++; /* Word 3 */ |
| 649 | *wptr++ = *dptr++; /* Word 4 */ |
| 650 | dptr++; /* Skip Word 5 in payload */ |
| 651 | *wptr++ = *dptr++; /* Word 6 */ |
| 652 | *wptr++ = *dptr++; /* Word 7 */ |
| 653 | dptr += 8; /* Skip Words 8-15 in payload */ |
| 654 | *wptr++ = *dptr++; /* Word 16 */ |
| 655 | *wptr++ = *dptr++; /* Word 17 */ |
| 656 | *wptr++ = *dptr++; /* Word 18 */ |
| 657 | *wptr++ = *dptr++; /* Word 19 */ |
| 658 | *wptr++ = *dptr++; /* Word 20 */ |
| 659 | *wptr++ = *dptr++; /* Word 21 */ |
| 660 | *wptr++ = *dptr++; /* Word 22 */ |
| 661 | *wptr = *dptr; /* Word 23 */ |
| 662 | } |
| 663 | |
James Smart | bd2cdd5 | 2017-02-12 13:52:33 -0800 | [diff] [blame] | 664 | #ifdef CONFIG_SCSI_LPFC_DEBUG_FS |
| 665 | static void |
| 666 | lpfc_nvme_ktime(struct lpfc_hba *phba, |
| 667 | struct lpfc_nvme_buf *lpfc_ncmd) |
| 668 | { |
| 669 | uint64_t seg1, seg2, seg3, seg4; |
| 670 | |
| 671 | if (!phba->ktime_on) |
| 672 | return; |
| 673 | if (!lpfc_ncmd->ts_last_cmd || |
| 674 | !lpfc_ncmd->ts_cmd_start || |
| 675 | !lpfc_ncmd->ts_cmd_wqput || |
| 676 | !lpfc_ncmd->ts_isr_cmpl || |
| 677 | !lpfc_ncmd->ts_data_nvme) |
| 678 | return; |
| 679 | if (lpfc_ncmd->ts_cmd_start < lpfc_ncmd->ts_last_cmd) |
| 680 | return; |
| 681 | if (lpfc_ncmd->ts_cmd_wqput < lpfc_ncmd->ts_cmd_start) |
| 682 | return; |
| 683 | if (lpfc_ncmd->ts_isr_cmpl < lpfc_ncmd->ts_cmd_wqput) |
| 684 | return; |
| 685 | if (lpfc_ncmd->ts_data_nvme < lpfc_ncmd->ts_isr_cmpl) |
| 686 | return; |
| 687 | /* |
| 688 | * Segment 1 - Time from Last FCP command cmpl is handed |
| 689 | * off to NVME Layer to start of next command. |
| 690 | * Segment 2 - Time from Driver receives a IO cmd start |
| 691 | * from NVME Layer to WQ put is done on IO cmd. |
| 692 | * Segment 3 - Time from Driver WQ put is done on IO cmd |
| 693 | * to MSI-X ISR for IO cmpl. |
| 694 | * Segment 4 - Time from MSI-X ISR for IO cmpl to when |
| 695 | * cmpl is handled off to the NVME Layer. |
| 696 | */ |
| 697 | seg1 = lpfc_ncmd->ts_cmd_start - lpfc_ncmd->ts_last_cmd; |
| 698 | if (seg1 > 5000000) /* 5 ms - for sequential IOs */ |
| 699 | return; |
| 700 | |
| 701 | /* Calculate times relative to start of IO */ |
| 702 | seg2 = (lpfc_ncmd->ts_cmd_wqput - lpfc_ncmd->ts_cmd_start); |
| 703 | seg3 = (lpfc_ncmd->ts_isr_cmpl - |
| 704 | lpfc_ncmd->ts_cmd_start) - seg2; |
| 705 | seg4 = (lpfc_ncmd->ts_data_nvme - |
| 706 | lpfc_ncmd->ts_cmd_start) - seg2 - seg3; |
| 707 | phba->ktime_data_samples++; |
| 708 | phba->ktime_seg1_total += seg1; |
| 709 | if (seg1 < phba->ktime_seg1_min) |
| 710 | phba->ktime_seg1_min = seg1; |
| 711 | else if (seg1 > phba->ktime_seg1_max) |
| 712 | phba->ktime_seg1_max = seg1; |
| 713 | phba->ktime_seg2_total += seg2; |
| 714 | if (seg2 < phba->ktime_seg2_min) |
| 715 | phba->ktime_seg2_min = seg2; |
| 716 | else if (seg2 > phba->ktime_seg2_max) |
| 717 | phba->ktime_seg2_max = seg2; |
| 718 | phba->ktime_seg3_total += seg3; |
| 719 | if (seg3 < phba->ktime_seg3_min) |
| 720 | phba->ktime_seg3_min = seg3; |
| 721 | else if (seg3 > phba->ktime_seg3_max) |
| 722 | phba->ktime_seg3_max = seg3; |
| 723 | phba->ktime_seg4_total += seg4; |
| 724 | if (seg4 < phba->ktime_seg4_min) |
| 725 | phba->ktime_seg4_min = seg4; |
| 726 | else if (seg4 > phba->ktime_seg4_max) |
| 727 | phba->ktime_seg4_max = seg4; |
| 728 | |
| 729 | lpfc_ncmd->ts_last_cmd = 0; |
| 730 | lpfc_ncmd->ts_cmd_start = 0; |
| 731 | lpfc_ncmd->ts_cmd_wqput = 0; |
| 732 | lpfc_ncmd->ts_isr_cmpl = 0; |
| 733 | lpfc_ncmd->ts_data_nvme = 0; |
| 734 | } |
| 735 | #endif |
| 736 | |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 737 | /** |
| 738 | * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO |
| 739 | * @lpfc_pnvme: Pointer to the driver's nvme instance data |
| 740 | * @lpfc_nvme_lport: Pointer to the driver's local port data |
| 741 | * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq |
| 742 | * |
| 743 | * Driver registers this routine as it io request handler. This |
| 744 | * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq |
| 745 | * data structure to the rport indicated in @lpfc_nvme_rport. |
| 746 | * |
| 747 | * Return value : |
| 748 | * 0 - Success |
| 749 | * TODO: What are the failure codes. |
| 750 | **/ |
| 751 | static void |
| 752 | lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn, |
| 753 | struct lpfc_wcqe_complete *wcqe) |
| 754 | { |
| 755 | struct lpfc_nvme_buf *lpfc_ncmd = |
| 756 | (struct lpfc_nvme_buf *)pwqeIn->context1; |
| 757 | struct lpfc_vport *vport = pwqeIn->vport; |
| 758 | struct nvmefc_fcp_req *nCmd; |
| 759 | struct nvme_fc_ersp_iu *ep; |
| 760 | struct nvme_fc_cmd_iu *cp; |
| 761 | struct lpfc_nvme_rport *rport; |
| 762 | struct lpfc_nodelist *ndlp; |
James Smart | bbe3012 | 2017-04-21 17:49:08 -0700 | [diff] [blame] | 763 | struct lpfc_nvme_fcpreq_priv *freqpriv; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 764 | unsigned long flags; |
| 765 | uint32_t code; |
| 766 | uint16_t cid, sqhd, data; |
| 767 | uint32_t *ptr; |
| 768 | |
| 769 | /* Sanity check on return of outstanding command */ |
| 770 | if (!lpfc_ncmd || !lpfc_ncmd->nvmeCmd || !lpfc_ncmd->nrport) { |
| 771 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, |
| 772 | "6071 Completion pointers bad on wqe %p.\n", |
| 773 | wcqe); |
| 774 | return; |
| 775 | } |
James Smart | 2cee780 | 2017-06-01 21:07:02 -0700 | [diff] [blame] | 776 | atomic_inc(&phba->fc4NvmeIoCmpls); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 777 | |
| 778 | nCmd = lpfc_ncmd->nvmeCmd; |
| 779 | rport = lpfc_ncmd->nrport; |
| 780 | |
James Smart | bd2cdd5 | 2017-02-12 13:52:33 -0800 | [diff] [blame] | 781 | lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n", |
| 782 | lpfc_ncmd->cur_iocbq.sli4_xritag, |
| 783 | bf_get(lpfc_wcqe_c_status, wcqe), wcqe->parameter); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 784 | /* |
| 785 | * Catch race where our node has transitioned, but the |
| 786 | * transport is still transitioning. |
| 787 | */ |
| 788 | ndlp = rport->ndlp; |
| 789 | if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { |
| 790 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, |
James Smart | c154e75 | 2017-04-21 16:04:48 -0700 | [diff] [blame] | 791 | "6061 rport %p, DID x%06x node not ready.\n", |
| 792 | rport, rport->remoteport->port_id); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 793 | |
| 794 | ndlp = lpfc_findnode_did(vport, rport->remoteport->port_id); |
| 795 | if (!ndlp) { |
| 796 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, |
| 797 | "6062 Ignoring NVME cmpl. No ndlp\n"); |
| 798 | goto out_err; |
| 799 | } |
| 800 | } |
| 801 | |
| 802 | code = bf_get(lpfc_wcqe_c_code, wcqe); |
| 803 | if (code == CQE_CODE_NVME_ERSP) { |
| 804 | /* For this type of CQE, we need to rebuild the rsp */ |
| 805 | ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr; |
| 806 | |
| 807 | /* |
| 808 | * Get Command Id from cmd to plug into response. This |
| 809 | * code is not needed in the next NVME Transport drop. |
| 810 | */ |
| 811 | cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr; |
| 812 | cid = cp->sqe.common.command_id; |
| 813 | |
| 814 | /* |
| 815 | * RSN is in CQE word 2 |
| 816 | * SQHD is in CQE Word 3 bits 15:0 |
| 817 | * Cmd Specific info is in CQE Word 1 |
| 818 | * and in CQE Word 0 bits 15:0 |
| 819 | */ |
| 820 | sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe); |
| 821 | |
| 822 | /* Now lets build the NVME ERSP IU */ |
| 823 | ep->iu_len = cpu_to_be16(8); |
| 824 | ep->rsn = wcqe->parameter; |
| 825 | ep->xfrd_len = cpu_to_be32(nCmd->payload_length); |
| 826 | ep->rsvd12 = 0; |
| 827 | ptr = (uint32_t *)&ep->cqe.result.u64; |
| 828 | *ptr++ = wcqe->total_data_placed; |
| 829 | data = bf_get(lpfc_wcqe_c_ersp0, wcqe); |
| 830 | *ptr = (uint32_t)data; |
| 831 | ep->cqe.sq_head = sqhd; |
| 832 | ep->cqe.sq_id = nCmd->sqid; |
| 833 | ep->cqe.command_id = cid; |
| 834 | ep->cqe.status = 0; |
| 835 | |
| 836 | lpfc_ncmd->status = IOSTAT_SUCCESS; |
| 837 | lpfc_ncmd->result = 0; |
| 838 | nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN; |
| 839 | nCmd->transferred_length = nCmd->payload_length; |
| 840 | } else { |
| 841 | lpfc_ncmd->status = (bf_get(lpfc_wcqe_c_status, wcqe) & |
| 842 | LPFC_IOCB_STATUS_MASK); |
| 843 | lpfc_ncmd->result = wcqe->parameter; |
| 844 | |
| 845 | /* For NVME, the only failure path that results in an |
| 846 | * IO error is when the adapter rejects it. All other |
| 847 | * conditions are a success case and resolved by the |
| 848 | * transport. |
| 849 | * IOSTAT_FCP_RSP_ERROR means: |
| 850 | * 1. Length of data received doesn't match total |
| 851 | * transfer length in WQE |
| 852 | * 2. If the RSP payload does NOT match these cases: |
| 853 | * a. RSP length 12/24 bytes and all zeros |
| 854 | * b. NVME ERSP |
| 855 | */ |
| 856 | switch (lpfc_ncmd->status) { |
| 857 | case IOSTAT_SUCCESS: |
| 858 | nCmd->transferred_length = wcqe->total_data_placed; |
| 859 | nCmd->rcv_rsplen = 0; |
| 860 | nCmd->status = 0; |
| 861 | break; |
| 862 | case IOSTAT_FCP_RSP_ERROR: |
| 863 | nCmd->transferred_length = wcqe->total_data_placed; |
| 864 | nCmd->rcv_rsplen = wcqe->parameter; |
| 865 | nCmd->status = 0; |
| 866 | /* Sanity check */ |
| 867 | if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) |
| 868 | break; |
| 869 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, |
| 870 | "6081 NVME Completion Protocol Error: " |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 871 | "xri %x status x%x result x%x " |
| 872 | "placed x%x\n", |
| 873 | lpfc_ncmd->cur_iocbq.sli4_xritag, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 874 | lpfc_ncmd->status, lpfc_ncmd->result, |
| 875 | wcqe->total_data_placed); |
| 876 | break; |
| 877 | default: |
| 878 | out_err: |
| 879 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 880 | "6072 NVME Completion Error: xri %x " |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 881 | "status x%x result x%x placed x%x\n", |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 882 | lpfc_ncmd->cur_iocbq.sli4_xritag, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 883 | lpfc_ncmd->status, lpfc_ncmd->result, |
| 884 | wcqe->total_data_placed); |
| 885 | nCmd->transferred_length = 0; |
| 886 | nCmd->rcv_rsplen = 0; |
James Smart | 8e009ce8 | 2017-09-07 16:27:29 -0700 | [diff] [blame] | 887 | nCmd->status = NVME_SC_INTERNAL; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 888 | } |
| 889 | } |
| 890 | |
| 891 | /* pick up SLI4 exhange busy condition */ |
| 892 | if (bf_get(lpfc_wcqe_c_xb, wcqe)) |
| 893 | lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; |
| 894 | else |
| 895 | lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; |
| 896 | |
| 897 | if (ndlp && NLP_CHK_NODE_ACT(ndlp)) |
| 898 | atomic_dec(&ndlp->cmd_pending); |
| 899 | |
| 900 | /* Update stats and complete the IO. There is |
| 901 | * no need for dma unprep because the nvme_transport |
| 902 | * owns the dma address. |
| 903 | */ |
James Smart | bd2cdd5 | 2017-02-12 13:52:33 -0800 | [diff] [blame] | 904 | #ifdef CONFIG_SCSI_LPFC_DEBUG_FS |
| 905 | if (phba->ktime_on) { |
| 906 | lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp; |
| 907 | lpfc_ncmd->ts_data_nvme = ktime_get_ns(); |
| 908 | phba->ktime_last_cmd = lpfc_ncmd->ts_data_nvme; |
| 909 | lpfc_nvme_ktime(phba, lpfc_ncmd); |
| 910 | } |
| 911 | if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) { |
| 912 | if (lpfc_ncmd->cpu != smp_processor_id()) |
| 913 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, |
| 914 | "6701 CPU Check cmpl: " |
| 915 | "cpu %d expect %d\n", |
| 916 | smp_processor_id(), lpfc_ncmd->cpu); |
| 917 | if (lpfc_ncmd->cpu < LPFC_CHECK_CPU_CNT) |
| 918 | phba->cpucheck_cmpl_io[lpfc_ncmd->cpu]++; |
| 919 | } |
| 920 | #endif |
James Smart | bbe3012 | 2017-04-21 17:49:08 -0700 | [diff] [blame] | 921 | freqpriv = nCmd->private; |
| 922 | freqpriv->nvme_buf = NULL; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 923 | nCmd->done(nCmd); |
| 924 | |
| 925 | spin_lock_irqsave(&phba->hbalock, flags); |
| 926 | lpfc_ncmd->nrport = NULL; |
| 927 | spin_unlock_irqrestore(&phba->hbalock, flags); |
| 928 | |
| 929 | lpfc_release_nvme_buf(phba, lpfc_ncmd); |
| 930 | } |
| 931 | |
| 932 | |
| 933 | /** |
| 934 | * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO |
| 935 | * @lpfc_pnvme: Pointer to the driver's nvme instance data |
| 936 | * @lpfc_nvme_lport: Pointer to the driver's local port data |
| 937 | * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq |
| 938 | * @lpfc_nvme_fcreq: IO request from nvme fc to driver. |
| 939 | * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue |
| 940 | * |
| 941 | * Driver registers this routine as it io request handler. This |
| 942 | * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq |
| 943 | * data structure to the rport indicated in @lpfc_nvme_rport. |
| 944 | * |
| 945 | * Return value : |
| 946 | * 0 - Success |
| 947 | * TODO: What are the failure codes. |
| 948 | **/ |
| 949 | static int |
| 950 | lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport, |
| 951 | struct lpfc_nvme_buf *lpfc_ncmd, |
| 952 | struct lpfc_nodelist *pnode) |
| 953 | { |
| 954 | struct lpfc_hba *phba = vport->phba; |
| 955 | struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; |
| 956 | struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq); |
| 957 | union lpfc_wqe128 *wqe = (union lpfc_wqe128 *)&pwqeq->wqe; |
| 958 | uint32_t req_len; |
| 959 | |
| 960 | if (!pnode || !NLP_CHK_NODE_ACT(pnode)) |
| 961 | return -EINVAL; |
| 962 | |
| 963 | /* |
| 964 | * There are three possibilities here - use scatter-gather segment, use |
| 965 | * the single mapping, or neither. |
| 966 | */ |
| 967 | wqe->fcp_iwrite.initial_xfer_len = 0; |
| 968 | if (nCmd->sg_cnt) { |
| 969 | if (nCmd->io_dir == NVMEFC_FCP_WRITE) { |
| 970 | /* Word 5 */ |
| 971 | if ((phba->cfg_nvme_enable_fb) && |
| 972 | (pnode->nlp_flag & NLP_FIRSTBURST)) { |
| 973 | req_len = lpfc_ncmd->nvmeCmd->payload_length; |
| 974 | if (req_len < pnode->nvme_fb_size) |
| 975 | wqe->fcp_iwrite.initial_xfer_len = |
| 976 | req_len; |
| 977 | else |
| 978 | wqe->fcp_iwrite.initial_xfer_len = |
| 979 | pnode->nvme_fb_size; |
| 980 | } |
| 981 | |
| 982 | /* Word 7 */ |
| 983 | bf_set(wqe_cmnd, &wqe->generic.wqe_com, |
| 984 | CMD_FCP_IWRITE64_WQE); |
| 985 | bf_set(wqe_pu, &wqe->generic.wqe_com, |
| 986 | PARM_READ_CHECK); |
| 987 | |
| 988 | /* Word 10 */ |
| 989 | bf_set(wqe_qosd, &wqe->fcp_iwrite.wqe_com, 0); |
| 990 | bf_set(wqe_iod, &wqe->fcp_iwrite.wqe_com, |
| 991 | LPFC_WQE_IOD_WRITE); |
| 992 | bf_set(wqe_lenloc, &wqe->fcp_iwrite.wqe_com, |
| 993 | LPFC_WQE_LENLOC_WORD4); |
| 994 | if (phba->cfg_nvme_oas) |
| 995 | bf_set(wqe_oas, &wqe->fcp_iwrite.wqe_com, 1); |
| 996 | |
| 997 | /* Word 11 */ |
| 998 | bf_set(wqe_cmd_type, &wqe->generic.wqe_com, |
| 999 | NVME_WRITE_CMD); |
| 1000 | |
James Smart | 2cee780 | 2017-06-01 21:07:02 -0700 | [diff] [blame] | 1001 | atomic_inc(&phba->fc4NvmeOutputRequests); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1002 | } else { |
| 1003 | /* Word 7 */ |
| 1004 | bf_set(wqe_cmnd, &wqe->generic.wqe_com, |
| 1005 | CMD_FCP_IREAD64_WQE); |
| 1006 | bf_set(wqe_pu, &wqe->generic.wqe_com, |
| 1007 | PARM_READ_CHECK); |
| 1008 | |
| 1009 | /* Word 10 */ |
| 1010 | bf_set(wqe_qosd, &wqe->fcp_iread.wqe_com, 0); |
| 1011 | bf_set(wqe_iod, &wqe->fcp_iread.wqe_com, |
| 1012 | LPFC_WQE_IOD_READ); |
| 1013 | bf_set(wqe_lenloc, &wqe->fcp_iread.wqe_com, |
| 1014 | LPFC_WQE_LENLOC_WORD4); |
| 1015 | if (phba->cfg_nvme_oas) |
| 1016 | bf_set(wqe_oas, &wqe->fcp_iread.wqe_com, 1); |
| 1017 | |
| 1018 | /* Word 11 */ |
| 1019 | bf_set(wqe_cmd_type, &wqe->generic.wqe_com, |
| 1020 | NVME_READ_CMD); |
| 1021 | |
James Smart | 2cee780 | 2017-06-01 21:07:02 -0700 | [diff] [blame] | 1022 | atomic_inc(&phba->fc4NvmeInputRequests); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1023 | } |
| 1024 | } else { |
| 1025 | /* Word 4 */ |
| 1026 | wqe->fcp_icmd.rsrvd4 = 0; |
| 1027 | |
| 1028 | /* Word 7 */ |
| 1029 | bf_set(wqe_cmnd, &wqe->generic.wqe_com, CMD_FCP_ICMND64_WQE); |
| 1030 | bf_set(wqe_pu, &wqe->generic.wqe_com, 0); |
| 1031 | |
| 1032 | /* Word 10 */ |
| 1033 | bf_set(wqe_qosd, &wqe->fcp_icmd.wqe_com, 1); |
| 1034 | bf_set(wqe_iod, &wqe->fcp_icmd.wqe_com, LPFC_WQE_IOD_WRITE); |
| 1035 | bf_set(wqe_lenloc, &wqe->fcp_icmd.wqe_com, |
| 1036 | LPFC_WQE_LENLOC_NONE); |
| 1037 | if (phba->cfg_nvme_oas) |
| 1038 | bf_set(wqe_oas, &wqe->fcp_icmd.wqe_com, 1); |
| 1039 | |
| 1040 | /* Word 11 */ |
| 1041 | bf_set(wqe_cmd_type, &wqe->generic.wqe_com, NVME_READ_CMD); |
| 1042 | |
James Smart | 2cee780 | 2017-06-01 21:07:02 -0700 | [diff] [blame] | 1043 | atomic_inc(&phba->fc4NvmeControlRequests); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1044 | } |
| 1045 | /* |
| 1046 | * Finish initializing those WQE fields that are independent |
| 1047 | * of the nvme_cmnd request_buffer |
| 1048 | */ |
| 1049 | |
| 1050 | /* Word 6 */ |
| 1051 | bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com, |
| 1052 | phba->sli4_hba.rpi_ids[pnode->nlp_rpi]); |
| 1053 | bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag); |
| 1054 | |
| 1055 | /* Word 7 */ |
| 1056 | /* Preserve Class data in the ndlp. */ |
| 1057 | bf_set(wqe_class, &wqe->generic.wqe_com, |
| 1058 | (pnode->nlp_fcp_info & 0x0f)); |
| 1059 | |
| 1060 | /* Word 8 */ |
| 1061 | wqe->generic.wqe_com.abort_tag = pwqeq->iotag; |
| 1062 | |
| 1063 | /* Word 9 */ |
| 1064 | bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag); |
| 1065 | |
| 1066 | /* Word 11 */ |
| 1067 | bf_set(wqe_cqid, &wqe->generic.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); |
| 1068 | |
| 1069 | pwqeq->vport = vport; |
| 1070 | return 0; |
| 1071 | } |
| 1072 | |
| 1073 | |
| 1074 | /** |
| 1075 | * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO |
| 1076 | * @lpfc_pnvme: Pointer to the driver's nvme instance data |
| 1077 | * @lpfc_nvme_lport: Pointer to the driver's local port data |
| 1078 | * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq |
| 1079 | * @lpfc_nvme_fcreq: IO request from nvme fc to driver. |
| 1080 | * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue |
| 1081 | * |
| 1082 | * Driver registers this routine as it io request handler. This |
| 1083 | * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq |
| 1084 | * data structure to the rport indicated in @lpfc_nvme_rport. |
| 1085 | * |
| 1086 | * Return value : |
| 1087 | * 0 - Success |
| 1088 | * TODO: What are the failure codes. |
| 1089 | **/ |
| 1090 | static int |
| 1091 | lpfc_nvme_prep_io_dma(struct lpfc_vport *vport, |
| 1092 | struct lpfc_nvme_buf *lpfc_ncmd) |
| 1093 | { |
| 1094 | struct lpfc_hba *phba = vport->phba; |
| 1095 | struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd; |
| 1096 | union lpfc_wqe128 *wqe = (union lpfc_wqe128 *)&lpfc_ncmd->cur_iocbq.wqe; |
| 1097 | struct sli4_sge *sgl = lpfc_ncmd->nvme_sgl; |
| 1098 | struct scatterlist *data_sg; |
| 1099 | struct sli4_sge *first_data_sgl; |
| 1100 | dma_addr_t physaddr; |
| 1101 | uint32_t num_bde = 0; |
| 1102 | uint32_t dma_len; |
| 1103 | uint32_t dma_offset = 0; |
| 1104 | int nseg, i; |
| 1105 | |
| 1106 | /* Fix up the command and response DMA stuff. */ |
| 1107 | lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd); |
| 1108 | |
| 1109 | /* |
| 1110 | * There are three possibilities here - use scatter-gather segment, use |
| 1111 | * the single mapping, or neither. |
| 1112 | */ |
| 1113 | if (nCmd->sg_cnt) { |
| 1114 | /* |
| 1115 | * Jump over the cmd and rsp SGEs. The fix routine |
| 1116 | * has already adjusted for this. |
| 1117 | */ |
| 1118 | sgl += 2; |
| 1119 | |
| 1120 | first_data_sgl = sgl; |
| 1121 | lpfc_ncmd->seg_cnt = nCmd->sg_cnt; |
James Smart | 4d4c4a4 | 2017-04-21 16:05:01 -0700 | [diff] [blame] | 1122 | if (lpfc_ncmd->seg_cnt > phba->cfg_nvme_seg_cnt) { |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1123 | lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, |
| 1124 | "6058 Too many sg segments from " |
| 1125 | "NVME Transport. Max %d, " |
| 1126 | "nvmeIO sg_cnt %d\n", |
James Smart | 4d4c4a4 | 2017-04-21 16:05:01 -0700 | [diff] [blame] | 1127 | phba->cfg_nvme_seg_cnt, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1128 | lpfc_ncmd->seg_cnt); |
| 1129 | lpfc_ncmd->seg_cnt = 0; |
| 1130 | return 1; |
| 1131 | } |
| 1132 | |
| 1133 | /* |
| 1134 | * The driver established a maximum scatter-gather segment count |
| 1135 | * during probe that limits the number of sg elements in any |
| 1136 | * single nvme command. Just run through the seg_cnt and format |
| 1137 | * the sge's. |
| 1138 | */ |
| 1139 | nseg = nCmd->sg_cnt; |
| 1140 | data_sg = nCmd->first_sgl; |
| 1141 | for (i = 0; i < nseg; i++) { |
| 1142 | if (data_sg == NULL) { |
| 1143 | lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, |
| 1144 | "6059 dptr err %d, nseg %d\n", |
| 1145 | i, nseg); |
| 1146 | lpfc_ncmd->seg_cnt = 0; |
| 1147 | return 1; |
| 1148 | } |
| 1149 | physaddr = data_sg->dma_address; |
| 1150 | dma_len = data_sg->length; |
| 1151 | sgl->addr_lo = cpu_to_le32(putPaddrLow(physaddr)); |
| 1152 | sgl->addr_hi = cpu_to_le32(putPaddrHigh(physaddr)); |
| 1153 | sgl->word2 = le32_to_cpu(sgl->word2); |
| 1154 | if ((num_bde + 1) == nseg) |
| 1155 | bf_set(lpfc_sli4_sge_last, sgl, 1); |
| 1156 | else |
| 1157 | bf_set(lpfc_sli4_sge_last, sgl, 0); |
| 1158 | bf_set(lpfc_sli4_sge_offset, sgl, dma_offset); |
| 1159 | bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA); |
| 1160 | sgl->word2 = cpu_to_le32(sgl->word2); |
| 1161 | sgl->sge_len = cpu_to_le32(dma_len); |
| 1162 | |
| 1163 | dma_offset += dma_len; |
| 1164 | data_sg = sg_next(data_sg); |
| 1165 | sgl++; |
| 1166 | } |
| 1167 | } else { |
| 1168 | /* For this clause to be valid, the payload_length |
| 1169 | * and sg_cnt must zero. |
| 1170 | */ |
| 1171 | if (nCmd->payload_length != 0) { |
| 1172 | lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, |
| 1173 | "6063 NVME DMA Prep Err: sg_cnt %d " |
| 1174 | "payload_length x%x\n", |
| 1175 | nCmd->sg_cnt, nCmd->payload_length); |
| 1176 | return 1; |
| 1177 | } |
| 1178 | } |
| 1179 | |
| 1180 | /* |
| 1181 | * Due to difference in data length between DIF/non-DIF paths, |
| 1182 | * we need to set word 4 of WQE here |
| 1183 | */ |
| 1184 | wqe->fcp_iread.total_xfer_len = nCmd->payload_length; |
| 1185 | return 0; |
| 1186 | } |
| 1187 | |
| 1188 | /** |
| 1189 | * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO |
| 1190 | * @lpfc_pnvme: Pointer to the driver's nvme instance data |
| 1191 | * @lpfc_nvme_lport: Pointer to the driver's local port data |
| 1192 | * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq |
| 1193 | * @lpfc_nvme_fcreq: IO request from nvme fc to driver. |
| 1194 | * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue |
| 1195 | * |
| 1196 | * Driver registers this routine as it io request handler. This |
| 1197 | * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq |
| 1198 | * data structure to the rport |
| 1199 | indicated in @lpfc_nvme_rport. |
| 1200 | * |
| 1201 | * Return value : |
| 1202 | * 0 - Success |
| 1203 | * TODO: What are the failure codes. |
| 1204 | **/ |
| 1205 | static int |
| 1206 | lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport, |
| 1207 | struct nvme_fc_remote_port *pnvme_rport, |
| 1208 | void *hw_queue_handle, |
| 1209 | struct nvmefc_fcp_req *pnvme_fcreq) |
| 1210 | { |
| 1211 | int ret = 0; |
| 1212 | struct lpfc_nvme_lport *lport; |
| 1213 | struct lpfc_vport *vport; |
| 1214 | struct lpfc_hba *phba; |
| 1215 | struct lpfc_nodelist *ndlp; |
| 1216 | struct lpfc_nvme_buf *lpfc_ncmd; |
| 1217 | struct lpfc_nvme_rport *rport; |
| 1218 | struct lpfc_nvme_qhandle *lpfc_queue_info; |
James Smart | bbe3012 | 2017-04-21 17:49:08 -0700 | [diff] [blame] | 1219 | struct lpfc_nvme_fcpreq_priv *freqpriv = pnvme_fcreq->private; |
James Smart | bd2cdd5 | 2017-02-12 13:52:33 -0800 | [diff] [blame] | 1220 | #ifdef CONFIG_SCSI_LPFC_DEBUG_FS |
| 1221 | uint64_t start = 0; |
| 1222 | #endif |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1223 | |
| 1224 | lport = (struct lpfc_nvme_lport *)pnvme_lport->private; |
| 1225 | vport = lport->vport; |
| 1226 | phba = vport->phba; |
| 1227 | |
James Smart | bd2cdd5 | 2017-02-12 13:52:33 -0800 | [diff] [blame] | 1228 | #ifdef CONFIG_SCSI_LPFC_DEBUG_FS |
| 1229 | if (phba->ktime_on) |
| 1230 | start = ktime_get_ns(); |
| 1231 | #endif |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1232 | rport = (struct lpfc_nvme_rport *)pnvme_rport->private; |
| 1233 | lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle; |
| 1234 | |
| 1235 | /* |
| 1236 | * Catch race where our node has transitioned, but the |
| 1237 | * transport is still transitioning. |
| 1238 | */ |
| 1239 | ndlp = rport->ndlp; |
| 1240 | if (!ndlp || !NLP_CHK_NODE_ACT(ndlp)) { |
| 1241 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, |
| 1242 | "6053 rport %p, ndlp %p, DID x%06x " |
| 1243 | "ndlp not ready.\n", |
| 1244 | rport, ndlp, pnvme_rport->port_id); |
| 1245 | |
| 1246 | ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id); |
| 1247 | if (!ndlp) { |
| 1248 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, |
| 1249 | "6066 Missing node for DID %x\n", |
| 1250 | pnvme_rport->port_id); |
| 1251 | ret = -ENODEV; |
| 1252 | goto out_fail; |
| 1253 | } |
| 1254 | } |
| 1255 | |
| 1256 | /* The remote node has to be a mapped target or it's an error. */ |
| 1257 | if ((ndlp->nlp_type & NLP_NVME_TARGET) && |
| 1258 | (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) { |
| 1259 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NODE | LOG_NVME_IOERR, |
| 1260 | "6036 rport %p, DID x%06x not ready for " |
| 1261 | "IO. State x%x, Type x%x\n", |
| 1262 | rport, pnvme_rport->port_id, |
| 1263 | ndlp->nlp_state, ndlp->nlp_type); |
| 1264 | ret = -ENODEV; |
| 1265 | goto out_fail; |
| 1266 | |
| 1267 | } |
| 1268 | |
| 1269 | /* The node is shared with FCP IO, make sure the IO pending count does |
| 1270 | * not exceed the programmed depth. |
| 1271 | */ |
| 1272 | if (atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) { |
Dick Kennedy | cd22d60 | 2017-08-23 16:55:35 -0700 | [diff] [blame] | 1273 | ret = -EBUSY; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1274 | goto out_fail; |
| 1275 | } |
| 1276 | |
| 1277 | lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp); |
| 1278 | if (lpfc_ncmd == NULL) { |
| 1279 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR, |
| 1280 | "6065 driver's buffer pool is empty, " |
| 1281 | "IO failed\n"); |
Dick Kennedy | cd22d60 | 2017-08-23 16:55:35 -0700 | [diff] [blame] | 1282 | ret = -EBUSY; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1283 | goto out_fail; |
| 1284 | } |
James Smart | bd2cdd5 | 2017-02-12 13:52:33 -0800 | [diff] [blame] | 1285 | #ifdef CONFIG_SCSI_LPFC_DEBUG_FS |
| 1286 | if (phba->ktime_on) { |
| 1287 | lpfc_ncmd->ts_cmd_start = start; |
| 1288 | lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd; |
| 1289 | } |
| 1290 | #endif |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1291 | |
| 1292 | /* |
| 1293 | * Store the data needed by the driver to issue, abort, and complete |
| 1294 | * an IO. |
| 1295 | * Do not let the IO hang out forever. There is no midlayer issuing |
| 1296 | * an abort so inform the FW of the maximum IO pending time. |
| 1297 | */ |
James Smart | bbe3012 | 2017-04-21 17:49:08 -0700 | [diff] [blame] | 1298 | freqpriv->nvme_buf = lpfc_ncmd; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1299 | lpfc_ncmd->nvmeCmd = pnvme_fcreq; |
| 1300 | lpfc_ncmd->nrport = rport; |
James Smart | 318083a | 2017-03-04 09:30:30 -0800 | [diff] [blame] | 1301 | lpfc_ncmd->ndlp = ndlp; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1302 | lpfc_ncmd->start_time = jiffies; |
| 1303 | |
| 1304 | lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp); |
| 1305 | ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd); |
| 1306 | if (ret) { |
| 1307 | ret = -ENOMEM; |
| 1308 | goto out_free_nvme_buf; |
| 1309 | } |
| 1310 | |
| 1311 | atomic_inc(&ndlp->cmd_pending); |
| 1312 | |
| 1313 | /* |
| 1314 | * Issue the IO on the WQ indicated by index in the hw_queue_handle. |
| 1315 | * This identfier was create in our hardware queue create callback |
| 1316 | * routine. The driver now is dependent on the IO queue steering from |
| 1317 | * the transport. We are trusting the upper NVME layers know which |
| 1318 | * index to use and that they have affinitized a CPU to this hardware |
| 1319 | * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ. |
| 1320 | */ |
| 1321 | lpfc_ncmd->cur_iocbq.hba_wqidx = lpfc_queue_info->index; |
| 1322 | |
James Smart | bd2cdd5 | 2017-02-12 13:52:33 -0800 | [diff] [blame] | 1323 | lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n", |
| 1324 | lpfc_ncmd->cur_iocbq.sli4_xritag, |
| 1325 | lpfc_queue_info->index, ndlp->nlp_DID); |
| 1326 | |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1327 | ret = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, &lpfc_ncmd->cur_iocbq); |
| 1328 | if (ret) { |
| 1329 | atomic_dec(&ndlp->cmd_pending); |
| 1330 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_IOERR, |
| 1331 | "6113 FCP could not issue WQE err %x " |
| 1332 | "sid: x%x did: x%x oxid: x%x\n", |
| 1333 | ret, vport->fc_myDID, ndlp->nlp_DID, |
| 1334 | lpfc_ncmd->cur_iocbq.sli4_xritag); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1335 | goto out_free_nvme_buf; |
| 1336 | } |
| 1337 | |
James Smart | bd2cdd5 | 2017-02-12 13:52:33 -0800 | [diff] [blame] | 1338 | #ifdef CONFIG_SCSI_LPFC_DEBUG_FS |
| 1339 | if (phba->ktime_on) |
| 1340 | lpfc_ncmd->ts_cmd_wqput = ktime_get_ns(); |
| 1341 | |
| 1342 | if (phba->cpucheck_on & LPFC_CHECK_NVME_IO) { |
| 1343 | lpfc_ncmd->cpu = smp_processor_id(); |
| 1344 | if (lpfc_ncmd->cpu != lpfc_queue_info->index) { |
| 1345 | /* Check for admin queue */ |
| 1346 | if (lpfc_queue_info->qidx) { |
| 1347 | lpfc_printf_vlog(vport, |
| 1348 | KERN_ERR, LOG_NVME_IOERR, |
| 1349 | "6702 CPU Check cmd: " |
| 1350 | "cpu %d wq %d\n", |
| 1351 | lpfc_ncmd->cpu, |
| 1352 | lpfc_queue_info->index); |
| 1353 | } |
| 1354 | lpfc_ncmd->cpu = lpfc_queue_info->index; |
| 1355 | } |
| 1356 | if (lpfc_ncmd->cpu < LPFC_CHECK_CPU_CNT) |
| 1357 | phba->cpucheck_xmt_io[lpfc_ncmd->cpu]++; |
| 1358 | } |
| 1359 | #endif |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1360 | return 0; |
| 1361 | |
| 1362 | out_free_nvme_buf: |
James Smart | 2cee780 | 2017-06-01 21:07:02 -0700 | [diff] [blame] | 1363 | if (lpfc_ncmd->nvmeCmd->sg_cnt) { |
| 1364 | if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE) |
| 1365 | atomic_dec(&phba->fc4NvmeOutputRequests); |
| 1366 | else |
| 1367 | atomic_dec(&phba->fc4NvmeInputRequests); |
| 1368 | } else |
| 1369 | atomic_dec(&phba->fc4NvmeControlRequests); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1370 | lpfc_release_nvme_buf(phba, lpfc_ncmd); |
| 1371 | out_fail: |
| 1372 | return ret; |
| 1373 | } |
| 1374 | |
| 1375 | /** |
| 1376 | * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request. |
| 1377 | * @phba: Pointer to HBA context object |
| 1378 | * @cmdiocb: Pointer to command iocb object. |
| 1379 | * @rspiocb: Pointer to response iocb object. |
| 1380 | * |
| 1381 | * This is the callback function for any NVME FCP IO that was aborted. |
| 1382 | * |
| 1383 | * Return value: |
| 1384 | * None |
| 1385 | **/ |
| 1386 | void |
| 1387 | lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb, |
| 1388 | struct lpfc_wcqe_complete *abts_cmpl) |
| 1389 | { |
| 1390 | lpfc_printf_log(phba, KERN_ERR, LOG_NVME, |
| 1391 | "6145 ABORT_XRI_CN completing on rpi x%x " |
| 1392 | "original iotag x%x, abort cmd iotag x%x " |
| 1393 | "req_tag x%x, status x%x, hwstatus x%x\n", |
| 1394 | cmdiocb->iocb.un.acxri.abortContextTag, |
| 1395 | cmdiocb->iocb.un.acxri.abortIoTag, |
| 1396 | cmdiocb->iotag, |
| 1397 | bf_get(lpfc_wcqe_c_request_tag, abts_cmpl), |
| 1398 | bf_get(lpfc_wcqe_c_status, abts_cmpl), |
| 1399 | bf_get(lpfc_wcqe_c_hw_status, abts_cmpl)); |
| 1400 | lpfc_sli_release_iocbq(phba, cmdiocb); |
| 1401 | } |
| 1402 | |
| 1403 | /** |
| 1404 | * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS |
| 1405 | * @lpfc_pnvme: Pointer to the driver's nvme instance data |
| 1406 | * @lpfc_nvme_lport: Pointer to the driver's local port data |
| 1407 | * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq |
| 1408 | * @lpfc_nvme_fcreq: IO request from nvme fc to driver. |
| 1409 | * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue |
| 1410 | * |
| 1411 | * Driver registers this routine as its nvme request io abort handler. This |
| 1412 | * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq |
| 1413 | * data structure to the rport indicated in @lpfc_nvme_rport. This routine |
| 1414 | * is executed asynchronously - one the target is validated as "MAPPED" and |
| 1415 | * ready for IO, the driver issues the abort request and returns. |
| 1416 | * |
| 1417 | * Return value: |
| 1418 | * None |
| 1419 | **/ |
| 1420 | static void |
| 1421 | lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport, |
| 1422 | struct nvme_fc_remote_port *pnvme_rport, |
| 1423 | void *hw_queue_handle, |
| 1424 | struct nvmefc_fcp_req *pnvme_fcreq) |
| 1425 | { |
| 1426 | struct lpfc_nvme_lport *lport; |
| 1427 | struct lpfc_vport *vport; |
| 1428 | struct lpfc_hba *phba; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1429 | struct lpfc_nvme_rport *rport; |
| 1430 | struct lpfc_nvme_buf *lpfc_nbuf; |
| 1431 | struct lpfc_iocbq *abts_buf; |
| 1432 | struct lpfc_iocbq *nvmereq_wqe; |
James Smart | bbe3012 | 2017-04-21 17:49:08 -0700 | [diff] [blame] | 1433 | struct lpfc_nvme_fcpreq_priv *freqpriv = pnvme_fcreq->private; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1434 | union lpfc_wqe *abts_wqe; |
| 1435 | unsigned long flags; |
| 1436 | int ret_val; |
| 1437 | |
| 1438 | lport = (struct lpfc_nvme_lport *)pnvme_lport->private; |
| 1439 | rport = (struct lpfc_nvme_rport *)pnvme_rport->private; |
| 1440 | vport = lport->vport; |
| 1441 | phba = vport->phba; |
| 1442 | |
| 1443 | /* Announce entry to new IO submit field. */ |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 1444 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1445 | "6002 Abort Request to rport DID x%06x " |
| 1446 | "for nvme_fc_req %p\n", |
| 1447 | pnvme_rport->port_id, |
| 1448 | pnvme_fcreq); |
| 1449 | |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1450 | /* If the hba is getting reset, this flag is set. It is |
| 1451 | * cleared when the reset is complete and rings reestablished. |
| 1452 | */ |
| 1453 | spin_lock_irqsave(&phba->hbalock, flags); |
| 1454 | /* driver queued commands are in process of being flushed */ |
| 1455 | if (phba->hba_flag & HBA_NVME_IOQ_FLUSH) { |
| 1456 | spin_unlock_irqrestore(&phba->hbalock, flags); |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 1457 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1458 | "6139 Driver in reset cleanup - flushing " |
| 1459 | "NVME Req now. hba_flag x%x\n", |
| 1460 | phba->hba_flag); |
| 1461 | return; |
| 1462 | } |
| 1463 | |
James Smart | bbe3012 | 2017-04-21 17:49:08 -0700 | [diff] [blame] | 1464 | lpfc_nbuf = freqpriv->nvme_buf; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1465 | if (!lpfc_nbuf) { |
| 1466 | spin_unlock_irqrestore(&phba->hbalock, flags); |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 1467 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1468 | "6140 NVME IO req has no matching lpfc nvme " |
| 1469 | "io buffer. Skipping abort req.\n"); |
| 1470 | return; |
| 1471 | } else if (!lpfc_nbuf->nvmeCmd) { |
| 1472 | spin_unlock_irqrestore(&phba->hbalock, flags); |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 1473 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1474 | "6141 lpfc NVME IO req has no nvme_fcreq " |
| 1475 | "io buffer. Skipping abort req.\n"); |
| 1476 | return; |
| 1477 | } |
James Smart | 2b7824d | 2017-04-21 16:04:59 -0700 | [diff] [blame] | 1478 | nvmereq_wqe = &lpfc_nbuf->cur_iocbq; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1479 | |
| 1480 | /* |
| 1481 | * The lpfc_nbuf and the mapped nvme_fcreq in the driver's |
| 1482 | * state must match the nvme_fcreq passed by the nvme |
| 1483 | * transport. If they don't match, it is likely the driver |
| 1484 | * has already completed the NVME IO and the nvme transport |
| 1485 | * has not seen it yet. |
| 1486 | */ |
| 1487 | if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) { |
| 1488 | spin_unlock_irqrestore(&phba->hbalock, flags); |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 1489 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1490 | "6143 NVME req mismatch: " |
| 1491 | "lpfc_nbuf %p nvmeCmd %p, " |
James Smart | 2b7824d | 2017-04-21 16:04:59 -0700 | [diff] [blame] | 1492 | "pnvme_fcreq %p. Skipping Abort xri x%x\n", |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1493 | lpfc_nbuf, lpfc_nbuf->nvmeCmd, |
James Smart | 2b7824d | 2017-04-21 16:04:59 -0700 | [diff] [blame] | 1494 | pnvme_fcreq, nvmereq_wqe->sli4_xritag); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1495 | return; |
| 1496 | } |
| 1497 | |
| 1498 | /* Don't abort IOs no longer on the pending queue. */ |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1499 | if (!(nvmereq_wqe->iocb_flag & LPFC_IO_ON_TXCMPLQ)) { |
| 1500 | spin_unlock_irqrestore(&phba->hbalock, flags); |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 1501 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1502 | "6142 NVME IO req %p not queued - skipping " |
James Smart | 2b7824d | 2017-04-21 16:04:59 -0700 | [diff] [blame] | 1503 | "abort req xri x%x\n", |
| 1504 | pnvme_fcreq, nvmereq_wqe->sli4_xritag); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1505 | return; |
| 1506 | } |
| 1507 | |
James Smart | bd2cdd5 | 2017-02-12 13:52:33 -0800 | [diff] [blame] | 1508 | lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n", |
| 1509 | nvmereq_wqe->sli4_xritag, |
James Smart | 00cefeb | 2017-06-21 10:37:58 -0700 | [diff] [blame] | 1510 | nvmereq_wqe->hba_wqidx, pnvme_rport->port_id); |
James Smart | bd2cdd5 | 2017-02-12 13:52:33 -0800 | [diff] [blame] | 1511 | |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1512 | /* Outstanding abort is in progress */ |
| 1513 | if (nvmereq_wqe->iocb_flag & LPFC_DRIVER_ABORTED) { |
| 1514 | spin_unlock_irqrestore(&phba->hbalock, flags); |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 1515 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1516 | "6144 Outstanding NVME I/O Abort Request " |
| 1517 | "still pending on nvme_fcreq %p, " |
James Smart | 2b7824d | 2017-04-21 16:04:59 -0700 | [diff] [blame] | 1518 | "lpfc_ncmd %p xri x%x\n", |
| 1519 | pnvme_fcreq, lpfc_nbuf, |
| 1520 | nvmereq_wqe->sli4_xritag); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1521 | return; |
| 1522 | } |
| 1523 | |
| 1524 | abts_buf = __lpfc_sli_get_iocbq(phba); |
| 1525 | if (!abts_buf) { |
| 1526 | spin_unlock_irqrestore(&phba->hbalock, flags); |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 1527 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1528 | "6136 No available abort wqes. Skipping " |
James Smart | 2b7824d | 2017-04-21 16:04:59 -0700 | [diff] [blame] | 1529 | "Abts req for nvme_fcreq %p xri x%x\n", |
| 1530 | pnvme_fcreq, nvmereq_wqe->sli4_xritag); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1531 | return; |
| 1532 | } |
| 1533 | |
| 1534 | /* Ready - mark outstanding as aborted by driver. */ |
| 1535 | nvmereq_wqe->iocb_flag |= LPFC_DRIVER_ABORTED; |
| 1536 | |
| 1537 | /* Complete prepping the abort wqe and issue to the FW. */ |
| 1538 | abts_wqe = &abts_buf->wqe; |
| 1539 | |
| 1540 | /* WQEs are reused. Clear stale data and set key fields to |
| 1541 | * zero like ia, iaab, iaar, xri_tag, and ctxt_tag. |
| 1542 | */ |
| 1543 | memset(abts_wqe, 0, sizeof(union lpfc_wqe)); |
| 1544 | bf_set(abort_cmd_criteria, &abts_wqe->abort_cmd, T_XRI_TAG); |
| 1545 | |
| 1546 | /* word 7 */ |
| 1547 | bf_set(wqe_ct, &abts_wqe->abort_cmd.wqe_com, 0); |
| 1548 | bf_set(wqe_cmnd, &abts_wqe->abort_cmd.wqe_com, CMD_ABORT_XRI_CX); |
| 1549 | bf_set(wqe_class, &abts_wqe->abort_cmd.wqe_com, |
| 1550 | nvmereq_wqe->iocb.ulpClass); |
| 1551 | |
| 1552 | /* word 8 - tell the FW to abort the IO associated with this |
| 1553 | * outstanding exchange ID. |
| 1554 | */ |
| 1555 | abts_wqe->abort_cmd.wqe_com.abort_tag = nvmereq_wqe->sli4_xritag; |
| 1556 | |
| 1557 | /* word 9 - this is the iotag for the abts_wqe completion. */ |
| 1558 | bf_set(wqe_reqtag, &abts_wqe->abort_cmd.wqe_com, |
| 1559 | abts_buf->iotag); |
| 1560 | |
| 1561 | /* word 10 */ |
| 1562 | bf_set(wqe_wqid, &abts_wqe->abort_cmd.wqe_com, nvmereq_wqe->hba_wqidx); |
| 1563 | bf_set(wqe_qosd, &abts_wqe->abort_cmd.wqe_com, 1); |
| 1564 | bf_set(wqe_lenloc, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_LENLOC_NONE); |
| 1565 | |
| 1566 | /* word 11 */ |
| 1567 | bf_set(wqe_cmd_type, &abts_wqe->abort_cmd.wqe_com, OTHER_COMMAND); |
| 1568 | bf_set(wqe_wqec, &abts_wqe->abort_cmd.wqe_com, 1); |
| 1569 | bf_set(wqe_cqid, &abts_wqe->abort_cmd.wqe_com, LPFC_WQE_CQ_ID_DEFAULT); |
| 1570 | |
| 1571 | /* ABTS WQE must go to the same WQ as the WQE to be aborted */ |
| 1572 | abts_buf->iocb_flag |= LPFC_IO_NVME; |
| 1573 | abts_buf->hba_wqidx = nvmereq_wqe->hba_wqidx; |
| 1574 | abts_buf->vport = vport; |
| 1575 | abts_buf->wqe_cmpl = lpfc_nvme_abort_fcreq_cmpl; |
| 1576 | ret_val = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, abts_buf); |
| 1577 | spin_unlock_irqrestore(&phba->hbalock, flags); |
Dick Kennedy | cd22d60 | 2017-08-23 16:55:35 -0700 | [diff] [blame] | 1578 | if (ret_val) { |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 1579 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_ABTS, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1580 | "6137 Failed abts issue_wqe with status x%x " |
| 1581 | "for nvme_fcreq %p.\n", |
| 1582 | ret_val, pnvme_fcreq); |
| 1583 | lpfc_sli_release_iocbq(phba, abts_buf); |
| 1584 | return; |
| 1585 | } |
| 1586 | |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 1587 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS, |
James Smart | 2b7824d | 2017-04-21 16:04:59 -0700 | [diff] [blame] | 1588 | "6138 Transport Abort NVME Request Issued for " |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1589 | "ox_id x%x on reqtag x%x\n", |
| 1590 | nvmereq_wqe->sli4_xritag, |
| 1591 | abts_buf->iotag); |
| 1592 | } |
| 1593 | |
| 1594 | /* Declare and initialization an instance of the FC NVME template. */ |
| 1595 | static struct nvme_fc_port_template lpfc_nvme_template = { |
| 1596 | /* initiator-based functions */ |
| 1597 | .localport_delete = lpfc_nvme_localport_delete, |
| 1598 | .remoteport_delete = lpfc_nvme_remoteport_delete, |
| 1599 | .create_queue = lpfc_nvme_create_queue, |
| 1600 | .delete_queue = lpfc_nvme_delete_queue, |
| 1601 | .ls_req = lpfc_nvme_ls_req, |
| 1602 | .fcp_io = lpfc_nvme_fcp_io_submit, |
| 1603 | .ls_abort = lpfc_nvme_ls_abort, |
| 1604 | .fcp_abort = lpfc_nvme_fcp_abort, |
| 1605 | |
| 1606 | .max_hw_queues = 1, |
| 1607 | .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS, |
| 1608 | .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS, |
| 1609 | .dma_boundary = 0xFFFFFFFF, |
| 1610 | |
| 1611 | /* Sizes of additional private data for data structures. |
| 1612 | * No use for the last two sizes at this time. |
| 1613 | */ |
| 1614 | .local_priv_sz = sizeof(struct lpfc_nvme_lport), |
| 1615 | .remote_priv_sz = sizeof(struct lpfc_nvme_rport), |
| 1616 | .lsrqst_priv_sz = 0, |
James Smart | bbe3012 | 2017-04-21 17:49:08 -0700 | [diff] [blame] | 1617 | .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv), |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1618 | }; |
| 1619 | |
| 1620 | /** |
| 1621 | * lpfc_sli4_post_nvme_sgl_block - post a block of nvme sgl list to firmware |
| 1622 | * @phba: pointer to lpfc hba data structure. |
| 1623 | * @nblist: pointer to nvme buffer list. |
| 1624 | * @count: number of scsi buffers on the list. |
| 1625 | * |
| 1626 | * This routine is invoked to post a block of @count scsi sgl pages from a |
| 1627 | * SCSI buffer list @nblist to the HBA using non-embedded mailbox command. |
| 1628 | * No Lock is held. |
| 1629 | * |
| 1630 | **/ |
| 1631 | static int |
| 1632 | lpfc_sli4_post_nvme_sgl_block(struct lpfc_hba *phba, |
| 1633 | struct list_head *nblist, |
| 1634 | int count) |
| 1635 | { |
| 1636 | struct lpfc_nvme_buf *lpfc_ncmd; |
| 1637 | struct lpfc_mbx_post_uembed_sgl_page1 *sgl; |
| 1638 | struct sgl_page_pairs *sgl_pg_pairs; |
| 1639 | void *viraddr; |
| 1640 | LPFC_MBOXQ_t *mbox; |
| 1641 | uint32_t reqlen, alloclen, pg_pairs; |
| 1642 | uint32_t mbox_tmo; |
| 1643 | uint16_t xritag_start = 0; |
| 1644 | int rc = 0; |
| 1645 | uint32_t shdr_status, shdr_add_status; |
| 1646 | dma_addr_t pdma_phys_bpl1; |
| 1647 | union lpfc_sli4_cfg_shdr *shdr; |
| 1648 | |
| 1649 | /* Calculate the requested length of the dma memory */ |
| 1650 | reqlen = count * sizeof(struct sgl_page_pairs) + |
| 1651 | sizeof(union lpfc_sli4_cfg_shdr) + sizeof(uint32_t); |
| 1652 | if (reqlen > SLI4_PAGE_SIZE) { |
| 1653 | lpfc_printf_log(phba, KERN_WARNING, LOG_INIT, |
| 1654 | "6118 Block sgl registration required DMA " |
| 1655 | "size (%d) great than a page\n", reqlen); |
| 1656 | return -ENOMEM; |
| 1657 | } |
| 1658 | mbox = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL); |
| 1659 | if (!mbox) { |
| 1660 | lpfc_printf_log(phba, KERN_ERR, LOG_INIT, |
| 1661 | "6119 Failed to allocate mbox cmd memory\n"); |
| 1662 | return -ENOMEM; |
| 1663 | } |
| 1664 | |
| 1665 | /* Allocate DMA memory and set up the non-embedded mailbox command */ |
| 1666 | alloclen = lpfc_sli4_config(phba, mbox, LPFC_MBOX_SUBSYSTEM_FCOE, |
| 1667 | LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES, reqlen, |
| 1668 | LPFC_SLI4_MBX_NEMBED); |
| 1669 | |
| 1670 | if (alloclen < reqlen) { |
| 1671 | lpfc_printf_log(phba, KERN_ERR, LOG_INIT, |
| 1672 | "6120 Allocated DMA memory size (%d) is " |
| 1673 | "less than the requested DMA memory " |
| 1674 | "size (%d)\n", alloclen, reqlen); |
| 1675 | lpfc_sli4_mbox_cmd_free(phba, mbox); |
| 1676 | return -ENOMEM; |
| 1677 | } |
| 1678 | |
| 1679 | /* Get the first SGE entry from the non-embedded DMA memory */ |
| 1680 | viraddr = mbox->sge_array->addr[0]; |
| 1681 | |
| 1682 | /* Set up the SGL pages in the non-embedded DMA pages */ |
| 1683 | sgl = (struct lpfc_mbx_post_uembed_sgl_page1 *)viraddr; |
| 1684 | sgl_pg_pairs = &sgl->sgl_pg_pairs; |
| 1685 | |
| 1686 | pg_pairs = 0; |
| 1687 | list_for_each_entry(lpfc_ncmd, nblist, list) { |
| 1688 | /* Set up the sge entry */ |
| 1689 | sgl_pg_pairs->sgl_pg0_addr_lo = |
| 1690 | cpu_to_le32(putPaddrLow(lpfc_ncmd->dma_phys_sgl)); |
| 1691 | sgl_pg_pairs->sgl_pg0_addr_hi = |
| 1692 | cpu_to_le32(putPaddrHigh(lpfc_ncmd->dma_phys_sgl)); |
| 1693 | if (phba->cfg_sg_dma_buf_size > SGL_PAGE_SIZE) |
| 1694 | pdma_phys_bpl1 = lpfc_ncmd->dma_phys_sgl + |
| 1695 | SGL_PAGE_SIZE; |
| 1696 | else |
| 1697 | pdma_phys_bpl1 = 0; |
| 1698 | sgl_pg_pairs->sgl_pg1_addr_lo = |
| 1699 | cpu_to_le32(putPaddrLow(pdma_phys_bpl1)); |
| 1700 | sgl_pg_pairs->sgl_pg1_addr_hi = |
| 1701 | cpu_to_le32(putPaddrHigh(pdma_phys_bpl1)); |
| 1702 | /* Keep the first xritag on the list */ |
| 1703 | if (pg_pairs == 0) |
| 1704 | xritag_start = lpfc_ncmd->cur_iocbq.sli4_xritag; |
| 1705 | sgl_pg_pairs++; |
| 1706 | pg_pairs++; |
| 1707 | } |
| 1708 | bf_set(lpfc_post_sgl_pages_xri, sgl, xritag_start); |
| 1709 | bf_set(lpfc_post_sgl_pages_xricnt, sgl, pg_pairs); |
| 1710 | /* Perform endian conversion if necessary */ |
| 1711 | sgl->word0 = cpu_to_le32(sgl->word0); |
| 1712 | |
| 1713 | if (!phba->sli4_hba.intr_enable) |
| 1714 | rc = lpfc_sli_issue_mbox(phba, mbox, MBX_POLL); |
| 1715 | else { |
| 1716 | mbox_tmo = lpfc_mbox_tmo_val(phba, mbox); |
| 1717 | rc = lpfc_sli_issue_mbox_wait(phba, mbox, mbox_tmo); |
| 1718 | } |
| 1719 | shdr = (union lpfc_sli4_cfg_shdr *)&sgl->cfg_shdr; |
| 1720 | shdr_status = bf_get(lpfc_mbox_hdr_status, &shdr->response); |
| 1721 | shdr_add_status = bf_get(lpfc_mbox_hdr_add_status, &shdr->response); |
| 1722 | if (rc != MBX_TIMEOUT) |
| 1723 | lpfc_sli4_mbox_cmd_free(phba, mbox); |
| 1724 | if (shdr_status || shdr_add_status || rc) { |
| 1725 | lpfc_printf_log(phba, KERN_ERR, LOG_SLI, |
| 1726 | "6125 POST_SGL_BLOCK mailbox command failed " |
| 1727 | "status x%x add_status x%x mbx status x%x\n", |
| 1728 | shdr_status, shdr_add_status, rc); |
| 1729 | rc = -ENXIO; |
| 1730 | } |
| 1731 | return rc; |
| 1732 | } |
| 1733 | |
| 1734 | /** |
| 1735 | * lpfc_post_nvme_sgl_list - Post blocks of nvme buffer sgls from a list |
| 1736 | * @phba: pointer to lpfc hba data structure. |
| 1737 | * @post_nblist: pointer to the nvme buffer list. |
| 1738 | * |
| 1739 | * This routine walks a list of nvme buffers that was passed in. It attempts |
| 1740 | * to construct blocks of nvme buffer sgls which contains contiguous xris and |
| 1741 | * uses the non-embedded SGL block post mailbox commands to post to the port. |
| 1742 | * For single NVME buffer sgl with non-contiguous xri, if any, it shall use |
| 1743 | * embedded SGL post mailbox command for posting. The @post_nblist passed in |
| 1744 | * must be local list, thus no lock is needed when manipulate the list. |
| 1745 | * |
| 1746 | * Returns: 0 = failure, non-zero number of successfully posted buffers. |
| 1747 | **/ |
| 1748 | static int |
| 1749 | lpfc_post_nvme_sgl_list(struct lpfc_hba *phba, |
| 1750 | struct list_head *post_nblist, int sb_count) |
| 1751 | { |
| 1752 | struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next; |
| 1753 | int status, sgl_size; |
| 1754 | int post_cnt = 0, block_cnt = 0, num_posting = 0, num_posted = 0; |
| 1755 | dma_addr_t pdma_phys_sgl1; |
| 1756 | int last_xritag = NO_XRI; |
| 1757 | int cur_xritag; |
| 1758 | LIST_HEAD(prep_nblist); |
| 1759 | LIST_HEAD(blck_nblist); |
| 1760 | LIST_HEAD(nvme_nblist); |
| 1761 | |
| 1762 | /* sanity check */ |
| 1763 | if (sb_count <= 0) |
| 1764 | return -EINVAL; |
| 1765 | |
| 1766 | sgl_size = phba->cfg_sg_dma_buf_size; |
| 1767 | |
| 1768 | list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, post_nblist, list) { |
| 1769 | list_del_init(&lpfc_ncmd->list); |
| 1770 | block_cnt++; |
| 1771 | if ((last_xritag != NO_XRI) && |
| 1772 | (lpfc_ncmd->cur_iocbq.sli4_xritag != last_xritag + 1)) { |
| 1773 | /* a hole in xri block, form a sgl posting block */ |
| 1774 | list_splice_init(&prep_nblist, &blck_nblist); |
| 1775 | post_cnt = block_cnt - 1; |
| 1776 | /* prepare list for next posting block */ |
| 1777 | list_add_tail(&lpfc_ncmd->list, &prep_nblist); |
| 1778 | block_cnt = 1; |
| 1779 | } else { |
| 1780 | /* prepare list for next posting block */ |
| 1781 | list_add_tail(&lpfc_ncmd->list, &prep_nblist); |
| 1782 | /* enough sgls for non-embed sgl mbox command */ |
| 1783 | if (block_cnt == LPFC_NEMBED_MBOX_SGL_CNT) { |
| 1784 | list_splice_init(&prep_nblist, &blck_nblist); |
| 1785 | post_cnt = block_cnt; |
| 1786 | block_cnt = 0; |
| 1787 | } |
| 1788 | } |
| 1789 | num_posting++; |
| 1790 | last_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag; |
| 1791 | |
| 1792 | /* end of repost sgl list condition for NVME buffers */ |
| 1793 | if (num_posting == sb_count) { |
| 1794 | if (post_cnt == 0) { |
| 1795 | /* last sgl posting block */ |
| 1796 | list_splice_init(&prep_nblist, &blck_nblist); |
| 1797 | post_cnt = block_cnt; |
| 1798 | } else if (block_cnt == 1) { |
| 1799 | /* last single sgl with non-contiguous xri */ |
| 1800 | if (sgl_size > SGL_PAGE_SIZE) |
| 1801 | pdma_phys_sgl1 = |
| 1802 | lpfc_ncmd->dma_phys_sgl + |
| 1803 | SGL_PAGE_SIZE; |
| 1804 | else |
| 1805 | pdma_phys_sgl1 = 0; |
| 1806 | cur_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag; |
| 1807 | status = lpfc_sli4_post_sgl(phba, |
| 1808 | lpfc_ncmd->dma_phys_sgl, |
| 1809 | pdma_phys_sgl1, cur_xritag); |
| 1810 | if (status) { |
| 1811 | /* failure, put on abort nvme list */ |
James Smart | 318083a | 2017-03-04 09:30:30 -0800 | [diff] [blame] | 1812 | lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1813 | } else { |
| 1814 | /* success, put on NVME buffer list */ |
James Smart | 318083a | 2017-03-04 09:30:30 -0800 | [diff] [blame] | 1815 | lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1816 | lpfc_ncmd->status = IOSTAT_SUCCESS; |
| 1817 | num_posted++; |
| 1818 | } |
| 1819 | /* success, put on NVME buffer sgl list */ |
| 1820 | list_add_tail(&lpfc_ncmd->list, &nvme_nblist); |
| 1821 | } |
| 1822 | } |
| 1823 | |
| 1824 | /* continue until a nembed page worth of sgls */ |
| 1825 | if (post_cnt == 0) |
| 1826 | continue; |
| 1827 | |
| 1828 | /* post block of NVME buffer list sgls */ |
| 1829 | status = lpfc_sli4_post_nvme_sgl_block(phba, &blck_nblist, |
| 1830 | post_cnt); |
| 1831 | |
| 1832 | /* don't reset xirtag due to hole in xri block */ |
| 1833 | if (block_cnt == 0) |
| 1834 | last_xritag = NO_XRI; |
| 1835 | |
| 1836 | /* reset NVME buffer post count for next round of posting */ |
| 1837 | post_cnt = 0; |
| 1838 | |
| 1839 | /* put posted NVME buffer-sgl posted on NVME buffer sgl list */ |
| 1840 | while (!list_empty(&blck_nblist)) { |
| 1841 | list_remove_head(&blck_nblist, lpfc_ncmd, |
| 1842 | struct lpfc_nvme_buf, list); |
| 1843 | if (status) { |
| 1844 | /* failure, put on abort nvme list */ |
James Smart | 318083a | 2017-03-04 09:30:30 -0800 | [diff] [blame] | 1845 | lpfc_ncmd->flags |= LPFC_SBUF_XBUSY; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1846 | } else { |
| 1847 | /* success, put on NVME buffer list */ |
James Smart | 318083a | 2017-03-04 09:30:30 -0800 | [diff] [blame] | 1848 | lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1849 | lpfc_ncmd->status = IOSTAT_SUCCESS; |
| 1850 | num_posted++; |
| 1851 | } |
| 1852 | list_add_tail(&lpfc_ncmd->list, &nvme_nblist); |
| 1853 | } |
| 1854 | } |
| 1855 | /* Push NVME buffers with sgl posted to the available list */ |
| 1856 | while (!list_empty(&nvme_nblist)) { |
| 1857 | list_remove_head(&nvme_nblist, lpfc_ncmd, |
| 1858 | struct lpfc_nvme_buf, list); |
| 1859 | lpfc_release_nvme_buf(phba, lpfc_ncmd); |
| 1860 | } |
| 1861 | return num_posted; |
| 1862 | } |
| 1863 | |
| 1864 | /** |
| 1865 | * lpfc_repost_nvme_sgl_list - Repost all the allocated nvme buffer sgls |
| 1866 | * @phba: pointer to lpfc hba data structure. |
| 1867 | * |
| 1868 | * This routine walks the list of nvme buffers that have been allocated and |
| 1869 | * repost them to the port by using SGL block post. This is needed after a |
| 1870 | * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine |
| 1871 | * is responsible for moving all nvme buffers on the lpfc_abts_nvme_sgl_list |
| 1872 | * to the lpfc_nvme_buf_list. If the repost fails, reject all nvme buffers. |
| 1873 | * |
| 1874 | * Returns: 0 = success, non-zero failure. |
| 1875 | **/ |
| 1876 | int |
| 1877 | lpfc_repost_nvme_sgl_list(struct lpfc_hba *phba) |
| 1878 | { |
| 1879 | LIST_HEAD(post_nblist); |
| 1880 | int num_posted, rc = 0; |
| 1881 | |
| 1882 | /* get all NVME buffers need to repost to a local list */ |
| 1883 | spin_lock_irq(&phba->nvme_buf_list_get_lock); |
| 1884 | spin_lock(&phba->nvme_buf_list_put_lock); |
| 1885 | list_splice_init(&phba->lpfc_nvme_buf_list_get, &post_nblist); |
| 1886 | list_splice(&phba->lpfc_nvme_buf_list_put, &post_nblist); |
| 1887 | spin_unlock(&phba->nvme_buf_list_put_lock); |
| 1888 | spin_unlock_irq(&phba->nvme_buf_list_get_lock); |
| 1889 | |
| 1890 | /* post the list of nvme buffer sgls to port if available */ |
| 1891 | if (!list_empty(&post_nblist)) { |
| 1892 | num_posted = lpfc_post_nvme_sgl_list(phba, &post_nblist, |
| 1893 | phba->sli4_hba.nvme_xri_cnt); |
| 1894 | /* failed to post any nvme buffer, return error */ |
| 1895 | if (num_posted == 0) |
| 1896 | rc = -EIO; |
| 1897 | } |
| 1898 | return rc; |
| 1899 | } |
| 1900 | |
| 1901 | /** |
| 1902 | * lpfc_new_nvme_buf - Scsi buffer allocator for HBA with SLI4 IF spec |
| 1903 | * @vport: The virtual port for which this call being executed. |
| 1904 | * @num_to_allocate: The requested number of buffers to allocate. |
| 1905 | * |
| 1906 | * This routine allocates nvme buffers for device with SLI-4 interface spec, |
| 1907 | * the nvme buffer contains all the necessary information needed to initiate |
| 1908 | * a NVME I/O. After allocating up to @num_to_allocate NVME buffers and put |
| 1909 | * them on a list, it post them to the port by using SGL block post. |
| 1910 | * |
| 1911 | * Return codes: |
| 1912 | * int - number of nvme buffers that were allocated and posted. |
| 1913 | * 0 = failure, less than num_to_alloc is a partial failure. |
| 1914 | **/ |
| 1915 | static int |
| 1916 | lpfc_new_nvme_buf(struct lpfc_vport *vport, int num_to_alloc) |
| 1917 | { |
| 1918 | struct lpfc_hba *phba = vport->phba; |
| 1919 | struct lpfc_nvme_buf *lpfc_ncmd; |
| 1920 | struct lpfc_iocbq *pwqeq; |
| 1921 | union lpfc_wqe128 *wqe; |
| 1922 | struct sli4_sge *sgl; |
| 1923 | dma_addr_t pdma_phys_sgl; |
| 1924 | uint16_t iotag, lxri = 0; |
| 1925 | int bcnt, num_posted, sgl_size; |
| 1926 | LIST_HEAD(prep_nblist); |
| 1927 | LIST_HEAD(post_nblist); |
| 1928 | LIST_HEAD(nvme_nblist); |
| 1929 | |
| 1930 | sgl_size = phba->cfg_sg_dma_buf_size; |
| 1931 | |
| 1932 | for (bcnt = 0; bcnt < num_to_alloc; bcnt++) { |
| 1933 | lpfc_ncmd = kzalloc(sizeof(struct lpfc_nvme_buf), GFP_KERNEL); |
| 1934 | if (!lpfc_ncmd) |
| 1935 | break; |
| 1936 | /* |
| 1937 | * Get memory from the pci pool to map the virt space to |
| 1938 | * pci bus space for an I/O. The DMA buffer includes the |
| 1939 | * number of SGE's necessary to support the sg_tablesize. |
| 1940 | */ |
Romain Perier | 771db5c | 2017-07-06 10:13:05 +0200 | [diff] [blame] | 1941 | lpfc_ncmd->data = dma_pool_alloc(phba->lpfc_sg_dma_buf_pool, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1942 | GFP_KERNEL, |
| 1943 | &lpfc_ncmd->dma_handle); |
| 1944 | if (!lpfc_ncmd->data) { |
| 1945 | kfree(lpfc_ncmd); |
| 1946 | break; |
| 1947 | } |
| 1948 | memset(lpfc_ncmd->data, 0, phba->cfg_sg_dma_buf_size); |
| 1949 | |
| 1950 | lxri = lpfc_sli4_next_xritag(phba); |
| 1951 | if (lxri == NO_XRI) { |
Romain Perier | 771db5c | 2017-07-06 10:13:05 +0200 | [diff] [blame] | 1952 | dma_pool_free(phba->lpfc_sg_dma_buf_pool, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1953 | lpfc_ncmd->data, lpfc_ncmd->dma_handle); |
| 1954 | kfree(lpfc_ncmd); |
| 1955 | break; |
| 1956 | } |
| 1957 | pwqeq = &(lpfc_ncmd->cur_iocbq); |
| 1958 | wqe = (union lpfc_wqe128 *)&pwqeq->wqe; |
| 1959 | |
| 1960 | /* Allocate iotag for lpfc_ncmd->cur_iocbq. */ |
| 1961 | iotag = lpfc_sli_next_iotag(phba, pwqeq); |
| 1962 | if (iotag == 0) { |
Romain Perier | 771db5c | 2017-07-06 10:13:05 +0200 | [diff] [blame] | 1963 | dma_pool_free(phba->lpfc_sg_dma_buf_pool, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 1964 | lpfc_ncmd->data, lpfc_ncmd->dma_handle); |
| 1965 | kfree(lpfc_ncmd); |
| 1966 | lpfc_printf_log(phba, KERN_ERR, LOG_NVME_IOERR, |
| 1967 | "6121 Failed to allocated IOTAG for" |
| 1968 | " XRI:0x%x\n", lxri); |
| 1969 | lpfc_sli4_free_xri(phba, lxri); |
| 1970 | break; |
| 1971 | } |
| 1972 | pwqeq->sli4_lxritag = lxri; |
| 1973 | pwqeq->sli4_xritag = phba->sli4_hba.xri_ids[lxri]; |
| 1974 | pwqeq->iocb_flag |= LPFC_IO_NVME; |
| 1975 | pwqeq->context1 = lpfc_ncmd; |
| 1976 | pwqeq->wqe_cmpl = lpfc_nvme_io_cmd_wqe_cmpl; |
| 1977 | |
| 1978 | /* Initialize local short-hand pointers. */ |
| 1979 | lpfc_ncmd->nvme_sgl = lpfc_ncmd->data; |
| 1980 | sgl = lpfc_ncmd->nvme_sgl; |
| 1981 | pdma_phys_sgl = lpfc_ncmd->dma_handle; |
| 1982 | lpfc_ncmd->dma_phys_sgl = pdma_phys_sgl; |
| 1983 | |
| 1984 | /* Rsp SGE will be filled in when we rcv an IO |
| 1985 | * from the NVME Layer to be sent. |
| 1986 | * The cmd is going to be embedded so we need a SKIP SGE. |
| 1987 | */ |
| 1988 | bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP); |
| 1989 | bf_set(lpfc_sli4_sge_last, sgl, 0); |
| 1990 | sgl->word2 = cpu_to_le32(sgl->word2); |
| 1991 | /* Fill in word 3 / sgl_len during cmd submission */ |
| 1992 | |
| 1993 | lpfc_ncmd->cur_iocbq.context1 = lpfc_ncmd; |
| 1994 | |
| 1995 | /* Word 7 */ |
| 1996 | bf_set(wqe_erp, &wqe->generic.wqe_com, 0); |
| 1997 | /* NVME upper layers will time things out, if needed */ |
| 1998 | bf_set(wqe_tmo, &wqe->generic.wqe_com, 0); |
| 1999 | |
| 2000 | /* Word 10 */ |
| 2001 | bf_set(wqe_ebde_cnt, &wqe->generic.wqe_com, 0); |
| 2002 | bf_set(wqe_dbde, &wqe->generic.wqe_com, 1); |
| 2003 | |
| 2004 | /* add the nvme buffer to a post list */ |
| 2005 | list_add_tail(&lpfc_ncmd->list, &post_nblist); |
| 2006 | spin_lock_irq(&phba->nvme_buf_list_get_lock); |
| 2007 | phba->sli4_hba.nvme_xri_cnt++; |
| 2008 | spin_unlock_irq(&phba->nvme_buf_list_get_lock); |
| 2009 | } |
| 2010 | lpfc_printf_log(phba, KERN_INFO, LOG_NVME, |
| 2011 | "6114 Allocate %d out of %d requested new NVME " |
| 2012 | "buffers\n", bcnt, num_to_alloc); |
| 2013 | |
| 2014 | /* post the list of nvme buffer sgls to port if available */ |
| 2015 | if (!list_empty(&post_nblist)) |
| 2016 | num_posted = lpfc_post_nvme_sgl_list(phba, |
| 2017 | &post_nblist, bcnt); |
| 2018 | else |
| 2019 | num_posted = 0; |
| 2020 | |
| 2021 | return num_posted; |
| 2022 | } |
| 2023 | |
| 2024 | /** |
| 2025 | * lpfc_get_nvme_buf - Get a nvme buffer from lpfc_nvme_buf_list of the HBA |
| 2026 | * @phba: The HBA for which this call is being executed. |
| 2027 | * |
| 2028 | * This routine removes a nvme buffer from head of @phba lpfc_nvme_buf_list list |
| 2029 | * and returns to caller. |
| 2030 | * |
| 2031 | * Return codes: |
| 2032 | * NULL - Error |
| 2033 | * Pointer to lpfc_nvme_buf - Success |
| 2034 | **/ |
| 2035 | static struct lpfc_nvme_buf * |
| 2036 | lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp) |
| 2037 | { |
| 2038 | struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next; |
| 2039 | unsigned long iflag = 0; |
| 2040 | int found = 0; |
| 2041 | |
| 2042 | spin_lock_irqsave(&phba->nvme_buf_list_get_lock, iflag); |
| 2043 | list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, |
| 2044 | &phba->lpfc_nvme_buf_list_get, list) { |
| 2045 | if (lpfc_test_rrq_active(phba, ndlp, |
| 2046 | lpfc_ncmd->cur_iocbq.sli4_lxritag)) |
| 2047 | continue; |
James Smart | bbe3012 | 2017-04-21 17:49:08 -0700 | [diff] [blame] | 2048 | list_del_init(&lpfc_ncmd->list); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2049 | found = 1; |
| 2050 | break; |
| 2051 | } |
| 2052 | if (!found) { |
| 2053 | spin_lock(&phba->nvme_buf_list_put_lock); |
| 2054 | list_splice(&phba->lpfc_nvme_buf_list_put, |
| 2055 | &phba->lpfc_nvme_buf_list_get); |
| 2056 | INIT_LIST_HEAD(&phba->lpfc_nvme_buf_list_put); |
| 2057 | spin_unlock(&phba->nvme_buf_list_put_lock); |
| 2058 | list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, |
| 2059 | &phba->lpfc_nvme_buf_list_get, list) { |
| 2060 | if (lpfc_test_rrq_active( |
| 2061 | phba, ndlp, lpfc_ncmd->cur_iocbq.sli4_lxritag)) |
| 2062 | continue; |
James Smart | bbe3012 | 2017-04-21 17:49:08 -0700 | [diff] [blame] | 2063 | list_del_init(&lpfc_ncmd->list); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2064 | found = 1; |
| 2065 | break; |
| 2066 | } |
| 2067 | } |
| 2068 | spin_unlock_irqrestore(&phba->nvme_buf_list_get_lock, iflag); |
| 2069 | if (!found) |
| 2070 | return NULL; |
| 2071 | return lpfc_ncmd; |
| 2072 | } |
| 2073 | |
| 2074 | /** |
| 2075 | * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list. |
| 2076 | * @phba: The Hba for which this call is being executed. |
| 2077 | * @lpfc_ncmd: The nvme buffer which is being released. |
| 2078 | * |
| 2079 | * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba |
| 2080 | * lpfc_nvme_buf_list list. For SLI4 XRI's are tied to the nvme buffer |
| 2081 | * and cannot be reused for at least RA_TOV amount of time if it was |
| 2082 | * aborted. |
| 2083 | **/ |
| 2084 | static void |
| 2085 | lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_nvme_buf *lpfc_ncmd) |
| 2086 | { |
| 2087 | unsigned long iflag = 0; |
| 2088 | |
| 2089 | lpfc_ncmd->nonsg_phys = 0; |
James Smart | 318083a | 2017-03-04 09:30:30 -0800 | [diff] [blame] | 2090 | if (lpfc_ncmd->flags & LPFC_SBUF_XBUSY) { |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 2091 | lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, |
| 2092 | "6310 XB release deferred for " |
| 2093 | "ox_id x%x on reqtag x%x\n", |
| 2094 | lpfc_ncmd->cur_iocbq.sli4_xritag, |
| 2095 | lpfc_ncmd->cur_iocbq.iotag); |
| 2096 | |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2097 | spin_lock_irqsave(&phba->sli4_hba.abts_nvme_buf_list_lock, |
| 2098 | iflag); |
| 2099 | lpfc_ncmd->nvmeCmd = NULL; |
| 2100 | list_add_tail(&lpfc_ncmd->list, |
| 2101 | &phba->sli4_hba.lpfc_abts_nvme_buf_list); |
| 2102 | spin_unlock_irqrestore(&phba->sli4_hba.abts_nvme_buf_list_lock, |
| 2103 | iflag); |
| 2104 | } else { |
| 2105 | lpfc_ncmd->nvmeCmd = NULL; |
| 2106 | lpfc_ncmd->cur_iocbq.iocb_flag = LPFC_IO_NVME; |
| 2107 | spin_lock_irqsave(&phba->nvme_buf_list_put_lock, iflag); |
| 2108 | list_add_tail(&lpfc_ncmd->list, &phba->lpfc_nvme_buf_list_put); |
| 2109 | spin_unlock_irqrestore(&phba->nvme_buf_list_put_lock, iflag); |
| 2110 | } |
| 2111 | } |
| 2112 | |
| 2113 | /** |
| 2114 | * lpfc_nvme_create_localport - Create/Bind an nvme localport instance. |
| 2115 | * @pvport - the lpfc_vport instance requesting a localport. |
| 2116 | * |
| 2117 | * This routine is invoked to create an nvme localport instance to bind |
| 2118 | * to the nvme_fc_transport. It is called once during driver load |
| 2119 | * like lpfc_create_shost after all other services are initialized. |
| 2120 | * It requires a vport, vpi, and wwns at call time. Other localport |
| 2121 | * parameters are modified as the driver's FCID and the Fabric WWN |
| 2122 | * are established. |
| 2123 | * |
| 2124 | * Return codes |
| 2125 | * 0 - successful |
| 2126 | * -ENOMEM - no heap memory available |
| 2127 | * other values - from nvme registration upcall |
| 2128 | **/ |
| 2129 | int |
| 2130 | lpfc_nvme_create_localport(struct lpfc_vport *vport) |
| 2131 | { |
James Smart | 166d721 | 2017-03-04 09:30:33 -0800 | [diff] [blame] | 2132 | int ret = 0; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2133 | struct lpfc_hba *phba = vport->phba; |
| 2134 | struct nvme_fc_port_info nfcp_info; |
| 2135 | struct nvme_fc_local_port *localport; |
| 2136 | struct lpfc_nvme_lport *lport; |
James Smart | 166d721 | 2017-03-04 09:30:33 -0800 | [diff] [blame] | 2137 | int len; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2138 | |
| 2139 | /* Initialize this localport instance. The vport wwn usage ensures |
| 2140 | * that NPIV is accounted for. |
| 2141 | */ |
| 2142 | memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info)); |
| 2143 | nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR; |
| 2144 | nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn); |
| 2145 | nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn); |
| 2146 | |
James Smart | 4d4c4a4 | 2017-04-21 16:05:01 -0700 | [diff] [blame] | 2147 | /* Limit to LPFC_MAX_NVME_SEG_CNT. |
| 2148 | * For now need + 1 to get around NVME transport logic. |
| 2149 | */ |
| 2150 | if (phba->cfg_sg_seg_cnt > LPFC_MAX_NVME_SEG_CNT) { |
| 2151 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_INIT, |
| 2152 | "6300 Reducing sg segment cnt to %d\n", |
| 2153 | LPFC_MAX_NVME_SEG_CNT); |
| 2154 | phba->cfg_nvme_seg_cnt = LPFC_MAX_NVME_SEG_CNT; |
| 2155 | } else { |
| 2156 | phba->cfg_nvme_seg_cnt = phba->cfg_sg_seg_cnt; |
| 2157 | } |
| 2158 | lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2159 | lpfc_nvme_template.max_hw_queues = phba->cfg_nvme_io_channel; |
| 2160 | |
| 2161 | /* localport is allocated from the stack, but the registration |
| 2162 | * call allocates heap memory as well as the private area. |
| 2163 | */ |
James Smart | 7d70803 | 2017-03-08 14:36:01 -0800 | [diff] [blame] | 2164 | #if (IS_ENABLED(CONFIG_NVME_FC)) |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2165 | ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template, |
| 2166 | &vport->phba->pcidev->dev, &localport); |
James Smart | 166d721 | 2017-03-04 09:30:33 -0800 | [diff] [blame] | 2167 | #else |
| 2168 | ret = -ENOMEM; |
| 2169 | #endif |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2170 | if (!ret) { |
| 2171 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC, |
| 2172 | "6005 Successfully registered local " |
| 2173 | "NVME port num %d, localP %p, private %p, " |
| 2174 | "sg_seg %d\n", |
| 2175 | localport->port_num, localport, |
| 2176 | localport->private, |
| 2177 | lpfc_nvme_template.max_sgl_segments); |
| 2178 | |
| 2179 | /* Private is our lport size declared in the template. */ |
| 2180 | lport = (struct lpfc_nvme_lport *)localport->private; |
| 2181 | vport->localport = localport; |
| 2182 | lport->vport = vport; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2183 | vport->nvmei_support = 1; |
Dick Kennedy | 6b486ce | 2017-08-23 16:55:45 -0700 | [diff] [blame] | 2184 | |
| 2185 | /* Don't post more new bufs if repost already recovered |
| 2186 | * the nvme sgls. |
| 2187 | */ |
| 2188 | if (phba->sli4_hba.nvme_xri_cnt == 0) { |
| 2189 | len = lpfc_new_nvme_buf(vport, |
| 2190 | phba->sli4_hba.nvme_xri_max); |
| 2191 | vport->phba->total_nvme_bufs += len; |
| 2192 | } |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2193 | } |
| 2194 | |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2195 | return ret; |
| 2196 | } |
| 2197 | |
| 2198 | /** |
| 2199 | * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport. |
| 2200 | * @pnvme: pointer to lpfc nvme data structure. |
| 2201 | * |
| 2202 | * This routine is invoked to destroy all lports bound to the phba. |
| 2203 | * The lport memory was allocated by the nvme fc transport and is |
| 2204 | * released there. This routine ensures all rports bound to the |
| 2205 | * lport have been disconnected. |
| 2206 | * |
| 2207 | **/ |
| 2208 | void |
| 2209 | lpfc_nvme_destroy_localport(struct lpfc_vport *vport) |
| 2210 | { |
James Smart | 7d70803 | 2017-03-08 14:36:01 -0800 | [diff] [blame] | 2211 | #if (IS_ENABLED(CONFIG_NVME_FC)) |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2212 | struct nvme_fc_local_port *localport; |
| 2213 | struct lpfc_nvme_lport *lport; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2214 | int ret; |
| 2215 | |
| 2216 | if (vport->nvmei_support == 0) |
| 2217 | return; |
| 2218 | |
| 2219 | localport = vport->localport; |
| 2220 | vport->localport = NULL; |
| 2221 | lport = (struct lpfc_nvme_lport *)localport->private; |
| 2222 | |
| 2223 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, |
| 2224 | "6011 Destroying NVME localport %p\n", |
| 2225 | localport); |
James Smart | 166d721 | 2017-03-04 09:30:33 -0800 | [diff] [blame] | 2226 | |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2227 | /* lport's rport list is clear. Unregister |
| 2228 | * lport and release resources. |
| 2229 | */ |
| 2230 | init_completion(&lport->lport_unreg_done); |
| 2231 | ret = nvme_fc_unregister_localport(localport); |
| 2232 | wait_for_completion_timeout(&lport->lport_unreg_done, 5); |
| 2233 | |
| 2234 | /* Regardless of the unregister upcall response, clear |
| 2235 | * nvmei_support. All rports are unregistered and the |
| 2236 | * driver will clean up. |
| 2237 | */ |
| 2238 | vport->nvmei_support = 0; |
| 2239 | if (ret == 0) { |
| 2240 | lpfc_printf_vlog(vport, |
| 2241 | KERN_INFO, LOG_NVME_DISC, |
| 2242 | "6009 Unregistered lport Success\n"); |
| 2243 | } else { |
| 2244 | lpfc_printf_vlog(vport, |
| 2245 | KERN_INFO, LOG_NVME_DISC, |
| 2246 | "6010 Unregistered lport " |
| 2247 | "Failed, status x%x\n", |
| 2248 | ret); |
| 2249 | } |
James Smart | 166d721 | 2017-03-04 09:30:33 -0800 | [diff] [blame] | 2250 | #endif |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2251 | } |
| 2252 | |
| 2253 | void |
| 2254 | lpfc_nvme_update_localport(struct lpfc_vport *vport) |
| 2255 | { |
James Smart | 4410a67 | 2017-04-21 16:04:57 -0700 | [diff] [blame] | 2256 | #if (IS_ENABLED(CONFIG_NVME_FC)) |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2257 | struct nvme_fc_local_port *localport; |
| 2258 | struct lpfc_nvme_lport *lport; |
| 2259 | |
| 2260 | localport = vport->localport; |
James Smart | 4410a67 | 2017-04-21 16:04:57 -0700 | [diff] [blame] | 2261 | if (!localport) { |
| 2262 | lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, |
| 2263 | "6710 Update NVME fail. No localport\n"); |
| 2264 | return; |
| 2265 | } |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2266 | lport = (struct lpfc_nvme_lport *)localport->private; |
James Smart | 4410a67 | 2017-04-21 16:04:57 -0700 | [diff] [blame] | 2267 | if (!lport) { |
| 2268 | lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME, |
| 2269 | "6171 Update NVME fail. localP %p, No lport\n", |
| 2270 | localport); |
| 2271 | return; |
| 2272 | } |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2273 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME, |
| 2274 | "6012 Update NVME lport %p did x%x\n", |
| 2275 | localport, vport->fc_myDID); |
| 2276 | |
| 2277 | localport->port_id = vport->fc_myDID; |
| 2278 | if (localport->port_id == 0) |
| 2279 | localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY; |
| 2280 | else |
| 2281 | localport->port_role = FC_PORT_ROLE_NVME_INITIATOR; |
| 2282 | |
| 2283 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, |
| 2284 | "6030 bound lport %p to DID x%06x\n", |
| 2285 | lport, localport->port_id); |
James Smart | 4410a67 | 2017-04-21 16:04:57 -0700 | [diff] [blame] | 2286 | #endif |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2287 | } |
| 2288 | |
| 2289 | int |
| 2290 | lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) |
| 2291 | { |
James Smart | 7d70803 | 2017-03-08 14:36:01 -0800 | [diff] [blame] | 2292 | #if (IS_ENABLED(CONFIG_NVME_FC)) |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2293 | int ret = 0; |
| 2294 | struct nvme_fc_local_port *localport; |
| 2295 | struct lpfc_nvme_lport *lport; |
| 2296 | struct lpfc_nvme_rport *rport; |
| 2297 | struct nvme_fc_remote_port *remote_port; |
| 2298 | struct nvme_fc_port_info rpinfo; |
| 2299 | |
| 2300 | lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC, |
| 2301 | "6006 Register NVME PORT. DID x%06x nlptype x%x\n", |
| 2302 | ndlp->nlp_DID, ndlp->nlp_type); |
| 2303 | |
| 2304 | localport = vport->localport; |
Dick Kennedy | bb6a8a2 | 2017-08-23 16:55:33 -0700 | [diff] [blame] | 2305 | if (!localport) |
| 2306 | return 0; |
| 2307 | |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2308 | lport = (struct lpfc_nvme_lport *)localport->private; |
| 2309 | |
James Smart | 7a06dcd | 2017-06-01 21:06:55 -0700 | [diff] [blame] | 2310 | /* NVME rports are not preserved across devloss. |
| 2311 | * Just register this instance. Note, rpinfo->dev_loss_tmo |
| 2312 | * is left 0 to indicate accept transport defaults. The |
| 2313 | * driver communicates port role capabilities consistent |
| 2314 | * with the PRLI response data. |
| 2315 | */ |
| 2316 | memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info)); |
| 2317 | rpinfo.port_id = ndlp->nlp_DID; |
| 2318 | if (ndlp->nlp_type & NLP_NVME_TARGET) |
| 2319 | rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET; |
| 2320 | if (ndlp->nlp_type & NLP_NVME_INITIATOR) |
| 2321 | rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2322 | |
James Smart | 7a06dcd | 2017-06-01 21:06:55 -0700 | [diff] [blame] | 2323 | if (ndlp->nlp_type & NLP_NVME_DISCOVERY) |
| 2324 | rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY; |
| 2325 | |
| 2326 | rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn); |
| 2327 | rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn); |
| 2328 | ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port); |
| 2329 | if (!ret) { |
| 2330 | /* If the ndlp already has an nrport, this is just |
| 2331 | * a resume of the existing rport. Else this is a |
| 2332 | * new rport. |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2333 | */ |
James Smart | 7a06dcd | 2017-06-01 21:06:55 -0700 | [diff] [blame] | 2334 | rport = remote_port->private; |
| 2335 | if (ndlp->nrport == rport) { |
| 2336 | lpfc_printf_vlog(ndlp->vport, KERN_INFO, |
| 2337 | LOG_NVME_DISC, |
| 2338 | "6014 Rebinding lport to " |
| 2339 | "rport wwpn 0x%llx, " |
| 2340 | "Data: x%x x%x x%x x%06x\n", |
| 2341 | remote_port->port_name, |
| 2342 | remote_port->port_id, |
| 2343 | remote_port->port_role, |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2344 | ndlp->nlp_type, |
| 2345 | ndlp->nlp_DID); |
James Smart | 7a06dcd | 2017-06-01 21:06:55 -0700 | [diff] [blame] | 2346 | } else { |
| 2347 | /* New rport. */ |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2348 | rport->remoteport = remote_port; |
| 2349 | rport->lport = lport; |
| 2350 | rport->ndlp = lpfc_nlp_get(ndlp); |
| 2351 | if (!rport->ndlp) |
| 2352 | return -1; |
| 2353 | ndlp->nrport = rport; |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2354 | lpfc_printf_vlog(vport, KERN_INFO, |
| 2355 | LOG_NVME_DISC | LOG_NODE, |
James Smart | 7a06dcd | 2017-06-01 21:06:55 -0700 | [diff] [blame] | 2356 | "6022 Binding new rport to " |
| 2357 | "lport %p Rport WWNN 0x%llx, " |
| 2358 | "Rport WWPN 0x%llx DID " |
| 2359 | "x%06x Role x%x\n", |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2360 | lport, |
| 2361 | rpinfo.node_name, rpinfo.port_name, |
| 2362 | rpinfo.port_id, rpinfo.port_role); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2363 | } |
| 2364 | } else { |
James Smart | 7a06dcd | 2017-06-01 21:06:55 -0700 | [diff] [blame] | 2365 | lpfc_printf_vlog(vport, KERN_ERR, |
| 2366 | LOG_NVME_DISC | LOG_NODE, |
| 2367 | "6031 RemotePort Registration failed " |
| 2368 | "err: %d, DID x%06x\n", |
| 2369 | ret, ndlp->nlp_DID); |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2370 | } |
James Smart | 7a06dcd | 2017-06-01 21:06:55 -0700 | [diff] [blame] | 2371 | |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2372 | return ret; |
James Smart | 166d721 | 2017-03-04 09:30:33 -0800 | [diff] [blame] | 2373 | #else |
| 2374 | return 0; |
| 2375 | #endif |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2376 | } |
| 2377 | |
| 2378 | /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport. |
| 2379 | * |
| 2380 | * There is no notion of Devloss or rport recovery from the current |
| 2381 | * nvme_transport perspective. Loss of an rport just means IO cannot |
| 2382 | * be sent and recovery is completely up to the initator. |
| 2383 | * For now, the driver just unbinds the DID and port_role so that |
| 2384 | * no further IO can be issued. Changes are planned for later. |
| 2385 | * |
| 2386 | * Notes - the ndlp reference count is not decremented here since |
| 2387 | * since there is no nvme_transport api for devloss. Node ref count |
| 2388 | * is only adjusted in driver unload. |
| 2389 | */ |
| 2390 | void |
| 2391 | lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp) |
| 2392 | { |
James Smart | 7d70803 | 2017-03-08 14:36:01 -0800 | [diff] [blame] | 2393 | #if (IS_ENABLED(CONFIG_NVME_FC)) |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2394 | int ret; |
| 2395 | struct nvme_fc_local_port *localport; |
| 2396 | struct lpfc_nvme_lport *lport; |
| 2397 | struct lpfc_nvme_rport *rport; |
| 2398 | struct nvme_fc_remote_port *remoteport; |
| 2399 | |
| 2400 | localport = vport->localport; |
| 2401 | |
| 2402 | /* This is fundamental error. The localport is always |
| 2403 | * available until driver unload. Just exit. |
| 2404 | */ |
| 2405 | if (!localport) |
| 2406 | return; |
| 2407 | |
| 2408 | lport = (struct lpfc_nvme_lport *)localport->private; |
| 2409 | if (!lport) |
| 2410 | goto input_err; |
| 2411 | |
| 2412 | rport = ndlp->nrport; |
| 2413 | if (!rport) |
| 2414 | goto input_err; |
| 2415 | |
| 2416 | remoteport = rport->remoteport; |
| 2417 | lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC, |
| 2418 | "6033 Unreg nvme remoteport %p, portname x%llx, " |
| 2419 | "port_id x%06x, portstate x%x port type x%x\n", |
| 2420 | remoteport, remoteport->port_name, |
| 2421 | remoteport->port_id, remoteport->port_state, |
| 2422 | ndlp->nlp_type); |
| 2423 | |
| 2424 | /* Sanity check ndlp type. Only call for NVME ports. Don't |
| 2425 | * clear any rport state until the transport calls back. |
| 2426 | */ |
| 2427 | if (ndlp->nlp_type & (NLP_NVME_TARGET | NLP_NVME_INITIATOR)) { |
| 2428 | init_completion(&rport->rport_unreg_done); |
James Smart | 7a06dcd | 2017-06-01 21:06:55 -0700 | [diff] [blame] | 2429 | |
| 2430 | /* No concern about the role change on the nvme remoteport. |
| 2431 | * The transport will update it. |
| 2432 | */ |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2433 | ret = nvme_fc_unregister_remoteport(remoteport); |
| 2434 | if (ret != 0) { |
| 2435 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, |
| 2436 | "6167 NVME unregister failed %d " |
| 2437 | "port_state x%x\n", |
| 2438 | ret, remoteport->port_state); |
| 2439 | } |
| 2440 | |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2441 | } |
| 2442 | return; |
| 2443 | |
| 2444 | input_err: |
James Smart | 166d721 | 2017-03-04 09:30:33 -0800 | [diff] [blame] | 2445 | #endif |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2446 | lpfc_printf_vlog(vport, KERN_ERR, LOG_NVME_DISC, |
James Smart | 2b7824d | 2017-04-21 16:04:59 -0700 | [diff] [blame] | 2447 | "6168 State error: lport %p, rport%p FCID x%06x\n", |
James Smart | 0164956 | 2017-02-12 13:52:32 -0800 | [diff] [blame] | 2448 | vport->localport, ndlp->rport, ndlp->nlp_DID); |
| 2449 | } |
James Smart | 318083a | 2017-03-04 09:30:30 -0800 | [diff] [blame] | 2450 | |
| 2451 | /** |
| 2452 | * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort |
| 2453 | * @phba: pointer to lpfc hba data structure. |
| 2454 | * @axri: pointer to the fcp xri abort wcqe structure. |
| 2455 | * |
| 2456 | * This routine is invoked by the worker thread to process a SLI4 fast-path |
| 2457 | * FCP aborted xri. |
| 2458 | **/ |
| 2459 | void |
| 2460 | lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba, |
| 2461 | struct sli4_wcqe_xri_aborted *axri) |
| 2462 | { |
| 2463 | uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri); |
| 2464 | uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri); |
| 2465 | struct lpfc_nvme_buf *lpfc_ncmd, *next_lpfc_ncmd; |
| 2466 | struct lpfc_nodelist *ndlp; |
| 2467 | unsigned long iflag = 0; |
| 2468 | int rrq_empty = 0; |
| 2469 | |
| 2470 | if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME)) |
| 2471 | return; |
| 2472 | spin_lock_irqsave(&phba->hbalock, iflag); |
| 2473 | spin_lock(&phba->sli4_hba.abts_nvme_buf_list_lock); |
| 2474 | list_for_each_entry_safe(lpfc_ncmd, next_lpfc_ncmd, |
| 2475 | &phba->sli4_hba.lpfc_abts_nvme_buf_list, |
| 2476 | list) { |
| 2477 | if (lpfc_ncmd->cur_iocbq.sli4_xritag == xri) { |
James Smart | bbe3012 | 2017-04-21 17:49:08 -0700 | [diff] [blame] | 2478 | list_del_init(&lpfc_ncmd->list); |
James Smart | 318083a | 2017-03-04 09:30:30 -0800 | [diff] [blame] | 2479 | lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY; |
| 2480 | lpfc_ncmd->status = IOSTAT_SUCCESS; |
| 2481 | spin_unlock( |
| 2482 | &phba->sli4_hba.abts_nvme_buf_list_lock); |
| 2483 | |
| 2484 | rrq_empty = list_empty(&phba->active_rrq_list); |
| 2485 | spin_unlock_irqrestore(&phba->hbalock, iflag); |
| 2486 | ndlp = lpfc_ncmd->ndlp; |
| 2487 | if (ndlp) { |
| 2488 | lpfc_set_rrq_active( |
| 2489 | phba, ndlp, |
| 2490 | lpfc_ncmd->cur_iocbq.sli4_lxritag, |
| 2491 | rxid, 1); |
| 2492 | lpfc_sli4_abts_err_handler(phba, ndlp, axri); |
| 2493 | } |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 2494 | |
| 2495 | lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, |
| 2496 | "6311 XRI Aborted xri x%x tag x%x " |
| 2497 | "released\n", |
| 2498 | xri, lpfc_ncmd->cur_iocbq.iotag); |
| 2499 | |
James Smart | 318083a | 2017-03-04 09:30:30 -0800 | [diff] [blame] | 2500 | lpfc_release_nvme_buf(phba, lpfc_ncmd); |
| 2501 | if (rrq_empty) |
| 2502 | lpfc_worker_wake_up(phba); |
| 2503 | return; |
| 2504 | } |
| 2505 | } |
| 2506 | spin_unlock(&phba->sli4_hba.abts_nvme_buf_list_lock); |
| 2507 | spin_unlock_irqrestore(&phba->hbalock, iflag); |
James Smart | 86c6737 | 2017-04-21 16:05:04 -0700 | [diff] [blame] | 2508 | |
| 2509 | lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS, |
| 2510 | "6312 XRI Aborted xri x%x not found\n", xri); |
| 2511 | |
James Smart | 318083a | 2017-03-04 09:30:30 -0800 | [diff] [blame] | 2512 | } |