\"Talpey, Thomas\ | f58851e | 2007-09-10 13:50:12 -0400 | [diff] [blame] | 1 | /* |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 2 | * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved. |
| 3 | * |
| 4 | * This software is available to you under a choice of one of two |
| 5 | * licenses. You may choose to be licensed under the terms of the GNU |
| 6 | * General Public License (GPL) Version 2, available from the file |
| 7 | * COPYING in the main directory of this source tree, or the BSD-type |
| 8 | * license below: |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or without |
| 11 | * modification, are permitted provided that the following conditions |
| 12 | * are met: |
| 13 | * |
| 14 | * Redistributions of source code must retain the above copyright |
| 15 | * notice, this list of conditions and the following disclaimer. |
| 16 | * |
| 17 | * Redistributions in binary form must reproduce the above |
| 18 | * copyright notice, this list of conditions and the following |
| 19 | * disclaimer in the documentation and/or other materials provided |
| 20 | * with the distribution. |
| 21 | * |
| 22 | * Neither the name of the Network Appliance, Inc. nor the names of |
| 23 | * its contributors may be used to endorse or promote products |
| 24 | * derived from this software without specific prior written |
| 25 | * permission. |
| 26 | * |
| 27 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 28 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 29 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 30 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 31 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 32 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 33 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 34 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 35 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 36 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 37 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
\"Talpey, Thomas\ | f58851e | 2007-09-10 13:50:12 -0400 | [diff] [blame] | 38 | */ |
| 39 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 40 | /* |
| 41 | * verbs.c |
| 42 | * |
| 43 | * Encapsulates the major functions managing: |
| 44 | * o adapters |
| 45 | * o endpoints |
| 46 | * o connections |
| 47 | * o buffer memory |
| 48 | */ |
| 49 | |
Alexey Dobriyan | a6b7a40 | 2011-06-06 10:43:46 +0000 | [diff] [blame] | 50 | #include <linux/interrupt.h> |
Tejun Heo | 5a0e3ad | 2010-03-24 17:04:11 +0900 | [diff] [blame] | 51 | #include <linux/slab.h> |
Chuck Lever | eba8ff6 | 2015-01-21 11:03:02 -0500 | [diff] [blame] | 52 | #include <linux/prefetch.h> |
Chuck Lever | 0dd39ca | 2015-03-30 14:33:43 -0400 | [diff] [blame] | 53 | #include <linux/sunrpc/addr.h> |
Chuck Lever | 05c9746 | 2016-09-06 11:22:58 -0400 | [diff] [blame] | 54 | #include <linux/sunrpc/svc_rdma.h> |
Chuck Lever | 65866f8 | 2014-05-28 10:33:59 -0400 | [diff] [blame] | 55 | #include <asm/bitops.h> |
Devesh Sharma | d0f36c4 | 2015-08-03 13:05:04 -0400 | [diff] [blame] | 56 | #include <linux/module.h> /* try_module_get()/module_put() */ |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 57 | |
\"Talpey, Thomas\ | f58851e | 2007-09-10 13:50:12 -0400 | [diff] [blame] | 58 | #include "xprt_rdma.h" |
| 59 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 60 | /* |
| 61 | * Globals/Macros |
| 62 | */ |
| 63 | |
Jeff Layton | f895b25 | 2014-11-17 16:58:04 -0500 | [diff] [blame] | 64 | #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 65 | # define RPCDBG_FACILITY RPCDBG_TRANS |
| 66 | #endif |
| 67 | |
| 68 | /* |
| 69 | * internal functions |
| 70 | */ |
| 71 | |
Chuck Lever | fe97b47 | 2015-10-24 17:27:10 -0400 | [diff] [blame] | 72 | static struct workqueue_struct *rpcrdma_receive_wq; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 73 | |
Chuck Lever | fe97b47 | 2015-10-24 17:27:10 -0400 | [diff] [blame] | 74 | int |
| 75 | rpcrdma_alloc_wq(void) |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 76 | { |
Chuck Lever | fe97b47 | 2015-10-24 17:27:10 -0400 | [diff] [blame] | 77 | struct workqueue_struct *recv_wq; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 78 | |
Chuck Lever | fe97b47 | 2015-10-24 17:27:10 -0400 | [diff] [blame] | 79 | recv_wq = alloc_workqueue("xprtrdma_receive", |
| 80 | WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_HIGHPRI, |
| 81 | 0); |
| 82 | if (!recv_wq) |
| 83 | return -ENOMEM; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 84 | |
Chuck Lever | fe97b47 | 2015-10-24 17:27:10 -0400 | [diff] [blame] | 85 | rpcrdma_receive_wq = recv_wq; |
| 86 | return 0; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 87 | } |
| 88 | |
Chuck Lever | fe97b47 | 2015-10-24 17:27:10 -0400 | [diff] [blame] | 89 | void |
| 90 | rpcrdma_destroy_wq(void) |
Chuck Lever | f1a03b7 | 2014-11-08 20:14:37 -0500 | [diff] [blame] | 91 | { |
Chuck Lever | fe97b47 | 2015-10-24 17:27:10 -0400 | [diff] [blame] | 92 | struct workqueue_struct *wq; |
Chuck Lever | f1a03b7 | 2014-11-08 20:14:37 -0500 | [diff] [blame] | 93 | |
Chuck Lever | fe97b47 | 2015-10-24 17:27:10 -0400 | [diff] [blame] | 94 | if (rpcrdma_receive_wq) { |
| 95 | wq = rpcrdma_receive_wq; |
| 96 | rpcrdma_receive_wq = NULL; |
| 97 | destroy_workqueue(wq); |
| 98 | } |
Chuck Lever | f1a03b7 | 2014-11-08 20:14:37 -0500 | [diff] [blame] | 99 | } |
| 100 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 101 | static void |
| 102 | rpcrdma_qp_async_error_upcall(struct ib_event *event, void *context) |
| 103 | { |
| 104 | struct rpcrdma_ep *ep = context; |
| 105 | |
Chuck Lever | 7ff11de | 2014-11-08 20:15:01 -0500 | [diff] [blame] | 106 | pr_err("RPC: %s: %s on device %s ep %p\n", |
Sagi Grimberg | 76357c7 | 2015-05-18 13:40:32 +0300 | [diff] [blame] | 107 | __func__, ib_event_msg(event->event), |
Chuck Lever | 7ff11de | 2014-11-08 20:15:01 -0500 | [diff] [blame] | 108 | event->device->name, context); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 109 | if (ep->rep_connected == 1) { |
| 110 | ep->rep_connected = -EIO; |
Chuck Lever | afadc46 | 2015-01-21 11:03:11 -0500 | [diff] [blame] | 111 | rpcrdma_conn_func(ep); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 112 | wake_up_all(&ep->rep_connect_wait); |
| 113 | } |
| 114 | } |
| 115 | |
Chuck Lever | 2fa8f88 | 2016-03-04 11:28:53 -0500 | [diff] [blame] | 116 | /** |
| 117 | * rpcrdma_wc_send - Invoked by RDMA provider for each polled Send WC |
| 118 | * @cq: completion queue (ignored) |
| 119 | * @wc: completed WR |
| 120 | * |
Chuck Lever | 4220a07 | 2015-10-24 17:26:45 -0400 | [diff] [blame] | 121 | */ |
| 122 | static void |
Chuck Lever | 2fa8f88 | 2016-03-04 11:28:53 -0500 | [diff] [blame] | 123 | rpcrdma_wc_send(struct ib_cq *cq, struct ib_wc *wc) |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 124 | { |
Chuck Lever | 2fa8f88 | 2016-03-04 11:28:53 -0500 | [diff] [blame] | 125 | /* WARNING: Only wr_cqe and status are reliable at this point */ |
| 126 | if (wc->status != IB_WC_SUCCESS && wc->status != IB_WC_WR_FLUSH_ERR) |
| 127 | pr_err("rpcrdma: Send: %s (%u/0x%x)\n", |
| 128 | ib_wc_status_msg(wc->status), |
| 129 | wc->status, wc->vendor_err); |
Chuck Lever | fc66448 | 2014-05-28 10:33:25 -0400 | [diff] [blame] | 130 | } |
| 131 | |
Chuck Lever | 23826c7 | 2016-03-04 11:28:27 -0500 | [diff] [blame] | 132 | /* Perform basic sanity checking to avoid using garbage |
| 133 | * to update the credit grant value. |
| 134 | */ |
| 135 | static void |
| 136 | rpcrdma_update_granted_credits(struct rpcrdma_rep *rep) |
| 137 | { |
| 138 | struct rpcrdma_msg *rmsgp = rdmab_to_msg(rep->rr_rdmabuf); |
| 139 | struct rpcrdma_buffer *buffer = &rep->rr_rxprt->rx_buf; |
| 140 | u32 credits; |
| 141 | |
| 142 | if (rep->rr_len < RPCRDMA_HDRLEN_ERR) |
| 143 | return; |
| 144 | |
| 145 | credits = be32_to_cpu(rmsgp->rm_credit); |
| 146 | if (credits == 0) |
| 147 | credits = 1; /* don't deadlock */ |
| 148 | else if (credits > buffer->rb_max_requests) |
| 149 | credits = buffer->rb_max_requests; |
| 150 | |
| 151 | atomic_set(&buffer->rb_credits, credits); |
| 152 | } |
| 153 | |
Chuck Lever | 552bf22 | 2016-03-04 11:28:36 -0500 | [diff] [blame] | 154 | /** |
Chuck Lever | 1519e96 | 2016-09-15 10:57:49 -0400 | [diff] [blame] | 155 | * rpcrdma_wc_receive - Invoked by RDMA provider for each polled Receive WC |
Chuck Lever | 552bf22 | 2016-03-04 11:28:36 -0500 | [diff] [blame] | 156 | * @cq: completion queue (ignored) |
| 157 | * @wc: completed WR |
| 158 | * |
| 159 | */ |
Chuck Lever | fe97b47 | 2015-10-24 17:27:10 -0400 | [diff] [blame] | 160 | static void |
Chuck Lever | 1519e96 | 2016-09-15 10:57:49 -0400 | [diff] [blame] | 161 | rpcrdma_wc_receive(struct ib_cq *cq, struct ib_wc *wc) |
Chuck Lever | fc66448 | 2014-05-28 10:33:25 -0400 | [diff] [blame] | 162 | { |
Chuck Lever | 552bf22 | 2016-03-04 11:28:36 -0500 | [diff] [blame] | 163 | struct ib_cqe *cqe = wc->wr_cqe; |
| 164 | struct rpcrdma_rep *rep = container_of(cqe, struct rpcrdma_rep, |
| 165 | rr_cqe); |
Chuck Lever | fc66448 | 2014-05-28 10:33:25 -0400 | [diff] [blame] | 166 | |
Chuck Lever | 8502427 | 2015-01-21 11:02:04 -0500 | [diff] [blame] | 167 | /* WARNING: Only wr_id and status are reliable at this point */ |
| 168 | if (wc->status != IB_WC_SUCCESS) |
| 169 | goto out_fail; |
Chuck Lever | fc66448 | 2014-05-28 10:33:25 -0400 | [diff] [blame] | 170 | |
Chuck Lever | 8502427 | 2015-01-21 11:02:04 -0500 | [diff] [blame] | 171 | /* status == SUCCESS means all fields in wc are trustworthy */ |
Chuck Lever | fc66448 | 2014-05-28 10:33:25 -0400 | [diff] [blame] | 172 | if (wc->opcode != IB_WC_RECV) |
| 173 | return; |
| 174 | |
Chuck Lever | 8502427 | 2015-01-21 11:02:04 -0500 | [diff] [blame] | 175 | dprintk("RPC: %s: rep %p opcode 'recv', length %u: success\n", |
| 176 | __func__, rep, wc->byte_len); |
| 177 | |
Chuck Lever | fc66448 | 2014-05-28 10:33:25 -0400 | [diff] [blame] | 178 | rep->rr_len = wc->byte_len; |
Chuck Lever | c8b920b | 2016-09-15 10:57:16 -0400 | [diff] [blame] | 179 | rep->rr_wc_flags = wc->wc_flags; |
| 180 | rep->rr_inv_rkey = wc->ex.invalidate_rkey; |
| 181 | |
Chuck Lever | 89e0d112 | 2015-05-26 11:51:56 -0400 | [diff] [blame] | 182 | ib_dma_sync_single_for_cpu(rep->rr_device, |
Chuck Lever | 6b1184c | 2015-01-21 11:04:25 -0500 | [diff] [blame] | 183 | rdmab_addr(rep->rr_rdmabuf), |
| 184 | rep->rr_len, DMA_FROM_DEVICE); |
Chuck Lever | 23826c7 | 2016-03-04 11:28:27 -0500 | [diff] [blame] | 185 | |
| 186 | rpcrdma_update_granted_credits(rep); |
Chuck Lever | fc66448 | 2014-05-28 10:33:25 -0400 | [diff] [blame] | 187 | |
| 188 | out_schedule: |
Chuck Lever | fe97b47 | 2015-10-24 17:27:10 -0400 | [diff] [blame] | 189 | queue_work(rpcrdma_receive_wq, &rep->rr_work); |
Chuck Lever | 8502427 | 2015-01-21 11:02:04 -0500 | [diff] [blame] | 190 | return; |
Chuck Lever | fe97b47 | 2015-10-24 17:27:10 -0400 | [diff] [blame] | 191 | |
Chuck Lever | 8502427 | 2015-01-21 11:02:04 -0500 | [diff] [blame] | 192 | out_fail: |
| 193 | if (wc->status != IB_WC_WR_FLUSH_ERR) |
Chuck Lever | 552bf22 | 2016-03-04 11:28:36 -0500 | [diff] [blame] | 194 | pr_err("rpcrdma: Recv: %s (%u/0x%x)\n", |
| 195 | ib_wc_status_msg(wc->status), |
| 196 | wc->status, wc->vendor_err); |
Chuck Lever | b0e178a | 2015-10-24 17:26:54 -0400 | [diff] [blame] | 197 | rep->rr_len = RPCRDMA_BAD_LEN; |
Chuck Lever | 8502427 | 2015-01-21 11:02:04 -0500 | [diff] [blame] | 198 | goto out_schedule; |
Chuck Lever | fc66448 | 2014-05-28 10:33:25 -0400 | [diff] [blame] | 199 | } |
| 200 | |
Chuck Lever | 87cfb9a | 2016-09-15 10:57:07 -0400 | [diff] [blame] | 201 | static void |
| 202 | rpcrdma_update_connect_private(struct rpcrdma_xprt *r_xprt, |
| 203 | struct rdma_conn_param *param) |
| 204 | { |
| 205 | struct rpcrdma_create_data_internal *cdata = &r_xprt->rx_data; |
| 206 | const struct rpcrdma_connect_private *pmsg = param->private_data; |
| 207 | unsigned int rsize, wsize; |
| 208 | |
Chuck Lever | c8b920b | 2016-09-15 10:57:16 -0400 | [diff] [blame] | 209 | /* Default settings for RPC-over-RDMA Version One */ |
| 210 | r_xprt->rx_ia.ri_reminv_expected = false; |
Chuck Lever | fab6c2c | 2017-02-08 16:59:54 -0500 | [diff] [blame] | 211 | r_xprt->rx_ia.ri_implicit_roundup = xprt_rdma_pad_optimize; |
Chuck Lever | 87cfb9a | 2016-09-15 10:57:07 -0400 | [diff] [blame] | 212 | rsize = RPCRDMA_V1_DEF_INLINE_SIZE; |
| 213 | wsize = RPCRDMA_V1_DEF_INLINE_SIZE; |
| 214 | |
| 215 | if (pmsg && |
| 216 | pmsg->cp_magic == rpcrdma_cmp_magic && |
| 217 | pmsg->cp_version == RPCRDMA_CMP_VERSION) { |
Chuck Lever | c8b920b | 2016-09-15 10:57:16 -0400 | [diff] [blame] | 218 | r_xprt->rx_ia.ri_reminv_expected = true; |
Chuck Lever | 73eea1c | 2017-02-08 17:00:02 -0500 | [diff] [blame] | 219 | r_xprt->rx_ia.ri_implicit_roundup = true; |
Chuck Lever | 87cfb9a | 2016-09-15 10:57:07 -0400 | [diff] [blame] | 220 | rsize = rpcrdma_decode_buffer_size(pmsg->cp_send_size); |
| 221 | wsize = rpcrdma_decode_buffer_size(pmsg->cp_recv_size); |
| 222 | } |
| 223 | |
| 224 | if (rsize < cdata->inline_rsize) |
| 225 | cdata->inline_rsize = rsize; |
| 226 | if (wsize < cdata->inline_wsize) |
| 227 | cdata->inline_wsize = wsize; |
Chuck Lever | d34b668 | 2016-11-29 10:53:13 -0500 | [diff] [blame] | 228 | dprintk("RPC: %s: max send %u, max recv %u\n", |
| 229 | __func__, cdata->inline_wsize, cdata->inline_rsize); |
Chuck Lever | 87cfb9a | 2016-09-15 10:57:07 -0400 | [diff] [blame] | 230 | rpcrdma_set_max_header_sizes(r_xprt); |
| 231 | } |
| 232 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 233 | static int |
| 234 | rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event) |
| 235 | { |
| 236 | struct rpcrdma_xprt *xprt = id->context; |
| 237 | struct rpcrdma_ia *ia = &xprt->rx_ia; |
| 238 | struct rpcrdma_ep *ep = &xprt->rx_ep; |
Jeff Layton | f895b25 | 2014-11-17 16:58:04 -0500 | [diff] [blame] | 239 | #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) |
Chuck Lever | 0dd39ca | 2015-03-30 14:33:43 -0400 | [diff] [blame] | 240 | struct sockaddr *sap = (struct sockaddr *)&ep->rep_remote_addr; |
Ingo Molnar | ff0db04 | 2008-11-25 16:58:42 -0800 | [diff] [blame] | 241 | #endif |
Chuck Lever | ce1ab9a | 2015-01-21 11:03:35 -0500 | [diff] [blame] | 242 | struct ib_qp_attr *attr = &ia->ri_qp_attr; |
| 243 | struct ib_qp_init_attr *iattr = &ia->ri_qp_init_attr; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 244 | int connstate = 0; |
| 245 | |
| 246 | switch (event->event) { |
| 247 | case RDMA_CM_EVENT_ADDR_RESOLVED: |
| 248 | case RDMA_CM_EVENT_ROUTE_RESOLVED: |
Tom Talpey | 5675add | 2008-10-09 15:01:41 -0400 | [diff] [blame] | 249 | ia->ri_async_rc = 0; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 250 | complete(&ia->ri_done); |
| 251 | break; |
| 252 | case RDMA_CM_EVENT_ADDR_ERROR: |
| 253 | ia->ri_async_rc = -EHOSTUNREACH; |
| 254 | dprintk("RPC: %s: CM address resolution error, ep 0x%p\n", |
| 255 | __func__, ep); |
| 256 | complete(&ia->ri_done); |
| 257 | break; |
| 258 | case RDMA_CM_EVENT_ROUTE_ERROR: |
| 259 | ia->ri_async_rc = -ENETUNREACH; |
| 260 | dprintk("RPC: %s: CM route resolution error, ep 0x%p\n", |
| 261 | __func__, ep); |
| 262 | complete(&ia->ri_done); |
| 263 | break; |
| 264 | case RDMA_CM_EVENT_ESTABLISHED: |
| 265 | connstate = 1; |
Chuck Lever | ce1ab9a | 2015-01-21 11:03:35 -0500 | [diff] [blame] | 266 | ib_query_qp(ia->ri_id->qp, attr, |
| 267 | IB_QP_MAX_QP_RD_ATOMIC | IB_QP_MAX_DEST_RD_ATOMIC, |
| 268 | iattr); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 269 | dprintk("RPC: %s: %d responder resources" |
| 270 | " (%d initiator)\n", |
Chuck Lever | ce1ab9a | 2015-01-21 11:03:35 -0500 | [diff] [blame] | 271 | __func__, attr->max_dest_rd_atomic, |
| 272 | attr->max_rd_atomic); |
Chuck Lever | 87cfb9a | 2016-09-15 10:57:07 -0400 | [diff] [blame] | 273 | rpcrdma_update_connect_private(xprt, &event->param.conn); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 274 | goto connected; |
| 275 | case RDMA_CM_EVENT_CONNECT_ERROR: |
| 276 | connstate = -ENOTCONN; |
| 277 | goto connected; |
| 278 | case RDMA_CM_EVENT_UNREACHABLE: |
| 279 | connstate = -ENETDOWN; |
| 280 | goto connected; |
| 281 | case RDMA_CM_EVENT_REJECTED: |
| 282 | connstate = -ECONNREFUSED; |
| 283 | goto connected; |
| 284 | case RDMA_CM_EVENT_DISCONNECTED: |
| 285 | connstate = -ECONNABORTED; |
| 286 | goto connected; |
| 287 | case RDMA_CM_EVENT_DEVICE_REMOVAL: |
| 288 | connstate = -ENODEV; |
| 289 | connected: |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 290 | dprintk("RPC: %s: %sconnected\n", |
| 291 | __func__, connstate > 0 ? "" : "dis"); |
Chuck Lever | 23826c7 | 2016-03-04 11:28:27 -0500 | [diff] [blame] | 292 | atomic_set(&xprt->rx_buf.rb_credits, 1); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 293 | ep->rep_connected = connstate; |
Chuck Lever | afadc46 | 2015-01-21 11:03:11 -0500 | [diff] [blame] | 294 | rpcrdma_conn_func(ep); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 295 | wake_up_all(&ep->rep_connect_wait); |
Chuck Lever | 8079fb7 | 2014-07-29 17:26:12 -0400 | [diff] [blame] | 296 | /*FALLTHROUGH*/ |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 297 | default: |
Chuck Lever | 0dd39ca | 2015-03-30 14:33:43 -0400 | [diff] [blame] | 298 | dprintk("RPC: %s: %pIS:%u (ep 0x%p): %s\n", |
| 299 | __func__, sap, rpc_get_port(sap), ep, |
Sagi Grimberg | 76357c7 | 2015-05-18 13:40:32 +0300 | [diff] [blame] | 300 | rdma_event_msg(event->event)); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 301 | break; |
| 302 | } |
| 303 | |
Jeff Layton | f895b25 | 2014-11-17 16:58:04 -0500 | [diff] [blame] | 304 | #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) |
Tom Talpey | b3cd8d4 | 2008-10-09 15:02:02 -0400 | [diff] [blame] | 305 | if (connstate == 1) { |
Chuck Lever | ce1ab9a | 2015-01-21 11:03:35 -0500 | [diff] [blame] | 306 | int ird = attr->max_dest_rd_atomic; |
Tom Talpey | b3cd8d4 | 2008-10-09 15:02:02 -0400 | [diff] [blame] | 307 | int tird = ep->rep_remote_cma.responder_resources; |
Chuck Lever | 0dd39ca | 2015-03-30 14:33:43 -0400 | [diff] [blame] | 308 | |
Chuck Lever | a0ce85f | 2015-03-30 14:34:21 -0400 | [diff] [blame] | 309 | pr_info("rpcrdma: connection to %pIS:%u on %s, memreg '%s', %d credits, %d responders%s\n", |
Chuck Lever | 0dd39ca | 2015-03-30 14:33:43 -0400 | [diff] [blame] | 310 | sap, rpc_get_port(sap), |
Chuck Lever | 89e0d112 | 2015-05-26 11:51:56 -0400 | [diff] [blame] | 311 | ia->ri_device->name, |
Chuck Lever | a0ce85f | 2015-03-30 14:34:21 -0400 | [diff] [blame] | 312 | ia->ri_ops->ro_displayname, |
Tom Talpey | b3cd8d4 | 2008-10-09 15:02:02 -0400 | [diff] [blame] | 313 | xprt->rx_buf.rb_max_requests, |
| 314 | ird, ird < 4 && ird < tird / 2 ? " (low!)" : ""); |
| 315 | } else if (connstate < 0) { |
Chuck Lever | 0dd39ca | 2015-03-30 14:33:43 -0400 | [diff] [blame] | 316 | pr_info("rpcrdma: connection to %pIS:%u closed (%d)\n", |
| 317 | sap, rpc_get_port(sap), connstate); |
Tom Talpey | b3cd8d4 | 2008-10-09 15:02:02 -0400 | [diff] [blame] | 318 | } |
| 319 | #endif |
| 320 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 321 | return 0; |
| 322 | } |
| 323 | |
Devesh Sharma | d0f36c4 | 2015-08-03 13:05:04 -0400 | [diff] [blame] | 324 | static void rpcrdma_destroy_id(struct rdma_cm_id *id) |
| 325 | { |
| 326 | if (id) { |
| 327 | module_put(id->device->owner); |
| 328 | rdma_destroy_id(id); |
| 329 | } |
| 330 | } |
| 331 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 332 | static struct rdma_cm_id * |
| 333 | rpcrdma_create_id(struct rpcrdma_xprt *xprt, |
| 334 | struct rpcrdma_ia *ia, struct sockaddr *addr) |
| 335 | { |
| 336 | struct rdma_cm_id *id; |
| 337 | int rc; |
| 338 | |
Tom Talpey | 1a95405 | 2008-10-09 15:01:31 -0400 | [diff] [blame] | 339 | init_completion(&ia->ri_done); |
| 340 | |
Guy Shapiro | fa20105 | 2015-10-22 15:20:10 +0300 | [diff] [blame] | 341 | id = rdma_create_id(&init_net, rpcrdma_conn_upcall, xprt, RDMA_PS_TCP, |
| 342 | IB_QPT_RC); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 343 | if (IS_ERR(id)) { |
| 344 | rc = PTR_ERR(id); |
| 345 | dprintk("RPC: %s: rdma_create_id() failed %i\n", |
| 346 | __func__, rc); |
| 347 | return id; |
| 348 | } |
| 349 | |
Tom Talpey | 5675add | 2008-10-09 15:01:41 -0400 | [diff] [blame] | 350 | ia->ri_async_rc = -ETIMEDOUT; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 351 | rc = rdma_resolve_addr(id, NULL, addr, RDMA_RESOLVE_TIMEOUT); |
| 352 | if (rc) { |
| 353 | dprintk("RPC: %s: rdma_resolve_addr() failed %i\n", |
| 354 | __func__, rc); |
| 355 | goto out; |
| 356 | } |
Tom Talpey | 5675add | 2008-10-09 15:01:41 -0400 | [diff] [blame] | 357 | wait_for_completion_interruptible_timeout(&ia->ri_done, |
| 358 | msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); |
Devesh Sharma | d0f36c4 | 2015-08-03 13:05:04 -0400 | [diff] [blame] | 359 | |
| 360 | /* FIXME: |
| 361 | * Until xprtrdma supports DEVICE_REMOVAL, the provider must |
| 362 | * be pinned while there are active NFS/RDMA mounts to prevent |
| 363 | * hangs and crashes at umount time. |
| 364 | */ |
| 365 | if (!ia->ri_async_rc && !try_module_get(id->device->owner)) { |
| 366 | dprintk("RPC: %s: Failed to get device module\n", |
| 367 | __func__); |
| 368 | ia->ri_async_rc = -ENODEV; |
| 369 | } |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 370 | rc = ia->ri_async_rc; |
| 371 | if (rc) |
| 372 | goto out; |
| 373 | |
Tom Talpey | 5675add | 2008-10-09 15:01:41 -0400 | [diff] [blame] | 374 | ia->ri_async_rc = -ETIMEDOUT; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 375 | rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT); |
| 376 | if (rc) { |
| 377 | dprintk("RPC: %s: rdma_resolve_route() failed %i\n", |
| 378 | __func__, rc); |
Devesh Sharma | d0f36c4 | 2015-08-03 13:05:04 -0400 | [diff] [blame] | 379 | goto put; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 380 | } |
Tom Talpey | 5675add | 2008-10-09 15:01:41 -0400 | [diff] [blame] | 381 | wait_for_completion_interruptible_timeout(&ia->ri_done, |
| 382 | msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 383 | rc = ia->ri_async_rc; |
| 384 | if (rc) |
Devesh Sharma | d0f36c4 | 2015-08-03 13:05:04 -0400 | [diff] [blame] | 385 | goto put; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 386 | |
| 387 | return id; |
Devesh Sharma | d0f36c4 | 2015-08-03 13:05:04 -0400 | [diff] [blame] | 388 | put: |
| 389 | module_put(id->device->owner); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 390 | out: |
| 391 | rdma_destroy_id(id); |
| 392 | return ERR_PTR(rc); |
| 393 | } |
| 394 | |
| 395 | /* |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 396 | * Exported functions. |
| 397 | */ |
| 398 | |
| 399 | /* |
| 400 | * Open and initialize an Interface Adapter. |
| 401 | * o initializes fields of struct rpcrdma_ia, including |
| 402 | * interface and provider attributes and protection zone. |
| 403 | */ |
| 404 | int |
| 405 | rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg) |
| 406 | { |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 407 | struct rpcrdma_ia *ia = &xprt->rx_ia; |
Chuck Lever | d1ed857 | 2015-08-03 13:03:30 -0400 | [diff] [blame] | 408 | int rc; |
| 409 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 410 | ia->ri_id = rpcrdma_create_id(xprt, ia, addr); |
| 411 | if (IS_ERR(ia->ri_id)) { |
| 412 | rc = PTR_ERR(ia->ri_id); |
| 413 | goto out1; |
| 414 | } |
Chuck Lever | 89e0d112 | 2015-05-26 11:51:56 -0400 | [diff] [blame] | 415 | ia->ri_device = ia->ri_id->device; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 416 | |
Christoph Hellwig | ed082d3 | 2016-09-05 12:56:17 +0200 | [diff] [blame] | 417 | ia->ri_pd = ib_alloc_pd(ia->ri_device, 0); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 418 | if (IS_ERR(ia->ri_pd)) { |
| 419 | rc = PTR_ERR(ia->ri_pd); |
Chuck Lever | b54054c | 2016-06-29 13:53:27 -0400 | [diff] [blame] | 420 | pr_err("rpcrdma: ib_alloc_pd() returned %d\n", rc); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 421 | goto out2; |
| 422 | } |
| 423 | |
Tom Talpey | bd7ed1d | 2008-10-09 15:00:09 -0400 | [diff] [blame] | 424 | switch (memreg) { |
Tom Talpey | 3197d309 | 2008-10-09 15:00:20 -0400 | [diff] [blame] | 425 | case RPCRDMA_FRMR: |
Chuck Lever | b54054c | 2016-06-29 13:53:27 -0400 | [diff] [blame] | 426 | if (frwr_is_supported(ia)) { |
| 427 | ia->ri_ops = &rpcrdma_frwr_memreg_ops; |
| 428 | break; |
| 429 | } |
| 430 | /*FALLTHROUGH*/ |
Tom Talpey | bd7ed1d | 2008-10-09 15:00:09 -0400 | [diff] [blame] | 431 | case RPCRDMA_MTHCAFMR: |
Chuck Lever | b54054c | 2016-06-29 13:53:27 -0400 | [diff] [blame] | 432 | if (fmr_is_supported(ia)) { |
| 433 | ia->ri_ops = &rpcrdma_fmr_memreg_ops; |
| 434 | break; |
| 435 | } |
| 436 | /*FALLTHROUGH*/ |
Tom Talpey | bd7ed1d | 2008-10-09 15:00:09 -0400 | [diff] [blame] | 437 | default: |
Chuck Lever | b54054c | 2016-06-29 13:53:27 -0400 | [diff] [blame] | 438 | pr_err("rpcrdma: Unsupported memory registration mode: %d\n", |
| 439 | memreg); |
| 440 | rc = -EINVAL; |
Chuck Lever | 5ae711a | 2015-01-21 11:03:19 -0500 | [diff] [blame] | 441 | goto out3; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 442 | } |
| 443 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 444 | return 0; |
Chuck Lever | 5ae711a | 2015-01-21 11:03:19 -0500 | [diff] [blame] | 445 | |
| 446 | out3: |
| 447 | ib_dealloc_pd(ia->ri_pd); |
| 448 | ia->ri_pd = NULL; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 449 | out2: |
Devesh Sharma | d0f36c4 | 2015-08-03 13:05:04 -0400 | [diff] [blame] | 450 | rpcrdma_destroy_id(ia->ri_id); |
Tom Talpey | fee08ca | 2008-10-09 15:01:00 -0400 | [diff] [blame] | 451 | ia->ri_id = NULL; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 452 | out1: |
| 453 | return rc; |
| 454 | } |
| 455 | |
| 456 | /* |
| 457 | * Clean up/close an IA. |
| 458 | * o if event handles and PD have been initialized, free them. |
| 459 | * o close the IA |
| 460 | */ |
| 461 | void |
| 462 | rpcrdma_ia_close(struct rpcrdma_ia *ia) |
| 463 | { |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 464 | dprintk("RPC: %s: entering\n", __func__); |
Tom Talpey | fee08ca | 2008-10-09 15:01:00 -0400 | [diff] [blame] | 465 | if (ia->ri_id != NULL && !IS_ERR(ia->ri_id)) { |
| 466 | if (ia->ri_id->qp) |
| 467 | rdma_destroy_qp(ia->ri_id); |
Devesh Sharma | d0f36c4 | 2015-08-03 13:05:04 -0400 | [diff] [blame] | 468 | rpcrdma_destroy_id(ia->ri_id); |
Tom Talpey | fee08ca | 2008-10-09 15:01:00 -0400 | [diff] [blame] | 469 | ia->ri_id = NULL; |
| 470 | } |
Chuck Lever | 6d44698 | 2015-05-26 11:51:27 -0400 | [diff] [blame] | 471 | |
| 472 | /* If the pd is still busy, xprtrdma missed freeing a resource */ |
| 473 | if (ia->ri_pd && !IS_ERR(ia->ri_pd)) |
Jason Gunthorpe | 7dd7864 | 2015-08-05 14:34:31 -0600 | [diff] [blame] | 474 | ib_dealloc_pd(ia->ri_pd); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 475 | } |
| 476 | |
| 477 | /* |
| 478 | * Create unconnected endpoint. |
| 479 | */ |
| 480 | int |
| 481 | rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia, |
Chuck Lever | 86840a6 | 2017-02-08 17:00:10 -0500 | [diff] [blame] | 482 | struct rpcrdma_create_data_internal *cdata) |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 483 | { |
Chuck Lever | 87cfb9a | 2016-09-15 10:57:07 -0400 | [diff] [blame] | 484 | struct rpcrdma_connect_private *pmsg = &ep->rep_cm_private; |
Chuck Lever | 86840a6 | 2017-02-08 17:00:10 -0500 | [diff] [blame] | 485 | unsigned int max_qp_wr, max_sge; |
Chuck Lever | fc66448 | 2014-05-28 10:33:25 -0400 | [diff] [blame] | 486 | struct ib_cq *sendcq, *recvcq; |
Chuck Lever | 2fa8f88 | 2016-03-04 11:28:53 -0500 | [diff] [blame] | 487 | int rc; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 488 | |
Chuck Lever | 87144ec | 2017-03-11 15:52:47 -0500 | [diff] [blame] | 489 | max_sge = min_t(unsigned int, ia->ri_device->attrs.max_sge, |
| 490 | RPCRDMA_MAX_SEND_SGES); |
Chuck Lever | 86840a6 | 2017-02-08 17:00:10 -0500 | [diff] [blame] | 491 | if (max_sge < RPCRDMA_MIN_SEND_SGES) { |
| 492 | pr_warn("rpcrdma: HCA provides only %d send SGEs\n", max_sge); |
Chuck Lever | b3221d6 | 2015-08-03 13:03:39 -0400 | [diff] [blame] | 493 | return -ENOMEM; |
| 494 | } |
Chuck Lever | 86840a6 | 2017-02-08 17:00:10 -0500 | [diff] [blame] | 495 | ia->ri_max_send_sges = max_sge - RPCRDMA_MIN_SEND_SGES; |
Chuck Lever | b3221d6 | 2015-08-03 13:03:39 -0400 | [diff] [blame] | 496 | |
Or Gerlitz | e3e45b1 | 2015-12-18 10:59:48 +0200 | [diff] [blame] | 497 | if (ia->ri_device->attrs.max_qp_wr <= RPCRDMA_BACKWARD_WRS) { |
Chuck Lever | 124fa17 | 2015-10-24 17:27:51 -0400 | [diff] [blame] | 498 | dprintk("RPC: %s: insufficient wqe's available\n", |
| 499 | __func__); |
| 500 | return -ENOMEM; |
| 501 | } |
Chuck Lever | 550d750 | 2016-05-02 14:41:47 -0400 | [diff] [blame] | 502 | max_qp_wr = ia->ri_device->attrs.max_qp_wr - RPCRDMA_BACKWARD_WRS - 1; |
Chuck Lever | 124fa17 | 2015-10-24 17:27:51 -0400 | [diff] [blame] | 503 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 504 | /* check provider's send/recv wr limits */ |
Chuck Lever | 124fa17 | 2015-10-24 17:27:51 -0400 | [diff] [blame] | 505 | if (cdata->max_requests > max_qp_wr) |
| 506 | cdata->max_requests = max_qp_wr; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 507 | |
| 508 | ep->rep_attr.event_handler = rpcrdma_qp_async_error_upcall; |
| 509 | ep->rep_attr.qp_context = ep; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 510 | ep->rep_attr.srq = NULL; |
| 511 | ep->rep_attr.cap.max_send_wr = cdata->max_requests; |
Chuck Lever | 124fa17 | 2015-10-24 17:27:51 -0400 | [diff] [blame] | 512 | ep->rep_attr.cap.max_send_wr += RPCRDMA_BACKWARD_WRS; |
Chuck Lever | 550d750 | 2016-05-02 14:41:47 -0400 | [diff] [blame] | 513 | ep->rep_attr.cap.max_send_wr += 1; /* drain cqe */ |
Chuck Lever | 3968cb5 | 2015-03-30 14:35:26 -0400 | [diff] [blame] | 514 | rc = ia->ri_ops->ro_open(ia, ep, cdata); |
| 515 | if (rc) |
| 516 | return rc; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 517 | ep->rep_attr.cap.max_recv_wr = cdata->max_requests; |
Chuck Lever | 124fa17 | 2015-10-24 17:27:51 -0400 | [diff] [blame] | 518 | ep->rep_attr.cap.max_recv_wr += RPCRDMA_BACKWARD_WRS; |
Chuck Lever | 550d750 | 2016-05-02 14:41:47 -0400 | [diff] [blame] | 519 | ep->rep_attr.cap.max_recv_wr += 1; /* drain cqe */ |
Chuck Lever | 86840a6 | 2017-02-08 17:00:10 -0500 | [diff] [blame] | 520 | ep->rep_attr.cap.max_send_sge = max_sge; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 521 | ep->rep_attr.cap.max_recv_sge = 1; |
| 522 | ep->rep_attr.cap.max_inline_data = 0; |
| 523 | ep->rep_attr.sq_sig_type = IB_SIGNAL_REQ_WR; |
| 524 | ep->rep_attr.qp_type = IB_QPT_RC; |
| 525 | ep->rep_attr.port_num = ~0; |
| 526 | |
| 527 | dprintk("RPC: %s: requested max: dtos: send %d recv %d; " |
| 528 | "iovs: send %d recv %d\n", |
| 529 | __func__, |
| 530 | ep->rep_attr.cap.max_send_wr, |
| 531 | ep->rep_attr.cap.max_recv_wr, |
| 532 | ep->rep_attr.cap.max_send_sge, |
| 533 | ep->rep_attr.cap.max_recv_sge); |
| 534 | |
| 535 | /* set trigger for requesting send completion */ |
Chuck Lever | fc66448 | 2014-05-28 10:33:25 -0400 | [diff] [blame] | 536 | ep->rep_cqinit = ep->rep_attr.cap.max_send_wr/2 - 1; |
Chuck Lever | 26ae9d1 | 2015-12-16 17:23:20 -0500 | [diff] [blame] | 537 | if (ep->rep_cqinit <= 2) |
| 538 | ep->rep_cqinit = 0; /* always signal? */ |
Chuck Lever | 8ade1c2 | 2016-11-29 10:52:16 -0500 | [diff] [blame] | 539 | rpcrdma_init_cqcount(ep, 0); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 540 | init_waitqueue_head(&ep->rep_connect_wait); |
Chuck Lever | 254f91e | 2014-05-28 10:32:17 -0400 | [diff] [blame] | 541 | INIT_DELAYED_WORK(&ep->rep_connect_worker, rpcrdma_connect_worker); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 542 | |
Chuck Lever | 2fa8f88 | 2016-03-04 11:28:53 -0500 | [diff] [blame] | 543 | sendcq = ib_alloc_cq(ia->ri_device, NULL, |
| 544 | ep->rep_attr.cap.max_send_wr + 1, |
| 545 | 0, IB_POLL_SOFTIRQ); |
Chuck Lever | fc66448 | 2014-05-28 10:33:25 -0400 | [diff] [blame] | 546 | if (IS_ERR(sendcq)) { |
| 547 | rc = PTR_ERR(sendcq); |
| 548 | dprintk("RPC: %s: failed to create send CQ: %i\n", |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 549 | __func__, rc); |
| 550 | goto out1; |
| 551 | } |
| 552 | |
Chuck Lever | 552bf22 | 2016-03-04 11:28:36 -0500 | [diff] [blame] | 553 | recvcq = ib_alloc_cq(ia->ri_device, NULL, |
| 554 | ep->rep_attr.cap.max_recv_wr + 1, |
| 555 | 0, IB_POLL_SOFTIRQ); |
Chuck Lever | fc66448 | 2014-05-28 10:33:25 -0400 | [diff] [blame] | 556 | if (IS_ERR(recvcq)) { |
| 557 | rc = PTR_ERR(recvcq); |
| 558 | dprintk("RPC: %s: failed to create recv CQ: %i\n", |
| 559 | __func__, rc); |
| 560 | goto out2; |
| 561 | } |
| 562 | |
Chuck Lever | fc66448 | 2014-05-28 10:33:25 -0400 | [diff] [blame] | 563 | ep->rep_attr.send_cq = sendcq; |
| 564 | ep->rep_attr.recv_cq = recvcq; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 565 | |
| 566 | /* Initialize cma parameters */ |
Chuck Lever | b2dde94 | 2016-05-02 14:43:03 -0400 | [diff] [blame] | 567 | memset(&ep->rep_remote_cma, 0, sizeof(ep->rep_remote_cma)); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 568 | |
Chuck Lever | 87cfb9a | 2016-09-15 10:57:07 -0400 | [diff] [blame] | 569 | /* Prepare RDMA-CM private message */ |
| 570 | pmsg->cp_magic = rpcrdma_cmp_magic; |
| 571 | pmsg->cp_version = RPCRDMA_CMP_VERSION; |
Chuck Lever | c8b920b | 2016-09-15 10:57:16 -0400 | [diff] [blame] | 572 | pmsg->cp_flags |= ia->ri_ops->ro_send_w_inv_ok; |
Chuck Lever | 87cfb9a | 2016-09-15 10:57:07 -0400 | [diff] [blame] | 573 | pmsg->cp_send_size = rpcrdma_encode_buffer_size(cdata->inline_wsize); |
| 574 | pmsg->cp_recv_size = rpcrdma_encode_buffer_size(cdata->inline_rsize); |
| 575 | ep->rep_remote_cma.private_data = pmsg; |
| 576 | ep->rep_remote_cma.private_data_len = sizeof(*pmsg); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 577 | |
| 578 | /* Client offers RDMA Read but does not initiate */ |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 579 | ep->rep_remote_cma.initiator_depth = 0; |
Or Gerlitz | e3e45b1 | 2015-12-18 10:59:48 +0200 | [diff] [blame] | 580 | if (ia->ri_device->attrs.max_qp_rd_atom > 32) /* arbitrary but <= 255 */ |
Tom Tucker | b334eaa | 2008-10-09 15:00:30 -0400 | [diff] [blame] | 581 | ep->rep_remote_cma.responder_resources = 32; |
| 582 | else |
Chuck Lever | 7bc7972 | 2015-01-21 11:03:27 -0500 | [diff] [blame] | 583 | ep->rep_remote_cma.responder_resources = |
Or Gerlitz | e3e45b1 | 2015-12-18 10:59:48 +0200 | [diff] [blame] | 584 | ia->ri_device->attrs.max_qp_rd_atom; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 585 | |
Chuck Lever | b2dde94 | 2016-05-02 14:43:03 -0400 | [diff] [blame] | 586 | /* Limit transport retries so client can detect server |
| 587 | * GID changes quickly. RPC layer handles re-establishing |
| 588 | * transport connection and retransmission. |
| 589 | */ |
| 590 | ep->rep_remote_cma.retry_count = 6; |
| 591 | |
| 592 | /* RPC-over-RDMA handles its own flow control. In addition, |
| 593 | * make all RNR NAKs visible so we know that RPC-over-RDMA |
| 594 | * flow control is working correctly (no NAKs should be seen). |
| 595 | */ |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 596 | ep->rep_remote_cma.flow_control = 0; |
| 597 | ep->rep_remote_cma.rnr_retry_count = 0; |
| 598 | |
| 599 | return 0; |
| 600 | |
| 601 | out2: |
Chuck Lever | 2fa8f88 | 2016-03-04 11:28:53 -0500 | [diff] [blame] | 602 | ib_free_cq(sendcq); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 603 | out1: |
| 604 | return rc; |
| 605 | } |
| 606 | |
| 607 | /* |
| 608 | * rpcrdma_ep_destroy |
| 609 | * |
| 610 | * Disconnect and destroy endpoint. After this, the only |
| 611 | * valid operations on the ep are to free it (if dynamically |
| 612 | * allocated) or re-create it. |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 613 | */ |
Chuck Lever | 7f1d541 | 2014-05-28 10:33:16 -0400 | [diff] [blame] | 614 | void |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 615 | rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) |
| 616 | { |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 617 | dprintk("RPC: %s: entering, connected is %d\n", |
| 618 | __func__, ep->rep_connected); |
| 619 | |
Chuck Lever | 254f91e | 2014-05-28 10:32:17 -0400 | [diff] [blame] | 620 | cancel_delayed_work_sync(&ep->rep_connect_worker); |
| 621 | |
Steve Wise | 72c0217 | 2015-09-21 12:24:23 -0500 | [diff] [blame] | 622 | if (ia->ri_id->qp) { |
Chuck Lever | 550d750 | 2016-05-02 14:41:47 -0400 | [diff] [blame] | 623 | rpcrdma_ep_disconnect(ep, ia); |
Tom Talpey | fee08ca | 2008-10-09 15:01:00 -0400 | [diff] [blame] | 624 | rdma_destroy_qp(ia->ri_id); |
| 625 | ia->ri_id->qp = NULL; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 626 | } |
| 627 | |
Chuck Lever | 552bf22 | 2016-03-04 11:28:36 -0500 | [diff] [blame] | 628 | ib_free_cq(ep->rep_attr.recv_cq); |
Chuck Lever | 2fa8f88 | 2016-03-04 11:28:53 -0500 | [diff] [blame] | 629 | ib_free_cq(ep->rep_attr.send_cq); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 630 | } |
| 631 | |
| 632 | /* |
| 633 | * Connect unconnected endpoint. |
| 634 | */ |
| 635 | int |
| 636 | rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) |
| 637 | { |
Chuck Lever | 73806c8 | 2014-07-29 17:23:25 -0400 | [diff] [blame] | 638 | struct rdma_cm_id *id, *old; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 639 | int rc = 0; |
| 640 | int retry_count = 0; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 641 | |
Tom Talpey | c055551 | 2008-10-10 11:32:45 -0400 | [diff] [blame] | 642 | if (ep->rep_connected != 0) { |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 643 | struct rpcrdma_xprt *xprt; |
| 644 | retry: |
Chuck Lever | ec62f40 | 2014-05-28 10:34:07 -0400 | [diff] [blame] | 645 | dprintk("RPC: %s: reconnecting...\n", __func__); |
Chuck Lever | 282191c | 2014-07-29 17:25:55 -0400 | [diff] [blame] | 646 | |
| 647 | rpcrdma_ep_disconnect(ep, ia); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 648 | |
| 649 | xprt = container_of(ia, struct rpcrdma_xprt, rx_ia); |
| 650 | id = rpcrdma_create_id(xprt, ia, |
| 651 | (struct sockaddr *)&xprt->rx_data.addr); |
| 652 | if (IS_ERR(id)) { |
Chuck Lever | ec62f40 | 2014-05-28 10:34:07 -0400 | [diff] [blame] | 653 | rc = -EHOSTUNREACH; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 654 | goto out; |
| 655 | } |
| 656 | /* TEMP TEMP TEMP - fail if new device: |
| 657 | * Deregister/remarshal *all* requests! |
| 658 | * Close and recreate adapter, pd, etc! |
| 659 | * Re-determine all attributes still sane! |
| 660 | * More stuff I haven't thought of! |
| 661 | * Rrrgh! |
| 662 | */ |
Chuck Lever | 89e0d112 | 2015-05-26 11:51:56 -0400 | [diff] [blame] | 663 | if (ia->ri_device != id->device) { |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 664 | printk("RPC: %s: can't reconnect on " |
| 665 | "different device!\n", __func__); |
Devesh Sharma | d0f36c4 | 2015-08-03 13:05:04 -0400 | [diff] [blame] | 666 | rpcrdma_destroy_id(id); |
Chuck Lever | ec62f40 | 2014-05-28 10:34:07 -0400 | [diff] [blame] | 667 | rc = -ENETUNREACH; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 668 | goto out; |
| 669 | } |
| 670 | /* END TEMP */ |
Chuck Lever | ec62f40 | 2014-05-28 10:34:07 -0400 | [diff] [blame] | 671 | rc = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr); |
| 672 | if (rc) { |
| 673 | dprintk("RPC: %s: rdma_create_qp failed %i\n", |
| 674 | __func__, rc); |
Devesh Sharma | d0f36c4 | 2015-08-03 13:05:04 -0400 | [diff] [blame] | 675 | rpcrdma_destroy_id(id); |
Chuck Lever | ec62f40 | 2014-05-28 10:34:07 -0400 | [diff] [blame] | 676 | rc = -ENETUNREACH; |
| 677 | goto out; |
| 678 | } |
Chuck Lever | 73806c8 | 2014-07-29 17:23:25 -0400 | [diff] [blame] | 679 | |
Chuck Lever | 73806c8 | 2014-07-29 17:23:25 -0400 | [diff] [blame] | 680 | old = ia->ri_id; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 681 | ia->ri_id = id; |
Chuck Lever | 73806c8 | 2014-07-29 17:23:25 -0400 | [diff] [blame] | 682 | |
| 683 | rdma_destroy_qp(old); |
Devesh Sharma | d0f36c4 | 2015-08-03 13:05:04 -0400 | [diff] [blame] | 684 | rpcrdma_destroy_id(old); |
Chuck Lever | ec62f40 | 2014-05-28 10:34:07 -0400 | [diff] [blame] | 685 | } else { |
| 686 | dprintk("RPC: %s: connecting...\n", __func__); |
| 687 | rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr); |
| 688 | if (rc) { |
| 689 | dprintk("RPC: %s: rdma_create_qp failed %i\n", |
| 690 | __func__, rc); |
| 691 | /* do not update ep->rep_connected */ |
| 692 | return -ENETUNREACH; |
| 693 | } |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 694 | } |
| 695 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 696 | ep->rep_connected = 0; |
| 697 | |
| 698 | rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma); |
| 699 | if (rc) { |
| 700 | dprintk("RPC: %s: rdma_connect() failed with %i\n", |
| 701 | __func__, rc); |
| 702 | goto out; |
| 703 | } |
| 704 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 705 | wait_event_interruptible(ep->rep_connect_wait, ep->rep_connected != 0); |
| 706 | |
| 707 | /* |
| 708 | * Check state. A non-peer reject indicates no listener |
| 709 | * (ECONNREFUSED), which may be a transient state. All |
| 710 | * others indicate a transport condition which has already |
| 711 | * undergone a best-effort. |
| 712 | */ |
Joe Perches | f64f9e7 | 2009-11-29 16:55:45 -0800 | [diff] [blame] | 713 | if (ep->rep_connected == -ECONNREFUSED && |
| 714 | ++retry_count <= RDMA_CONNECT_RETRY_MAX) { |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 715 | dprintk("RPC: %s: non-peer_reject, retry\n", __func__); |
| 716 | goto retry; |
| 717 | } |
| 718 | if (ep->rep_connected <= 0) { |
| 719 | /* Sometimes, the only way to reliably connect to remote |
| 720 | * CMs is to use same nonzero values for ORD and IRD. */ |
Tom Tucker | b334eaa | 2008-10-09 15:00:30 -0400 | [diff] [blame] | 721 | if (retry_count++ <= RDMA_CONNECT_RETRY_MAX + 1 && |
| 722 | (ep->rep_remote_cma.responder_resources == 0 || |
| 723 | ep->rep_remote_cma.initiator_depth != |
| 724 | ep->rep_remote_cma.responder_resources)) { |
| 725 | if (ep->rep_remote_cma.responder_resources == 0) |
| 726 | ep->rep_remote_cma.responder_resources = 1; |
| 727 | ep->rep_remote_cma.initiator_depth = |
| 728 | ep->rep_remote_cma.responder_resources; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 729 | goto retry; |
Tom Tucker | b334eaa | 2008-10-09 15:00:30 -0400 | [diff] [blame] | 730 | } |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 731 | rc = ep->rep_connected; |
| 732 | } else { |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 733 | struct rpcrdma_xprt *r_xprt; |
| 734 | unsigned int extras; |
| 735 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 736 | dprintk("RPC: %s: connected\n", __func__); |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 737 | |
| 738 | r_xprt = container_of(ia, struct rpcrdma_xprt, rx_ia); |
| 739 | extras = r_xprt->rx_buf.rb_bc_srv_max_requests; |
| 740 | |
| 741 | if (extras) { |
| 742 | rc = rpcrdma_ep_post_extra_recv(r_xprt, extras); |
Dan Carpenter | 38b95bc | 2015-11-05 11:37:08 +0300 | [diff] [blame] | 743 | if (rc) { |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 744 | pr_warn("%s: rpcrdma_ep_post_extra_recv: %i\n", |
| 745 | __func__, rc); |
| 746 | rc = 0; |
Dan Carpenter | 38b95bc | 2015-11-05 11:37:08 +0300 | [diff] [blame] | 747 | } |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 748 | } |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 749 | } |
| 750 | |
| 751 | out: |
| 752 | if (rc) |
| 753 | ep->rep_connected = rc; |
| 754 | return rc; |
| 755 | } |
| 756 | |
| 757 | /* |
| 758 | * rpcrdma_ep_disconnect |
| 759 | * |
| 760 | * This is separate from destroy to facilitate the ability |
| 761 | * to reconnect without recreating the endpoint. |
| 762 | * |
| 763 | * This call is not reentrant, and must not be made in parallel |
| 764 | * on the same endpoint. |
| 765 | */ |
Chuck Lever | 282191c | 2014-07-29 17:25:55 -0400 | [diff] [blame] | 766 | void |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 767 | rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) |
| 768 | { |
| 769 | int rc; |
| 770 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 771 | rc = rdma_disconnect(ia->ri_id); |
| 772 | if (!rc) { |
| 773 | /* returns without wait if not connected */ |
| 774 | wait_event_interruptible(ep->rep_connect_wait, |
| 775 | ep->rep_connected != 1); |
| 776 | dprintk("RPC: %s: after wait, %sconnected\n", __func__, |
| 777 | (ep->rep_connected == 1) ? "still " : "dis"); |
| 778 | } else { |
| 779 | dprintk("RPC: %s: rdma_disconnect %i\n", __func__, rc); |
| 780 | ep->rep_connected = rc; |
| 781 | } |
Chuck Lever | 550d750 | 2016-05-02 14:41:47 -0400 | [diff] [blame] | 782 | |
| 783 | ib_drain_qp(ia->ri_id->qp); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 784 | } |
| 785 | |
Chuck Lever | 505bbe6 | 2016-06-29 13:52:54 -0400 | [diff] [blame] | 786 | static void |
| 787 | rpcrdma_mr_recovery_worker(struct work_struct *work) |
| 788 | { |
| 789 | struct rpcrdma_buffer *buf = container_of(work, struct rpcrdma_buffer, |
| 790 | rb_recovery_worker.work); |
| 791 | struct rpcrdma_mw *mw; |
| 792 | |
| 793 | spin_lock(&buf->rb_recovery_lock); |
| 794 | while (!list_empty(&buf->rb_stale_mrs)) { |
| 795 | mw = list_first_entry(&buf->rb_stale_mrs, |
| 796 | struct rpcrdma_mw, mw_list); |
| 797 | list_del_init(&mw->mw_list); |
| 798 | spin_unlock(&buf->rb_recovery_lock); |
| 799 | |
| 800 | dprintk("RPC: %s: recovering MR %p\n", __func__, mw); |
| 801 | mw->mw_xprt->rx_ia.ri_ops->ro_recover_mr(mw); |
| 802 | |
| 803 | spin_lock(&buf->rb_recovery_lock); |
kbuild test robot | 53d7852 | 2016-07-16 06:02:05 +0800 | [diff] [blame] | 804 | } |
Chuck Lever | 505bbe6 | 2016-06-29 13:52:54 -0400 | [diff] [blame] | 805 | spin_unlock(&buf->rb_recovery_lock); |
| 806 | } |
| 807 | |
| 808 | void |
| 809 | rpcrdma_defer_mr_recovery(struct rpcrdma_mw *mw) |
| 810 | { |
| 811 | struct rpcrdma_xprt *r_xprt = mw->mw_xprt; |
| 812 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; |
| 813 | |
| 814 | spin_lock(&buf->rb_recovery_lock); |
| 815 | list_add(&mw->mw_list, &buf->rb_stale_mrs); |
| 816 | spin_unlock(&buf->rb_recovery_lock); |
| 817 | |
| 818 | schedule_delayed_work(&buf->rb_recovery_worker, 0); |
| 819 | } |
| 820 | |
Chuck Lever | e2ac236 | 2016-06-29 13:54:00 -0400 | [diff] [blame] | 821 | static void |
| 822 | rpcrdma_create_mrs(struct rpcrdma_xprt *r_xprt) |
| 823 | { |
| 824 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; |
| 825 | struct rpcrdma_ia *ia = &r_xprt->rx_ia; |
| 826 | unsigned int count; |
| 827 | LIST_HEAD(free); |
| 828 | LIST_HEAD(all); |
| 829 | |
| 830 | for (count = 0; count < 32; count++) { |
| 831 | struct rpcrdma_mw *mw; |
| 832 | int rc; |
| 833 | |
| 834 | mw = kzalloc(sizeof(*mw), GFP_KERNEL); |
| 835 | if (!mw) |
| 836 | break; |
| 837 | |
| 838 | rc = ia->ri_ops->ro_init_mr(ia, mw); |
| 839 | if (rc) { |
| 840 | kfree(mw); |
| 841 | break; |
| 842 | } |
| 843 | |
| 844 | mw->mw_xprt = r_xprt; |
| 845 | |
| 846 | list_add(&mw->mw_list, &free); |
| 847 | list_add(&mw->mw_all, &all); |
| 848 | } |
| 849 | |
| 850 | spin_lock(&buf->rb_mwlock); |
| 851 | list_splice(&free, &buf->rb_mws); |
| 852 | list_splice(&all, &buf->rb_all); |
| 853 | r_xprt->rx_stats.mrs_allocated += count; |
| 854 | spin_unlock(&buf->rb_mwlock); |
| 855 | |
| 856 | dprintk("RPC: %s: created %u MRs\n", __func__, count); |
| 857 | } |
| 858 | |
| 859 | static void |
| 860 | rpcrdma_mr_refresh_worker(struct work_struct *work) |
| 861 | { |
| 862 | struct rpcrdma_buffer *buf = container_of(work, struct rpcrdma_buffer, |
| 863 | rb_refresh_worker.work); |
| 864 | struct rpcrdma_xprt *r_xprt = container_of(buf, struct rpcrdma_xprt, |
| 865 | rx_buf); |
| 866 | |
| 867 | rpcrdma_create_mrs(r_xprt); |
| 868 | } |
| 869 | |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 870 | struct rpcrdma_req * |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 871 | rpcrdma_create_req(struct rpcrdma_xprt *r_xprt) |
| 872 | { |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 873 | struct rpcrdma_buffer *buffer = &r_xprt->rx_buf; |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 874 | struct rpcrdma_req *req; |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 875 | |
Chuck Lever | 85275c8 | 2015-01-21 11:04:16 -0500 | [diff] [blame] | 876 | req = kzalloc(sizeof(*req), GFP_KERNEL); |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 877 | if (req == NULL) |
Chuck Lever | 85275c8 | 2015-01-21 11:04:16 -0500 | [diff] [blame] | 878 | return ERR_PTR(-ENOMEM); |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 879 | |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 880 | INIT_LIST_HEAD(&req->rl_free); |
| 881 | spin_lock(&buffer->rb_reqslock); |
| 882 | list_add(&req->rl_all, &buffer->rb_allreqs); |
| 883 | spin_unlock(&buffer->rb_reqslock); |
Chuck Lever | 2fa8f88 | 2016-03-04 11:28:53 -0500 | [diff] [blame] | 884 | req->rl_cqe.done = rpcrdma_wc_send; |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 885 | req->rl_buffer = &r_xprt->rx_buf; |
Chuck Lever | 9d6b040 | 2016-06-29 13:54:16 -0400 | [diff] [blame] | 886 | INIT_LIST_HEAD(&req->rl_registered); |
Chuck Lever | 90aab60 | 2016-09-15 10:56:43 -0400 | [diff] [blame] | 887 | req->rl_send_wr.next = NULL; |
| 888 | req->rl_send_wr.wr_cqe = &req->rl_cqe; |
Chuck Lever | 655fec6 | 2016-09-15 10:57:24 -0400 | [diff] [blame] | 889 | req->rl_send_wr.sg_list = req->rl_send_sge; |
Chuck Lever | 90aab60 | 2016-09-15 10:56:43 -0400 | [diff] [blame] | 890 | req->rl_send_wr.opcode = IB_WR_SEND; |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 891 | return req; |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 892 | } |
| 893 | |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 894 | struct rpcrdma_rep * |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 895 | rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt) |
| 896 | { |
| 897 | struct rpcrdma_create_data_internal *cdata = &r_xprt->rx_data; |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 898 | struct rpcrdma_ia *ia = &r_xprt->rx_ia; |
| 899 | struct rpcrdma_rep *rep; |
| 900 | int rc; |
| 901 | |
| 902 | rc = -ENOMEM; |
Chuck Lever | 6b1184c | 2015-01-21 11:04:25 -0500 | [diff] [blame] | 903 | rep = kzalloc(sizeof(*rep), GFP_KERNEL); |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 904 | if (rep == NULL) |
| 905 | goto out; |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 906 | |
Chuck Lever | 13650c2 | 2016-09-15 10:56:26 -0400 | [diff] [blame] | 907 | rep->rr_rdmabuf = rpcrdma_alloc_regbuf(cdata->inline_rsize, |
Chuck Lever | 99ef4db | 2016-09-15 10:56:10 -0400 | [diff] [blame] | 908 | DMA_FROM_DEVICE, GFP_KERNEL); |
Chuck Lever | 6b1184c | 2015-01-21 11:04:25 -0500 | [diff] [blame] | 909 | if (IS_ERR(rep->rr_rdmabuf)) { |
| 910 | rc = PTR_ERR(rep->rr_rdmabuf); |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 911 | goto out_free; |
Chuck Lever | 6b1184c | 2015-01-21 11:04:25 -0500 | [diff] [blame] | 912 | } |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 913 | |
Chuck Lever | 89e0d112 | 2015-05-26 11:51:56 -0400 | [diff] [blame] | 914 | rep->rr_device = ia->ri_device; |
Chuck Lever | 1519e96 | 2016-09-15 10:57:49 -0400 | [diff] [blame] | 915 | rep->rr_cqe.done = rpcrdma_wc_receive; |
Chuck Lever | fed171b | 2015-05-26 11:51:37 -0400 | [diff] [blame] | 916 | rep->rr_rxprt = r_xprt; |
Chuck Lever | 496b77a | 2016-09-15 10:57:57 -0400 | [diff] [blame] | 917 | INIT_WORK(&rep->rr_work, rpcrdma_reply_handler); |
Chuck Lever | 6ea8e71 | 2016-09-15 10:56:51 -0400 | [diff] [blame] | 918 | rep->rr_recv_wr.next = NULL; |
| 919 | rep->rr_recv_wr.wr_cqe = &rep->rr_cqe; |
| 920 | rep->rr_recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov; |
| 921 | rep->rr_recv_wr.num_sge = 1; |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 922 | return rep; |
| 923 | |
| 924 | out_free: |
| 925 | kfree(rep); |
| 926 | out: |
| 927 | return ERR_PTR(rc); |
| 928 | } |
| 929 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 930 | int |
Chuck Lever | ac920d0 | 2015-01-21 11:03:44 -0500 | [diff] [blame] | 931 | rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt) |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 932 | { |
Chuck Lever | ac920d0 | 2015-01-21 11:03:44 -0500 | [diff] [blame] | 933 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 934 | int i, rc; |
| 935 | |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 936 | buf->rb_max_requests = r_xprt->rx_data.max_requests; |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 937 | buf->rb_bc_srv_max_requests = 0; |
Chuck Lever | 23826c7 | 2016-03-04 11:28:27 -0500 | [diff] [blame] | 938 | atomic_set(&buf->rb_credits, 1); |
Chuck Lever | e2ac236 | 2016-06-29 13:54:00 -0400 | [diff] [blame] | 939 | spin_lock_init(&buf->rb_mwlock); |
Chuck Lever | 505bbe6 | 2016-06-29 13:52:54 -0400 | [diff] [blame] | 940 | spin_lock_init(&buf->rb_lock); |
| 941 | spin_lock_init(&buf->rb_recovery_lock); |
Chuck Lever | e2ac236 | 2016-06-29 13:54:00 -0400 | [diff] [blame] | 942 | INIT_LIST_HEAD(&buf->rb_mws); |
| 943 | INIT_LIST_HEAD(&buf->rb_all); |
Chuck Lever | 505bbe6 | 2016-06-29 13:52:54 -0400 | [diff] [blame] | 944 | INIT_LIST_HEAD(&buf->rb_stale_mrs); |
Chuck Lever | e2ac236 | 2016-06-29 13:54:00 -0400 | [diff] [blame] | 945 | INIT_DELAYED_WORK(&buf->rb_refresh_worker, |
| 946 | rpcrdma_mr_refresh_worker); |
Chuck Lever | 505bbe6 | 2016-06-29 13:52:54 -0400 | [diff] [blame] | 947 | INIT_DELAYED_WORK(&buf->rb_recovery_worker, |
| 948 | rpcrdma_mr_recovery_worker); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 949 | |
Chuck Lever | e2ac236 | 2016-06-29 13:54:00 -0400 | [diff] [blame] | 950 | rpcrdma_create_mrs(r_xprt); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 951 | |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 952 | INIT_LIST_HEAD(&buf->rb_send_bufs); |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 953 | INIT_LIST_HEAD(&buf->rb_allreqs); |
| 954 | spin_lock_init(&buf->rb_reqslock); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 955 | for (i = 0; i < buf->rb_max_requests; i++) { |
| 956 | struct rpcrdma_req *req; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 957 | |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 958 | req = rpcrdma_create_req(r_xprt); |
| 959 | if (IS_ERR(req)) { |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 960 | dprintk("RPC: %s: request buffer %d alloc" |
| 961 | " failed\n", __func__, i); |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 962 | rc = PTR_ERR(req); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 963 | goto out; |
| 964 | } |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 965 | req->rl_backchannel = false; |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 966 | list_add(&req->rl_free, &buf->rb_send_bufs); |
| 967 | } |
| 968 | |
| 969 | INIT_LIST_HEAD(&buf->rb_recv_bufs); |
Chuck Lever | 05c9746 | 2016-09-06 11:22:58 -0400 | [diff] [blame] | 970 | for (i = 0; i < buf->rb_max_requests + RPCRDMA_MAX_BC_REQUESTS; i++) { |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 971 | struct rpcrdma_rep *rep; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 972 | |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 973 | rep = rpcrdma_create_rep(r_xprt); |
| 974 | if (IS_ERR(rep)) { |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 975 | dprintk("RPC: %s: reply buffer %d alloc failed\n", |
| 976 | __func__, i); |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 977 | rc = PTR_ERR(rep); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 978 | goto out; |
| 979 | } |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 980 | list_add(&rep->rr_list, &buf->rb_recv_bufs); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 981 | } |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 982 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 983 | return 0; |
| 984 | out: |
| 985 | rpcrdma_buffer_destroy(buf); |
| 986 | return rc; |
| 987 | } |
| 988 | |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 989 | static struct rpcrdma_req * |
| 990 | rpcrdma_buffer_get_req_locked(struct rpcrdma_buffer *buf) |
| 991 | { |
| 992 | struct rpcrdma_req *req; |
| 993 | |
| 994 | req = list_first_entry(&buf->rb_send_bufs, |
| 995 | struct rpcrdma_req, rl_free); |
| 996 | list_del(&req->rl_free); |
| 997 | return req; |
| 998 | } |
| 999 | |
| 1000 | static struct rpcrdma_rep * |
| 1001 | rpcrdma_buffer_get_rep_locked(struct rpcrdma_buffer *buf) |
| 1002 | { |
| 1003 | struct rpcrdma_rep *rep; |
| 1004 | |
| 1005 | rep = list_first_entry(&buf->rb_recv_bufs, |
| 1006 | struct rpcrdma_rep, rr_list); |
| 1007 | list_del(&rep->rr_list); |
| 1008 | return rep; |
| 1009 | } |
| 1010 | |
Chuck Lever | 2e84522 | 2014-07-29 17:25:38 -0400 | [diff] [blame] | 1011 | static void |
Chuck Lever | 13650c2 | 2016-09-15 10:56:26 -0400 | [diff] [blame] | 1012 | rpcrdma_destroy_rep(struct rpcrdma_rep *rep) |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 1013 | { |
Chuck Lever | 13650c2 | 2016-09-15 10:56:26 -0400 | [diff] [blame] | 1014 | rpcrdma_free_regbuf(rep->rr_rdmabuf); |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 1015 | kfree(rep); |
| 1016 | } |
| 1017 | |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 1018 | void |
Chuck Lever | 13650c2 | 2016-09-15 10:56:26 -0400 | [diff] [blame] | 1019 | rpcrdma_destroy_req(struct rpcrdma_req *req) |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 1020 | { |
Chuck Lever | 13650c2 | 2016-09-15 10:56:26 -0400 | [diff] [blame] | 1021 | rpcrdma_free_regbuf(req->rl_recvbuf); |
| 1022 | rpcrdma_free_regbuf(req->rl_sendbuf); |
| 1023 | rpcrdma_free_regbuf(req->rl_rdmabuf); |
Chuck Lever | 1392402 | 2015-01-21 11:03:52 -0500 | [diff] [blame] | 1024 | kfree(req); |
| 1025 | } |
| 1026 | |
Chuck Lever | e2ac236 | 2016-06-29 13:54:00 -0400 | [diff] [blame] | 1027 | static void |
| 1028 | rpcrdma_destroy_mrs(struct rpcrdma_buffer *buf) |
| 1029 | { |
| 1030 | struct rpcrdma_xprt *r_xprt = container_of(buf, struct rpcrdma_xprt, |
| 1031 | rx_buf); |
| 1032 | struct rpcrdma_ia *ia = rdmab_to_ia(buf); |
| 1033 | struct rpcrdma_mw *mw; |
| 1034 | unsigned int count; |
| 1035 | |
| 1036 | count = 0; |
| 1037 | spin_lock(&buf->rb_mwlock); |
| 1038 | while (!list_empty(&buf->rb_all)) { |
| 1039 | mw = list_entry(buf->rb_all.next, struct rpcrdma_mw, mw_all); |
| 1040 | list_del(&mw->mw_all); |
| 1041 | |
| 1042 | spin_unlock(&buf->rb_mwlock); |
| 1043 | ia->ri_ops->ro_release_mr(mw); |
| 1044 | count++; |
| 1045 | spin_lock(&buf->rb_mwlock); |
| 1046 | } |
| 1047 | spin_unlock(&buf->rb_mwlock); |
| 1048 | r_xprt->rx_stats.mrs_allocated = 0; |
| 1049 | |
| 1050 | dprintk("RPC: %s: released %u MRs\n", __func__, count); |
| 1051 | } |
| 1052 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1053 | void |
| 1054 | rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf) |
| 1055 | { |
Chuck Lever | 505bbe6 | 2016-06-29 13:52:54 -0400 | [diff] [blame] | 1056 | cancel_delayed_work_sync(&buf->rb_recovery_worker); |
Chuck Lever | abdb881 | 2017-04-11 13:22:29 -0400 | [diff] [blame] | 1057 | cancel_delayed_work_sync(&buf->rb_refresh_worker); |
Chuck Lever | 505bbe6 | 2016-06-29 13:52:54 -0400 | [diff] [blame] | 1058 | |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 1059 | while (!list_empty(&buf->rb_recv_bufs)) { |
| 1060 | struct rpcrdma_rep *rep; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1061 | |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 1062 | rep = rpcrdma_buffer_get_rep_locked(buf); |
Chuck Lever | 13650c2 | 2016-09-15 10:56:26 -0400 | [diff] [blame] | 1063 | rpcrdma_destroy_rep(rep); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1064 | } |
Chuck Lever | 05c9746 | 2016-09-06 11:22:58 -0400 | [diff] [blame] | 1065 | buf->rb_send_count = 0; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1066 | |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 1067 | spin_lock(&buf->rb_reqslock); |
| 1068 | while (!list_empty(&buf->rb_allreqs)) { |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 1069 | struct rpcrdma_req *req; |
Allen Andrews | 4034ba0 | 2014-05-28 10:32:09 -0400 | [diff] [blame] | 1070 | |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 1071 | req = list_first_entry(&buf->rb_allreqs, |
| 1072 | struct rpcrdma_req, rl_all); |
| 1073 | list_del(&req->rl_all); |
| 1074 | |
| 1075 | spin_unlock(&buf->rb_reqslock); |
Chuck Lever | 13650c2 | 2016-09-15 10:56:26 -0400 | [diff] [blame] | 1076 | rpcrdma_destroy_req(req); |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 1077 | spin_lock(&buf->rb_reqslock); |
Chuck Lever | 9f9d802 | 2014-07-29 17:24:45 -0400 | [diff] [blame] | 1078 | } |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 1079 | spin_unlock(&buf->rb_reqslock); |
Chuck Lever | 05c9746 | 2016-09-06 11:22:58 -0400 | [diff] [blame] | 1080 | buf->rb_recv_count = 0; |
Chuck Lever | 9f9d802 | 2014-07-29 17:24:45 -0400 | [diff] [blame] | 1081 | |
Chuck Lever | e2ac236 | 2016-06-29 13:54:00 -0400 | [diff] [blame] | 1082 | rpcrdma_destroy_mrs(buf); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1083 | } |
| 1084 | |
Chuck Lever | 346aa66 | 2015-05-26 11:52:06 -0400 | [diff] [blame] | 1085 | struct rpcrdma_mw * |
| 1086 | rpcrdma_get_mw(struct rpcrdma_xprt *r_xprt) |
Chuck Lever | c2922c0 | 2014-07-29 17:24:36 -0400 | [diff] [blame] | 1087 | { |
Chuck Lever | 346aa66 | 2015-05-26 11:52:06 -0400 | [diff] [blame] | 1088 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; |
| 1089 | struct rpcrdma_mw *mw = NULL; |
Chuck Lever | 346aa66 | 2015-05-26 11:52:06 -0400 | [diff] [blame] | 1090 | |
Chuck Lever | 58d1dcf | 2015-05-26 11:53:13 -0400 | [diff] [blame] | 1091 | spin_lock(&buf->rb_mwlock); |
Chuck Lever | 346aa66 | 2015-05-26 11:52:06 -0400 | [diff] [blame] | 1092 | if (!list_empty(&buf->rb_mws)) { |
| 1093 | mw = list_first_entry(&buf->rb_mws, |
| 1094 | struct rpcrdma_mw, mw_list); |
| 1095 | list_del_init(&mw->mw_list); |
Chuck Lever | c2922c0 | 2014-07-29 17:24:36 -0400 | [diff] [blame] | 1096 | } |
Chuck Lever | 58d1dcf | 2015-05-26 11:53:13 -0400 | [diff] [blame] | 1097 | spin_unlock(&buf->rb_mwlock); |
Chuck Lever | 346aa66 | 2015-05-26 11:52:06 -0400 | [diff] [blame] | 1098 | |
| 1099 | if (!mw) |
Chuck Lever | e2ac236 | 2016-06-29 13:54:00 -0400 | [diff] [blame] | 1100 | goto out_nomws; |
Chuck Lever | 346aa66 | 2015-05-26 11:52:06 -0400 | [diff] [blame] | 1101 | return mw; |
Chuck Lever | e2ac236 | 2016-06-29 13:54:00 -0400 | [diff] [blame] | 1102 | |
| 1103 | out_nomws: |
| 1104 | dprintk("RPC: %s: no MWs available\n", __func__); |
| 1105 | schedule_delayed_work(&buf->rb_refresh_worker, 0); |
| 1106 | |
| 1107 | /* Allow the reply handler and refresh worker to run */ |
| 1108 | cond_resched(); |
| 1109 | |
| 1110 | return NULL; |
Chuck Lever | c2922c0 | 2014-07-29 17:24:36 -0400 | [diff] [blame] | 1111 | } |
| 1112 | |
Chuck Lever | 346aa66 | 2015-05-26 11:52:06 -0400 | [diff] [blame] | 1113 | void |
| 1114 | rpcrdma_put_mw(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mw *mw) |
Chuck Lever | c2922c0 | 2014-07-29 17:24:36 -0400 | [diff] [blame] | 1115 | { |
Chuck Lever | 346aa66 | 2015-05-26 11:52:06 -0400 | [diff] [blame] | 1116 | struct rpcrdma_buffer *buf = &r_xprt->rx_buf; |
Chuck Lever | c2922c0 | 2014-07-29 17:24:36 -0400 | [diff] [blame] | 1117 | |
Chuck Lever | 58d1dcf | 2015-05-26 11:53:13 -0400 | [diff] [blame] | 1118 | spin_lock(&buf->rb_mwlock); |
Chuck Lever | 346aa66 | 2015-05-26 11:52:06 -0400 | [diff] [blame] | 1119 | list_add_tail(&mw->mw_list, &buf->rb_mws); |
Chuck Lever | 58d1dcf | 2015-05-26 11:53:13 -0400 | [diff] [blame] | 1120 | spin_unlock(&buf->rb_mwlock); |
Chuck Lever | c2922c0 | 2014-07-29 17:24:36 -0400 | [diff] [blame] | 1121 | } |
| 1122 | |
Chuck Lever | 05c9746 | 2016-09-06 11:22:58 -0400 | [diff] [blame] | 1123 | static struct rpcrdma_rep * |
| 1124 | rpcrdma_buffer_get_rep(struct rpcrdma_buffer *buffers) |
| 1125 | { |
| 1126 | /* If an RPC previously completed without a reply (say, a |
| 1127 | * credential problem or a soft timeout occurs) then hold off |
| 1128 | * on supplying more Receive buffers until the number of new |
| 1129 | * pending RPCs catches up to the number of posted Receives. |
| 1130 | */ |
| 1131 | if (unlikely(buffers->rb_send_count < buffers->rb_recv_count)) |
| 1132 | return NULL; |
| 1133 | |
| 1134 | if (unlikely(list_empty(&buffers->rb_recv_bufs))) |
| 1135 | return NULL; |
| 1136 | buffers->rb_recv_count++; |
| 1137 | return rpcrdma_buffer_get_rep_locked(buffers); |
| 1138 | } |
| 1139 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1140 | /* |
| 1141 | * Get a set of request/reply buffers. |
Chuck Lever | 78d506e | 2016-09-06 11:22:49 -0400 | [diff] [blame] | 1142 | * |
| 1143 | * Reply buffer (if available) is attached to send buffer upon return. |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1144 | */ |
| 1145 | struct rpcrdma_req * |
| 1146 | rpcrdma_buffer_get(struct rpcrdma_buffer *buffers) |
| 1147 | { |
| 1148 | struct rpcrdma_req *req; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1149 | |
Chuck Lever | a5b027e | 2015-10-24 17:27:27 -0400 | [diff] [blame] | 1150 | spin_lock(&buffers->rb_lock); |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 1151 | if (list_empty(&buffers->rb_send_bufs)) |
| 1152 | goto out_reqbuf; |
Chuck Lever | 05c9746 | 2016-09-06 11:22:58 -0400 | [diff] [blame] | 1153 | buffers->rb_send_count++; |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 1154 | req = rpcrdma_buffer_get_req_locked(buffers); |
Chuck Lever | 05c9746 | 2016-09-06 11:22:58 -0400 | [diff] [blame] | 1155 | req->rl_reply = rpcrdma_buffer_get_rep(buffers); |
Chuck Lever | a5b027e | 2015-10-24 17:27:27 -0400 | [diff] [blame] | 1156 | spin_unlock(&buffers->rb_lock); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1157 | return req; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1158 | |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 1159 | out_reqbuf: |
Chuck Lever | a5b027e | 2015-10-24 17:27:27 -0400 | [diff] [blame] | 1160 | spin_unlock(&buffers->rb_lock); |
Chuck Lever | 78d506e | 2016-09-06 11:22:49 -0400 | [diff] [blame] | 1161 | pr_warn("RPC: %s: out of request buffers\n", __func__); |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 1162 | return NULL; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1163 | } |
| 1164 | |
| 1165 | /* |
| 1166 | * Put request/reply buffers back into pool. |
| 1167 | * Pre-decrement counter/array index. |
| 1168 | */ |
| 1169 | void |
| 1170 | rpcrdma_buffer_put(struct rpcrdma_req *req) |
| 1171 | { |
| 1172 | struct rpcrdma_buffer *buffers = req->rl_buffer; |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 1173 | struct rpcrdma_rep *rep = req->rl_reply; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1174 | |
Chuck Lever | 90aab60 | 2016-09-15 10:56:43 -0400 | [diff] [blame] | 1175 | req->rl_send_wr.num_sge = 0; |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 1176 | req->rl_reply = NULL; |
| 1177 | |
Chuck Lever | a5b027e | 2015-10-24 17:27:27 -0400 | [diff] [blame] | 1178 | spin_lock(&buffers->rb_lock); |
Chuck Lever | 05c9746 | 2016-09-06 11:22:58 -0400 | [diff] [blame] | 1179 | buffers->rb_send_count--; |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 1180 | list_add_tail(&req->rl_free, &buffers->rb_send_bufs); |
Chuck Lever | 05c9746 | 2016-09-06 11:22:58 -0400 | [diff] [blame] | 1181 | if (rep) { |
| 1182 | buffers->rb_recv_count--; |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 1183 | list_add_tail(&rep->rr_list, &buffers->rb_recv_bufs); |
Chuck Lever | 05c9746 | 2016-09-06 11:22:58 -0400 | [diff] [blame] | 1184 | } |
Chuck Lever | a5b027e | 2015-10-24 17:27:27 -0400 | [diff] [blame] | 1185 | spin_unlock(&buffers->rb_lock); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1186 | } |
| 1187 | |
| 1188 | /* |
| 1189 | * Recover reply buffers from pool. |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 1190 | * This happens when recovering from disconnect. |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1191 | */ |
| 1192 | void |
| 1193 | rpcrdma_recv_buffer_get(struct rpcrdma_req *req) |
| 1194 | { |
| 1195 | struct rpcrdma_buffer *buffers = req->rl_buffer; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1196 | |
Chuck Lever | a5b027e | 2015-10-24 17:27:27 -0400 | [diff] [blame] | 1197 | spin_lock(&buffers->rb_lock); |
Chuck Lever | 05c9746 | 2016-09-06 11:22:58 -0400 | [diff] [blame] | 1198 | req->rl_reply = rpcrdma_buffer_get_rep(buffers); |
Chuck Lever | a5b027e | 2015-10-24 17:27:27 -0400 | [diff] [blame] | 1199 | spin_unlock(&buffers->rb_lock); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1200 | } |
| 1201 | |
| 1202 | /* |
| 1203 | * Put reply buffers back into pool when not attached to |
| 1204 | * request. This happens in error conditions. |
| 1205 | */ |
| 1206 | void |
| 1207 | rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep) |
| 1208 | { |
Chuck Lever | fed171b | 2015-05-26 11:51:37 -0400 | [diff] [blame] | 1209 | struct rpcrdma_buffer *buffers = &rep->rr_rxprt->rx_buf; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1210 | |
Chuck Lever | a5b027e | 2015-10-24 17:27:27 -0400 | [diff] [blame] | 1211 | spin_lock(&buffers->rb_lock); |
Chuck Lever | 05c9746 | 2016-09-06 11:22:58 -0400 | [diff] [blame] | 1212 | buffers->rb_recv_count--; |
Chuck Lever | 1e465fd | 2015-10-24 17:27:02 -0400 | [diff] [blame] | 1213 | list_add_tail(&rep->rr_list, &buffers->rb_recv_bufs); |
Chuck Lever | a5b027e | 2015-10-24 17:27:27 -0400 | [diff] [blame] | 1214 | spin_unlock(&buffers->rb_lock); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1215 | } |
| 1216 | |
Chuck Lever | 9128c3e | 2015-01-21 11:04:00 -0500 | [diff] [blame] | 1217 | /** |
Chuck Lever | 99ef4db | 2016-09-15 10:56:10 -0400 | [diff] [blame] | 1218 | * rpcrdma_alloc_regbuf - allocate and DMA-map memory for SEND/RECV buffers |
Chuck Lever | 9128c3e | 2015-01-21 11:04:00 -0500 | [diff] [blame] | 1219 | * @size: size of buffer to be allocated, in bytes |
Chuck Lever | 99ef4db | 2016-09-15 10:56:10 -0400 | [diff] [blame] | 1220 | * @direction: direction of data movement |
Chuck Lever | 9128c3e | 2015-01-21 11:04:00 -0500 | [diff] [blame] | 1221 | * @flags: GFP flags |
| 1222 | * |
Chuck Lever | 54cbd6b | 2016-09-15 10:56:18 -0400 | [diff] [blame] | 1223 | * Returns an ERR_PTR, or a pointer to a regbuf, a buffer that |
| 1224 | * can be persistently DMA-mapped for I/O. |
Chuck Lever | 9128c3e | 2015-01-21 11:04:00 -0500 | [diff] [blame] | 1225 | * |
| 1226 | * xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for |
Chuck Lever | 99ef4db | 2016-09-15 10:56:10 -0400 | [diff] [blame] | 1227 | * receiving the payload of RDMA RECV operations. During Long Calls |
| 1228 | * or Replies they may be registered externally via ro_map. |
Chuck Lever | 9128c3e | 2015-01-21 11:04:00 -0500 | [diff] [blame] | 1229 | */ |
| 1230 | struct rpcrdma_regbuf * |
Chuck Lever | 13650c2 | 2016-09-15 10:56:26 -0400 | [diff] [blame] | 1231 | rpcrdma_alloc_regbuf(size_t size, enum dma_data_direction direction, |
| 1232 | gfp_t flags) |
Chuck Lever | 9128c3e | 2015-01-21 11:04:00 -0500 | [diff] [blame] | 1233 | { |
| 1234 | struct rpcrdma_regbuf *rb; |
Chuck Lever | 9128c3e | 2015-01-21 11:04:00 -0500 | [diff] [blame] | 1235 | |
Chuck Lever | 9128c3e | 2015-01-21 11:04:00 -0500 | [diff] [blame] | 1236 | rb = kmalloc(sizeof(*rb) + size, flags); |
| 1237 | if (rb == NULL) |
Chuck Lever | 54cbd6b | 2016-09-15 10:56:18 -0400 | [diff] [blame] | 1238 | return ERR_PTR(-ENOMEM); |
Chuck Lever | 9128c3e | 2015-01-21 11:04:00 -0500 | [diff] [blame] | 1239 | |
Chuck Lever | 54cbd6b | 2016-09-15 10:56:18 -0400 | [diff] [blame] | 1240 | rb->rg_device = NULL; |
Chuck Lever | 99ef4db | 2016-09-15 10:56:10 -0400 | [diff] [blame] | 1241 | rb->rg_direction = direction; |
Chuck Lever | 54cbd6b | 2016-09-15 10:56:18 -0400 | [diff] [blame] | 1242 | rb->rg_iov.length = size; |
Chuck Lever | 9128c3e | 2015-01-21 11:04:00 -0500 | [diff] [blame] | 1243 | |
| 1244 | return rb; |
Chuck Lever | 54cbd6b | 2016-09-15 10:56:18 -0400 | [diff] [blame] | 1245 | } |
Chuck Lever | 9128c3e | 2015-01-21 11:04:00 -0500 | [diff] [blame] | 1246 | |
Chuck Lever | 54cbd6b | 2016-09-15 10:56:18 -0400 | [diff] [blame] | 1247 | /** |
| 1248 | * __rpcrdma_map_regbuf - DMA-map a regbuf |
| 1249 | * @ia: controlling rpcrdma_ia |
| 1250 | * @rb: regbuf to be mapped |
| 1251 | */ |
| 1252 | bool |
| 1253 | __rpcrdma_dma_map_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb) |
| 1254 | { |
| 1255 | if (rb->rg_direction == DMA_NONE) |
| 1256 | return false; |
| 1257 | |
| 1258 | rb->rg_iov.addr = ib_dma_map_single(ia->ri_device, |
| 1259 | (void *)rb->rg_base, |
| 1260 | rdmab_length(rb), |
| 1261 | rb->rg_direction); |
| 1262 | if (ib_dma_mapping_error(ia->ri_device, rdmab_addr(rb))) |
| 1263 | return false; |
| 1264 | |
| 1265 | rb->rg_device = ia->ri_device; |
| 1266 | rb->rg_iov.lkey = ia->ri_pd->local_dma_lkey; |
| 1267 | return true; |
| 1268 | } |
| 1269 | |
| 1270 | static void |
| 1271 | rpcrdma_dma_unmap_regbuf(struct rpcrdma_regbuf *rb) |
| 1272 | { |
| 1273 | if (!rpcrdma_regbuf_is_mapped(rb)) |
| 1274 | return; |
| 1275 | |
| 1276 | ib_dma_unmap_single(rb->rg_device, rdmab_addr(rb), |
| 1277 | rdmab_length(rb), rb->rg_direction); |
| 1278 | rb->rg_device = NULL; |
Chuck Lever | 9128c3e | 2015-01-21 11:04:00 -0500 | [diff] [blame] | 1279 | } |
| 1280 | |
| 1281 | /** |
| 1282 | * rpcrdma_free_regbuf - deregister and free registered buffer |
Chuck Lever | 9128c3e | 2015-01-21 11:04:00 -0500 | [diff] [blame] | 1283 | * @rb: regbuf to be deregistered and freed |
| 1284 | */ |
| 1285 | void |
Chuck Lever | 13650c2 | 2016-09-15 10:56:26 -0400 | [diff] [blame] | 1286 | rpcrdma_free_regbuf(struct rpcrdma_regbuf *rb) |
Chuck Lever | 9128c3e | 2015-01-21 11:04:00 -0500 | [diff] [blame] | 1287 | { |
Chuck Lever | e531dca | 2015-08-03 13:03:20 -0400 | [diff] [blame] | 1288 | if (!rb) |
| 1289 | return; |
| 1290 | |
Chuck Lever | 54cbd6b | 2016-09-15 10:56:18 -0400 | [diff] [blame] | 1291 | rpcrdma_dma_unmap_regbuf(rb); |
Chuck Lever | e531dca | 2015-08-03 13:03:20 -0400 | [diff] [blame] | 1292 | kfree(rb); |
Chuck Lever | 9128c3e | 2015-01-21 11:04:00 -0500 | [diff] [blame] | 1293 | } |
| 1294 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1295 | /* |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1296 | * Prepost any receive buffer, then post send. |
| 1297 | * |
| 1298 | * Receive buffer is donated to hardware, reclaimed upon recv completion. |
| 1299 | */ |
| 1300 | int |
| 1301 | rpcrdma_ep_post(struct rpcrdma_ia *ia, |
| 1302 | struct rpcrdma_ep *ep, |
| 1303 | struct rpcrdma_req *req) |
| 1304 | { |
Chuck Lever | 90aab60 | 2016-09-15 10:56:43 -0400 | [diff] [blame] | 1305 | struct ib_send_wr *send_wr = &req->rl_send_wr; |
| 1306 | struct ib_send_wr *send_wr_fail; |
Chuck Lever | 655fec6 | 2016-09-15 10:57:24 -0400 | [diff] [blame] | 1307 | int rc; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1308 | |
Chuck Lever | 90aab60 | 2016-09-15 10:56:43 -0400 | [diff] [blame] | 1309 | if (req->rl_reply) { |
| 1310 | rc = rpcrdma_ep_post_recv(ia, req->rl_reply); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1311 | if (rc) |
Chuck Lever | 7a89f9c | 2016-06-29 13:53:43 -0400 | [diff] [blame] | 1312 | return rc; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1313 | req->rl_reply = NULL; |
| 1314 | } |
| 1315 | |
Chuck Lever | b3221d6 | 2015-08-03 13:03:39 -0400 | [diff] [blame] | 1316 | dprintk("RPC: %s: posting %d s/g entries\n", |
Chuck Lever | 90aab60 | 2016-09-15 10:56:43 -0400 | [diff] [blame] | 1317 | __func__, send_wr->num_sge); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1318 | |
Chuck Lever | 8ade1c2 | 2016-11-29 10:52:16 -0500 | [diff] [blame] | 1319 | rpcrdma_set_signaled(ep, send_wr); |
Chuck Lever | 90aab60 | 2016-09-15 10:56:43 -0400 | [diff] [blame] | 1320 | rc = ib_post_send(ia->ri_id->qp, send_wr, &send_wr_fail); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1321 | if (rc) |
Chuck Lever | 7a89f9c | 2016-06-29 13:53:43 -0400 | [diff] [blame] | 1322 | goto out_postsend_err; |
| 1323 | return 0; |
| 1324 | |
| 1325 | out_postsend_err: |
| 1326 | pr_err("rpcrdma: RDMA Send ib_post_send returned %i\n", rc); |
| 1327 | return -ENOTCONN; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1328 | } |
| 1329 | |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1330 | int |
| 1331 | rpcrdma_ep_post_recv(struct rpcrdma_ia *ia, |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1332 | struct rpcrdma_rep *rep) |
| 1333 | { |
Chuck Lever | 6ea8e71 | 2016-09-15 10:56:51 -0400 | [diff] [blame] | 1334 | struct ib_recv_wr *recv_wr_fail; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1335 | int rc; |
| 1336 | |
Chuck Lever | 54cbd6b | 2016-09-15 10:56:18 -0400 | [diff] [blame] | 1337 | if (!rpcrdma_dma_map_regbuf(ia, rep->rr_rdmabuf)) |
| 1338 | goto out_map; |
Chuck Lever | 6ea8e71 | 2016-09-15 10:56:51 -0400 | [diff] [blame] | 1339 | rc = ib_post_recv(ia->ri_id->qp, &rep->rr_recv_wr, &recv_wr_fail); |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1340 | if (rc) |
Chuck Lever | 7a89f9c | 2016-06-29 13:53:43 -0400 | [diff] [blame] | 1341 | goto out_postrecv; |
| 1342 | return 0; |
| 1343 | |
Chuck Lever | 54cbd6b | 2016-09-15 10:56:18 -0400 | [diff] [blame] | 1344 | out_map: |
| 1345 | pr_err("rpcrdma: failed to DMA map the Receive buffer\n"); |
| 1346 | return -EIO; |
| 1347 | |
Chuck Lever | 7a89f9c | 2016-06-29 13:53:43 -0400 | [diff] [blame] | 1348 | out_postrecv: |
| 1349 | pr_err("rpcrdma: ib_post_recv returned %i\n", rc); |
| 1350 | return -ENOTCONN; |
\"Talpey, Thomas\ | c56c65f | 2007-09-10 13:51:18 -0400 | [diff] [blame] | 1351 | } |
Chuck Lever | 43e9598 | 2014-07-29 17:23:34 -0400 | [diff] [blame] | 1352 | |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 1353 | /** |
| 1354 | * rpcrdma_ep_post_extra_recv - Post buffers for incoming backchannel requests |
| 1355 | * @r_xprt: transport associated with these backchannel resources |
| 1356 | * @min_reqs: minimum number of incoming requests expected |
| 1357 | * |
| 1358 | * Returns zero if all requested buffers were posted, or a negative errno. |
| 1359 | */ |
| 1360 | int |
| 1361 | rpcrdma_ep_post_extra_recv(struct rpcrdma_xprt *r_xprt, unsigned int count) |
| 1362 | { |
| 1363 | struct rpcrdma_buffer *buffers = &r_xprt->rx_buf; |
| 1364 | struct rpcrdma_ia *ia = &r_xprt->rx_ia; |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 1365 | struct rpcrdma_rep *rep; |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 1366 | int rc; |
| 1367 | |
| 1368 | while (count--) { |
Chuck Lever | 9b06688 | 2015-12-16 17:22:06 -0500 | [diff] [blame] | 1369 | spin_lock(&buffers->rb_lock); |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 1370 | if (list_empty(&buffers->rb_recv_bufs)) |
| 1371 | goto out_reqbuf; |
| 1372 | rep = rpcrdma_buffer_get_rep_locked(buffers); |
Chuck Lever | 9b06688 | 2015-12-16 17:22:06 -0500 | [diff] [blame] | 1373 | spin_unlock(&buffers->rb_lock); |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 1374 | |
Chuck Lever | b157380 | 2016-09-15 10:56:35 -0400 | [diff] [blame] | 1375 | rc = rpcrdma_ep_post_recv(ia, rep); |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 1376 | if (rc) |
| 1377 | goto out_rc; |
| 1378 | } |
| 1379 | |
| 1380 | return 0; |
| 1381 | |
| 1382 | out_reqbuf: |
Chuck Lever | 9b06688 | 2015-12-16 17:22:06 -0500 | [diff] [blame] | 1383 | spin_unlock(&buffers->rb_lock); |
Chuck Lever | f531a5d | 2015-10-24 17:27:43 -0400 | [diff] [blame] | 1384 | pr_warn("%s: no extra receive buffers\n", __func__); |
| 1385 | return -ENOMEM; |
| 1386 | |
| 1387 | out_rc: |
| 1388 | rpcrdma_recv_buffer_put(rep); |
| 1389 | return rc; |
| 1390 | } |