blob: 878f1bfb1db98f6972e641130d36a80cdf97faf8 [file] [log] [blame]
\"Talpey, Thomas\f58851e2007-09-10 13:50:12 -04001/*
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04002 * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the BSD-type
8 * license below:
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 *
14 * Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 *
17 * Redistributions in binary form must reproduce the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer in the documentation and/or other materials provided
20 * with the distribution.
21 *
22 * Neither the name of the Network Appliance, Inc. nor the names of
23 * its contributors may be used to endorse or promote products
24 * derived from this software without specific prior written
25 * permission.
26 *
27 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
28 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
29 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
30 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
31 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
32 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
33 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
34 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
35 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
36 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\"Talpey, Thomas\f58851e2007-09-10 13:50:12 -040038 */
39
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -040040/*
41 * verbs.c
42 *
43 * Encapsulates the major functions managing:
44 * o adapters
45 * o endpoints
46 * o connections
47 * o buffer memory
48 */
49
Alexey Dobriyana6b7a402011-06-06 10:43:46 +000050#include <linux/interrupt.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090051#include <linux/slab.h>
Chuck Levereba8ff62015-01-21 11:03:02 -050052#include <linux/prefetch.h>
Chuck Lever0dd39ca2015-03-30 14:33:43 -040053#include <linux/sunrpc/addr.h>
Chuck Lever65866f82014-05-28 10:33:59 -040054#include <asm/bitops.h>
Devesh Sharmad0f36c42015-08-03 13:05:04 -040055#include <linux/module.h> /* try_module_get()/module_put() */
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -040056
\"Talpey, Thomas\f58851e2007-09-10 13:50:12 -040057#include "xprt_rdma.h"
58
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -040059/*
60 * Globals/Macros
61 */
62
Jeff Laytonf895b252014-11-17 16:58:04 -050063#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -040064# define RPCDBG_FACILITY RPCDBG_TRANS
65#endif
66
67/*
68 * internal functions
69 */
70
Chuck Leverfe97b472015-10-24 17:27:10 -040071static struct workqueue_struct *rpcrdma_receive_wq;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -040072
Chuck Leverfe97b472015-10-24 17:27:10 -040073int
74rpcrdma_alloc_wq(void)
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -040075{
Chuck Leverfe97b472015-10-24 17:27:10 -040076 struct workqueue_struct *recv_wq;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -040077
Chuck Leverfe97b472015-10-24 17:27:10 -040078 recv_wq = alloc_workqueue("xprtrdma_receive",
79 WQ_MEM_RECLAIM | WQ_UNBOUND | WQ_HIGHPRI,
80 0);
81 if (!recv_wq)
82 return -ENOMEM;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -040083
Chuck Leverfe97b472015-10-24 17:27:10 -040084 rpcrdma_receive_wq = recv_wq;
85 return 0;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -040086}
87
Chuck Leverfe97b472015-10-24 17:27:10 -040088void
89rpcrdma_destroy_wq(void)
Chuck Leverf1a03b72014-11-08 20:14:37 -050090{
Chuck Leverfe97b472015-10-24 17:27:10 -040091 struct workqueue_struct *wq;
Chuck Leverf1a03b72014-11-08 20:14:37 -050092
Chuck Leverfe97b472015-10-24 17:27:10 -040093 if (rpcrdma_receive_wq) {
94 wq = rpcrdma_receive_wq;
95 rpcrdma_receive_wq = NULL;
96 destroy_workqueue(wq);
97 }
Chuck Leverf1a03b72014-11-08 20:14:37 -050098}
99
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400100static void
101rpcrdma_qp_async_error_upcall(struct ib_event *event, void *context)
102{
103 struct rpcrdma_ep *ep = context;
104
Chuck Lever7ff11de2014-11-08 20:15:01 -0500105 pr_err("RPC: %s: %s on device %s ep %p\n",
Sagi Grimberg76357c72015-05-18 13:40:32 +0300106 __func__, ib_event_msg(event->event),
Chuck Lever7ff11de2014-11-08 20:15:01 -0500107 event->device->name, context);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400108 if (ep->rep_connected == 1) {
109 ep->rep_connected = -EIO;
Chuck Leverafadc462015-01-21 11:03:11 -0500110 rpcrdma_conn_func(ep);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400111 wake_up_all(&ep->rep_connect_wait);
112 }
113}
114
115static void
116rpcrdma_cq_async_error_upcall(struct ib_event *event, void *context)
117{
118 struct rpcrdma_ep *ep = context;
119
Chuck Lever7ff11de2014-11-08 20:15:01 -0500120 pr_err("RPC: %s: %s on device %s ep %p\n",
Sagi Grimberg76357c72015-05-18 13:40:32 +0300121 __func__, ib_event_msg(event->event),
Chuck Lever7ff11de2014-11-08 20:15:01 -0500122 event->device->name, context);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400123 if (ep->rep_connected == 1) {
124 ep->rep_connected = -EIO;
Chuck Leverafadc462015-01-21 11:03:11 -0500125 rpcrdma_conn_func(ep);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400126 wake_up_all(&ep->rep_connect_wait);
127 }
128}
129
Chuck Leverfc664482014-05-28 10:33:25 -0400130static void
131rpcrdma_sendcq_process_wc(struct ib_wc *wc)
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400132{
Chuck Lever85024272015-01-21 11:02:04 -0500133 /* WARNING: Only wr_id and status are reliable at this point */
Chuck Levere46ac342015-03-30 14:35:35 -0400134 if (wc->wr_id == RPCRDMA_IGNORE_COMPLETION) {
135 if (wc->status != IB_WC_SUCCESS &&
136 wc->status != IB_WC_WR_FLUSH_ERR)
Chuck Lever85024272015-01-21 11:02:04 -0500137 pr_err("RPC: %s: SEND: %s\n",
Sagi Grimberg76357c72015-05-18 13:40:32 +0300138 __func__, ib_wc_status_msg(wc->status));
Chuck Lever85024272015-01-21 11:02:04 -0500139 } else {
140 struct rpcrdma_mw *r;
141
142 r = (struct rpcrdma_mw *)(unsigned long)wc->wr_id;
Chuck Levere46ac342015-03-30 14:35:35 -0400143 r->mw_sendcompletion(wc);
Chuck Lever85024272015-01-21 11:02:04 -0500144 }
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400145}
146
Chuck Lever4220a072015-10-24 17:26:45 -0400147/* The common case is a single send completion is waiting. By
148 * passing two WC entries to ib_poll_cq, a return code of 1
149 * means there is exactly one WC waiting and no more. We don't
150 * have to invoke ib_poll_cq again to know that the CQ has been
151 * properly drained.
152 */
153static void
154rpcrdma_sendcq_poll(struct ib_cq *cq)
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400155{
Chuck Lever4220a072015-10-24 17:26:45 -0400156 struct ib_wc *pos, wcs[2];
157 int count, rc;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400158
Chuck Lever1c00dd02014-05-28 10:33:42 -0400159 do {
Chuck Lever4220a072015-10-24 17:26:45 -0400160 pos = wcs;
Chuck Lever1c00dd02014-05-28 10:33:42 -0400161
Chuck Lever4220a072015-10-24 17:26:45 -0400162 rc = ib_poll_cq(cq, ARRAY_SIZE(wcs), pos);
163 if (rc < 0)
164 break;
Chuck Lever1c00dd02014-05-28 10:33:42 -0400165
166 count = rc;
167 while (count-- > 0)
Chuck Lever4220a072015-10-24 17:26:45 -0400168 rpcrdma_sendcq_process_wc(pos++);
169 } while (rc == ARRAY_SIZE(wcs));
170 return;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400171}
172
Chuck Lever7b3d7702015-10-24 17:26:37 -0400173/* Handle provider send completion upcalls.
Chuck Leverfc664482014-05-28 10:33:25 -0400174 */
175static void
176rpcrdma_sendcq_upcall(struct ib_cq *cq, void *cq_context)
177{
Chuck Lever7b3d7702015-10-24 17:26:37 -0400178 do {
Chuck Lever4220a072015-10-24 17:26:45 -0400179 rpcrdma_sendcq_poll(cq);
Chuck Lever7b3d7702015-10-24 17:26:37 -0400180 } while (ib_req_notify_cq(cq, IB_CQ_NEXT_COMP |
181 IB_CQ_REPORT_MISSED_EVENTS) > 0);
Chuck Leverfc664482014-05-28 10:33:25 -0400182}
183
184static void
Chuck Leverfe97b472015-10-24 17:27:10 -0400185rpcrdma_receive_worker(struct work_struct *work)
186{
187 struct rpcrdma_rep *rep =
188 container_of(work, struct rpcrdma_rep, rr_work);
189
190 rpcrdma_reply_handler(rep);
191}
192
193static void
194rpcrdma_recvcq_process_wc(struct ib_wc *wc)
Chuck Leverfc664482014-05-28 10:33:25 -0400195{
196 struct rpcrdma_rep *rep =
197 (struct rpcrdma_rep *)(unsigned long)wc->wr_id;
198
Chuck Lever85024272015-01-21 11:02:04 -0500199 /* WARNING: Only wr_id and status are reliable at this point */
200 if (wc->status != IB_WC_SUCCESS)
201 goto out_fail;
Chuck Leverfc664482014-05-28 10:33:25 -0400202
Chuck Lever85024272015-01-21 11:02:04 -0500203 /* status == SUCCESS means all fields in wc are trustworthy */
Chuck Leverfc664482014-05-28 10:33:25 -0400204 if (wc->opcode != IB_WC_RECV)
205 return;
206
Chuck Lever85024272015-01-21 11:02:04 -0500207 dprintk("RPC: %s: rep %p opcode 'recv', length %u: success\n",
208 __func__, rep, wc->byte_len);
209
Chuck Leverfc664482014-05-28 10:33:25 -0400210 rep->rr_len = wc->byte_len;
Chuck Lever89e0d1122015-05-26 11:51:56 -0400211 ib_dma_sync_single_for_cpu(rep->rr_device,
Chuck Lever6b1184c2015-01-21 11:04:25 -0500212 rdmab_addr(rep->rr_rdmabuf),
213 rep->rr_len, DMA_FROM_DEVICE);
214 prefetch(rdmab_to_msg(rep->rr_rdmabuf));
Chuck Leverfc664482014-05-28 10:33:25 -0400215
216out_schedule:
Chuck Leverfe97b472015-10-24 17:27:10 -0400217 queue_work(rpcrdma_receive_wq, &rep->rr_work);
Chuck Lever85024272015-01-21 11:02:04 -0500218 return;
Chuck Leverfe97b472015-10-24 17:27:10 -0400219
Chuck Lever85024272015-01-21 11:02:04 -0500220out_fail:
221 if (wc->status != IB_WC_WR_FLUSH_ERR)
222 pr_err("RPC: %s: rep %p: %s\n",
Sagi Grimberg76357c72015-05-18 13:40:32 +0300223 __func__, rep, ib_wc_status_msg(wc->status));
Chuck Leverb0e178a2015-10-24 17:26:54 -0400224 rep->rr_len = RPCRDMA_BAD_LEN;
Chuck Lever85024272015-01-21 11:02:04 -0500225 goto out_schedule;
Chuck Leverfc664482014-05-28 10:33:25 -0400226}
227
Chuck Lever4220a072015-10-24 17:26:45 -0400228/* The wc array is on stack: automatic memory is always CPU-local.
229 *
230 * struct ib_wc is 64 bytes, making the poll array potentially
231 * large. But this is at the bottom of the call chain. Further
232 * substantial work is done in another thread.
233 */
234static void
235rpcrdma_recvcq_poll(struct ib_cq *cq)
Chuck Leverfc664482014-05-28 10:33:25 -0400236{
Chuck Lever4220a072015-10-24 17:26:45 -0400237 struct ib_wc *pos, wcs[4];
Chuck Lever4220a072015-10-24 17:26:45 -0400238 int count, rc;
Chuck Leverfc664482014-05-28 10:33:25 -0400239
Chuck Lever1c00dd02014-05-28 10:33:42 -0400240 do {
Chuck Lever4220a072015-10-24 17:26:45 -0400241 pos = wcs;
Chuck Lever1c00dd02014-05-28 10:33:42 -0400242
Chuck Lever4220a072015-10-24 17:26:45 -0400243 rc = ib_poll_cq(cq, ARRAY_SIZE(wcs), pos);
244 if (rc < 0)
245 break;
Chuck Lever1c00dd02014-05-28 10:33:42 -0400246
247 count = rc;
248 while (count-- > 0)
Chuck Leverfe97b472015-10-24 17:27:10 -0400249 rpcrdma_recvcq_process_wc(pos++);
Chuck Lever4220a072015-10-24 17:26:45 -0400250 } while (rc == ARRAY_SIZE(wcs));
Chuck Leverfc664482014-05-28 10:33:25 -0400251}
252
Chuck Lever7b3d7702015-10-24 17:26:37 -0400253/* Handle provider receive completion upcalls.
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400254 */
255static void
Chuck Leverfc664482014-05-28 10:33:25 -0400256rpcrdma_recvcq_upcall(struct ib_cq *cq, void *cq_context)
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400257{
Chuck Lever7b3d7702015-10-24 17:26:37 -0400258 do {
Chuck Lever4220a072015-10-24 17:26:45 -0400259 rpcrdma_recvcq_poll(cq);
Chuck Lever7b3d7702015-10-24 17:26:37 -0400260 } while (ib_req_notify_cq(cq, IB_CQ_NEXT_COMP |
261 IB_CQ_REPORT_MISSED_EVENTS) > 0);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400262}
263
Chuck Levera7bc2112014-07-29 17:23:52 -0400264static void
265rpcrdma_flush_cqs(struct rpcrdma_ep *ep)
266{
Chuck Lever5c166be2014-11-08 20:14:45 -0500267 struct ib_wc wc;
Chuck Lever5c166be2014-11-08 20:14:45 -0500268
269 while (ib_poll_cq(ep->rep_attr.recv_cq, 1, &wc) > 0)
Chuck Leverfe97b472015-10-24 17:27:10 -0400270 rpcrdma_recvcq_process_wc(&wc);
Chuck Lever5c166be2014-11-08 20:14:45 -0500271 while (ib_poll_cq(ep->rep_attr.send_cq, 1, &wc) > 0)
272 rpcrdma_sendcq_process_wc(&wc);
Chuck Levera7bc2112014-07-29 17:23:52 -0400273}
274
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400275static int
276rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event)
277{
278 struct rpcrdma_xprt *xprt = id->context;
279 struct rpcrdma_ia *ia = &xprt->rx_ia;
280 struct rpcrdma_ep *ep = &xprt->rx_ep;
Jeff Laytonf895b252014-11-17 16:58:04 -0500281#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
Chuck Lever0dd39ca2015-03-30 14:33:43 -0400282 struct sockaddr *sap = (struct sockaddr *)&ep->rep_remote_addr;
Ingo Molnarff0db042008-11-25 16:58:42 -0800283#endif
Chuck Leverce1ab9a2015-01-21 11:03:35 -0500284 struct ib_qp_attr *attr = &ia->ri_qp_attr;
285 struct ib_qp_init_attr *iattr = &ia->ri_qp_init_attr;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400286 int connstate = 0;
287
288 switch (event->event) {
289 case RDMA_CM_EVENT_ADDR_RESOLVED:
290 case RDMA_CM_EVENT_ROUTE_RESOLVED:
Tom Talpey5675add2008-10-09 15:01:41 -0400291 ia->ri_async_rc = 0;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400292 complete(&ia->ri_done);
293 break;
294 case RDMA_CM_EVENT_ADDR_ERROR:
295 ia->ri_async_rc = -EHOSTUNREACH;
296 dprintk("RPC: %s: CM address resolution error, ep 0x%p\n",
297 __func__, ep);
298 complete(&ia->ri_done);
299 break;
300 case RDMA_CM_EVENT_ROUTE_ERROR:
301 ia->ri_async_rc = -ENETUNREACH;
302 dprintk("RPC: %s: CM route resolution error, ep 0x%p\n",
303 __func__, ep);
304 complete(&ia->ri_done);
305 break;
306 case RDMA_CM_EVENT_ESTABLISHED:
307 connstate = 1;
Chuck Leverce1ab9a2015-01-21 11:03:35 -0500308 ib_query_qp(ia->ri_id->qp, attr,
309 IB_QP_MAX_QP_RD_ATOMIC | IB_QP_MAX_DEST_RD_ATOMIC,
310 iattr);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400311 dprintk("RPC: %s: %d responder resources"
312 " (%d initiator)\n",
Chuck Leverce1ab9a2015-01-21 11:03:35 -0500313 __func__, attr->max_dest_rd_atomic,
314 attr->max_rd_atomic);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400315 goto connected;
316 case RDMA_CM_EVENT_CONNECT_ERROR:
317 connstate = -ENOTCONN;
318 goto connected;
319 case RDMA_CM_EVENT_UNREACHABLE:
320 connstate = -ENETDOWN;
321 goto connected;
322 case RDMA_CM_EVENT_REJECTED:
323 connstate = -ECONNREFUSED;
324 goto connected;
325 case RDMA_CM_EVENT_DISCONNECTED:
326 connstate = -ECONNABORTED;
327 goto connected;
328 case RDMA_CM_EVENT_DEVICE_REMOVAL:
329 connstate = -ENODEV;
330connected:
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400331 dprintk("RPC: %s: %sconnected\n",
332 __func__, connstate > 0 ? "" : "dis");
333 ep->rep_connected = connstate;
Chuck Leverafadc462015-01-21 11:03:11 -0500334 rpcrdma_conn_func(ep);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400335 wake_up_all(&ep->rep_connect_wait);
Chuck Lever8079fb72014-07-29 17:26:12 -0400336 /*FALLTHROUGH*/
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400337 default:
Chuck Lever0dd39ca2015-03-30 14:33:43 -0400338 dprintk("RPC: %s: %pIS:%u (ep 0x%p): %s\n",
339 __func__, sap, rpc_get_port(sap), ep,
Sagi Grimberg76357c72015-05-18 13:40:32 +0300340 rdma_event_msg(event->event));
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400341 break;
342 }
343
Jeff Laytonf895b252014-11-17 16:58:04 -0500344#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
Tom Talpeyb3cd8d42008-10-09 15:02:02 -0400345 if (connstate == 1) {
Chuck Leverce1ab9a2015-01-21 11:03:35 -0500346 int ird = attr->max_dest_rd_atomic;
Tom Talpeyb3cd8d42008-10-09 15:02:02 -0400347 int tird = ep->rep_remote_cma.responder_resources;
Chuck Lever0dd39ca2015-03-30 14:33:43 -0400348
Chuck Levera0ce85f2015-03-30 14:34:21 -0400349 pr_info("rpcrdma: connection to %pIS:%u on %s, memreg '%s', %d credits, %d responders%s\n",
Chuck Lever0dd39ca2015-03-30 14:33:43 -0400350 sap, rpc_get_port(sap),
Chuck Lever89e0d1122015-05-26 11:51:56 -0400351 ia->ri_device->name,
Chuck Levera0ce85f2015-03-30 14:34:21 -0400352 ia->ri_ops->ro_displayname,
Tom Talpeyb3cd8d42008-10-09 15:02:02 -0400353 xprt->rx_buf.rb_max_requests,
354 ird, ird < 4 && ird < tird / 2 ? " (low!)" : "");
355 } else if (connstate < 0) {
Chuck Lever0dd39ca2015-03-30 14:33:43 -0400356 pr_info("rpcrdma: connection to %pIS:%u closed (%d)\n",
357 sap, rpc_get_port(sap), connstate);
Tom Talpeyb3cd8d42008-10-09 15:02:02 -0400358 }
359#endif
360
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400361 return 0;
362}
363
Devesh Sharmad0f36c42015-08-03 13:05:04 -0400364static void rpcrdma_destroy_id(struct rdma_cm_id *id)
365{
366 if (id) {
367 module_put(id->device->owner);
368 rdma_destroy_id(id);
369 }
370}
371
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400372static struct rdma_cm_id *
373rpcrdma_create_id(struct rpcrdma_xprt *xprt,
374 struct rpcrdma_ia *ia, struct sockaddr *addr)
375{
376 struct rdma_cm_id *id;
377 int rc;
378
Tom Talpey1a954052008-10-09 15:01:31 -0400379 init_completion(&ia->ri_done);
380
Guy Shapirofa201052015-10-22 15:20:10 +0300381 id = rdma_create_id(&init_net, rpcrdma_conn_upcall, xprt, RDMA_PS_TCP,
382 IB_QPT_RC);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400383 if (IS_ERR(id)) {
384 rc = PTR_ERR(id);
385 dprintk("RPC: %s: rdma_create_id() failed %i\n",
386 __func__, rc);
387 return id;
388 }
389
Tom Talpey5675add2008-10-09 15:01:41 -0400390 ia->ri_async_rc = -ETIMEDOUT;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400391 rc = rdma_resolve_addr(id, NULL, addr, RDMA_RESOLVE_TIMEOUT);
392 if (rc) {
393 dprintk("RPC: %s: rdma_resolve_addr() failed %i\n",
394 __func__, rc);
395 goto out;
396 }
Tom Talpey5675add2008-10-09 15:01:41 -0400397 wait_for_completion_interruptible_timeout(&ia->ri_done,
398 msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1);
Devesh Sharmad0f36c42015-08-03 13:05:04 -0400399
400 /* FIXME:
401 * Until xprtrdma supports DEVICE_REMOVAL, the provider must
402 * be pinned while there are active NFS/RDMA mounts to prevent
403 * hangs and crashes at umount time.
404 */
405 if (!ia->ri_async_rc && !try_module_get(id->device->owner)) {
406 dprintk("RPC: %s: Failed to get device module\n",
407 __func__);
408 ia->ri_async_rc = -ENODEV;
409 }
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400410 rc = ia->ri_async_rc;
411 if (rc)
412 goto out;
413
Tom Talpey5675add2008-10-09 15:01:41 -0400414 ia->ri_async_rc = -ETIMEDOUT;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400415 rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT);
416 if (rc) {
417 dprintk("RPC: %s: rdma_resolve_route() failed %i\n",
418 __func__, rc);
Devesh Sharmad0f36c42015-08-03 13:05:04 -0400419 goto put;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400420 }
Tom Talpey5675add2008-10-09 15:01:41 -0400421 wait_for_completion_interruptible_timeout(&ia->ri_done,
422 msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400423 rc = ia->ri_async_rc;
424 if (rc)
Devesh Sharmad0f36c42015-08-03 13:05:04 -0400425 goto put;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400426
427 return id;
Devesh Sharmad0f36c42015-08-03 13:05:04 -0400428put:
429 module_put(id->device->owner);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400430out:
431 rdma_destroy_id(id);
432 return ERR_PTR(rc);
433}
434
435/*
436 * Drain any cq, prior to teardown.
437 */
438static void
439rpcrdma_clean_cq(struct ib_cq *cq)
440{
441 struct ib_wc wc;
442 int count = 0;
443
444 while (1 == ib_poll_cq(cq, 1, &wc))
445 ++count;
446
447 if (count)
448 dprintk("RPC: %s: flushed %d events (last 0x%x)\n",
449 __func__, count, wc.opcode);
450}
451
452/*
453 * Exported functions.
454 */
455
456/*
457 * Open and initialize an Interface Adapter.
458 * o initializes fields of struct rpcrdma_ia, including
459 * interface and provider attributes and protection zone.
460 */
461int
462rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg)
463{
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400464 struct rpcrdma_ia *ia = &xprt->rx_ia;
Chuck Leverd1ed8572015-08-03 13:03:30 -0400465 int rc;
466
467 ia->ri_dma_mr = NULL;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400468
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400469 ia->ri_id = rpcrdma_create_id(xprt, ia, addr);
470 if (IS_ERR(ia->ri_id)) {
471 rc = PTR_ERR(ia->ri_id);
472 goto out1;
473 }
Chuck Lever89e0d1122015-05-26 11:51:56 -0400474 ia->ri_device = ia->ri_id->device;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400475
Chuck Lever89e0d1122015-05-26 11:51:56 -0400476 ia->ri_pd = ib_alloc_pd(ia->ri_device);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400477 if (IS_ERR(ia->ri_pd)) {
478 rc = PTR_ERR(ia->ri_pd);
479 dprintk("RPC: %s: ib_alloc_pd() failed %i\n",
480 __func__, rc);
481 goto out2;
482 }
483
Chuck Leverf10eafd2014-05-28 10:32:51 -0400484 if (memreg == RPCRDMA_FRMR) {
Or Gerlitze3e45b12015-12-18 10:59:48 +0200485 if (!(ia->ri_device->attrs.device_cap_flags &
486 IB_DEVICE_MEM_MGT_EXTENSIONS) ||
487 (ia->ri_device->attrs.max_fast_reg_page_list_len == 0)) {
Tom Talpey3197d3092008-10-09 15:00:20 -0400488 dprintk("RPC: %s: FRMR registration "
Chuck Leverf10eafd2014-05-28 10:32:51 -0400489 "not supported by HCA\n", __func__);
490 memreg = RPCRDMA_MTHCAFMR;
Tom Talpey3197d3092008-10-09 15:00:20 -0400491 }
Chuck Leverf10eafd2014-05-28 10:32:51 -0400492 }
493 if (memreg == RPCRDMA_MTHCAFMR) {
Chuck Lever89e0d1122015-05-26 11:51:56 -0400494 if (!ia->ri_device->alloc_fmr) {
Chuck Leverf10eafd2014-05-28 10:32:51 -0400495 dprintk("RPC: %s: MTHCAFMR registration "
496 "not supported by HCA\n", __func__);
Sagi Grimbergf022fa82015-10-06 19:52:37 +0300497 rc = -EINVAL;
Chuck Leverd2310932015-08-03 13:03:09 -0400498 goto out3;
Chuck Leverf10eafd2014-05-28 10:32:51 -0400499 }
Tom Talpeybd7ed1d2008-10-09 15:00:09 -0400500 }
501
Tom Talpeybd7ed1d2008-10-09 15:00:09 -0400502 switch (memreg) {
Tom Talpey3197d3092008-10-09 15:00:20 -0400503 case RPCRDMA_FRMR:
Chuck Levera0ce85f2015-03-30 14:34:21 -0400504 ia->ri_ops = &rpcrdma_frwr_memreg_ops;
Tom Talpeybd7ed1d2008-10-09 15:00:09 -0400505 break;
Tom Talpeybd7ed1d2008-10-09 15:00:09 -0400506 case RPCRDMA_ALLPHYSICAL:
Chuck Levera0ce85f2015-03-30 14:34:21 -0400507 ia->ri_ops = &rpcrdma_physical_memreg_ops;
Chuck Leverd1ed8572015-08-03 13:03:30 -0400508 break;
Tom Talpeybd7ed1d2008-10-09 15:00:09 -0400509 case RPCRDMA_MTHCAFMR:
Chuck Levera0ce85f2015-03-30 14:34:21 -0400510 ia->ri_ops = &rpcrdma_fmr_memreg_ops;
Tom Talpeybd7ed1d2008-10-09 15:00:09 -0400511 break;
512 default:
Chuck Levercdd9ade2014-05-28 10:33:00 -0400513 printk(KERN_ERR "RPC: Unsupported memory "
514 "registration mode: %d\n", memreg);
515 rc = -ENOMEM;
Chuck Lever5ae711a2015-01-21 11:03:19 -0500516 goto out3;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400517 }
Chuck Levera0ce85f2015-03-30 14:34:21 -0400518 dprintk("RPC: %s: memory registration strategy is '%s'\n",
519 __func__, ia->ri_ops->ro_displayname);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400520
Chuck Lever73806c82014-07-29 17:23:25 -0400521 rwlock_init(&ia->ri_qplock);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400522 return 0;
Chuck Lever5ae711a2015-01-21 11:03:19 -0500523
524out3:
525 ib_dealloc_pd(ia->ri_pd);
526 ia->ri_pd = NULL;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400527out2:
Devesh Sharmad0f36c42015-08-03 13:05:04 -0400528 rpcrdma_destroy_id(ia->ri_id);
Tom Talpeyfee08ca2008-10-09 15:01:00 -0400529 ia->ri_id = NULL;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400530out1:
531 return rc;
532}
533
534/*
535 * Clean up/close an IA.
536 * o if event handles and PD have been initialized, free them.
537 * o close the IA
538 */
539void
540rpcrdma_ia_close(struct rpcrdma_ia *ia)
541{
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400542 dprintk("RPC: %s: entering\n", __func__);
Tom Talpeyfee08ca2008-10-09 15:01:00 -0400543 if (ia->ri_id != NULL && !IS_ERR(ia->ri_id)) {
544 if (ia->ri_id->qp)
545 rdma_destroy_qp(ia->ri_id);
Devesh Sharmad0f36c42015-08-03 13:05:04 -0400546 rpcrdma_destroy_id(ia->ri_id);
Tom Talpeyfee08ca2008-10-09 15:01:00 -0400547 ia->ri_id = NULL;
548 }
Chuck Lever6d446982015-05-26 11:51:27 -0400549
550 /* If the pd is still busy, xprtrdma missed freeing a resource */
551 if (ia->ri_pd && !IS_ERR(ia->ri_pd))
Jason Gunthorpe7dd78642015-08-05 14:34:31 -0600552 ib_dealloc_pd(ia->ri_pd);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400553}
554
555/*
556 * Create unconnected endpoint.
557 */
558int
559rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
560 struct rpcrdma_create_data_internal *cdata)
561{
Chuck Leverfc664482014-05-28 10:33:25 -0400562 struct ib_cq *sendcq, *recvcq;
Matan Barak8e372102015-06-11 16:35:21 +0300563 struct ib_cq_init_attr cq_attr = {};
Chuck Lever124fa172015-10-24 17:27:51 -0400564 unsigned int max_qp_wr;
Chuck Lever5d40a8a2007-10-26 13:30:54 -0400565 int rc, err;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400566
Or Gerlitze3e45b12015-12-18 10:59:48 +0200567 if (ia->ri_device->attrs.max_sge < RPCRDMA_MAX_IOVS) {
Chuck Leverb3221d62015-08-03 13:03:39 -0400568 dprintk("RPC: %s: insufficient sge's available\n",
569 __func__);
570 return -ENOMEM;
571 }
572
Or Gerlitze3e45b12015-12-18 10:59:48 +0200573 if (ia->ri_device->attrs.max_qp_wr <= RPCRDMA_BACKWARD_WRS) {
Chuck Lever124fa172015-10-24 17:27:51 -0400574 dprintk("RPC: %s: insufficient wqe's available\n",
575 __func__);
576 return -ENOMEM;
577 }
Or Gerlitze3e45b12015-12-18 10:59:48 +0200578 max_qp_wr = ia->ri_device->attrs.max_qp_wr - RPCRDMA_BACKWARD_WRS;
Chuck Lever124fa172015-10-24 17:27:51 -0400579
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400580 /* check provider's send/recv wr limits */
Chuck Lever124fa172015-10-24 17:27:51 -0400581 if (cdata->max_requests > max_qp_wr)
582 cdata->max_requests = max_qp_wr;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400583
584 ep->rep_attr.event_handler = rpcrdma_qp_async_error_upcall;
585 ep->rep_attr.qp_context = ep;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400586 ep->rep_attr.srq = NULL;
587 ep->rep_attr.cap.max_send_wr = cdata->max_requests;
Chuck Lever124fa172015-10-24 17:27:51 -0400588 ep->rep_attr.cap.max_send_wr += RPCRDMA_BACKWARD_WRS;
Chuck Lever3968cb52015-03-30 14:35:26 -0400589 rc = ia->ri_ops->ro_open(ia, ep, cdata);
590 if (rc)
591 return rc;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400592 ep->rep_attr.cap.max_recv_wr = cdata->max_requests;
Chuck Lever124fa172015-10-24 17:27:51 -0400593 ep->rep_attr.cap.max_recv_wr += RPCRDMA_BACKWARD_WRS;
Chuck Leverb3221d62015-08-03 13:03:39 -0400594 ep->rep_attr.cap.max_send_sge = RPCRDMA_MAX_IOVS;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400595 ep->rep_attr.cap.max_recv_sge = 1;
596 ep->rep_attr.cap.max_inline_data = 0;
597 ep->rep_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
598 ep->rep_attr.qp_type = IB_QPT_RC;
599 ep->rep_attr.port_num = ~0;
600
601 dprintk("RPC: %s: requested max: dtos: send %d recv %d; "
602 "iovs: send %d recv %d\n",
603 __func__,
604 ep->rep_attr.cap.max_send_wr,
605 ep->rep_attr.cap.max_recv_wr,
606 ep->rep_attr.cap.max_send_sge,
607 ep->rep_attr.cap.max_recv_sge);
608
609 /* set trigger for requesting send completion */
Chuck Leverfc664482014-05-28 10:33:25 -0400610 ep->rep_cqinit = ep->rep_attr.cap.max_send_wr/2 - 1;
Chuck Lever26ae9d12015-12-16 17:23:20 -0500611 if (ep->rep_cqinit <= 2)
612 ep->rep_cqinit = 0; /* always signal? */
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400613 INIT_CQCOUNT(ep);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400614 init_waitqueue_head(&ep->rep_connect_wait);
Chuck Lever254f91e2014-05-28 10:32:17 -0400615 INIT_DELAYED_WORK(&ep->rep_connect_worker, rpcrdma_connect_worker);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400616
Matan Barak8e372102015-06-11 16:35:21 +0300617 cq_attr.cqe = ep->rep_attr.cap.max_send_wr + 1;
Chuck Lever89e0d1122015-05-26 11:51:56 -0400618 sendcq = ib_create_cq(ia->ri_device, rpcrdma_sendcq_upcall,
Chuck Lever4220a072015-10-24 17:26:45 -0400619 rpcrdma_cq_async_error_upcall, NULL, &cq_attr);
Chuck Leverfc664482014-05-28 10:33:25 -0400620 if (IS_ERR(sendcq)) {
621 rc = PTR_ERR(sendcq);
622 dprintk("RPC: %s: failed to create send CQ: %i\n",
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400623 __func__, rc);
624 goto out1;
625 }
626
Chuck Leverfc664482014-05-28 10:33:25 -0400627 rc = ib_req_notify_cq(sendcq, IB_CQ_NEXT_COMP);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400628 if (rc) {
629 dprintk("RPC: %s: ib_req_notify_cq failed: %i\n",
630 __func__, rc);
631 goto out2;
632 }
633
Matan Barak8e372102015-06-11 16:35:21 +0300634 cq_attr.cqe = ep->rep_attr.cap.max_recv_wr + 1;
Chuck Lever89e0d1122015-05-26 11:51:56 -0400635 recvcq = ib_create_cq(ia->ri_device, rpcrdma_recvcq_upcall,
Chuck Lever4220a072015-10-24 17:26:45 -0400636 rpcrdma_cq_async_error_upcall, NULL, &cq_attr);
Chuck Leverfc664482014-05-28 10:33:25 -0400637 if (IS_ERR(recvcq)) {
638 rc = PTR_ERR(recvcq);
639 dprintk("RPC: %s: failed to create recv CQ: %i\n",
640 __func__, rc);
641 goto out2;
642 }
643
644 rc = ib_req_notify_cq(recvcq, IB_CQ_NEXT_COMP);
645 if (rc) {
646 dprintk("RPC: %s: ib_req_notify_cq failed: %i\n",
647 __func__, rc);
648 ib_destroy_cq(recvcq);
649 goto out2;
650 }
651
652 ep->rep_attr.send_cq = sendcq;
653 ep->rep_attr.recv_cq = recvcq;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400654
655 /* Initialize cma parameters */
656
657 /* RPC/RDMA does not use private data */
658 ep->rep_remote_cma.private_data = NULL;
659 ep->rep_remote_cma.private_data_len = 0;
660
661 /* Client offers RDMA Read but does not initiate */
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400662 ep->rep_remote_cma.initiator_depth = 0;
Or Gerlitze3e45b12015-12-18 10:59:48 +0200663 if (ia->ri_device->attrs.max_qp_rd_atom > 32) /* arbitrary but <= 255 */
Tom Tuckerb334eaa2008-10-09 15:00:30 -0400664 ep->rep_remote_cma.responder_resources = 32;
665 else
Chuck Lever7bc79722015-01-21 11:03:27 -0500666 ep->rep_remote_cma.responder_resources =
Or Gerlitze3e45b12015-12-18 10:59:48 +0200667 ia->ri_device->attrs.max_qp_rd_atom;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400668
669 ep->rep_remote_cma.retry_count = 7;
670 ep->rep_remote_cma.flow_control = 0;
671 ep->rep_remote_cma.rnr_retry_count = 0;
672
673 return 0;
674
675out2:
Chuck Leverfc664482014-05-28 10:33:25 -0400676 err = ib_destroy_cq(sendcq);
Chuck Lever5d40a8a2007-10-26 13:30:54 -0400677 if (err)
678 dprintk("RPC: %s: ib_destroy_cq returned %i\n",
679 __func__, err);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400680out1:
Chuck Leverd1ed8572015-08-03 13:03:30 -0400681 if (ia->ri_dma_mr)
682 ib_dereg_mr(ia->ri_dma_mr);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400683 return rc;
684}
685
686/*
687 * rpcrdma_ep_destroy
688 *
689 * Disconnect and destroy endpoint. After this, the only
690 * valid operations on the ep are to free it (if dynamically
691 * allocated) or re-create it.
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400692 */
Chuck Lever7f1d5412014-05-28 10:33:16 -0400693void
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400694rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
695{
696 int rc;
697
698 dprintk("RPC: %s: entering, connected is %d\n",
699 __func__, ep->rep_connected);
700
Chuck Lever254f91e2014-05-28 10:32:17 -0400701 cancel_delayed_work_sync(&ep->rep_connect_worker);
702
Steve Wise72c02172015-09-21 12:24:23 -0500703 if (ia->ri_id->qp)
Chuck Lever282191c2014-07-29 17:25:55 -0400704 rpcrdma_ep_disconnect(ep, ia);
Steve Wise72c02172015-09-21 12:24:23 -0500705
706 rpcrdma_clean_cq(ep->rep_attr.recv_cq);
707 rpcrdma_clean_cq(ep->rep_attr.send_cq);
708
709 if (ia->ri_id->qp) {
Tom Talpeyfee08ca2008-10-09 15:01:00 -0400710 rdma_destroy_qp(ia->ri_id);
711 ia->ri_id->qp = NULL;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400712 }
713
Chuck Leverfc664482014-05-28 10:33:25 -0400714 rc = ib_destroy_cq(ep->rep_attr.recv_cq);
715 if (rc)
716 dprintk("RPC: %s: ib_destroy_cq returned %i\n",
717 __func__, rc);
718
Chuck Leverfc664482014-05-28 10:33:25 -0400719 rc = ib_destroy_cq(ep->rep_attr.send_cq);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400720 if (rc)
721 dprintk("RPC: %s: ib_destroy_cq returned %i\n",
722 __func__, rc);
Chuck Leverd1ed8572015-08-03 13:03:30 -0400723
724 if (ia->ri_dma_mr) {
725 rc = ib_dereg_mr(ia->ri_dma_mr);
726 dprintk("RPC: %s: ib_dereg_mr returned %i\n",
727 __func__, rc);
728 }
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400729}
730
731/*
732 * Connect unconnected endpoint.
733 */
734int
735rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
736{
Chuck Lever73806c82014-07-29 17:23:25 -0400737 struct rdma_cm_id *id, *old;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400738 int rc = 0;
739 int retry_count = 0;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400740
Tom Talpeyc0555512008-10-10 11:32:45 -0400741 if (ep->rep_connected != 0) {
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400742 struct rpcrdma_xprt *xprt;
743retry:
Chuck Leverec62f402014-05-28 10:34:07 -0400744 dprintk("RPC: %s: reconnecting...\n", __func__);
Chuck Lever282191c2014-07-29 17:25:55 -0400745
746 rpcrdma_ep_disconnect(ep, ia);
Chuck Levera7bc2112014-07-29 17:23:52 -0400747 rpcrdma_flush_cqs(ep);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400748
749 xprt = container_of(ia, struct rpcrdma_xprt, rx_ia);
750 id = rpcrdma_create_id(xprt, ia,
751 (struct sockaddr *)&xprt->rx_data.addr);
752 if (IS_ERR(id)) {
Chuck Leverec62f402014-05-28 10:34:07 -0400753 rc = -EHOSTUNREACH;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400754 goto out;
755 }
756 /* TEMP TEMP TEMP - fail if new device:
757 * Deregister/remarshal *all* requests!
758 * Close and recreate adapter, pd, etc!
759 * Re-determine all attributes still sane!
760 * More stuff I haven't thought of!
761 * Rrrgh!
762 */
Chuck Lever89e0d1122015-05-26 11:51:56 -0400763 if (ia->ri_device != id->device) {
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400764 printk("RPC: %s: can't reconnect on "
765 "different device!\n", __func__);
Devesh Sharmad0f36c42015-08-03 13:05:04 -0400766 rpcrdma_destroy_id(id);
Chuck Leverec62f402014-05-28 10:34:07 -0400767 rc = -ENETUNREACH;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400768 goto out;
769 }
770 /* END TEMP */
Chuck Leverec62f402014-05-28 10:34:07 -0400771 rc = rdma_create_qp(id, ia->ri_pd, &ep->rep_attr);
772 if (rc) {
773 dprintk("RPC: %s: rdma_create_qp failed %i\n",
774 __func__, rc);
Devesh Sharmad0f36c42015-08-03 13:05:04 -0400775 rpcrdma_destroy_id(id);
Chuck Leverec62f402014-05-28 10:34:07 -0400776 rc = -ENETUNREACH;
777 goto out;
778 }
Chuck Lever73806c82014-07-29 17:23:25 -0400779
780 write_lock(&ia->ri_qplock);
781 old = ia->ri_id;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400782 ia->ri_id = id;
Chuck Lever73806c82014-07-29 17:23:25 -0400783 write_unlock(&ia->ri_qplock);
784
785 rdma_destroy_qp(old);
Devesh Sharmad0f36c42015-08-03 13:05:04 -0400786 rpcrdma_destroy_id(old);
Chuck Leverec62f402014-05-28 10:34:07 -0400787 } else {
788 dprintk("RPC: %s: connecting...\n", __func__);
789 rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr);
790 if (rc) {
791 dprintk("RPC: %s: rdma_create_qp failed %i\n",
792 __func__, rc);
793 /* do not update ep->rep_connected */
794 return -ENETUNREACH;
795 }
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400796 }
797
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400798 ep->rep_connected = 0;
799
800 rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma);
801 if (rc) {
802 dprintk("RPC: %s: rdma_connect() failed with %i\n",
803 __func__, rc);
804 goto out;
805 }
806
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400807 wait_event_interruptible(ep->rep_connect_wait, ep->rep_connected != 0);
808
809 /*
810 * Check state. A non-peer reject indicates no listener
811 * (ECONNREFUSED), which may be a transient state. All
812 * others indicate a transport condition which has already
813 * undergone a best-effort.
814 */
Joe Perchesf64f9e72009-11-29 16:55:45 -0800815 if (ep->rep_connected == -ECONNREFUSED &&
816 ++retry_count <= RDMA_CONNECT_RETRY_MAX) {
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400817 dprintk("RPC: %s: non-peer_reject, retry\n", __func__);
818 goto retry;
819 }
820 if (ep->rep_connected <= 0) {
821 /* Sometimes, the only way to reliably connect to remote
822 * CMs is to use same nonzero values for ORD and IRD. */
Tom Tuckerb334eaa2008-10-09 15:00:30 -0400823 if (retry_count++ <= RDMA_CONNECT_RETRY_MAX + 1 &&
824 (ep->rep_remote_cma.responder_resources == 0 ||
825 ep->rep_remote_cma.initiator_depth !=
826 ep->rep_remote_cma.responder_resources)) {
827 if (ep->rep_remote_cma.responder_resources == 0)
828 ep->rep_remote_cma.responder_resources = 1;
829 ep->rep_remote_cma.initiator_depth =
830 ep->rep_remote_cma.responder_resources;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400831 goto retry;
Tom Tuckerb334eaa2008-10-09 15:00:30 -0400832 }
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400833 rc = ep->rep_connected;
834 } else {
Chuck Leverf531a5d2015-10-24 17:27:43 -0400835 struct rpcrdma_xprt *r_xprt;
836 unsigned int extras;
837
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400838 dprintk("RPC: %s: connected\n", __func__);
Chuck Leverf531a5d2015-10-24 17:27:43 -0400839
840 r_xprt = container_of(ia, struct rpcrdma_xprt, rx_ia);
841 extras = r_xprt->rx_buf.rb_bc_srv_max_requests;
842
843 if (extras) {
844 rc = rpcrdma_ep_post_extra_recv(r_xprt, extras);
Dan Carpenter38b95bc2015-11-05 11:37:08 +0300845 if (rc) {
Chuck Leverf531a5d2015-10-24 17:27:43 -0400846 pr_warn("%s: rpcrdma_ep_post_extra_recv: %i\n",
847 __func__, rc);
848 rc = 0;
Dan Carpenter38b95bc2015-11-05 11:37:08 +0300849 }
Chuck Leverf531a5d2015-10-24 17:27:43 -0400850 }
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400851 }
852
853out:
854 if (rc)
855 ep->rep_connected = rc;
856 return rc;
857}
858
859/*
860 * rpcrdma_ep_disconnect
861 *
862 * This is separate from destroy to facilitate the ability
863 * to reconnect without recreating the endpoint.
864 *
865 * This call is not reentrant, and must not be made in parallel
866 * on the same endpoint.
867 */
Chuck Lever282191c2014-07-29 17:25:55 -0400868void
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400869rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
870{
871 int rc;
872
Chuck Levera7bc2112014-07-29 17:23:52 -0400873 rpcrdma_flush_cqs(ep);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400874 rc = rdma_disconnect(ia->ri_id);
875 if (!rc) {
876 /* returns without wait if not connected */
877 wait_event_interruptible(ep->rep_connect_wait,
878 ep->rep_connected != 1);
879 dprintk("RPC: %s: after wait, %sconnected\n", __func__,
880 (ep->rep_connected == 1) ? "still " : "dis");
881 } else {
882 dprintk("RPC: %s: rdma_disconnect %i\n", __func__, rc);
883 ep->rep_connected = rc;
884 }
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400885}
886
Chuck Leverf531a5d2015-10-24 17:27:43 -0400887struct rpcrdma_req *
Chuck Lever13924022015-01-21 11:03:52 -0500888rpcrdma_create_req(struct rpcrdma_xprt *r_xprt)
889{
Chuck Leverf531a5d2015-10-24 17:27:43 -0400890 struct rpcrdma_buffer *buffer = &r_xprt->rx_buf;
Chuck Lever13924022015-01-21 11:03:52 -0500891 struct rpcrdma_req *req;
Chuck Lever13924022015-01-21 11:03:52 -0500892
Chuck Lever85275c82015-01-21 11:04:16 -0500893 req = kzalloc(sizeof(*req), GFP_KERNEL);
Chuck Lever13924022015-01-21 11:03:52 -0500894 if (req == NULL)
Chuck Lever85275c82015-01-21 11:04:16 -0500895 return ERR_PTR(-ENOMEM);
Chuck Lever13924022015-01-21 11:03:52 -0500896
Chuck Leverf531a5d2015-10-24 17:27:43 -0400897 INIT_LIST_HEAD(&req->rl_free);
898 spin_lock(&buffer->rb_reqslock);
899 list_add(&req->rl_all, &buffer->rb_allreqs);
900 spin_unlock(&buffer->rb_reqslock);
Chuck Lever13924022015-01-21 11:03:52 -0500901 req->rl_buffer = &r_xprt->rx_buf;
902 return req;
Chuck Lever13924022015-01-21 11:03:52 -0500903}
904
Chuck Leverf531a5d2015-10-24 17:27:43 -0400905struct rpcrdma_rep *
Chuck Lever13924022015-01-21 11:03:52 -0500906rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt)
907{
908 struct rpcrdma_create_data_internal *cdata = &r_xprt->rx_data;
Chuck Lever13924022015-01-21 11:03:52 -0500909 struct rpcrdma_ia *ia = &r_xprt->rx_ia;
910 struct rpcrdma_rep *rep;
911 int rc;
912
913 rc = -ENOMEM;
Chuck Lever6b1184c2015-01-21 11:04:25 -0500914 rep = kzalloc(sizeof(*rep), GFP_KERNEL);
Chuck Lever13924022015-01-21 11:03:52 -0500915 if (rep == NULL)
916 goto out;
Chuck Lever13924022015-01-21 11:03:52 -0500917
Chuck Lever6b1184c2015-01-21 11:04:25 -0500918 rep->rr_rdmabuf = rpcrdma_alloc_regbuf(ia, cdata->inline_rsize,
919 GFP_KERNEL);
920 if (IS_ERR(rep->rr_rdmabuf)) {
921 rc = PTR_ERR(rep->rr_rdmabuf);
Chuck Lever13924022015-01-21 11:03:52 -0500922 goto out_free;
Chuck Lever6b1184c2015-01-21 11:04:25 -0500923 }
Chuck Lever13924022015-01-21 11:03:52 -0500924
Chuck Lever89e0d1122015-05-26 11:51:56 -0400925 rep->rr_device = ia->ri_device;
Chuck Leverfed171b2015-05-26 11:51:37 -0400926 rep->rr_rxprt = r_xprt;
Chuck Leverfe97b472015-10-24 17:27:10 -0400927 INIT_WORK(&rep->rr_work, rpcrdma_receive_worker);
Chuck Lever13924022015-01-21 11:03:52 -0500928 return rep;
929
930out_free:
931 kfree(rep);
932out:
933 return ERR_PTR(rc);
934}
935
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400936int
Chuck Leverac920d02015-01-21 11:03:44 -0500937rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt)
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400938{
Chuck Leverac920d02015-01-21 11:03:44 -0500939 struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
940 struct rpcrdma_ia *ia = &r_xprt->rx_ia;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400941 int i, rc;
942
Chuck Lever1e465fd2015-10-24 17:27:02 -0400943 buf->rb_max_requests = r_xprt->rx_data.max_requests;
Chuck Leverf531a5d2015-10-24 17:27:43 -0400944 buf->rb_bc_srv_max_requests = 0;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400945 spin_lock_init(&buf->rb_lock);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400946
Chuck Lever91e70e72015-03-30 14:34:58 -0400947 rc = ia->ri_ops->ro_init(r_xprt);
948 if (rc)
949 goto out;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400950
Chuck Lever1e465fd2015-10-24 17:27:02 -0400951 INIT_LIST_HEAD(&buf->rb_send_bufs);
Chuck Leverf531a5d2015-10-24 17:27:43 -0400952 INIT_LIST_HEAD(&buf->rb_allreqs);
953 spin_lock_init(&buf->rb_reqslock);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400954 for (i = 0; i < buf->rb_max_requests; i++) {
955 struct rpcrdma_req *req;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400956
Chuck Lever13924022015-01-21 11:03:52 -0500957 req = rpcrdma_create_req(r_xprt);
958 if (IS_ERR(req)) {
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400959 dprintk("RPC: %s: request buffer %d alloc"
960 " failed\n", __func__, i);
Chuck Lever13924022015-01-21 11:03:52 -0500961 rc = PTR_ERR(req);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400962 goto out;
963 }
Chuck Leverf531a5d2015-10-24 17:27:43 -0400964 req->rl_backchannel = false;
Chuck Lever1e465fd2015-10-24 17:27:02 -0400965 list_add(&req->rl_free, &buf->rb_send_bufs);
966 }
967
968 INIT_LIST_HEAD(&buf->rb_recv_bufs);
969 for (i = 0; i < buf->rb_max_requests + 2; i++) {
970 struct rpcrdma_rep *rep;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400971
Chuck Lever13924022015-01-21 11:03:52 -0500972 rep = rpcrdma_create_rep(r_xprt);
973 if (IS_ERR(rep)) {
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400974 dprintk("RPC: %s: reply buffer %d alloc failed\n",
975 __func__, i);
Chuck Lever13924022015-01-21 11:03:52 -0500976 rc = PTR_ERR(rep);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400977 goto out;
978 }
Chuck Lever1e465fd2015-10-24 17:27:02 -0400979 list_add(&rep->rr_list, &buf->rb_recv_bufs);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400980 }
Chuck Lever13924022015-01-21 11:03:52 -0500981
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -0400982 return 0;
983out:
984 rpcrdma_buffer_destroy(buf);
985 return rc;
986}
987
Chuck Lever1e465fd2015-10-24 17:27:02 -0400988static struct rpcrdma_req *
989rpcrdma_buffer_get_req_locked(struct rpcrdma_buffer *buf)
990{
991 struct rpcrdma_req *req;
992
993 req = list_first_entry(&buf->rb_send_bufs,
994 struct rpcrdma_req, rl_free);
995 list_del(&req->rl_free);
996 return req;
997}
998
999static struct rpcrdma_rep *
1000rpcrdma_buffer_get_rep_locked(struct rpcrdma_buffer *buf)
1001{
1002 struct rpcrdma_rep *rep;
1003
1004 rep = list_first_entry(&buf->rb_recv_bufs,
1005 struct rpcrdma_rep, rr_list);
1006 list_del(&rep->rr_list);
1007 return rep;
1008}
1009
Chuck Lever2e845222014-07-29 17:25:38 -04001010static void
Chuck Lever13924022015-01-21 11:03:52 -05001011rpcrdma_destroy_rep(struct rpcrdma_ia *ia, struct rpcrdma_rep *rep)
1012{
Chuck Lever6b1184c2015-01-21 11:04:25 -05001013 rpcrdma_free_regbuf(ia, rep->rr_rdmabuf);
Chuck Lever13924022015-01-21 11:03:52 -05001014 kfree(rep);
1015}
1016
Chuck Leverf531a5d2015-10-24 17:27:43 -04001017void
Chuck Lever13924022015-01-21 11:03:52 -05001018rpcrdma_destroy_req(struct rpcrdma_ia *ia, struct rpcrdma_req *req)
1019{
Chuck Lever0ca77dc2015-01-21 11:04:08 -05001020 rpcrdma_free_regbuf(ia, req->rl_sendbuf);
Chuck Lever85275c82015-01-21 11:04:16 -05001021 rpcrdma_free_regbuf(ia, req->rl_rdmabuf);
Chuck Lever13924022015-01-21 11:03:52 -05001022 kfree(req);
1023}
1024
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001025void
1026rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf)
1027{
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001028 struct rpcrdma_ia *ia = rdmab_to_ia(buf);
1029
Chuck Lever1e465fd2015-10-24 17:27:02 -04001030 while (!list_empty(&buf->rb_recv_bufs)) {
1031 struct rpcrdma_rep *rep;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001032
Chuck Lever1e465fd2015-10-24 17:27:02 -04001033 rep = rpcrdma_buffer_get_rep_locked(buf);
1034 rpcrdma_destroy_rep(ia, rep);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001035 }
1036
Chuck Leverf531a5d2015-10-24 17:27:43 -04001037 spin_lock(&buf->rb_reqslock);
1038 while (!list_empty(&buf->rb_allreqs)) {
Chuck Lever1e465fd2015-10-24 17:27:02 -04001039 struct rpcrdma_req *req;
Allen Andrews4034ba02014-05-28 10:32:09 -04001040
Chuck Leverf531a5d2015-10-24 17:27:43 -04001041 req = list_first_entry(&buf->rb_allreqs,
1042 struct rpcrdma_req, rl_all);
1043 list_del(&req->rl_all);
1044
1045 spin_unlock(&buf->rb_reqslock);
Chuck Lever1e465fd2015-10-24 17:27:02 -04001046 rpcrdma_destroy_req(ia, req);
Chuck Leverf531a5d2015-10-24 17:27:43 -04001047 spin_lock(&buf->rb_reqslock);
Chuck Lever9f9d8022014-07-29 17:24:45 -04001048 }
Chuck Leverf531a5d2015-10-24 17:27:43 -04001049 spin_unlock(&buf->rb_reqslock);
Chuck Lever9f9d8022014-07-29 17:24:45 -04001050
1051 ia->ri_ops->ro_destroy(buf);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001052}
1053
Chuck Lever346aa662015-05-26 11:52:06 -04001054struct rpcrdma_mw *
1055rpcrdma_get_mw(struct rpcrdma_xprt *r_xprt)
Chuck Leverc2922c02014-07-29 17:24:36 -04001056{
Chuck Lever346aa662015-05-26 11:52:06 -04001057 struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
1058 struct rpcrdma_mw *mw = NULL;
Chuck Lever346aa662015-05-26 11:52:06 -04001059
Chuck Lever58d1dcf2015-05-26 11:53:13 -04001060 spin_lock(&buf->rb_mwlock);
Chuck Lever346aa662015-05-26 11:52:06 -04001061 if (!list_empty(&buf->rb_mws)) {
1062 mw = list_first_entry(&buf->rb_mws,
1063 struct rpcrdma_mw, mw_list);
1064 list_del_init(&mw->mw_list);
Chuck Leverc2922c02014-07-29 17:24:36 -04001065 }
Chuck Lever58d1dcf2015-05-26 11:53:13 -04001066 spin_unlock(&buf->rb_mwlock);
Chuck Lever346aa662015-05-26 11:52:06 -04001067
1068 if (!mw)
1069 pr_err("RPC: %s: no MWs available\n", __func__);
1070 return mw;
Chuck Leverc2922c02014-07-29 17:24:36 -04001071}
1072
Chuck Lever346aa662015-05-26 11:52:06 -04001073void
1074rpcrdma_put_mw(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mw *mw)
Chuck Leverc2922c02014-07-29 17:24:36 -04001075{
Chuck Lever346aa662015-05-26 11:52:06 -04001076 struct rpcrdma_buffer *buf = &r_xprt->rx_buf;
Chuck Leverc2922c02014-07-29 17:24:36 -04001077
Chuck Lever58d1dcf2015-05-26 11:53:13 -04001078 spin_lock(&buf->rb_mwlock);
Chuck Lever346aa662015-05-26 11:52:06 -04001079 list_add_tail(&mw->mw_list, &buf->rb_mws);
Chuck Lever58d1dcf2015-05-26 11:53:13 -04001080 spin_unlock(&buf->rb_mwlock);
Chuck Leverc2922c02014-07-29 17:24:36 -04001081}
1082
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001083/*
1084 * Get a set of request/reply buffers.
1085 *
Chuck Lever1e465fd2015-10-24 17:27:02 -04001086 * Reply buffer (if available) is attached to send buffer upon return.
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001087 */
1088struct rpcrdma_req *
1089rpcrdma_buffer_get(struct rpcrdma_buffer *buffers)
1090{
1091 struct rpcrdma_req *req;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001092
Chuck Levera5b027e2015-10-24 17:27:27 -04001093 spin_lock(&buffers->rb_lock);
Chuck Lever1e465fd2015-10-24 17:27:02 -04001094 if (list_empty(&buffers->rb_send_bufs))
1095 goto out_reqbuf;
1096 req = rpcrdma_buffer_get_req_locked(buffers);
1097 if (list_empty(&buffers->rb_recv_bufs))
1098 goto out_repbuf;
1099 req->rl_reply = rpcrdma_buffer_get_rep_locked(buffers);
Chuck Levera5b027e2015-10-24 17:27:27 -04001100 spin_unlock(&buffers->rb_lock);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001101 return req;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001102
Chuck Lever1e465fd2015-10-24 17:27:02 -04001103out_reqbuf:
Chuck Levera5b027e2015-10-24 17:27:27 -04001104 spin_unlock(&buffers->rb_lock);
Chuck Lever1e465fd2015-10-24 17:27:02 -04001105 pr_warn("RPC: %s: out of request buffers\n", __func__);
1106 return NULL;
1107out_repbuf:
Chuck Levera5b027e2015-10-24 17:27:27 -04001108 spin_unlock(&buffers->rb_lock);
Chuck Lever1e465fd2015-10-24 17:27:02 -04001109 pr_warn("RPC: %s: out of reply buffers\n", __func__);
1110 req->rl_reply = NULL;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001111 return req;
1112}
1113
1114/*
1115 * Put request/reply buffers back into pool.
1116 * Pre-decrement counter/array index.
1117 */
1118void
1119rpcrdma_buffer_put(struct rpcrdma_req *req)
1120{
1121 struct rpcrdma_buffer *buffers = req->rl_buffer;
Chuck Lever1e465fd2015-10-24 17:27:02 -04001122 struct rpcrdma_rep *rep = req->rl_reply;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001123
Chuck Lever1e465fd2015-10-24 17:27:02 -04001124 req->rl_niovs = 0;
1125 req->rl_reply = NULL;
1126
Chuck Levera5b027e2015-10-24 17:27:27 -04001127 spin_lock(&buffers->rb_lock);
Chuck Lever1e465fd2015-10-24 17:27:02 -04001128 list_add_tail(&req->rl_free, &buffers->rb_send_bufs);
1129 if (rep)
1130 list_add_tail(&rep->rr_list, &buffers->rb_recv_bufs);
Chuck Levera5b027e2015-10-24 17:27:27 -04001131 spin_unlock(&buffers->rb_lock);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001132}
1133
1134/*
1135 * Recover reply buffers from pool.
Chuck Lever1e465fd2015-10-24 17:27:02 -04001136 * This happens when recovering from disconnect.
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001137 */
1138void
1139rpcrdma_recv_buffer_get(struct rpcrdma_req *req)
1140{
1141 struct rpcrdma_buffer *buffers = req->rl_buffer;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001142
Chuck Levera5b027e2015-10-24 17:27:27 -04001143 spin_lock(&buffers->rb_lock);
Chuck Lever1e465fd2015-10-24 17:27:02 -04001144 if (!list_empty(&buffers->rb_recv_bufs))
1145 req->rl_reply = rpcrdma_buffer_get_rep_locked(buffers);
Chuck Levera5b027e2015-10-24 17:27:27 -04001146 spin_unlock(&buffers->rb_lock);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001147}
1148
1149/*
1150 * Put reply buffers back into pool when not attached to
1151 * request. This happens in error conditions.
1152 */
1153void
1154rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep)
1155{
Chuck Leverfed171b2015-05-26 11:51:37 -04001156 struct rpcrdma_buffer *buffers = &rep->rr_rxprt->rx_buf;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001157
Chuck Levera5b027e2015-10-24 17:27:27 -04001158 spin_lock(&buffers->rb_lock);
Chuck Lever1e465fd2015-10-24 17:27:02 -04001159 list_add_tail(&rep->rr_list, &buffers->rb_recv_bufs);
Chuck Levera5b027e2015-10-24 17:27:27 -04001160 spin_unlock(&buffers->rb_lock);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001161}
1162
1163/*
1164 * Wrappers for internal-use kmalloc memory registration, used by buffer code.
1165 */
1166
Chuck Leverd6547882015-03-30 14:35:44 -04001167void
1168rpcrdma_mapping_error(struct rpcrdma_mr_seg *seg)
1169{
1170 dprintk("RPC: map_one: offset %p iova %llx len %zu\n",
1171 seg->mr_offset,
1172 (unsigned long long)seg->mr_dma, seg->mr_dmalen);
1173}
1174
Chuck Lever9128c3e2015-01-21 11:04:00 -05001175/**
1176 * rpcrdma_alloc_regbuf - kmalloc and register memory for SEND/RECV buffers
1177 * @ia: controlling rpcrdma_ia
1178 * @size: size of buffer to be allocated, in bytes
1179 * @flags: GFP flags
1180 *
1181 * Returns pointer to private header of an area of internally
1182 * registered memory, or an ERR_PTR. The registered buffer follows
1183 * the end of the private header.
1184 *
1185 * xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for
1186 * receiving the payload of RDMA RECV operations. regbufs are not
1187 * used for RDMA READ/WRITE operations, thus are registered only for
1188 * LOCAL access.
1189 */
1190struct rpcrdma_regbuf *
1191rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, gfp_t flags)
1192{
1193 struct rpcrdma_regbuf *rb;
Chuck Levere531dca2015-08-03 13:03:20 -04001194 struct ib_sge *iov;
Chuck Lever9128c3e2015-01-21 11:04:00 -05001195
Chuck Lever9128c3e2015-01-21 11:04:00 -05001196 rb = kmalloc(sizeof(*rb) + size, flags);
1197 if (rb == NULL)
1198 goto out;
1199
Chuck Levere531dca2015-08-03 13:03:20 -04001200 iov = &rb->rg_iov;
1201 iov->addr = ib_dma_map_single(ia->ri_device,
1202 (void *)rb->rg_base, size,
1203 DMA_BIDIRECTIONAL);
1204 if (ib_dma_mapping_error(ia->ri_device, iov->addr))
Chuck Lever9128c3e2015-01-21 11:04:00 -05001205 goto out_free;
1206
Chuck Levere531dca2015-08-03 13:03:20 -04001207 iov->length = size;
Chuck Leverbb6c96d2015-09-24 10:34:21 +03001208 iov->lkey = ia->ri_pd->local_dma_lkey;
Chuck Levere531dca2015-08-03 13:03:20 -04001209 rb->rg_size = size;
1210 rb->rg_owner = NULL;
Chuck Lever9128c3e2015-01-21 11:04:00 -05001211 return rb;
1212
1213out_free:
1214 kfree(rb);
1215out:
Chuck Levere531dca2015-08-03 13:03:20 -04001216 return ERR_PTR(-ENOMEM);
Chuck Lever9128c3e2015-01-21 11:04:00 -05001217}
1218
1219/**
1220 * rpcrdma_free_regbuf - deregister and free registered buffer
1221 * @ia: controlling rpcrdma_ia
1222 * @rb: regbuf to be deregistered and freed
1223 */
1224void
1225rpcrdma_free_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)
1226{
Chuck Levere531dca2015-08-03 13:03:20 -04001227 struct ib_sge *iov;
1228
1229 if (!rb)
1230 return;
1231
1232 iov = &rb->rg_iov;
1233 ib_dma_unmap_single(ia->ri_device,
1234 iov->addr, iov->length, DMA_BIDIRECTIONAL);
1235 kfree(rb);
Chuck Lever9128c3e2015-01-21 11:04:00 -05001236}
1237
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001238/*
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001239 * Prepost any receive buffer, then post send.
1240 *
1241 * Receive buffer is donated to hardware, reclaimed upon recv completion.
1242 */
1243int
1244rpcrdma_ep_post(struct rpcrdma_ia *ia,
1245 struct rpcrdma_ep *ep,
1246 struct rpcrdma_req *req)
1247{
Chuck Leverb3221d62015-08-03 13:03:39 -04001248 struct ib_device *device = ia->ri_device;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001249 struct ib_send_wr send_wr, *send_wr_fail;
1250 struct rpcrdma_rep *rep = req->rl_reply;
Chuck Leverb3221d62015-08-03 13:03:39 -04001251 struct ib_sge *iov = req->rl_send_iov;
1252 int i, rc;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001253
1254 if (rep) {
1255 rc = rpcrdma_ep_post_recv(ia, ep, rep);
1256 if (rc)
1257 goto out;
1258 req->rl_reply = NULL;
1259 }
1260
1261 send_wr.next = NULL;
Chuck Levere46ac342015-03-30 14:35:35 -04001262 send_wr.wr_id = RPCRDMA_IGNORE_COMPLETION;
Chuck Leverb3221d62015-08-03 13:03:39 -04001263 send_wr.sg_list = iov;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001264 send_wr.num_sge = req->rl_niovs;
1265 send_wr.opcode = IB_WR_SEND;
Chuck Leverb3221d62015-08-03 13:03:39 -04001266
1267 for (i = 0; i < send_wr.num_sge; i++)
1268 ib_dma_sync_single_for_device(device, iov[i].addr,
1269 iov[i].length, DMA_TO_DEVICE);
1270 dprintk("RPC: %s: posting %d s/g entries\n",
1271 __func__, send_wr.num_sge);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001272
1273 if (DECR_CQCOUNT(ep) > 0)
1274 send_wr.send_flags = 0;
1275 else { /* Provider must take a send completion every now and then */
1276 INIT_CQCOUNT(ep);
1277 send_wr.send_flags = IB_SEND_SIGNALED;
1278 }
1279
1280 rc = ib_post_send(ia->ri_id->qp, &send_wr, &send_wr_fail);
1281 if (rc)
1282 dprintk("RPC: %s: ib_post_send returned %i\n", __func__,
1283 rc);
1284out:
1285 return rc;
1286}
1287
1288/*
1289 * (Re)post a receive buffer.
1290 */
1291int
1292rpcrdma_ep_post_recv(struct rpcrdma_ia *ia,
1293 struct rpcrdma_ep *ep,
1294 struct rpcrdma_rep *rep)
1295{
1296 struct ib_recv_wr recv_wr, *recv_wr_fail;
1297 int rc;
1298
1299 recv_wr.next = NULL;
1300 recv_wr.wr_id = (u64) (unsigned long) rep;
Chuck Lever6b1184c2015-01-21 11:04:25 -05001301 recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov;
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001302 recv_wr.num_sge = 1;
1303
Chuck Lever89e0d1122015-05-26 11:51:56 -04001304 ib_dma_sync_single_for_cpu(ia->ri_device,
Chuck Lever6b1184c2015-01-21 11:04:25 -05001305 rdmab_addr(rep->rr_rdmabuf),
1306 rdmab_length(rep->rr_rdmabuf),
1307 DMA_BIDIRECTIONAL);
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001308
\"Talpey, Thomas\c56c65f2007-09-10 13:51:18 -04001309 rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail);
1310
1311 if (rc)
1312 dprintk("RPC: %s: ib_post_recv returned %i\n", __func__,
1313 rc);
1314 return rc;
1315}
Chuck Lever43e95982014-07-29 17:23:34 -04001316
Chuck Leverf531a5d2015-10-24 17:27:43 -04001317/**
1318 * rpcrdma_ep_post_extra_recv - Post buffers for incoming backchannel requests
1319 * @r_xprt: transport associated with these backchannel resources
1320 * @min_reqs: minimum number of incoming requests expected
1321 *
1322 * Returns zero if all requested buffers were posted, or a negative errno.
1323 */
1324int
1325rpcrdma_ep_post_extra_recv(struct rpcrdma_xprt *r_xprt, unsigned int count)
1326{
1327 struct rpcrdma_buffer *buffers = &r_xprt->rx_buf;
1328 struct rpcrdma_ia *ia = &r_xprt->rx_ia;
1329 struct rpcrdma_ep *ep = &r_xprt->rx_ep;
1330 struct rpcrdma_rep *rep;
Chuck Leverf531a5d2015-10-24 17:27:43 -04001331 int rc;
1332
1333 while (count--) {
Chuck Lever9b066882015-12-16 17:22:06 -05001334 spin_lock(&buffers->rb_lock);
Chuck Leverf531a5d2015-10-24 17:27:43 -04001335 if (list_empty(&buffers->rb_recv_bufs))
1336 goto out_reqbuf;
1337 rep = rpcrdma_buffer_get_rep_locked(buffers);
Chuck Lever9b066882015-12-16 17:22:06 -05001338 spin_unlock(&buffers->rb_lock);
Chuck Leverf531a5d2015-10-24 17:27:43 -04001339
1340 rc = rpcrdma_ep_post_recv(ia, ep, rep);
1341 if (rc)
1342 goto out_rc;
1343 }
1344
1345 return 0;
1346
1347out_reqbuf:
Chuck Lever9b066882015-12-16 17:22:06 -05001348 spin_unlock(&buffers->rb_lock);
Chuck Leverf531a5d2015-10-24 17:27:43 -04001349 pr_warn("%s: no extra receive buffers\n", __func__);
1350 return -ENOMEM;
1351
1352out_rc:
1353 rpcrdma_recv_buffer_put(rep);
1354 return rc;
1355}
1356
Chuck Lever1c9351e2015-03-30 14:34:30 -04001357/* How many chunk list items fit within our inline buffers?
Chuck Lever43e95982014-07-29 17:23:34 -04001358 */
Chuck Lever1c9351e2015-03-30 14:34:30 -04001359unsigned int
1360rpcrdma_max_segments(struct rpcrdma_xprt *r_xprt)
Chuck Lever43e95982014-07-29 17:23:34 -04001361{
1362 struct rpcrdma_create_data_internal *cdata = &r_xprt->rx_data;
Chuck Lever1c9351e2015-03-30 14:34:30 -04001363 int bytes, segments;
Chuck Lever43e95982014-07-29 17:23:34 -04001364
Chuck Lever1c9351e2015-03-30 14:34:30 -04001365 bytes = min_t(unsigned int, cdata->inline_wsize, cdata->inline_rsize);
1366 bytes -= RPCRDMA_HDRLEN_MIN;
1367 if (bytes < sizeof(struct rpcrdma_segment) * 2) {
1368 pr_warn("RPC: %s: inline threshold too small\n",
1369 __func__);
1370 return 0;
Chuck Lever43e95982014-07-29 17:23:34 -04001371 }
Chuck Lever1c9351e2015-03-30 14:34:30 -04001372
1373 segments = 1 << (fls(bytes / sizeof(struct rpcrdma_segment)) - 1);
1374 dprintk("RPC: %s: max chunk list size = %d segments\n",
1375 __func__, segments);
1376 return segments;
Chuck Lever43e95982014-07-29 17:23:34 -04001377}