\"Talpey, Thomas\ | f58851e | 2007-09-10 13:50:12 -0400 | [diff] [blame] | 1 | /* |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 2 | * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved. |
| 3 | * |
| 4 | * This software is available to you under a choice of one of two |
| 5 | * licenses. You may choose to be licensed under the terms of the GNU |
| 6 | * General Public License (GPL) Version 2, available from the file |
| 7 | * COPYING in the main directory of this source tree, or the BSD-type |
| 8 | * license below: |
| 9 | * |
| 10 | * Redistribution and use in source and binary forms, with or without |
| 11 | * modification, are permitted provided that the following conditions |
| 12 | * are met: |
| 13 | * |
| 14 | * Redistributions of source code must retain the above copyright |
| 15 | * notice, this list of conditions and the following disclaimer. |
| 16 | * |
| 17 | * Redistributions in binary form must reproduce the above |
| 18 | * copyright notice, this list of conditions and the following |
| 19 | * disclaimer in the documentation and/or other materials provided |
| 20 | * with the distribution. |
| 21 | * |
| 22 | * Neither the name of the Network Appliance, Inc. nor the names of |
| 23 | * its contributors may be used to endorse or promote products |
| 24 | * derived from this software without specific prior written |
| 25 | * permission. |
| 26 | * |
| 27 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 28 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 29 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 30 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 31 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 32 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 33 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 34 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 35 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 36 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 37 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 38 | */ |
| 39 | |
| 40 | /* |
| 41 | * rpc_rdma.c |
| 42 | * |
| 43 | * This file contains the guts of the RPC RDMA protocol, and |
| 44 | * does marshaling/unmarshaling, etc. It is also where interfacing |
| 45 | * to the Linux RPC framework lives. |
\"Talpey, Thomas\ | f58851e | 2007-09-10 13:50:12 -0400 | [diff] [blame] | 46 | */ |
| 47 | |
| 48 | #include "xprt_rdma.h" |
| 49 | |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 50 | #include <linux/highmem.h> |
| 51 | |
Jeff Layton | f895b25 | 2014-11-17 16:58:04 -0500 | [diff] [blame] | 52 | #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 53 | # define RPCDBG_FACILITY RPCDBG_TRANS |
| 54 | #endif |
| 55 | |
Jeff Layton | f895b25 | 2014-11-17 16:58:04 -0500 | [diff] [blame] | 56 | #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 57 | static const char transfertypes[][12] = { |
| 58 | "pure inline", /* no chunks */ |
| 59 | " read chunk", /* some argument via rdma read */ |
| 60 | "*read chunk", /* entire request via rdma read */ |
| 61 | "write chunk", /* some result via rdma write */ |
| 62 | "reply chunk" /* entire reply via rdma write */ |
| 63 | }; |
| 64 | #endif |
| 65 | |
| 66 | /* |
| 67 | * Chunk assembly from upper layer xdr_buf. |
| 68 | * |
| 69 | * Prepare the passed-in xdr_buf into representation as RPC/RDMA chunk |
| 70 | * elements. Segments are then coalesced when registered, if possible |
| 71 | * within the selected memreg mode. |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 72 | * |
| 73 | * Returns positive number of segments converted, or a negative errno. |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 74 | */ |
| 75 | |
| 76 | static int |
Chuck Lever | 2a428b2 | 2007-10-26 13:30:43 -0400 | [diff] [blame] | 77 | rpcrdma_convert_iovs(struct xdr_buf *xdrbuf, unsigned int pos, |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 78 | enum rpcrdma_chunktype type, struct rpcrdma_mr_seg *seg, int nsegs) |
| 79 | { |
| 80 | int len, n = 0, p; |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 81 | int page_base; |
| 82 | struct page **ppages; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 83 | |
| 84 | if (pos == 0 && xdrbuf->head[0].iov_len) { |
| 85 | seg[n].mr_page = NULL; |
| 86 | seg[n].mr_offset = xdrbuf->head[0].iov_base; |
| 87 | seg[n].mr_len = xdrbuf->head[0].iov_len; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 88 | ++n; |
| 89 | } |
| 90 | |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 91 | len = xdrbuf->page_len; |
| 92 | ppages = xdrbuf->pages + (xdrbuf->page_base >> PAGE_SHIFT); |
| 93 | page_base = xdrbuf->page_base & ~PAGE_MASK; |
| 94 | p = 0; |
| 95 | while (len && n < nsegs) { |
Shirley Ma | 196c699 | 2014-05-28 10:34:24 -0400 | [diff] [blame] | 96 | if (!ppages[p]) { |
| 97 | /* alloc the pagelist for receiving buffer */ |
| 98 | ppages[p] = alloc_page(GFP_ATOMIC); |
| 99 | if (!ppages[p]) |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 100 | return -ENOMEM; |
Shirley Ma | 196c699 | 2014-05-28 10:34:24 -0400 | [diff] [blame] | 101 | } |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 102 | seg[n].mr_page = ppages[p]; |
| 103 | seg[n].mr_offset = (void *)(unsigned long) page_base; |
| 104 | seg[n].mr_len = min_t(u32, PAGE_SIZE - page_base, len); |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 105 | if (seg[n].mr_len > PAGE_SIZE) |
| 106 | return -EIO; |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 107 | len -= seg[n].mr_len; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 108 | ++n; |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 109 | ++p; |
| 110 | page_base = 0; /* page offset only applies to first page */ |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 111 | } |
| 112 | |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 113 | /* Message overflows the seg array */ |
| 114 | if (len && n == nsegs) |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 115 | return -EIO; |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 116 | |
James Lentini | 50e1092 | 2007-12-10 11:24:48 -0500 | [diff] [blame] | 117 | if (xdrbuf->tail[0].iov_len) { |
Tom Talpey | 9191ca3 | 2008-10-09 15:01:11 -0400 | [diff] [blame] | 118 | /* the rpcrdma protocol allows us to omit any trailing |
| 119 | * xdr pad bytes, saving the server an RDMA operation. */ |
| 120 | if (xdrbuf->tail[0].iov_len < 4 && xprt_rdma_pad_optimize) |
| 121 | return n; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 122 | if (n == nsegs) |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 123 | /* Tail remains, but we're out of segments */ |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 124 | return -EIO; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 125 | seg[n].mr_page = NULL; |
| 126 | seg[n].mr_offset = xdrbuf->tail[0].iov_base; |
| 127 | seg[n].mr_len = xdrbuf->tail[0].iov_len; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 128 | ++n; |
| 129 | } |
| 130 | |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 131 | return n; |
| 132 | } |
| 133 | |
| 134 | /* |
| 135 | * Create read/write chunk lists, and reply chunks, for RDMA |
| 136 | * |
| 137 | * Assume check against THRESHOLD has been done, and chunks are required. |
| 138 | * Assume only encoding one list entry for read|write chunks. The NFSv3 |
| 139 | * protocol is simple enough to allow this as it only has a single "bulk |
| 140 | * result" in each procedure - complicated NFSv4 COMPOUNDs are not. (The |
| 141 | * RDMA/Sessions NFSv4 proposal addresses this for future v4 revs.) |
| 142 | * |
| 143 | * When used for a single reply chunk (which is a special write |
| 144 | * chunk used for the entire reply, rather than just the data), it |
| 145 | * is used primarily for READDIR and READLINK which would otherwise |
| 146 | * be severely size-limited by a small rdma inline read max. The server |
| 147 | * response will come back as an RDMA Write, followed by a message |
| 148 | * of type RDMA_NOMSG carrying the xid and length. As a result, reply |
| 149 | * chunks do not provide data alignment, however they do not require |
| 150 | * "fixup" (moving the response to the upper layer buffer) either. |
| 151 | * |
| 152 | * Encoding key for single-list chunks (HLOO = Handle32 Length32 Offset64): |
| 153 | * |
| 154 | * Read chunklist (a linked list): |
| 155 | * N elements, position P (same P for all chunks of same arg!): |
| 156 | * 1 - PHLOO - 1 - PHLOO - ... - 1 - PHLOO - 0 |
| 157 | * |
| 158 | * Write chunklist (a list of (one) counted array): |
| 159 | * N elements: |
| 160 | * 1 - N - HLOO - HLOO - ... - HLOO - 0 |
| 161 | * |
| 162 | * Reply chunk (a counted array): |
| 163 | * N elements: |
| 164 | * 1 - N - HLOO - HLOO - ... - HLOO |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 165 | * |
| 166 | * Returns positive RPC/RDMA header size, or negative errno. |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 167 | */ |
| 168 | |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 169 | static ssize_t |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 170 | rpcrdma_create_chunks(struct rpc_rqst *rqst, struct xdr_buf *target, |
| 171 | struct rpcrdma_msg *headerp, enum rpcrdma_chunktype type) |
| 172 | { |
| 173 | struct rpcrdma_req *req = rpcr_to_rdmar(rqst); |
Trond Myklebust | a4f0835 | 2013-01-08 09:10:21 -0500 | [diff] [blame] | 174 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt); |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 175 | int n, nsegs, nchunks = 0; |
Chuck Lever | 2a428b2 | 2007-10-26 13:30:43 -0400 | [diff] [blame] | 176 | unsigned int pos; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 177 | struct rpcrdma_mr_seg *seg = req->rl_segments; |
| 178 | struct rpcrdma_read_chunk *cur_rchunk = NULL; |
| 179 | struct rpcrdma_write_array *warray = NULL; |
| 180 | struct rpcrdma_write_chunk *cur_wchunk = NULL; |
Al Viro | 2d8a972 | 2007-10-29 04:37:58 +0000 | [diff] [blame] | 181 | __be32 *iptr = headerp->rm_body.rm_chunks; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 182 | |
| 183 | if (type == rpcrdma_readch || type == rpcrdma_areadch) { |
| 184 | /* a read chunk - server will RDMA Read our memory */ |
| 185 | cur_rchunk = (struct rpcrdma_read_chunk *) iptr; |
| 186 | } else { |
| 187 | /* a write or reply chunk - server will RDMA Write our memory */ |
| 188 | *iptr++ = xdr_zero; /* encode a NULL read chunk list */ |
| 189 | if (type == rpcrdma_replych) |
| 190 | *iptr++ = xdr_zero; /* a NULL write chunk list */ |
| 191 | warray = (struct rpcrdma_write_array *) iptr; |
| 192 | cur_wchunk = (struct rpcrdma_write_chunk *) (warray + 1); |
| 193 | } |
| 194 | |
| 195 | if (type == rpcrdma_replych || type == rpcrdma_areadch) |
| 196 | pos = 0; |
| 197 | else |
| 198 | pos = target->head[0].iov_len; |
| 199 | |
| 200 | nsegs = rpcrdma_convert_iovs(target, pos, type, seg, RPCRDMA_MAX_SEGS); |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 201 | if (nsegs < 0) |
| 202 | return nsegs; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 203 | |
| 204 | do { |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 205 | n = rpcrdma_register_external(seg, nsegs, |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 206 | cur_wchunk != NULL, r_xprt); |
| 207 | if (n <= 0) |
| 208 | goto out; |
| 209 | if (cur_rchunk) { /* read */ |
| 210 | cur_rchunk->rc_discrim = xdr_one; |
| 211 | /* all read chunks have the same "position" */ |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 212 | cur_rchunk->rc_position = cpu_to_be32(pos); |
| 213 | cur_rchunk->rc_target.rs_handle = |
| 214 | cpu_to_be32(seg->mr_rkey); |
| 215 | cur_rchunk->rc_target.rs_length = |
| 216 | cpu_to_be32(seg->mr_len); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 217 | xdr_encode_hyper( |
Al Viro | 2d8a972 | 2007-10-29 04:37:58 +0000 | [diff] [blame] | 218 | (__be32 *)&cur_rchunk->rc_target.rs_offset, |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 219 | seg->mr_base); |
| 220 | dprintk("RPC: %s: read chunk " |
Chuck Lever | 2a428b2 | 2007-10-26 13:30:43 -0400 | [diff] [blame] | 221 | "elem %d@0x%llx:0x%x pos %u (%s)\n", __func__, |
Stephen Rothwell | e08a132 | 2007-10-30 00:44:32 -0700 | [diff] [blame] | 222 | seg->mr_len, (unsigned long long)seg->mr_base, |
| 223 | seg->mr_rkey, pos, n < nsegs ? "more" : "last"); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 224 | cur_rchunk++; |
| 225 | r_xprt->rx_stats.read_chunk_count++; |
| 226 | } else { /* write/reply */ |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 227 | cur_wchunk->wc_target.rs_handle = |
| 228 | cpu_to_be32(seg->mr_rkey); |
| 229 | cur_wchunk->wc_target.rs_length = |
| 230 | cpu_to_be32(seg->mr_len); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 231 | xdr_encode_hyper( |
Al Viro | 2d8a972 | 2007-10-29 04:37:58 +0000 | [diff] [blame] | 232 | (__be32 *)&cur_wchunk->wc_target.rs_offset, |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 233 | seg->mr_base); |
| 234 | dprintk("RPC: %s: %s chunk " |
| 235 | "elem %d@0x%llx:0x%x (%s)\n", __func__, |
| 236 | (type == rpcrdma_replych) ? "reply" : "write", |
Stephen Rothwell | e08a132 | 2007-10-30 00:44:32 -0700 | [diff] [blame] | 237 | seg->mr_len, (unsigned long long)seg->mr_base, |
| 238 | seg->mr_rkey, n < nsegs ? "more" : "last"); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 239 | cur_wchunk++; |
| 240 | if (type == rpcrdma_replych) |
| 241 | r_xprt->rx_stats.reply_chunk_count++; |
| 242 | else |
| 243 | r_xprt->rx_stats.write_chunk_count++; |
| 244 | r_xprt->rx_stats.total_rdma_request += seg->mr_len; |
| 245 | } |
| 246 | nchunks++; |
| 247 | seg += n; |
| 248 | nsegs -= n; |
| 249 | } while (nsegs); |
| 250 | |
| 251 | /* success. all failures return above */ |
| 252 | req->rl_nchunks = nchunks; |
| 253 | |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 254 | /* |
| 255 | * finish off header. If write, marshal discrim and nchunks. |
| 256 | */ |
| 257 | if (cur_rchunk) { |
Al Viro | 2d8a972 | 2007-10-29 04:37:58 +0000 | [diff] [blame] | 258 | iptr = (__be32 *) cur_rchunk; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 259 | *iptr++ = xdr_zero; /* finish the read chunk list */ |
| 260 | *iptr++ = xdr_zero; /* encode a NULL write chunk list */ |
| 261 | *iptr++ = xdr_zero; /* encode a NULL reply chunk */ |
| 262 | } else { |
| 263 | warray->wc_discrim = xdr_one; |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 264 | warray->wc_nchunks = cpu_to_be32(nchunks); |
Al Viro | 2d8a972 | 2007-10-29 04:37:58 +0000 | [diff] [blame] | 265 | iptr = (__be32 *) cur_wchunk; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 266 | if (type == rpcrdma_writech) { |
| 267 | *iptr++ = xdr_zero; /* finish the write chunk list */ |
| 268 | *iptr++ = xdr_zero; /* encode a NULL reply chunk */ |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | /* |
| 273 | * Return header size. |
| 274 | */ |
| 275 | return (unsigned char *)iptr - (unsigned char *)headerp; |
| 276 | |
| 277 | out: |
Chuck Lever | 539431a | 2014-07-29 17:24:01 -0400 | [diff] [blame] | 278 | if (r_xprt->rx_ia.ri_memreg_strategy != RPCRDMA_FRMR) { |
| 279 | for (pos = 0; nchunks--;) |
| 280 | pos += rpcrdma_deregister_external( |
| 281 | &req->rl_segments[pos], r_xprt); |
| 282 | } |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 283 | return n; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 284 | } |
| 285 | |
| 286 | /* |
Chuck Lever | 6ab5994 | 2014-07-29 17:23:43 -0400 | [diff] [blame] | 287 | * Marshal chunks. This routine returns the header length |
| 288 | * consumed by marshaling. |
| 289 | * |
| 290 | * Returns positive RPC/RDMA header size, or negative errno. |
| 291 | */ |
| 292 | |
| 293 | ssize_t |
| 294 | rpcrdma_marshal_chunks(struct rpc_rqst *rqst, ssize_t result) |
| 295 | { |
| 296 | struct rpcrdma_req *req = rpcr_to_rdmar(rqst); |
| 297 | struct rpcrdma_msg *headerp = (struct rpcrdma_msg *)req->rl_base; |
| 298 | |
| 299 | if (req->rl_rtype != rpcrdma_noch) |
| 300 | result = rpcrdma_create_chunks(rqst, &rqst->rq_snd_buf, |
| 301 | headerp, req->rl_rtype); |
| 302 | else if (req->rl_wtype != rpcrdma_noch) |
| 303 | result = rpcrdma_create_chunks(rqst, &rqst->rq_rcv_buf, |
| 304 | headerp, req->rl_wtype); |
| 305 | return result; |
| 306 | } |
| 307 | |
| 308 | /* |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 309 | * Copy write data inline. |
| 310 | * This function is used for "small" requests. Data which is passed |
| 311 | * to RPC via iovecs (or page list) is copied directly into the |
| 312 | * pre-registered memory buffer for this request. For small amounts |
| 313 | * of data, this is efficient. The cutoff value is tunable. |
| 314 | */ |
| 315 | static int |
| 316 | rpcrdma_inline_pullup(struct rpc_rqst *rqst, int pad) |
| 317 | { |
| 318 | int i, npages, curlen; |
| 319 | int copy_len; |
| 320 | unsigned char *srcp, *destp; |
| 321 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt); |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 322 | int page_base; |
| 323 | struct page **ppages; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 324 | |
| 325 | destp = rqst->rq_svec[0].iov_base; |
| 326 | curlen = rqst->rq_svec[0].iov_len; |
| 327 | destp += curlen; |
| 328 | /* |
| 329 | * Do optional padding where it makes sense. Alignment of write |
| 330 | * payload can help the server, if our setting is accurate. |
| 331 | */ |
| 332 | pad -= (curlen + 36/*sizeof(struct rpcrdma_msg_padded)*/); |
| 333 | if (pad < 0 || rqst->rq_slen - curlen < RPCRDMA_INLINE_PAD_THRESH) |
| 334 | pad = 0; /* don't pad this request */ |
| 335 | |
| 336 | dprintk("RPC: %s: pad %d destp 0x%p len %d hdrlen %d\n", |
| 337 | __func__, pad, destp, rqst->rq_slen, curlen); |
| 338 | |
| 339 | copy_len = rqst->rq_snd_buf.page_len; |
Tom Talpey | b38ab40 | 2009-03-11 14:37:55 -0400 | [diff] [blame] | 340 | |
| 341 | if (rqst->rq_snd_buf.tail[0].iov_len) { |
| 342 | curlen = rqst->rq_snd_buf.tail[0].iov_len; |
| 343 | if (destp + copy_len != rqst->rq_snd_buf.tail[0].iov_base) { |
| 344 | memmove(destp + copy_len, |
| 345 | rqst->rq_snd_buf.tail[0].iov_base, curlen); |
| 346 | r_xprt->rx_stats.pullup_copy_count += curlen; |
| 347 | } |
| 348 | dprintk("RPC: %s: tail destp 0x%p len %d\n", |
| 349 | __func__, destp + copy_len, curlen); |
| 350 | rqst->rq_svec[0].iov_len += curlen; |
| 351 | } |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 352 | r_xprt->rx_stats.pullup_copy_count += copy_len; |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 353 | |
| 354 | page_base = rqst->rq_snd_buf.page_base; |
| 355 | ppages = rqst->rq_snd_buf.pages + (page_base >> PAGE_SHIFT); |
| 356 | page_base &= ~PAGE_MASK; |
| 357 | npages = PAGE_ALIGN(page_base+copy_len) >> PAGE_SHIFT; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 358 | for (i = 0; copy_len && i < npages; i++) { |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 359 | curlen = PAGE_SIZE - page_base; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 360 | if (curlen > copy_len) |
| 361 | curlen = copy_len; |
| 362 | dprintk("RPC: %s: page %d destp 0x%p len %d curlen %d\n", |
| 363 | __func__, i, destp, copy_len, curlen); |
Cong Wang | b854178 | 2011-11-25 23:14:40 +0800 | [diff] [blame] | 364 | srcp = kmap_atomic(ppages[i]); |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 365 | memcpy(destp, srcp+page_base, curlen); |
Cong Wang | b854178 | 2011-11-25 23:14:40 +0800 | [diff] [blame] | 366 | kunmap_atomic(srcp); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 367 | rqst->rq_svec[0].iov_len += curlen; |
| 368 | destp += curlen; |
| 369 | copy_len -= curlen; |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 370 | page_base = 0; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 371 | } |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 372 | /* header now contains entire send message */ |
| 373 | return pad; |
| 374 | } |
| 375 | |
| 376 | /* |
| 377 | * Marshal a request: the primary job of this routine is to choose |
| 378 | * the transfer modes. See comments below. |
| 379 | * |
| 380 | * Uses multiple RDMA IOVs for a request: |
| 381 | * [0] -- RPC RDMA header, which uses memory from the *start* of the |
| 382 | * preregistered buffer that already holds the RPC data in |
| 383 | * its middle. |
| 384 | * [1] -- the RPC header/data, marshaled by RPC and the NFS protocol. |
| 385 | * [2] -- optional padding. |
| 386 | * [3] -- if padded, header only in [1] and data here. |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 387 | * |
| 388 | * Returns zero on success, otherwise a negative errno. |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 389 | */ |
| 390 | |
| 391 | int |
| 392 | rpcrdma_marshal_req(struct rpc_rqst *rqst) |
| 393 | { |
Trond Myklebust | a4f0835 | 2013-01-08 09:10:21 -0500 | [diff] [blame] | 394 | struct rpc_xprt *xprt = rqst->rq_xprt; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 395 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); |
| 396 | struct rpcrdma_req *req = rpcr_to_rdmar(rqst); |
| 397 | char *base; |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 398 | size_t rpclen, padlen; |
| 399 | ssize_t hdrlen; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 400 | struct rpcrdma_msg *headerp; |
| 401 | |
| 402 | /* |
| 403 | * rpclen gets amount of data in first buffer, which is the |
| 404 | * pre-registered buffer. |
| 405 | */ |
| 406 | base = rqst->rq_svec[0].iov_base; |
| 407 | rpclen = rqst->rq_svec[0].iov_len; |
| 408 | |
| 409 | /* build RDMA header in private area at front */ |
| 410 | headerp = (struct rpcrdma_msg *) req->rl_base; |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 411 | /* don't byte-swap XID, it's already done in request */ |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 412 | headerp->rm_xid = rqst->rq_xid; |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 413 | headerp->rm_vers = rpcrdma_version; |
| 414 | headerp->rm_credit = cpu_to_be32(r_xprt->rx_buf.rb_max_requests); |
| 415 | headerp->rm_type = rdma_msg; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 416 | |
| 417 | /* |
| 418 | * Chunks needed for results? |
| 419 | * |
| 420 | * o If the expected result is under the inline threshold, all ops |
| 421 | * return as inline (but see later). |
| 422 | * o Large non-read ops return as a single reply chunk. |
| 423 | * o Large read ops return data as write chunk(s), header as inline. |
| 424 | * |
| 425 | * Note: the NFS code sending down multiple result segments implies |
| 426 | * the op is one of read, readdir[plus], readlink or NFSv4 getacl. |
| 427 | */ |
| 428 | |
| 429 | /* |
| 430 | * This code can handle read chunks, write chunks OR reply |
| 431 | * chunks -- only one type. If the request is too big to fit |
| 432 | * inline, then we will choose read chunks. If the request is |
| 433 | * a READ, then use write chunks to separate the file data |
| 434 | * into pages; otherwise use reply chunks. |
| 435 | */ |
| 436 | if (rqst->rq_rcv_buf.buflen <= RPCRDMA_INLINE_READ_THRESHOLD(rqst)) |
Chuck Lever | 6ab5994 | 2014-07-29 17:23:43 -0400 | [diff] [blame] | 437 | req->rl_wtype = rpcrdma_noch; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 438 | else if (rqst->rq_rcv_buf.page_len == 0) |
Chuck Lever | 6ab5994 | 2014-07-29 17:23:43 -0400 | [diff] [blame] | 439 | req->rl_wtype = rpcrdma_replych; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 440 | else if (rqst->rq_rcv_buf.flags & XDRBUF_READ) |
Chuck Lever | 6ab5994 | 2014-07-29 17:23:43 -0400 | [diff] [blame] | 441 | req->rl_wtype = rpcrdma_writech; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 442 | else |
Chuck Lever | 6ab5994 | 2014-07-29 17:23:43 -0400 | [diff] [blame] | 443 | req->rl_wtype = rpcrdma_replych; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 444 | |
| 445 | /* |
| 446 | * Chunks needed for arguments? |
| 447 | * |
| 448 | * o If the total request is under the inline threshold, all ops |
| 449 | * are sent as inline. |
| 450 | * o Large non-write ops are sent with the entire message as a |
| 451 | * single read chunk (protocol 0-position special case). |
| 452 | * o Large write ops transmit data as read chunk(s), header as |
| 453 | * inline. |
| 454 | * |
| 455 | * Note: the NFS code sending down multiple argument segments |
| 456 | * implies the op is a write. |
| 457 | * TBD check NFSv4 setacl |
| 458 | */ |
| 459 | if (rqst->rq_snd_buf.len <= RPCRDMA_INLINE_WRITE_THRESHOLD(rqst)) |
Chuck Lever | 6ab5994 | 2014-07-29 17:23:43 -0400 | [diff] [blame] | 460 | req->rl_rtype = rpcrdma_noch; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 461 | else if (rqst->rq_snd_buf.page_len == 0) |
Chuck Lever | 6ab5994 | 2014-07-29 17:23:43 -0400 | [diff] [blame] | 462 | req->rl_rtype = rpcrdma_areadch; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 463 | else |
Chuck Lever | 6ab5994 | 2014-07-29 17:23:43 -0400 | [diff] [blame] | 464 | req->rl_rtype = rpcrdma_readch; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 465 | |
| 466 | /* The following simplification is not true forever */ |
Chuck Lever | 6ab5994 | 2014-07-29 17:23:43 -0400 | [diff] [blame] | 467 | if (req->rl_rtype != rpcrdma_noch && req->rl_wtype == rpcrdma_replych) |
| 468 | req->rl_wtype = rpcrdma_noch; |
| 469 | if (req->rl_rtype != rpcrdma_noch && req->rl_wtype != rpcrdma_noch) { |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 470 | dprintk("RPC: %s: cannot marshal multiple chunk lists\n", |
| 471 | __func__); |
| 472 | return -EIO; |
| 473 | } |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 474 | |
Chuck Lever | f284648 | 2015-01-21 11:02:29 -0500 | [diff] [blame] | 475 | hdrlen = RPCRDMA_HDRLEN_MIN; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 476 | padlen = 0; |
| 477 | |
| 478 | /* |
| 479 | * Pull up any extra send data into the preregistered buffer. |
| 480 | * When padding is in use and applies to the transfer, insert |
| 481 | * it and change the message type. |
| 482 | */ |
Chuck Lever | 6ab5994 | 2014-07-29 17:23:43 -0400 | [diff] [blame] | 483 | if (req->rl_rtype == rpcrdma_noch) { |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 484 | |
| 485 | padlen = rpcrdma_inline_pullup(rqst, |
| 486 | RPCRDMA_INLINE_PAD_VALUE(rqst)); |
| 487 | |
| 488 | if (padlen) { |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 489 | headerp->rm_type = rdma_msgp; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 490 | headerp->rm_body.rm_padded.rm_align = |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 491 | cpu_to_be32(RPCRDMA_INLINE_PAD_VALUE(rqst)); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 492 | headerp->rm_body.rm_padded.rm_thresh = |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 493 | cpu_to_be32(RPCRDMA_INLINE_PAD_THRESH); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 494 | headerp->rm_body.rm_padded.rm_pempty[0] = xdr_zero; |
| 495 | headerp->rm_body.rm_padded.rm_pempty[1] = xdr_zero; |
| 496 | headerp->rm_body.rm_padded.rm_pempty[2] = xdr_zero; |
| 497 | hdrlen += 2 * sizeof(u32); /* extra words in padhdr */ |
Chuck Lever | 6ab5994 | 2014-07-29 17:23:43 -0400 | [diff] [blame] | 498 | if (req->rl_wtype != rpcrdma_noch) { |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 499 | dprintk("RPC: %s: invalid chunk list\n", |
| 500 | __func__); |
| 501 | return -EIO; |
| 502 | } |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 503 | } else { |
| 504 | headerp->rm_body.rm_nochunks.rm_empty[0] = xdr_zero; |
| 505 | headerp->rm_body.rm_nochunks.rm_empty[1] = xdr_zero; |
| 506 | headerp->rm_body.rm_nochunks.rm_empty[2] = xdr_zero; |
| 507 | /* new length after pullup */ |
| 508 | rpclen = rqst->rq_svec[0].iov_len; |
| 509 | /* |
| 510 | * Currently we try to not actually use read inline. |
| 511 | * Reply chunks have the desirable property that |
| 512 | * they land, packed, directly in the target buffers |
| 513 | * without headers, so they require no fixup. The |
| 514 | * additional RDMA Write op sends the same amount |
| 515 | * of data, streams on-the-wire and adds no overhead |
| 516 | * on receive. Therefore, we request a reply chunk |
| 517 | * for non-writes wherever feasible and efficient. |
| 518 | */ |
Chuck Lever | 6ab5994 | 2014-07-29 17:23:43 -0400 | [diff] [blame] | 519 | if (req->rl_wtype == rpcrdma_noch) |
| 520 | req->rl_wtype = rpcrdma_replych; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 521 | } |
| 522 | } |
| 523 | |
Chuck Lever | 6ab5994 | 2014-07-29 17:23:43 -0400 | [diff] [blame] | 524 | hdrlen = rpcrdma_marshal_chunks(rqst, hdrlen); |
Chuck Lever | c93c622 | 2014-05-28 10:35:14 -0400 | [diff] [blame] | 525 | if (hdrlen < 0) |
| 526 | return hdrlen; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 527 | |
Tom Talpey | 5f37d56 | 2008-10-09 15:01:52 -0400 | [diff] [blame] | 528 | dprintk("RPC: %s: %s: hdrlen %zd rpclen %zd padlen %zd" |
| 529 | " headerp 0x%p base 0x%p lkey 0x%x\n", |
Chuck Lever | 6ab5994 | 2014-07-29 17:23:43 -0400 | [diff] [blame] | 530 | __func__, transfertypes[req->rl_wtype], hdrlen, rpclen, padlen, |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 531 | headerp, base, req->rl_iov.lkey); |
| 532 | |
| 533 | /* |
| 534 | * initialize send_iov's - normally only two: rdma chunk header and |
| 535 | * single preregistered RPC header buffer, but if padding is present, |
| 536 | * then use a preregistered (and zeroed) pad buffer between the RPC |
| 537 | * header and any write data. In all non-rdma cases, any following |
| 538 | * data has been copied into the RPC header buffer. |
| 539 | */ |
| 540 | req->rl_send_iov[0].addr = req->rl_iov.addr; |
| 541 | req->rl_send_iov[0].length = hdrlen; |
| 542 | req->rl_send_iov[0].lkey = req->rl_iov.lkey; |
| 543 | |
| 544 | req->rl_send_iov[1].addr = req->rl_iov.addr + (base - req->rl_base); |
| 545 | req->rl_send_iov[1].length = rpclen; |
| 546 | req->rl_send_iov[1].lkey = req->rl_iov.lkey; |
| 547 | |
| 548 | req->rl_niovs = 2; |
| 549 | |
| 550 | if (padlen) { |
| 551 | struct rpcrdma_ep *ep = &r_xprt->rx_ep; |
| 552 | |
| 553 | req->rl_send_iov[2].addr = ep->rep_pad.addr; |
| 554 | req->rl_send_iov[2].length = padlen; |
| 555 | req->rl_send_iov[2].lkey = ep->rep_pad.lkey; |
| 556 | |
| 557 | req->rl_send_iov[3].addr = req->rl_send_iov[1].addr + rpclen; |
| 558 | req->rl_send_iov[3].length = rqst->rq_slen - rpclen; |
| 559 | req->rl_send_iov[3].lkey = req->rl_iov.lkey; |
| 560 | |
| 561 | req->rl_niovs = 4; |
| 562 | } |
| 563 | |
| 564 | return 0; |
| 565 | } |
| 566 | |
| 567 | /* |
| 568 | * Chase down a received write or reply chunklist to get length |
| 569 | * RDMA'd by server. See map at rpcrdma_create_chunks()! :-) |
| 570 | */ |
| 571 | static int |
Chuck Lever | d4b37ff | 2007-10-26 13:30:49 -0400 | [diff] [blame] | 572 | rpcrdma_count_chunks(struct rpcrdma_rep *rep, unsigned int max, int wrchunk, __be32 **iptrp) |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 573 | { |
| 574 | unsigned int i, total_len; |
| 575 | struct rpcrdma_write_chunk *cur_wchunk; |
| 576 | |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 577 | i = be32_to_cpu(**iptrp); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 578 | if (i > max) |
| 579 | return -1; |
| 580 | cur_wchunk = (struct rpcrdma_write_chunk *) (*iptrp + 1); |
| 581 | total_len = 0; |
| 582 | while (i--) { |
| 583 | struct rpcrdma_segment *seg = &cur_wchunk->wc_target; |
| 584 | ifdebug(FACILITY) { |
| 585 | u64 off; |
Al Viro | 2d8a972 | 2007-10-29 04:37:58 +0000 | [diff] [blame] | 586 | xdr_decode_hyper((__be32 *)&seg->rs_offset, &off); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 587 | dprintk("RPC: %s: chunk %d@0x%llx:0x%x\n", |
| 588 | __func__, |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 589 | be32_to_cpu(seg->rs_length), |
Stephen Rothwell | e08a132 | 2007-10-30 00:44:32 -0700 | [diff] [blame] | 590 | (unsigned long long)off, |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 591 | be32_to_cpu(seg->rs_handle)); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 592 | } |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 593 | total_len += be32_to_cpu(seg->rs_length); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 594 | ++cur_wchunk; |
| 595 | } |
| 596 | /* check and adjust for properly terminated write chunk */ |
| 597 | if (wrchunk) { |
Al Viro | 2d8a972 | 2007-10-29 04:37:58 +0000 | [diff] [blame] | 598 | __be32 *w = (__be32 *) cur_wchunk; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 599 | if (*w++ != xdr_zero) |
| 600 | return -1; |
| 601 | cur_wchunk = (struct rpcrdma_write_chunk *) w; |
| 602 | } |
| 603 | if ((char *) cur_wchunk > rep->rr_base + rep->rr_len) |
| 604 | return -1; |
| 605 | |
Al Viro | 2d8a972 | 2007-10-29 04:37:58 +0000 | [diff] [blame] | 606 | *iptrp = (__be32 *) cur_wchunk; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 607 | return total_len; |
| 608 | } |
| 609 | |
| 610 | /* |
| 611 | * Scatter inline received data back into provided iov's. |
| 612 | */ |
| 613 | static void |
Tom Talpey | 9191ca3 | 2008-10-09 15:01:11 -0400 | [diff] [blame] | 614 | rpcrdma_inline_fixup(struct rpc_rqst *rqst, char *srcp, int copy_len, int pad) |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 615 | { |
| 616 | int i, npages, curlen, olen; |
| 617 | char *destp; |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 618 | struct page **ppages; |
| 619 | int page_base; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 620 | |
| 621 | curlen = rqst->rq_rcv_buf.head[0].iov_len; |
| 622 | if (curlen > copy_len) { /* write chunk header fixup */ |
| 623 | curlen = copy_len; |
| 624 | rqst->rq_rcv_buf.head[0].iov_len = curlen; |
| 625 | } |
| 626 | |
| 627 | dprintk("RPC: %s: srcp 0x%p len %d hdrlen %d\n", |
| 628 | __func__, srcp, copy_len, curlen); |
| 629 | |
| 630 | /* Shift pointer for first receive segment only */ |
| 631 | rqst->rq_rcv_buf.head[0].iov_base = srcp; |
| 632 | srcp += curlen; |
| 633 | copy_len -= curlen; |
| 634 | |
| 635 | olen = copy_len; |
| 636 | i = 0; |
| 637 | rpcx_to_rdmax(rqst->rq_xprt)->rx_stats.fixup_copy_count += olen; |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 638 | page_base = rqst->rq_rcv_buf.page_base; |
| 639 | ppages = rqst->rq_rcv_buf.pages + (page_base >> PAGE_SHIFT); |
| 640 | page_base &= ~PAGE_MASK; |
| 641 | |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 642 | if (copy_len && rqst->rq_rcv_buf.page_len) { |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 643 | npages = PAGE_ALIGN(page_base + |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 644 | rqst->rq_rcv_buf.page_len) >> PAGE_SHIFT; |
| 645 | for (; i < npages; i++) { |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 646 | curlen = PAGE_SIZE - page_base; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 647 | if (curlen > copy_len) |
| 648 | curlen = copy_len; |
| 649 | dprintk("RPC: %s: page %d" |
| 650 | " srcp 0x%p len %d curlen %d\n", |
| 651 | __func__, i, srcp, copy_len, curlen); |
Cong Wang | b854178 | 2011-11-25 23:14:40 +0800 | [diff] [blame] | 652 | destp = kmap_atomic(ppages[i]); |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 653 | memcpy(destp + page_base, srcp, curlen); |
| 654 | flush_dcache_page(ppages[i]); |
Cong Wang | b854178 | 2011-11-25 23:14:40 +0800 | [diff] [blame] | 655 | kunmap_atomic(destp); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 656 | srcp += curlen; |
| 657 | copy_len -= curlen; |
| 658 | if (copy_len == 0) |
| 659 | break; |
Tom Tucker | bd7ea31 | 2011-02-09 19:45:28 +0000 | [diff] [blame] | 660 | page_base = 0; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 661 | } |
Chuck Lever | 2b7bbc9 | 2014-03-12 12:51:30 -0400 | [diff] [blame] | 662 | } |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 663 | |
| 664 | if (copy_len && rqst->rq_rcv_buf.tail[0].iov_len) { |
| 665 | curlen = copy_len; |
| 666 | if (curlen > rqst->rq_rcv_buf.tail[0].iov_len) |
| 667 | curlen = rqst->rq_rcv_buf.tail[0].iov_len; |
| 668 | if (rqst->rq_rcv_buf.tail[0].iov_base != srcp) |
Tom Talpey | b38ab40 | 2009-03-11 14:37:55 -0400 | [diff] [blame] | 669 | memmove(rqst->rq_rcv_buf.tail[0].iov_base, srcp, curlen); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 670 | dprintk("RPC: %s: tail srcp 0x%p len %d curlen %d\n", |
| 671 | __func__, srcp, copy_len, curlen); |
| 672 | rqst->rq_rcv_buf.tail[0].iov_len = curlen; |
| 673 | copy_len -= curlen; ++i; |
| 674 | } else |
| 675 | rqst->rq_rcv_buf.tail[0].iov_len = 0; |
| 676 | |
Tom Talpey | 9191ca3 | 2008-10-09 15:01:11 -0400 | [diff] [blame] | 677 | if (pad) { |
| 678 | /* implicit padding on terminal chunk */ |
| 679 | unsigned char *p = rqst->rq_rcv_buf.tail[0].iov_base; |
| 680 | while (pad--) |
| 681 | p[rqst->rq_rcv_buf.tail[0].iov_len++] = 0; |
| 682 | } |
| 683 | |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 684 | if (copy_len) |
| 685 | dprintk("RPC: %s: %d bytes in" |
| 686 | " %d extra segments (%d lost)\n", |
| 687 | __func__, olen, i, copy_len); |
| 688 | |
| 689 | /* TBD avoid a warning from call_decode() */ |
| 690 | rqst->rq_private_buf = rqst->rq_rcv_buf; |
| 691 | } |
| 692 | |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 693 | void |
Chuck Lever | 254f91e | 2014-05-28 10:32:17 -0400 | [diff] [blame] | 694 | rpcrdma_connect_worker(struct work_struct *work) |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 695 | { |
Chuck Lever | 254f91e | 2014-05-28 10:32:17 -0400 | [diff] [blame] | 696 | struct rpcrdma_ep *ep = |
| 697 | container_of(work, struct rpcrdma_ep, rep_connect_worker.work); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 698 | struct rpc_xprt *xprt = ep->rep_xprt; |
| 699 | |
| 700 | spin_lock_bh(&xprt->transport_lock); |
Tom Talpey | 575448b | 2008-10-09 15:00:40 -0400 | [diff] [blame] | 701 | if (++xprt->connect_cookie == 0) /* maintain a reserved value */ |
| 702 | ++xprt->connect_cookie; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 703 | if (ep->rep_connected > 0) { |
| 704 | if (!xprt_test_and_set_connected(xprt)) |
| 705 | xprt_wake_pending_tasks(xprt, 0); |
| 706 | } else { |
| 707 | if (xprt_test_and_clear_connected(xprt)) |
Tom Talpey | 926449b | 2008-10-09 15:01:21 -0400 | [diff] [blame] | 708 | xprt_wake_pending_tasks(xprt, -ENOTCONN); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 709 | } |
| 710 | spin_unlock_bh(&xprt->transport_lock); |
| 711 | } |
| 712 | |
| 713 | /* |
Chuck Lever | 254f91e | 2014-05-28 10:32:17 -0400 | [diff] [blame] | 714 | * This function is called when an async event is posted to |
| 715 | * the connection which changes the connection state. All it |
| 716 | * does at this point is mark the connection up/down, the rpc |
| 717 | * timers do the rest. |
| 718 | */ |
| 719 | void |
| 720 | rpcrdma_conn_func(struct rpcrdma_ep *ep) |
| 721 | { |
| 722 | schedule_delayed_work(&ep->rep_connect_worker, 0); |
| 723 | } |
| 724 | |
| 725 | /* |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 726 | * Called as a tasklet to do req/reply match and complete a request |
| 727 | * Errors must result in the RPC task either being awakened, or |
| 728 | * allowed to timeout, to discover the errors at that time. |
| 729 | */ |
| 730 | void |
| 731 | rpcrdma_reply_handler(struct rpcrdma_rep *rep) |
| 732 | { |
| 733 | struct rpcrdma_msg *headerp; |
| 734 | struct rpcrdma_req *req; |
| 735 | struct rpc_rqst *rqst; |
| 736 | struct rpc_xprt *xprt = rep->rr_xprt; |
| 737 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); |
Al Viro | 2d8a972 | 2007-10-29 04:37:58 +0000 | [diff] [blame] | 738 | __be32 *iptr; |
Chuck Lever | eba8ff6 | 2015-01-21 11:03:02 -0500 | [diff] [blame^] | 739 | int credits, rdmalen, status; |
Chuck Lever | e7ce710 | 2014-05-28 10:34:57 -0400 | [diff] [blame] | 740 | unsigned long cwnd; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 741 | |
| 742 | /* Check status. If bad, signal disconnect and return rep to pool */ |
| 743 | if (rep->rr_len == ~0U) { |
| 744 | rpcrdma_recv_buffer_put(rep); |
| 745 | if (r_xprt->rx_ep.rep_connected == 1) { |
| 746 | r_xprt->rx_ep.rep_connected = -EIO; |
| 747 | rpcrdma_conn_func(&r_xprt->rx_ep); |
| 748 | } |
| 749 | return; |
| 750 | } |
Chuck Lever | f284648 | 2015-01-21 11:02:29 -0500 | [diff] [blame] | 751 | if (rep->rr_len < RPCRDMA_HDRLEN_MIN) { |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 752 | dprintk("RPC: %s: short/invalid reply\n", __func__); |
| 753 | goto repost; |
| 754 | } |
| 755 | headerp = (struct rpcrdma_msg *) rep->rr_base; |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 756 | if (headerp->rm_vers != rpcrdma_version) { |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 757 | dprintk("RPC: %s: invalid version %d\n", |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 758 | __func__, be32_to_cpu(headerp->rm_vers)); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 759 | goto repost; |
| 760 | } |
| 761 | |
| 762 | /* Get XID and try for a match. */ |
| 763 | spin_lock(&xprt->transport_lock); |
| 764 | rqst = xprt_lookup_rqst(xprt, headerp->rm_xid); |
| 765 | if (rqst == NULL) { |
| 766 | spin_unlock(&xprt->transport_lock); |
| 767 | dprintk("RPC: %s: reply 0x%p failed " |
| 768 | "to match any request xid 0x%08x len %d\n", |
Chuck Lever | 052151a | 2015-01-21 11:02:21 -0500 | [diff] [blame] | 769 | __func__, rep, be32_to_cpu(headerp->rm_xid), |
| 770 | rep->rr_len); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 771 | repost: |
| 772 | r_xprt->rx_stats.bad_reply_count++; |
| 773 | rep->rr_func = rpcrdma_reply_handler; |
| 774 | if (rpcrdma_ep_post_recv(&r_xprt->rx_ia, &r_xprt->rx_ep, rep)) |
| 775 | rpcrdma_recv_buffer_put(rep); |
| 776 | |
| 777 | return; |
| 778 | } |
| 779 | |
| 780 | /* get request object */ |
| 781 | req = rpcr_to_rdmar(rqst); |
Tom Tucker | 4a6862b | 2012-02-20 13:07:42 -0600 | [diff] [blame] | 782 | if (req->rl_reply) { |
| 783 | spin_unlock(&xprt->transport_lock); |
| 784 | dprintk("RPC: %s: duplicate reply 0x%p to RPC " |
| 785 | "request 0x%p: xid 0x%08x\n", __func__, rep, req, |
Chuck Lever | 052151a | 2015-01-21 11:02:21 -0500 | [diff] [blame] | 786 | be32_to_cpu(headerp->rm_xid)); |
Tom Tucker | 4a6862b | 2012-02-20 13:07:42 -0600 | [diff] [blame] | 787 | goto repost; |
| 788 | } |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 789 | |
| 790 | dprintk("RPC: %s: reply 0x%p completes request 0x%p\n" |
| 791 | " RPC request 0x%p xid 0x%08x\n", |
Chuck Lever | 052151a | 2015-01-21 11:02:21 -0500 | [diff] [blame] | 792 | __func__, rep, req, rqst, |
| 793 | be32_to_cpu(headerp->rm_xid)); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 794 | |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 795 | /* from here on, the reply is no longer an orphan */ |
| 796 | req->rl_reply = rep; |
Chuck Lever | 1890697 | 2014-05-28 10:34:41 -0400 | [diff] [blame] | 797 | xprt->reestablish_timeout = 0; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 798 | |
| 799 | /* check for expected message types */ |
| 800 | /* The order of some of these tests is important. */ |
| 801 | switch (headerp->rm_type) { |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 802 | case rdma_msg: |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 803 | /* never expect read chunks */ |
| 804 | /* never expect reply chunks (two ways to check) */ |
| 805 | /* never expect write chunks without having offered RDMA */ |
| 806 | if (headerp->rm_body.rm_chunks[0] != xdr_zero || |
| 807 | (headerp->rm_body.rm_chunks[1] == xdr_zero && |
| 808 | headerp->rm_body.rm_chunks[2] != xdr_zero) || |
| 809 | (headerp->rm_body.rm_chunks[1] != xdr_zero && |
| 810 | req->rl_nchunks == 0)) |
| 811 | goto badheader; |
| 812 | if (headerp->rm_body.rm_chunks[1] != xdr_zero) { |
| 813 | /* count any expected write chunks in read reply */ |
| 814 | /* start at write chunk array count */ |
| 815 | iptr = &headerp->rm_body.rm_chunks[2]; |
| 816 | rdmalen = rpcrdma_count_chunks(rep, |
| 817 | req->rl_nchunks, 1, &iptr); |
| 818 | /* check for validity, and no reply chunk after */ |
| 819 | if (rdmalen < 0 || *iptr++ != xdr_zero) |
| 820 | goto badheader; |
| 821 | rep->rr_len -= |
| 822 | ((unsigned char *)iptr - (unsigned char *)headerp); |
| 823 | status = rep->rr_len + rdmalen; |
| 824 | r_xprt->rx_stats.total_rdma_reply += rdmalen; |
Tom Talpey | 9191ca3 | 2008-10-09 15:01:11 -0400 | [diff] [blame] | 825 | /* special case - last chunk may omit padding */ |
| 826 | if (rdmalen &= 3) { |
| 827 | rdmalen = 4 - rdmalen; |
| 828 | status += rdmalen; |
| 829 | } |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 830 | } else { |
| 831 | /* else ordinary inline */ |
Tom Talpey | 9191ca3 | 2008-10-09 15:01:11 -0400 | [diff] [blame] | 832 | rdmalen = 0; |
Chuck Lever | f284648 | 2015-01-21 11:02:29 -0500 | [diff] [blame] | 833 | iptr = (__be32 *)((unsigned char *)headerp + |
| 834 | RPCRDMA_HDRLEN_MIN); |
| 835 | rep->rr_len -= RPCRDMA_HDRLEN_MIN; |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 836 | status = rep->rr_len; |
| 837 | } |
| 838 | /* Fix up the rpc results for upper layer */ |
Tom Talpey | 9191ca3 | 2008-10-09 15:01:11 -0400 | [diff] [blame] | 839 | rpcrdma_inline_fixup(rqst, (char *)iptr, rep->rr_len, rdmalen); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 840 | break; |
| 841 | |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 842 | case rdma_nomsg: |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 843 | /* never expect read or write chunks, always reply chunks */ |
| 844 | if (headerp->rm_body.rm_chunks[0] != xdr_zero || |
| 845 | headerp->rm_body.rm_chunks[1] != xdr_zero || |
| 846 | headerp->rm_body.rm_chunks[2] != xdr_one || |
| 847 | req->rl_nchunks == 0) |
| 848 | goto badheader; |
Chuck Lever | f284648 | 2015-01-21 11:02:29 -0500 | [diff] [blame] | 849 | iptr = (__be32 *)((unsigned char *)headerp + |
| 850 | RPCRDMA_HDRLEN_MIN); |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 851 | rdmalen = rpcrdma_count_chunks(rep, req->rl_nchunks, 0, &iptr); |
| 852 | if (rdmalen < 0) |
| 853 | goto badheader; |
| 854 | r_xprt->rx_stats.total_rdma_reply += rdmalen; |
| 855 | /* Reply chunk buffer already is the reply vector - no fixup. */ |
| 856 | status = rdmalen; |
| 857 | break; |
| 858 | |
| 859 | badheader: |
| 860 | default: |
| 861 | dprintk("%s: invalid rpcrdma reply header (type %d):" |
| 862 | " chunks[012] == %d %d %d" |
| 863 | " expected chunks <= %d\n", |
Chuck Lever | 284f490 | 2015-01-21 11:02:13 -0500 | [diff] [blame] | 864 | __func__, be32_to_cpu(headerp->rm_type), |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 865 | headerp->rm_body.rm_chunks[0], |
| 866 | headerp->rm_body.rm_chunks[1], |
| 867 | headerp->rm_body.rm_chunks[2], |
| 868 | req->rl_nchunks); |
| 869 | status = -EIO; |
| 870 | r_xprt->rx_stats.bad_reply_count++; |
| 871 | break; |
| 872 | } |
| 873 | |
Chuck Lever | eba8ff6 | 2015-01-21 11:03:02 -0500 | [diff] [blame^] | 874 | credits = be32_to_cpu(headerp->rm_credit); |
| 875 | if (credits == 0) |
| 876 | credits = 1; /* don't deadlock */ |
| 877 | else if (credits > r_xprt->rx_buf.rb_max_requests) |
| 878 | credits = r_xprt->rx_buf.rb_max_requests; |
| 879 | |
Chuck Lever | e7ce710 | 2014-05-28 10:34:57 -0400 | [diff] [blame] | 880 | cwnd = xprt->cwnd; |
Chuck Lever | eba8ff6 | 2015-01-21 11:03:02 -0500 | [diff] [blame^] | 881 | xprt->cwnd = credits << RPC_CWNDSHIFT; |
Chuck Lever | e7ce710 | 2014-05-28 10:34:57 -0400 | [diff] [blame] | 882 | if (xprt->cwnd > cwnd) |
| 883 | xprt_release_rqst_cong(rqst->rq_task); |
| 884 | |
\"Talpey, Thomas\ | e960182 | 2007-09-10 13:50:42 -0400 | [diff] [blame] | 885 | dprintk("RPC: %s: xprt_complete_rqst(0x%p, 0x%p, %d)\n", |
| 886 | __func__, xprt, rqst, status); |
| 887 | xprt_complete_rqst(rqst->rq_task, status); |
| 888 | spin_unlock(&xprt->transport_lock); |
| 889 | } |