| /* |
| * Copyright (c) 2014 Open Grid Computing, Inc. All rights reserved. |
| * Copyright (c) 2005-2006 Network Appliance, Inc. All rights reserved. |
| * |
| * This software is available to you under a choice of one of two |
| * licenses. You may choose to be licensed under the terms of the GNU |
| * General Public License (GPL) Version 2, available from the file |
| * COPYING in the main directory of this source tree, or the BSD-type |
| * license below: |
| * |
| * Redistribution and use in source and binary forms, with or without |
| * modification, are permitted provided that the following conditions |
| * are met: |
| * |
| * Redistributions of source code must retain the above copyright |
| * notice, this list of conditions and the following disclaimer. |
| * |
| * Redistributions in binary form must reproduce the above |
| * copyright notice, this list of conditions and the following |
| * disclaimer in the documentation and/or other materials provided |
| * with the distribution. |
| * |
| * Neither the name of the Network Appliance, Inc. nor the names of |
| * its contributors may be used to endorse or promote products |
| * derived from this software without specific prior written |
| * permission. |
| * |
| * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| * |
| * Author: Tom Tucker <tom@opengridcomputing.com> |
| */ |
| |
| #include <linux/sunrpc/debug.h> |
| #include <linux/sunrpc/rpc_rdma.h> |
| #include <linux/spinlock.h> |
| #include <linux/highmem.h> |
| #include <asm/unaligned.h> |
| #include <rdma/ib_verbs.h> |
| #include <rdma/rdma_cm.h> |
| #include <linux/sunrpc/svc_rdma.h> |
| |
| #define RPCDBG_FACILITY RPCDBG_SVCXPRT |
| |
| /* |
| * Replace the pages in the rq_argpages array with the pages from the SGE in |
| * the RDMA_RECV completion. The SGL should contain full pages up until the |
| * last one. |
| */ |
| static void rdma_build_arg_xdr(struct svc_rqst *rqstp, |
| struct svc_rdma_op_ctxt *ctxt, |
| u32 byte_count) |
| { |
| struct page *page; |
| u32 bc; |
| int sge_no; |
| |
| /* Swap the page in the SGE with the page in argpages */ |
| page = ctxt->pages[0]; |
| put_page(rqstp->rq_pages[0]); |
| rqstp->rq_pages[0] = page; |
| |
| /* Set up the XDR head */ |
| rqstp->rq_arg.head[0].iov_base = page_address(page); |
| rqstp->rq_arg.head[0].iov_len = |
| min_t(size_t, byte_count, ctxt->sge[0].length); |
| rqstp->rq_arg.len = byte_count; |
| rqstp->rq_arg.buflen = byte_count; |
| |
| /* Compute bytes past head in the SGL */ |
| bc = byte_count - rqstp->rq_arg.head[0].iov_len; |
| |
| /* If data remains, store it in the pagelist */ |
| rqstp->rq_arg.page_len = bc; |
| rqstp->rq_arg.page_base = 0; |
| rqstp->rq_arg.pages = &rqstp->rq_pages[1]; |
| sge_no = 1; |
| while (bc && sge_no < ctxt->count) { |
| page = ctxt->pages[sge_no]; |
| put_page(rqstp->rq_pages[sge_no]); |
| rqstp->rq_pages[sge_no] = page; |
| bc -= min_t(u32, bc, ctxt->sge[sge_no].length); |
| rqstp->rq_arg.buflen += ctxt->sge[sge_no].length; |
| sge_no++; |
| } |
| rqstp->rq_respages = &rqstp->rq_pages[sge_no]; |
| rqstp->rq_next_page = rqstp->rq_respages + 1; |
| |
| /* If not all pages were used from the SGL, free the remaining ones */ |
| bc = sge_no; |
| while (sge_no < ctxt->count) { |
| page = ctxt->pages[sge_no++]; |
| put_page(page); |
| } |
| ctxt->count = bc; |
| |
| /* Set up tail */ |
| rqstp->rq_arg.tail[0].iov_base = NULL; |
| rqstp->rq_arg.tail[0].iov_len = 0; |
| } |
| |
| static int rdma_read_max_sge(struct svcxprt_rdma *xprt, int sge_count) |
| { |
| if (rdma_node_get_transport(xprt->sc_cm_id->device->node_type) == |
| RDMA_TRANSPORT_IWARP) |
| return 1; |
| else |
| return min_t(int, sge_count, xprt->sc_max_sge); |
| } |
| |
| /* Issue an RDMA_READ using the local lkey to map the data sink */ |
| int rdma_read_chunk_lcl(struct svcxprt_rdma *xprt, |
| struct svc_rqst *rqstp, |
| struct svc_rdma_op_ctxt *head, |
| int *page_no, |
| u32 *page_offset, |
| u32 rs_handle, |
| u32 rs_length, |
| u64 rs_offset, |
| bool last) |
| { |
| struct ib_send_wr read_wr; |
| int pages_needed = PAGE_ALIGN(*page_offset + rs_length) >> PAGE_SHIFT; |
| struct svc_rdma_op_ctxt *ctxt = svc_rdma_get_context(xprt); |
| int ret, read, pno; |
| u32 pg_off = *page_offset; |
| u32 pg_no = *page_no; |
| |
| ctxt->direction = DMA_FROM_DEVICE; |
| ctxt->read_hdr = head; |
| pages_needed = |
| min_t(int, pages_needed, rdma_read_max_sge(xprt, pages_needed)); |
| read = min_t(int, pages_needed << PAGE_SHIFT, rs_length); |
| |
| for (pno = 0; pno < pages_needed; pno++) { |
| int len = min_t(int, rs_length, PAGE_SIZE - pg_off); |
| |
| head->arg.pages[pg_no] = rqstp->rq_arg.pages[pg_no]; |
| head->arg.page_len += len; |
| head->arg.len += len; |
| if (!pg_off) |
| head->count++; |
| rqstp->rq_respages = &rqstp->rq_arg.pages[pg_no+1]; |
| rqstp->rq_next_page = rqstp->rq_respages + 1; |
| ctxt->sge[pno].addr = |
| ib_dma_map_page(xprt->sc_cm_id->device, |
| head->arg.pages[pg_no], pg_off, |
| PAGE_SIZE - pg_off, |
| DMA_FROM_DEVICE); |
| ret = ib_dma_mapping_error(xprt->sc_cm_id->device, |
| ctxt->sge[pno].addr); |
| if (ret) |
| goto err; |
| atomic_inc(&xprt->sc_dma_used); |
| |
| /* The lkey here is either a local dma lkey or a dma_mr lkey */ |
| ctxt->sge[pno].lkey = xprt->sc_dma_lkey; |
| ctxt->sge[pno].length = len; |
| ctxt->count++; |
| |
| /* adjust offset and wrap to next page if needed */ |
| pg_off += len; |
| if (pg_off == PAGE_SIZE) { |
| pg_off = 0; |
| pg_no++; |
| } |
| rs_length -= len; |
| } |
| |
| if (last && rs_length == 0) |
| set_bit(RDMACTXT_F_LAST_CTXT, &ctxt->flags); |
| else |
| clear_bit(RDMACTXT_F_LAST_CTXT, &ctxt->flags); |
| |
| memset(&read_wr, 0, sizeof(read_wr)); |
| read_wr.wr_id = (unsigned long)ctxt; |
| read_wr.opcode = IB_WR_RDMA_READ; |
| ctxt->wr_op = read_wr.opcode; |
| read_wr.send_flags = IB_SEND_SIGNALED; |
| read_wr.wr.rdma.rkey = rs_handle; |
| read_wr.wr.rdma.remote_addr = rs_offset; |
| read_wr.sg_list = ctxt->sge; |
| read_wr.num_sge = pages_needed; |
| |
| ret = svc_rdma_send(xprt, &read_wr); |
| if (ret) { |
| pr_err("svcrdma: Error %d posting RDMA_READ\n", ret); |
| set_bit(XPT_CLOSE, &xprt->sc_xprt.xpt_flags); |
| goto err; |
| } |
| |
| /* return current location in page array */ |
| *page_no = pg_no; |
| *page_offset = pg_off; |
| ret = read; |
| atomic_inc(&rdma_stat_read); |
| return ret; |
| err: |
| svc_rdma_unmap_dma(ctxt); |
| svc_rdma_put_context(ctxt, 0); |
| return ret; |
| } |
| |
| /* Issue an RDMA_READ using an FRMR to map the data sink */ |
| int rdma_read_chunk_frmr(struct svcxprt_rdma *xprt, |
| struct svc_rqst *rqstp, |
| struct svc_rdma_op_ctxt *head, |
| int *page_no, |
| u32 *page_offset, |
| u32 rs_handle, |
| u32 rs_length, |
| u64 rs_offset, |
| bool last) |
| { |
| struct ib_send_wr read_wr; |
| struct ib_send_wr inv_wr; |
| struct ib_send_wr fastreg_wr; |
| u8 key; |
| int pages_needed = PAGE_ALIGN(*page_offset + rs_length) >> PAGE_SHIFT; |
| struct svc_rdma_op_ctxt *ctxt = svc_rdma_get_context(xprt); |
| struct svc_rdma_fastreg_mr *frmr = svc_rdma_get_frmr(xprt); |
| int ret, read, pno; |
| u32 pg_off = *page_offset; |
| u32 pg_no = *page_no; |
| |
| if (IS_ERR(frmr)) |
| return -ENOMEM; |
| |
| ctxt->direction = DMA_FROM_DEVICE; |
| ctxt->frmr = frmr; |
| pages_needed = min_t(int, pages_needed, xprt->sc_frmr_pg_list_len); |
| read = min_t(int, pages_needed << PAGE_SHIFT, rs_length); |
| |
| frmr->kva = page_address(rqstp->rq_arg.pages[pg_no]); |
| frmr->direction = DMA_FROM_DEVICE; |
| frmr->access_flags = (IB_ACCESS_LOCAL_WRITE|IB_ACCESS_REMOTE_WRITE); |
| frmr->map_len = pages_needed << PAGE_SHIFT; |
| frmr->page_list_len = pages_needed; |
| |
| for (pno = 0; pno < pages_needed; pno++) { |
| int len = min_t(int, rs_length, PAGE_SIZE - pg_off); |
| |
| head->arg.pages[pg_no] = rqstp->rq_arg.pages[pg_no]; |
| head->arg.page_len += len; |
| head->arg.len += len; |
| if (!pg_off) |
| head->count++; |
| rqstp->rq_respages = &rqstp->rq_arg.pages[pg_no+1]; |
| rqstp->rq_next_page = rqstp->rq_respages + 1; |
| frmr->page_list->page_list[pno] = |
| ib_dma_map_page(xprt->sc_cm_id->device, |
| head->arg.pages[pg_no], 0, |
| PAGE_SIZE, DMA_FROM_DEVICE); |
| ret = ib_dma_mapping_error(xprt->sc_cm_id->device, |
| frmr->page_list->page_list[pno]); |
| if (ret) |
| goto err; |
| atomic_inc(&xprt->sc_dma_used); |
| |
| /* adjust offset and wrap to next page if needed */ |
| pg_off += len; |
| if (pg_off == PAGE_SIZE) { |
| pg_off = 0; |
| pg_no++; |
| } |
| rs_length -= len; |
| } |
| |
| if (last && rs_length == 0) |
| set_bit(RDMACTXT_F_LAST_CTXT, &ctxt->flags); |
| else |
| clear_bit(RDMACTXT_F_LAST_CTXT, &ctxt->flags); |
| |
| /* Bump the key */ |
| key = (u8)(frmr->mr->lkey & 0x000000FF); |
| ib_update_fast_reg_key(frmr->mr, ++key); |
| |
| ctxt->sge[0].addr = (unsigned long)frmr->kva + *page_offset; |
| ctxt->sge[0].lkey = frmr->mr->lkey; |
| ctxt->sge[0].length = read; |
| ctxt->count = 1; |
| ctxt->read_hdr = head; |
| |
| /* Prepare FASTREG WR */ |
| memset(&fastreg_wr, 0, sizeof(fastreg_wr)); |
| fastreg_wr.opcode = IB_WR_FAST_REG_MR; |
| fastreg_wr.send_flags = IB_SEND_SIGNALED; |
| fastreg_wr.wr.fast_reg.iova_start = (unsigned long)frmr->kva; |
| fastreg_wr.wr.fast_reg.page_list = frmr->page_list; |
| fastreg_wr.wr.fast_reg.page_list_len = frmr->page_list_len; |
| fastreg_wr.wr.fast_reg.page_shift = PAGE_SHIFT; |
| fastreg_wr.wr.fast_reg.length = frmr->map_len; |
| fastreg_wr.wr.fast_reg.access_flags = frmr->access_flags; |
| fastreg_wr.wr.fast_reg.rkey = frmr->mr->lkey; |
| fastreg_wr.next = &read_wr; |
| |
| /* Prepare RDMA_READ */ |
| memset(&read_wr, 0, sizeof(read_wr)); |
| read_wr.send_flags = IB_SEND_SIGNALED; |
| read_wr.wr.rdma.rkey = rs_handle; |
| read_wr.wr.rdma.remote_addr = rs_offset; |
| read_wr.sg_list = ctxt->sge; |
| read_wr.num_sge = 1; |
| if (xprt->sc_dev_caps & SVCRDMA_DEVCAP_READ_W_INV) { |
| read_wr.opcode = IB_WR_RDMA_READ_WITH_INV; |
| read_wr.wr_id = (unsigned long)ctxt; |
| read_wr.ex.invalidate_rkey = ctxt->frmr->mr->lkey; |
| } else { |
| read_wr.opcode = IB_WR_RDMA_READ; |
| read_wr.next = &inv_wr; |
| /* Prepare invalidate */ |
| memset(&inv_wr, 0, sizeof(inv_wr)); |
| inv_wr.wr_id = (unsigned long)ctxt; |
| inv_wr.opcode = IB_WR_LOCAL_INV; |
| inv_wr.send_flags = IB_SEND_SIGNALED | IB_SEND_FENCE; |
| inv_wr.ex.invalidate_rkey = frmr->mr->lkey; |
| } |
| ctxt->wr_op = read_wr.opcode; |
| |
| /* Post the chain */ |
| ret = svc_rdma_send(xprt, &fastreg_wr); |
| if (ret) { |
| pr_err("svcrdma: Error %d posting RDMA_READ\n", ret); |
| set_bit(XPT_CLOSE, &xprt->sc_xprt.xpt_flags); |
| goto err; |
| } |
| |
| /* return current location in page array */ |
| *page_no = pg_no; |
| *page_offset = pg_off; |
| ret = read; |
| atomic_inc(&rdma_stat_read); |
| return ret; |
| err: |
| svc_rdma_unmap_dma(ctxt); |
| svc_rdma_put_context(ctxt, 0); |
| svc_rdma_put_frmr(xprt, frmr); |
| return ret; |
| } |
| |
| static unsigned int |
| rdma_rcl_chunk_count(struct rpcrdma_read_chunk *ch) |
| { |
| unsigned int count; |
| |
| for (count = 0; ch->rc_discrim != xdr_zero; ch++) |
| count++; |
| return count; |
| } |
| |
| static int rdma_read_chunks(struct svcxprt_rdma *xprt, |
| struct rpcrdma_msg *rmsgp, |
| struct svc_rqst *rqstp, |
| struct svc_rdma_op_ctxt *head) |
| { |
| int page_no, ret; |
| struct rpcrdma_read_chunk *ch; |
| u32 handle, page_offset, byte_count; |
| u32 position; |
| u64 rs_offset; |
| bool last; |
| |
| /* If no read list is present, return 0 */ |
| ch = svc_rdma_get_read_chunk(rmsgp); |
| if (!ch) |
| return 0; |
| |
| if (rdma_rcl_chunk_count(ch) > RPCSVC_MAXPAGES) |
| return -EINVAL; |
| |
| /* The request is completed when the RDMA_READs complete. The |
| * head context keeps all the pages that comprise the |
| * request. |
| */ |
| head->arg.head[0] = rqstp->rq_arg.head[0]; |
| head->arg.tail[0] = rqstp->rq_arg.tail[0]; |
| head->arg.pages = &head->pages[head->count]; |
| head->hdr_count = head->count; |
| head->arg.page_base = 0; |
| head->arg.page_len = 0; |
| head->arg.len = rqstp->rq_arg.len; |
| head->arg.buflen = rqstp->rq_arg.buflen; |
| |
| ch = (struct rpcrdma_read_chunk *)&rmsgp->rm_body.rm_chunks[0]; |
| position = be32_to_cpu(ch->rc_position); |
| |
| ret = 0; |
| page_no = 0; |
| page_offset = 0; |
| for (; ch->rc_discrim != xdr_zero; ch++) { |
| if (be32_to_cpu(ch->rc_position) != position) |
| goto err; |
| |
| handle = be32_to_cpu(ch->rc_target.rs_handle), |
| byte_count = be32_to_cpu(ch->rc_target.rs_length); |
| xdr_decode_hyper((__be32 *)&ch->rc_target.rs_offset, |
| &rs_offset); |
| |
| while (byte_count > 0) { |
| last = (ch + 1)->rc_discrim == xdr_zero; |
| ret = xprt->sc_reader(xprt, rqstp, head, |
| &page_no, &page_offset, |
| handle, byte_count, |
| rs_offset, last); |
| if (ret < 0) |
| goto err; |
| byte_count -= ret; |
| rs_offset += ret; |
| head->arg.buflen += ret; |
| } |
| } |
| ret = 1; |
| err: |
| /* Detach arg pages. svc_recv will replenish them */ |
| for (page_no = 0; |
| &rqstp->rq_pages[page_no] < rqstp->rq_respages; page_no++) |
| rqstp->rq_pages[page_no] = NULL; |
| |
| return ret; |
| } |
| |
| /* |
| * To avoid a separate RDMA READ just for a handful of zero bytes, |
| * RFC 5666 section 3.7 allows the client to omit the XDR zero pad |
| * in chunk lists. |
| */ |
| static void |
| rdma_fix_xdr_pad(struct xdr_buf *buf) |
| { |
| unsigned int page_len = buf->page_len; |
| unsigned int size = (XDR_QUADLEN(page_len) << 2) - page_len; |
| unsigned int offset, pg_no; |
| char *p; |
| |
| if (size == 0) |
| return; |
| |
| pg_no = page_len >> PAGE_SHIFT; |
| offset = page_len & ~PAGE_MASK; |
| p = page_address(buf->pages[pg_no]); |
| memset(p + offset, 0, size); |
| |
| buf->page_len += size; |
| buf->buflen += size; |
| buf->len += size; |
| } |
| |
| static int rdma_read_complete(struct svc_rqst *rqstp, |
| struct svc_rdma_op_ctxt *head) |
| { |
| int page_no; |
| int ret; |
| |
| /* Copy RPC pages */ |
| for (page_no = 0; page_no < head->count; page_no++) { |
| put_page(rqstp->rq_pages[page_no]); |
| rqstp->rq_pages[page_no] = head->pages[page_no]; |
| } |
| /* Point rq_arg.pages past header */ |
| rdma_fix_xdr_pad(&head->arg); |
| rqstp->rq_arg.pages = &rqstp->rq_pages[head->hdr_count]; |
| rqstp->rq_arg.page_len = head->arg.page_len; |
| rqstp->rq_arg.page_base = head->arg.page_base; |
| |
| /* rq_respages starts after the last arg page */ |
| rqstp->rq_respages = &rqstp->rq_arg.pages[page_no]; |
| rqstp->rq_next_page = rqstp->rq_respages + 1; |
| |
| /* Rebuild rq_arg head and tail. */ |
| rqstp->rq_arg.head[0] = head->arg.head[0]; |
| rqstp->rq_arg.tail[0] = head->arg.tail[0]; |
| rqstp->rq_arg.len = head->arg.len; |
| rqstp->rq_arg.buflen = head->arg.buflen; |
| |
| /* Free the context */ |
| svc_rdma_put_context(head, 0); |
| |
| /* XXX: What should this be? */ |
| rqstp->rq_prot = IPPROTO_MAX; |
| svc_xprt_copy_addrs(rqstp, rqstp->rq_xprt); |
| |
| ret = rqstp->rq_arg.head[0].iov_len |
| + rqstp->rq_arg.page_len |
| + rqstp->rq_arg.tail[0].iov_len; |
| dprintk("svcrdma: deferred read ret=%d, rq_arg.len=%u, " |
| "rq_arg.head[0].iov_base=%p, rq_arg.head[0].iov_len=%zu\n", |
| ret, rqstp->rq_arg.len, rqstp->rq_arg.head[0].iov_base, |
| rqstp->rq_arg.head[0].iov_len); |
| |
| return ret; |
| } |
| |
| /* |
| * Set up the rqstp thread context to point to the RQ buffer. If |
| * necessary, pull additional data from the client with an RDMA_READ |
| * request. |
| */ |
| int svc_rdma_recvfrom(struct svc_rqst *rqstp) |
| { |
| struct svc_xprt *xprt = rqstp->rq_xprt; |
| struct svcxprt_rdma *rdma_xprt = |
| container_of(xprt, struct svcxprt_rdma, sc_xprt); |
| struct svc_rdma_op_ctxt *ctxt = NULL; |
| struct rpcrdma_msg *rmsgp; |
| int ret = 0; |
| int len; |
| |
| dprintk("svcrdma: rqstp=%p\n", rqstp); |
| |
| spin_lock_bh(&rdma_xprt->sc_rq_dto_lock); |
| if (!list_empty(&rdma_xprt->sc_read_complete_q)) { |
| ctxt = list_entry(rdma_xprt->sc_read_complete_q.next, |
| struct svc_rdma_op_ctxt, |
| dto_q); |
| list_del_init(&ctxt->dto_q); |
| spin_unlock_bh(&rdma_xprt->sc_rq_dto_lock); |
| return rdma_read_complete(rqstp, ctxt); |
| } else if (!list_empty(&rdma_xprt->sc_rq_dto_q)) { |
| ctxt = list_entry(rdma_xprt->sc_rq_dto_q.next, |
| struct svc_rdma_op_ctxt, |
| dto_q); |
| list_del_init(&ctxt->dto_q); |
| } else { |
| atomic_inc(&rdma_stat_rq_starve); |
| clear_bit(XPT_DATA, &xprt->xpt_flags); |
| ctxt = NULL; |
| } |
| spin_unlock_bh(&rdma_xprt->sc_rq_dto_lock); |
| if (!ctxt) { |
| /* This is the EAGAIN path. The svc_recv routine will |
| * return -EAGAIN, the nfsd thread will go to call into |
| * svc_recv again and we shouldn't be on the active |
| * transport list |
| */ |
| if (test_bit(XPT_CLOSE, &xprt->xpt_flags)) |
| goto close_out; |
| |
| goto out; |
| } |
| dprintk("svcrdma: processing ctxt=%p on xprt=%p, rqstp=%p, status=%d\n", |
| ctxt, rdma_xprt, rqstp, ctxt->wc_status); |
| atomic_inc(&rdma_stat_recv); |
| |
| /* Build up the XDR from the receive buffers. */ |
| rdma_build_arg_xdr(rqstp, ctxt, ctxt->byte_len); |
| |
| /* Decode the RDMA header. */ |
| len = svc_rdma_xdr_decode_req(&rmsgp, rqstp); |
| rqstp->rq_xprt_hlen = len; |
| |
| /* If the request is invalid, reply with an error */ |
| if (len < 0) { |
| if (len == -ENOSYS) |
| svc_rdma_send_error(rdma_xprt, rmsgp, ERR_VERS); |
| goto close_out; |
| } |
| |
| /* Read read-list data. */ |
| ret = rdma_read_chunks(rdma_xprt, rmsgp, rqstp, ctxt); |
| if (ret > 0) { |
| /* read-list posted, defer until data received from client. */ |
| goto defer; |
| } else if (ret < 0) { |
| /* Post of read-list failed, free context. */ |
| svc_rdma_put_context(ctxt, 1); |
| return 0; |
| } |
| |
| ret = rqstp->rq_arg.head[0].iov_len |
| + rqstp->rq_arg.page_len |
| + rqstp->rq_arg.tail[0].iov_len; |
| svc_rdma_put_context(ctxt, 0); |
| out: |
| dprintk("svcrdma: ret=%d, rq_arg.len=%u, " |
| "rq_arg.head[0].iov_base=%p, rq_arg.head[0].iov_len=%zd\n", |
| ret, rqstp->rq_arg.len, |
| rqstp->rq_arg.head[0].iov_base, |
| rqstp->rq_arg.head[0].iov_len); |
| rqstp->rq_prot = IPPROTO_MAX; |
| svc_xprt_copy_addrs(rqstp, xprt); |
| return ret; |
| |
| close_out: |
| if (ctxt) |
| svc_rdma_put_context(ctxt, 1); |
| dprintk("svcrdma: transport %p is closing\n", xprt); |
| /* |
| * Set the close bit and enqueue it. svc_recv will see the |
| * close bit and call svc_xprt_delete |
| */ |
| set_bit(XPT_CLOSE, &xprt->xpt_flags); |
| defer: |
| return 0; |
| } |