blob: bdc38f423ca2f8e5908b3229a2cf1e04d2701a38 [file] [log] [blame]
Or Gerlitze85b24b2006-05-11 10:02:19 +03001/*
2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
Or Gerlitz28f292e2013-05-08 12:21:18 +00003 * Copyright (c) 2013 Mellanox Technologies. All rights reserved.
Or Gerlitze85b24b2006-05-11 10:02:19 +03004 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
Or Gerlitze85b24b2006-05-11 10:02:19 +030032 */
33#include <linux/kernel.h>
34#include <linux/slab.h>
35#include <linux/mm.h>
Or Gerlitze85b24b2006-05-11 10:02:19 +030036#include <linux/scatterlist.h>
37#include <linux/kfifo.h>
38#include <scsi/scsi_cmnd.h>
39#include <scsi/scsi_host.h>
40
41#include "iscsi_iser.h"
42
Or Gerlitze85b24b2006-05-11 10:02:19 +030043/* Register user buffer memory and initialize passive rdma
44 * dto descriptor. Total data size is stored in
Mike Christie2261ec32008-05-21 15:54:11 -050045 * iser_task->data[ISER_DIR_IN].data_len
Or Gerlitze85b24b2006-05-11 10:02:19 +030046 */
Mike Christie2261ec32008-05-21 15:54:11 -050047static int iser_prepare_read_cmd(struct iscsi_task *task,
Or Gerlitze85b24b2006-05-11 10:02:19 +030048 unsigned int edtl)
49
50{
Mike Christie2261ec32008-05-21 15:54:11 -050051 struct iscsi_iser_task *iser_task = task->dd_data;
Sagi Grimbergb4e155f2013-07-28 12:35:39 +030052 struct iser_device *device = iser_task->iser_conn->ib_conn->device;
Or Gerlitze85b24b2006-05-11 10:02:19 +030053 struct iser_regd_buf *regd_buf;
54 int err;
Mike Christie2261ec32008-05-21 15:54:11 -050055 struct iser_hdr *hdr = &iser_task->desc.iser_header;
56 struct iser_data_buf *buf_in = &iser_task->data[ISER_DIR_IN];
Or Gerlitze85b24b2006-05-11 10:02:19 +030057
Mike Christie2261ec32008-05-21 15:54:11 -050058 err = iser_dma_map_task_data(iser_task,
Or Gerlitze85b24b2006-05-11 10:02:19 +030059 buf_in,
60 ISER_DIR_IN,
61 DMA_FROM_DEVICE);
62 if (err)
63 return err;
64
Mike Christie2261ec32008-05-21 15:54:11 -050065 if (edtl > iser_task->data[ISER_DIR_IN].data_len) {
Or Gerlitze85b24b2006-05-11 10:02:19 +030066 iser_err("Total data length: %ld, less than EDTL: "
67 "%d, in READ cmd BHS itt: %d, conn: 0x%p\n",
Mike Christie2261ec32008-05-21 15:54:11 -050068 iser_task->data[ISER_DIR_IN].data_len, edtl,
69 task->itt, iser_task->iser_conn);
Or Gerlitze85b24b2006-05-11 10:02:19 +030070 return -EINVAL;
71 }
72
Sagi Grimbergb4e155f2013-07-28 12:35:39 +030073 err = device->iser_reg_rdma_mem(iser_task, ISER_DIR_IN);
Or Gerlitze85b24b2006-05-11 10:02:19 +030074 if (err) {
75 iser_err("Failed to set up Data-IN RDMA\n");
76 return err;
77 }
Mike Christie2261ec32008-05-21 15:54:11 -050078 regd_buf = &iser_task->rdma_regd[ISER_DIR_IN];
Or Gerlitze85b24b2006-05-11 10:02:19 +030079
80 hdr->flags |= ISER_RSV;
81 hdr->read_stag = cpu_to_be32(regd_buf->reg.rkey);
82 hdr->read_va = cpu_to_be64(regd_buf->reg.va);
83
84 iser_dbg("Cmd itt:%d READ tags RKEY:%#.4X VA:%#llX\n",
Mike Christie2261ec32008-05-21 15:54:11 -050085 task->itt, regd_buf->reg.rkey,
Or Gerlitze85b24b2006-05-11 10:02:19 +030086 (unsigned long long)regd_buf->reg.va);
87
88 return 0;
89}
90
91/* Register user buffer memory and initialize passive rdma
92 * dto descriptor. Total data size is stored in
Mike Christie2261ec32008-05-21 15:54:11 -050093 * task->data[ISER_DIR_OUT].data_len
Or Gerlitze85b24b2006-05-11 10:02:19 +030094 */
95static int
Mike Christie2261ec32008-05-21 15:54:11 -050096iser_prepare_write_cmd(struct iscsi_task *task,
Or Gerlitze85b24b2006-05-11 10:02:19 +030097 unsigned int imm_sz,
98 unsigned int unsol_sz,
99 unsigned int edtl)
100{
Mike Christie2261ec32008-05-21 15:54:11 -0500101 struct iscsi_iser_task *iser_task = task->dd_data;
Sagi Grimbergb4e155f2013-07-28 12:35:39 +0300102 struct iser_device *device = iser_task->iser_conn->ib_conn->device;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300103 struct iser_regd_buf *regd_buf;
104 int err;
Mike Christie2261ec32008-05-21 15:54:11 -0500105 struct iser_hdr *hdr = &iser_task->desc.iser_header;
106 struct iser_data_buf *buf_out = &iser_task->data[ISER_DIR_OUT];
Or Gerlitzf19624a2010-02-08 13:19:56 +0000107 struct ib_sge *tx_dsg = &iser_task->desc.tx_sg[1];
Or Gerlitze85b24b2006-05-11 10:02:19 +0300108
Mike Christie2261ec32008-05-21 15:54:11 -0500109 err = iser_dma_map_task_data(iser_task,
Or Gerlitze85b24b2006-05-11 10:02:19 +0300110 buf_out,
111 ISER_DIR_OUT,
112 DMA_TO_DEVICE);
113 if (err)
114 return err;
115
Mike Christie2261ec32008-05-21 15:54:11 -0500116 if (edtl > iser_task->data[ISER_DIR_OUT].data_len) {
Or Gerlitze85b24b2006-05-11 10:02:19 +0300117 iser_err("Total data length: %ld, less than EDTL: %d, "
118 "in WRITE cmd BHS itt: %d, conn: 0x%p\n",
Mike Christie2261ec32008-05-21 15:54:11 -0500119 iser_task->data[ISER_DIR_OUT].data_len,
120 edtl, task->itt, task->conn);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300121 return -EINVAL;
122 }
123
Sagi Grimbergb4e155f2013-07-28 12:35:39 +0300124 err = device->iser_reg_rdma_mem(iser_task, ISER_DIR_OUT);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300125 if (err != 0) {
126 iser_err("Failed to register write cmd RDMA mem\n");
127 return err;
128 }
129
Mike Christie2261ec32008-05-21 15:54:11 -0500130 regd_buf = &iser_task->rdma_regd[ISER_DIR_OUT];
Or Gerlitze85b24b2006-05-11 10:02:19 +0300131
132 if (unsol_sz < edtl) {
133 hdr->flags |= ISER_WSV;
134 hdr->write_stag = cpu_to_be32(regd_buf->reg.rkey);
135 hdr->write_va = cpu_to_be64(regd_buf->reg.va + unsol_sz);
136
137 iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X "
138 "VA:%#llX + unsol:%d\n",
Mike Christie2261ec32008-05-21 15:54:11 -0500139 task->itt, regd_buf->reg.rkey,
Or Gerlitze85b24b2006-05-11 10:02:19 +0300140 (unsigned long long)regd_buf->reg.va, unsol_sz);
141 }
142
143 if (imm_sz > 0) {
144 iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n",
Mike Christie2261ec32008-05-21 15:54:11 -0500145 task->itt, imm_sz);
Or Gerlitzf19624a2010-02-08 13:19:56 +0000146 tx_dsg->addr = regd_buf->reg.va;
147 tx_dsg->length = imm_sz;
148 tx_dsg->lkey = regd_buf->reg.lkey;
149 iser_task->desc.num_sge = 2;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300150 }
151
152 return 0;
153}
154
Or Gerlitze85b24b2006-05-11 10:02:19 +0300155/* creates a new tx descriptor and adds header regd buffer */
Or Gerlitzf19624a2010-02-08 13:19:56 +0000156static void iser_create_send_desc(struct iser_conn *ib_conn,
157 struct iser_tx_desc *tx_desc)
Or Gerlitze85b24b2006-05-11 10:02:19 +0300158{
Or Gerlitzf19624a2010-02-08 13:19:56 +0000159 struct iser_device *device = ib_conn->device;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300160
Or Gerlitzf19624a2010-02-08 13:19:56 +0000161 ib_dma_sync_single_for_cpu(device->ib_device,
162 tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300163
164 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
165 tx_desc->iser_header.flags = ISER_VER;
166
Or Gerlitzf19624a2010-02-08 13:19:56 +0000167 tx_desc->num_sge = 1;
168
169 if (tx_desc->tx_sg[0].lkey != device->mr->lkey) {
170 tx_desc->tx_sg[0].lkey = device->mr->lkey;
171 iser_dbg("sdesc %p lkey mismatch, fixing\n", tx_desc);
172 }
Or Gerlitze85b24b2006-05-11 10:02:19 +0300173}
174
Shlomo Pongratz986db0d2013-07-28 12:35:37 +0300175static void iser_free_login_buf(struct iser_conn *ib_conn)
176{
177 if (!ib_conn->login_buf)
178 return;
179
180 if (ib_conn->login_req_dma)
181 ib_dma_unmap_single(ib_conn->device->ib_device,
182 ib_conn->login_req_dma,
183 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE);
184
185 if (ib_conn->login_resp_dma)
186 ib_dma_unmap_single(ib_conn->device->ib_device,
187 ib_conn->login_resp_dma,
188 ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE);
189
190 kfree(ib_conn->login_buf);
191
192 /* make sure we never redo any unmapping */
193 ib_conn->login_req_dma = 0;
194 ib_conn->login_resp_dma = 0;
195 ib_conn->login_buf = NULL;
196}
197
198static int iser_alloc_login_buf(struct iser_conn *ib_conn)
199{
200 struct iser_device *device;
201 int req_err, resp_err;
202
203 BUG_ON(ib_conn->device == NULL);
204
205 device = ib_conn->device;
206
207 ib_conn->login_buf = kmalloc(ISCSI_DEF_MAX_RECV_SEG_LEN +
208 ISER_RX_LOGIN_SIZE, GFP_KERNEL);
209 if (!ib_conn->login_buf)
210 goto out_err;
211
212 ib_conn->login_req_buf = ib_conn->login_buf;
213 ib_conn->login_resp_buf = ib_conn->login_buf +
214 ISCSI_DEF_MAX_RECV_SEG_LEN;
215
216 ib_conn->login_req_dma = ib_dma_map_single(ib_conn->device->ib_device,
217 (void *)ib_conn->login_req_buf,
218 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE);
219
220 ib_conn->login_resp_dma = ib_dma_map_single(ib_conn->device->ib_device,
221 (void *)ib_conn->login_resp_buf,
222 ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE);
223
224 req_err = ib_dma_mapping_error(device->ib_device,
225 ib_conn->login_req_dma);
226 resp_err = ib_dma_mapping_error(device->ib_device,
227 ib_conn->login_resp_dma);
228
229 if (req_err || resp_err) {
230 if (req_err)
231 ib_conn->login_req_dma = 0;
232 if (resp_err)
233 ib_conn->login_resp_dma = 0;
234 goto free_login_buf;
235 }
236 return 0;
237
238free_login_buf:
239 iser_free_login_buf(ib_conn);
240
241out_err:
242 iser_err("unable to alloc or map login buf\n");
243 return -ENOMEM;
244}
Or Gerlitzf19624a2010-02-08 13:19:56 +0000245
Shlomo Pongratzb7f04512013-07-28 12:35:38 +0300246int iser_alloc_rx_descriptors(struct iser_conn *ib_conn, struct iscsi_session *session)
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000247{
248 int i, j;
249 u64 dma_addr;
250 struct iser_rx_desc *rx_desc;
251 struct ib_sge *rx_sg;
252 struct iser_device *device = ib_conn->device;
253
Shlomo Pongratzb7f04512013-07-28 12:35:38 +0300254 ib_conn->qp_max_recv_dtos = session->cmds_max;
255 ib_conn->qp_max_recv_dtos_mask = session->cmds_max - 1; /* cmds_max is 2^N */
256 ib_conn->min_posted_rx = ib_conn->qp_max_recv_dtos >> 2;
257
Sagi Grimbergb4e155f2013-07-28 12:35:39 +0300258 if (device->iser_alloc_rdma_reg_res(ib_conn, session->scsi_cmds_max))
259 goto create_rdma_reg_res_failed;
Shlomo Pongratz986db0d2013-07-28 12:35:37 +0300260
261 if (iser_alloc_login_buf(ib_conn))
262 goto alloc_login_buf_fail;
263
Shlomo Pongratzb7f04512013-07-28 12:35:38 +0300264 ib_conn->rx_descs = kmalloc(session->cmds_max *
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000265 sizeof(struct iser_rx_desc), GFP_KERNEL);
266 if (!ib_conn->rx_descs)
267 goto rx_desc_alloc_fail;
268
269 rx_desc = ib_conn->rx_descs;
270
Shlomo Pongratzb7f04512013-07-28 12:35:38 +0300271 for (i = 0; i < ib_conn->qp_max_recv_dtos; i++, rx_desc++) {
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000272 dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc,
273 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
274 if (ib_dma_mapping_error(device->ib_device, dma_addr))
275 goto rx_desc_dma_map_failed;
276
277 rx_desc->dma_addr = dma_addr;
278
279 rx_sg = &rx_desc->rx_sg;
280 rx_sg->addr = rx_desc->dma_addr;
281 rx_sg->length = ISER_RX_PAYLOAD_SIZE;
282 rx_sg->lkey = device->mr->lkey;
283 }
284
285 ib_conn->rx_desc_head = 0;
286 return 0;
287
288rx_desc_dma_map_failed:
289 rx_desc = ib_conn->rx_descs;
290 for (j = 0; j < i; j++, rx_desc++)
291 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
Shlomo Pongratz986db0d2013-07-28 12:35:37 +0300292 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000293 kfree(ib_conn->rx_descs);
294 ib_conn->rx_descs = NULL;
295rx_desc_alloc_fail:
Shlomo Pongratz986db0d2013-07-28 12:35:37 +0300296 iser_free_login_buf(ib_conn);
297alloc_login_buf_fail:
Sagi Grimbergb4e155f2013-07-28 12:35:39 +0300298 device->iser_free_rdma_reg_res(ib_conn);
299create_rdma_reg_res_failed:
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000300 iser_err("failed allocating rx descriptors / data buffers\n");
301 return -ENOMEM;
302}
303
304void iser_free_rx_descriptors(struct iser_conn *ib_conn)
305{
306 int i;
307 struct iser_rx_desc *rx_desc;
308 struct iser_device *device = ib_conn->device;
309
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000310 if (!ib_conn->rx_descs)
Shlomo Pongratz986db0d2013-07-28 12:35:37 +0300311 goto free_login_buf;
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000312
Sagi Grimbergb4e155f2013-07-28 12:35:39 +0300313 if (device && device->iser_free_rdma_reg_res)
314 device->iser_free_rdma_reg_res(ib_conn);
315
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000316 rx_desc = ib_conn->rx_descs;
Shlomo Pongratzb7f04512013-07-28 12:35:38 +0300317 for (i = 0; i < ib_conn->qp_max_recv_dtos; i++, rx_desc++)
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000318 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
Shlomo Pongratz986db0d2013-07-28 12:35:37 +0300319 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000320 kfree(ib_conn->rx_descs);
Shlomo Pongratz986db0d2013-07-28 12:35:37 +0300321 /* make sure we never redo any unmapping */
322 ib_conn->rx_descs = NULL;
323
324free_login_buf:
325 iser_free_login_buf(ib_conn);
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000326}
327
Or Gerlitz89e984e2012-03-05 18:21:44 +0200328static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req)
Or Gerlitze85b24b2006-05-11 10:02:19 +0300329{
330 struct iscsi_iser_conn *iser_conn = conn->dd_data;
331
Or Gerlitz89e984e2012-03-05 18:21:44 +0200332 iser_dbg("req op %x flags %x\n", req->opcode, req->flags);
333 /* check if this is the last login - going to full feature phase */
334 if ((req->flags & ISCSI_FULL_FEATURE_PHASE) != ISCSI_FULL_FEATURE_PHASE)
335 return 0;
336
337 /*
338 * Check that there is one posted recv buffer (for the last login
339 * response) and no posted send buffers left - they must have been
340 * consumed during previous login phases.
341 */
342 WARN_ON(iser_conn->ib_conn->post_recv_buf_count != 1);
343 WARN_ON(atomic_read(&iser_conn->ib_conn->post_send_buf_count) != 0);
344
Shlomo Pongratzb7f04512013-07-28 12:35:38 +0300345 iser_dbg("Initially post: %d\n", iser_conn->ib_conn->min_posted_rx);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300346 /* Initial post receive buffers */
Shlomo Pongratzb7f04512013-07-28 12:35:38 +0300347 if (iser_post_recvm(iser_conn->ib_conn,
348 iser_conn->ib_conn->min_posted_rx))
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000349 return -ENOMEM;
350
Or Gerlitze85b24b2006-05-11 10:02:19 +0300351 return 0;
352}
353
Or Gerlitze85b24b2006-05-11 10:02:19 +0300354/**
355 * iser_send_command - send command PDU
356 */
Mike Christie2747fdb2008-05-21 15:54:08 -0500357int iser_send_command(struct iscsi_conn *conn,
Mike Christie2261ec32008-05-21 15:54:11 -0500358 struct iscsi_task *task)
Or Gerlitze85b24b2006-05-11 10:02:19 +0300359{
360 struct iscsi_iser_conn *iser_conn = conn->dd_data;
Mike Christie2261ec32008-05-21 15:54:11 -0500361 struct iscsi_iser_task *iser_task = task->dd_data;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300362 unsigned long edtl;
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000363 int err;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300364 struct iser_data_buf *data_buf;
Nicholas Bellinger12352182011-05-27 11:16:33 +0000365 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr;
Mike Christie2261ec32008-05-21 15:54:11 -0500366 struct scsi_cmnd *sc = task->sc;
Or Gerlitzf19624a2010-02-08 13:19:56 +0000367 struct iser_tx_desc *tx_desc = &iser_task->desc;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300368
Or Gerlitze85b24b2006-05-11 10:02:19 +0300369 edtl = ntohl(hdr->data_length);
370
371 /* build the tx desc regd header and add it to the tx desc dto */
Or Gerlitzf19624a2010-02-08 13:19:56 +0000372 tx_desc->type = ISCSI_TX_SCSI_COMMAND;
373 iser_create_send_desc(iser_conn->ib_conn, tx_desc);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300374
375 if (hdr->flags & ISCSI_FLAG_CMD_READ)
Mike Christie2261ec32008-05-21 15:54:11 -0500376 data_buf = &iser_task->data[ISER_DIR_IN];
Or Gerlitze85b24b2006-05-11 10:02:19 +0300377 else
Mike Christie2261ec32008-05-21 15:54:11 -0500378 data_buf = &iser_task->data[ISER_DIR_OUT];
Or Gerlitze85b24b2006-05-11 10:02:19 +0300379
FUJITA Tomonorida9c0c72007-06-01 18:56:21 +0900380 if (scsi_sg_count(sc)) { /* using a scatter list */
381 data_buf->buf = scsi_sglist(sc);
382 data_buf->size = scsi_sg_count(sc);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300383 }
384
FUJITA Tomonorida9c0c72007-06-01 18:56:21 +0900385 data_buf->data_len = scsi_bufflen(sc);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300386
387 if (hdr->flags & ISCSI_FLAG_CMD_READ) {
Mike Christie2261ec32008-05-21 15:54:11 -0500388 err = iser_prepare_read_cmd(task, edtl);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300389 if (err)
390 goto send_command_error;
391 }
392 if (hdr->flags & ISCSI_FLAG_CMD_WRITE) {
Mike Christie2261ec32008-05-21 15:54:11 -0500393 err = iser_prepare_write_cmd(task,
394 task->imm_count,
395 task->imm_count +
Mike Christie0f9c7442008-12-02 00:32:06 -0600396 task->unsol_r2t.data_length,
Or Gerlitze85b24b2006-05-11 10:02:19 +0300397 edtl);
398 if (err)
399 goto send_command_error;
400 }
401
Mike Christie2261ec32008-05-21 15:54:11 -0500402 iser_task->status = ISER_TASK_STATUS_STARTED;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300403
Or Gerlitzf19624a2010-02-08 13:19:56 +0000404 err = iser_post_send(iser_conn->ib_conn, tx_desc);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300405 if (!err)
406 return 0;
407
408send_command_error:
Mike Christie2261ec32008-05-21 15:54:11 -0500409 iser_err("conn %p failed task->itt %d err %d\n",conn, task->itt, err);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300410 return err;
411}
412
413/**
414 * iser_send_data_out - send data out PDU
415 */
Mike Christie2747fdb2008-05-21 15:54:08 -0500416int iser_send_data_out(struct iscsi_conn *conn,
Mike Christie2261ec32008-05-21 15:54:11 -0500417 struct iscsi_task *task,
Or Gerlitze85b24b2006-05-11 10:02:19 +0300418 struct iscsi_data *hdr)
419{
420 struct iscsi_iser_conn *iser_conn = conn->dd_data;
Mike Christie2261ec32008-05-21 15:54:11 -0500421 struct iscsi_iser_task *iser_task = task->dd_data;
Or Gerlitzf19624a2010-02-08 13:19:56 +0000422 struct iser_tx_desc *tx_desc = NULL;
423 struct iser_regd_buf *regd_buf;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300424 unsigned long buf_offset;
425 unsigned long data_seg_len;
Erez Zilber0a22ab92008-04-16 21:09:35 -0700426 uint32_t itt;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300427 int err = 0;
Or Gerlitzf19624a2010-02-08 13:19:56 +0000428 struct ib_sge *tx_dsg;
429
Erez Zilber0a22ab92008-04-16 21:09:35 -0700430 itt = (__force uint32_t)hdr->itt;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300431 data_seg_len = ntoh24(hdr->dlength);
432 buf_offset = ntohl(hdr->offset);
433
434 iser_dbg("%s itt %d dseg_len %d offset %d\n",
435 __func__,(int)itt,(int)data_seg_len,(int)buf_offset);
436
Or Gerlitz528f4e82010-02-08 13:20:43 +0000437 tx_desc = kmem_cache_zalloc(ig.desc_cache, GFP_ATOMIC);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300438 if (tx_desc == NULL) {
439 iser_err("Failed to alloc desc for post dataout\n");
440 return -ENOMEM;
441 }
442
443 tx_desc->type = ISCSI_TX_DATAOUT;
Or Gerlitzf19624a2010-02-08 13:19:56 +0000444 tx_desc->iser_header.flags = ISER_VER;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300445 memcpy(&tx_desc->iscsi_header, hdr, sizeof(struct iscsi_hdr));
446
Or Gerlitzf19624a2010-02-08 13:19:56 +0000447 /* build the tx desc */
448 iser_initialize_task_headers(task, tx_desc);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300449
Or Gerlitzf19624a2010-02-08 13:19:56 +0000450 regd_buf = &iser_task->rdma_regd[ISER_DIR_OUT];
451 tx_dsg = &tx_desc->tx_sg[1];
452 tx_dsg->addr = regd_buf->reg.va + buf_offset;
453 tx_dsg->length = data_seg_len;
454 tx_dsg->lkey = regd_buf->reg.lkey;
455 tx_desc->num_sge = 2;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300456
Mike Christie2261ec32008-05-21 15:54:11 -0500457 if (buf_offset + data_seg_len > iser_task->data[ISER_DIR_OUT].data_len) {
Or Gerlitze85b24b2006-05-11 10:02:19 +0300458 iser_err("Offset:%ld & DSL:%ld in Data-Out "
459 "inconsistent with total len:%ld, itt:%d\n",
460 buf_offset, data_seg_len,
Mike Christie2261ec32008-05-21 15:54:11 -0500461 iser_task->data[ISER_DIR_OUT].data_len, itt);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300462 err = -EINVAL;
463 goto send_data_out_error;
464 }
465 iser_dbg("data-out itt: %d, offset: %ld, sz: %ld\n",
466 itt, buf_offset, data_seg_len);
467
468
Or Gerlitzf19624a2010-02-08 13:19:56 +0000469 err = iser_post_send(iser_conn->ib_conn, tx_desc);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300470 if (!err)
471 return 0;
472
473send_data_out_error:
Or Gerlitze85b24b2006-05-11 10:02:19 +0300474 kmem_cache_free(ig.desc_cache, tx_desc);
475 iser_err("conn %p failed err %d\n",conn, err);
476 return err;
477}
478
479int iser_send_control(struct iscsi_conn *conn,
Mike Christie2261ec32008-05-21 15:54:11 -0500480 struct iscsi_task *task)
Or Gerlitze85b24b2006-05-11 10:02:19 +0300481{
482 struct iscsi_iser_conn *iser_conn = conn->dd_data;
Mike Christie2261ec32008-05-21 15:54:11 -0500483 struct iscsi_iser_task *iser_task = task->dd_data;
Or Gerlitzf19624a2010-02-08 13:19:56 +0000484 struct iser_tx_desc *mdesc = &iser_task->desc;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300485 unsigned long data_seg_len;
Or Gerlitzf19624a2010-02-08 13:19:56 +0000486 int err = 0;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300487 struct iser_device *device;
Or Gerlitz2c4ce602011-11-04 00:19:46 +0200488 struct iser_conn *ib_conn = iser_conn->ib_conn;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300489
Or Gerlitze85b24b2006-05-11 10:02:19 +0300490 /* build the tx desc regd header and add it to the tx desc dto */
491 mdesc->type = ISCSI_TX_CONTROL;
Or Gerlitzf19624a2010-02-08 13:19:56 +0000492 iser_create_send_desc(iser_conn->ib_conn, mdesc);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300493
494 device = iser_conn->ib_conn->device;
495
Mike Christie2261ec32008-05-21 15:54:11 -0500496 data_seg_len = ntoh24(task->hdr->dlength);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300497
498 if (data_seg_len > 0) {
Or Gerlitzf19624a2010-02-08 13:19:56 +0000499 struct ib_sge *tx_dsg = &mdesc->tx_sg[1];
500 if (task != conn->login_task) {
501 iser_err("data present on non login task!!!\n");
502 goto send_control_error;
503 }
Or Gerlitz2c4ce602011-11-04 00:19:46 +0200504
505 ib_dma_sync_single_for_cpu(device->ib_device,
506 ib_conn->login_req_dma, task->data_count,
507 DMA_TO_DEVICE);
508
509 memcpy(iser_conn->ib_conn->login_req_buf, task->data,
Or Gerlitzf19624a2010-02-08 13:19:56 +0000510 task->data_count);
Or Gerlitz2c4ce602011-11-04 00:19:46 +0200511
512 ib_dma_sync_single_for_device(device->ib_device,
513 ib_conn->login_req_dma, task->data_count,
514 DMA_TO_DEVICE);
515
516 tx_dsg->addr = iser_conn->ib_conn->login_req_dma;
Or Gerlitz200ae1a2011-08-01 21:14:09 +0000517 tx_dsg->length = task->data_count;
Or Gerlitzf19624a2010-02-08 13:19:56 +0000518 tx_dsg->lkey = device->mr->lkey;
519 mdesc->num_sge = 2;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300520 }
521
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000522 if (task == conn->login_task) {
523 err = iser_post_recvl(iser_conn->ib_conn);
524 if (err)
525 goto send_control_error;
Or Gerlitz89e984e2012-03-05 18:21:44 +0200526 err = iser_post_rx_bufs(conn, task->hdr);
527 if (err)
528 goto send_control_error;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300529 }
530
Or Gerlitzf19624a2010-02-08 13:19:56 +0000531 err = iser_post_send(iser_conn->ib_conn, mdesc);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300532 if (!err)
533 return 0;
534
535send_control_error:
Or Gerlitze85b24b2006-05-11 10:02:19 +0300536 iser_err("conn %p failed err %d\n",conn, err);
537 return err;
538}
539
540/**
541 * iser_rcv_dto_completion - recv DTO completion
542 */
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000543void iser_rcv_completion(struct iser_rx_desc *rx_desc,
544 unsigned long rx_xfer_len,
545 struct iser_conn *ib_conn)
Or Gerlitze85b24b2006-05-11 10:02:19 +0300546{
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000547 struct iscsi_iser_conn *conn = ib_conn->iser_conn;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300548 struct iscsi_hdr *hdr;
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000549 u64 rx_dma;
550 int rx_buflen, outstanding, count, err;
551
552 /* differentiate between login to all other PDUs */
Or Gerlitz2c4ce602011-11-04 00:19:46 +0200553 if ((char *)rx_desc == ib_conn->login_resp_buf) {
554 rx_dma = ib_conn->login_resp_dma;
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000555 rx_buflen = ISER_RX_LOGIN_SIZE;
556 } else {
557 rx_dma = rx_desc->dma_addr;
558 rx_buflen = ISER_RX_PAYLOAD_SIZE;
559 }
560
561 ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, rx_dma,
562 rx_buflen, DMA_FROM_DEVICE);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300563
564 hdr = &rx_desc->iscsi_header;
565
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000566 iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode,
567 hdr->itt, (int)(rx_xfer_len - ISER_HEADERS_LEN));
Or Gerlitze85b24b2006-05-11 10:02:19 +0300568
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000569 iscsi_iser_recv(conn->iscsi_conn, hdr,
570 rx_desc->data, rx_xfer_len - ISER_HEADERS_LEN);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300571
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000572 ib_dma_sync_single_for_device(ib_conn->device->ib_device, rx_dma,
573 rx_buflen, DMA_FROM_DEVICE);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300574
575 /* decrementing conn->post_recv_buf_count only --after-- freeing the *
576 * task eliminates the need to worry on tasks which are completed in *
577 * parallel to the execution of iser_conn_term. So the code that waits *
578 * for the posted rx bufs refcount to become zero handles everything */
Or Gerlitz704315f2010-02-08 13:18:39 +0000579 conn->ib_conn->post_recv_buf_count--;
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000580
Or Gerlitz2c4ce602011-11-04 00:19:46 +0200581 if (rx_dma == ib_conn->login_resp_dma)
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000582 return;
583
Or Gerlitz704315f2010-02-08 13:18:39 +0000584 outstanding = ib_conn->post_recv_buf_count;
Shlomo Pongratzb7f04512013-07-28 12:35:38 +0300585 if (outstanding + ib_conn->min_posted_rx <= ib_conn->qp_max_recv_dtos) {
586 count = min(ib_conn->qp_max_recv_dtos - outstanding,
587 ib_conn->min_posted_rx);
Or Gerlitzbcc60c32010-02-08 13:17:42 +0000588 err = iser_post_recvm(ib_conn, count);
589 if (err)
590 iser_err("posting %d rx bufs err %d\n", count, err);
591 }
Or Gerlitze85b24b2006-05-11 10:02:19 +0300592}
593
Or Gerlitzf19624a2010-02-08 13:19:56 +0000594void iser_snd_completion(struct iser_tx_desc *tx_desc,
595 struct iser_conn *ib_conn)
Or Gerlitze85b24b2006-05-11 10:02:19 +0300596{
Mike Christie2261ec32008-05-21 15:54:11 -0500597 struct iscsi_task *task;
Or Gerlitzf19624a2010-02-08 13:19:56 +0000598 struct iser_device *device = ib_conn->device;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300599
Or Gerlitzf19624a2010-02-08 13:19:56 +0000600 if (tx_desc->type == ISCSI_TX_DATAOUT) {
601 ib_dma_unmap_single(device->ib_device, tx_desc->dma_addr,
602 ISER_HEADERS_LEN, DMA_TO_DEVICE);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300603 kmem_cache_free(ig.desc_cache, tx_desc);
Or Gerlitzf19624a2010-02-08 13:19:56 +0000604 }
Or Gerlitze85b24b2006-05-11 10:02:19 +0300605
Erez Zilber87e8df72006-09-27 15:27:10 +0300606 atomic_dec(&ib_conn->post_send_buf_count);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300607
Or Gerlitze85b24b2006-05-11 10:02:19 +0300608 if (tx_desc->type == ISCSI_TX_CONTROL) {
609 /* this arithmetic is legal by libiscsi dd_data allocation */
Mike Christie2261ec32008-05-21 15:54:11 -0500610 task = (void *) ((long)(void *)tx_desc -
611 sizeof(struct iscsi_task));
612 if (task->hdr->itt == RESERVED_ITT)
613 iscsi_put_task(task);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300614 }
615}
616
Mike Christie2261ec32008-05-21 15:54:11 -0500617void iser_task_rdma_init(struct iscsi_iser_task *iser_task)
Or Gerlitze85b24b2006-05-11 10:02:19 +0300618
619{
Mike Christie2261ec32008-05-21 15:54:11 -0500620 iser_task->status = ISER_TASK_STATUS_INIT;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300621
Mike Christie2261ec32008-05-21 15:54:11 -0500622 iser_task->dir[ISER_DIR_IN] = 0;
623 iser_task->dir[ISER_DIR_OUT] = 0;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300624
Mike Christie2261ec32008-05-21 15:54:11 -0500625 iser_task->data[ISER_DIR_IN].data_len = 0;
626 iser_task->data[ISER_DIR_OUT].data_len = 0;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300627
Mike Christie2261ec32008-05-21 15:54:11 -0500628 memset(&iser_task->rdma_regd[ISER_DIR_IN], 0,
Or Gerlitze85b24b2006-05-11 10:02:19 +0300629 sizeof(struct iser_regd_buf));
Mike Christie2261ec32008-05-21 15:54:11 -0500630 memset(&iser_task->rdma_regd[ISER_DIR_OUT], 0,
Or Gerlitze85b24b2006-05-11 10:02:19 +0300631 sizeof(struct iser_regd_buf));
632}
633
Mike Christie2261ec32008-05-21 15:54:11 -0500634void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task)
Or Gerlitze85b24b2006-05-11 10:02:19 +0300635{
Sagi Grimbergb4e155f2013-07-28 12:35:39 +0300636 struct iser_device *device = iser_task->iser_conn->ib_conn->device;
Erez Zilber74a20782006-09-27 16:43:06 +0300637 int is_rdma_aligned = 1;
Or Gerlitze85b24b2006-05-11 10:02:19 +0300638
639 /* if we were reading, copy back to unaligned sglist,
640 * anyway dma_unmap and free the copy
641 */
Mike Christie2261ec32008-05-21 15:54:11 -0500642 if (iser_task->data_copy[ISER_DIR_IN].copy_buf != NULL) {
Erez Zilber74a20782006-09-27 16:43:06 +0300643 is_rdma_aligned = 0;
Mike Christie2261ec32008-05-21 15:54:11 -0500644 iser_finalize_rdma_unaligned_sg(iser_task, ISER_DIR_IN);
Erez Zilber74a20782006-09-27 16:43:06 +0300645 }
Mike Christie2261ec32008-05-21 15:54:11 -0500646 if (iser_task->data_copy[ISER_DIR_OUT].copy_buf != NULL) {
Erez Zilber74a20782006-09-27 16:43:06 +0300647 is_rdma_aligned = 0;
Mike Christie2261ec32008-05-21 15:54:11 -0500648 iser_finalize_rdma_unaligned_sg(iser_task, ISER_DIR_OUT);
Erez Zilber74a20782006-09-27 16:43:06 +0300649 }
Or Gerlitze85b24b2006-05-11 10:02:19 +0300650
Sagi Grimbergb4e155f2013-07-28 12:35:39 +0300651 if (iser_task->dir[ISER_DIR_IN])
652 device->iser_unreg_rdma_mem(iser_task, ISER_DIR_IN);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300653
Sagi Grimbergb4e155f2013-07-28 12:35:39 +0300654 if (iser_task->dir[ISER_DIR_OUT])
655 device->iser_unreg_rdma_mem(iser_task, ISER_DIR_OUT);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300656
Erez Zilber74a20782006-09-27 16:43:06 +0300657 /* if the data was unaligned, it was already unmapped and then copied */
658 if (is_rdma_aligned)
Mike Christie2261ec32008-05-21 15:54:11 -0500659 iser_dma_unmap_task_data(iser_task);
Or Gerlitze85b24b2006-05-11 10:02:19 +0300660}