drivers/scsi/cxgb3i/cxgb3i_pdu.c - kernel/msm-4.9 - Gitiles

 /*
  * cxgb3i_pdu.c: Chelsio S3xx iSCSI driver.
  *
  * Copyright (c) 2008 Chelsio Communications, Inc.
  * Copyright (c) 2008 Mike Christie
  * Copyright (c) 2008 Red Hat, Inc.  All rights reserved.
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation.
  *
  * Written by: Karen Xie (kxie@chelsio.com)
  */

 #include <linux/skbuff.h>
 #include <linux/crypto.h>
 #include <scsi/scsi_cmnd.h>
 #include <scsi/scsi_host.h>

 #include "cxgb3i.h"
 #include "cxgb3i_pdu.h"

 #ifdef __DEBUG_CXGB3I_RX__
 #define cxgb3i_rx_debug		cxgb3i_log_debug
 #else
 #define cxgb3i_rx_debug(fmt...)
 #endif

 #ifdef __DEBUG_CXGB3I_TX__
 #define cxgb3i_tx_debug		cxgb3i_log_debug
 #else
 #define cxgb3i_tx_debug(fmt...)
 #endif

 static struct page *pad_page;

 /*
  * pdu receive, interact with libiscsi_tcp
  */
 static inline int read_pdu_skb(struct iscsi_conn *conn, struct sk_buff *skb,
 			       unsigned int offset, int offloaded)
 {
 	int status = 0;
 	int bytes_read;

 	bytes_read = iscsi_tcp_recv_skb(conn, skb, offset, offloaded, &status);
 	switch (status) {
 	case ISCSI_TCP_CONN_ERR:
 		return -EIO;
 	case ISCSI_TCP_SUSPENDED:
 		/* no transfer - just have caller flush queue */
 		return bytes_read;
 	case ISCSI_TCP_SKB_DONE:
 		/*
 		 * pdus should always fit in the skb and we should get
 		 * segment done notifcation.
 		 */
 		iscsi_conn_printk(KERN_ERR, conn, "Invalid pdu or skb.");
 		return -EFAULT;
 	case ISCSI_TCP_SEGMENT_DONE:
 		return bytes_read;
 	default:
 		iscsi_conn_printk(KERN_ERR, conn, "Invalid iscsi_tcp_recv_skb "
 				  "status %d\n", status);
 		return -EINVAL;
 	}
 }

 static int cxgb3i_conn_read_pdu_skb(struct iscsi_conn *conn,
 				    struct sk_buff *skb)
 {
 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 	bool offloaded = 0;
 	unsigned int offset;
 	int rc;

 	cxgb3i_rx_debug("conn 0x%p, skb 0x%p, len %u, flag 0x%x.\n",
 			conn, skb, skb->len, skb_ulp_mode(skb));

 	if (!iscsi_tcp_recv_segment_is_hdr(tcp_conn)) {
 		iscsi_conn_failure(conn, ISCSI_ERR_PROTO);
 		return -EIO;
 	}

 	if (conn->hdrdgst_en && (skb_ulp_mode(skb) & ULP2_FLAG_HCRC_ERROR)) {
 		iscsi_conn_failure(conn, ISCSI_ERR_HDR_DGST);
 		return -EIO;
 	}

 	if (conn->datadgst_en && (skb_ulp_mode(skb) & ULP2_FLAG_DCRC_ERROR)) {
 		iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
 		return -EIO;
 	}

 	/* iscsi hdr */
 	rc = read_pdu_skb(conn, skb, 0, 0);
 	if (rc <= 0)
 		return rc;

 	if (iscsi_tcp_recv_segment_is_hdr(tcp_conn))
 		return 0;

 	offset = rc;
 	if (conn->hdrdgst_en)
 		offset += ISCSI_DIGEST_SIZE;

 	/* iscsi data */
 	if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED) {
 		cxgb3i_rx_debug("skb 0x%p, opcode 0x%x, data %u, ddp'ed, "
 				"itt 0x%x.\n",
 				skb,
 				tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK,
 				tcp_conn->in.datalen,
 				ntohl(tcp_conn->in.hdr->itt));
 		offloaded = 1;
 	} else {
 		cxgb3i_rx_debug("skb 0x%p, opcode 0x%x, data %u, NOT ddp'ed, "
 				"itt 0x%x.\n",
 				skb,
 				tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK,
 				tcp_conn->in.datalen,
 				ntohl(tcp_conn->in.hdr->itt));
 		offset += sizeof(struct cpl_iscsi_hdr_norss);
 	}

 	rc = read_pdu_skb(conn, skb, offset, offloaded);
 	if (rc < 0)
 		return rc;
 	else
 		return 0;
 }

 /*
  * pdu transmit, interact with libiscsi_tcp
  */
 static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc)
 {
 	u8 submode = 0;

 	if (hcrc)
 		submode |= 1;
 	if (dcrc)
 		submode |= 2;
 	skb_ulp_mode(skb) = (ULP_MODE_ISCSI << 4) | submode;
 }

 void cxgb3i_conn_cleanup_task(struct iscsi_task *task)
 {
 	struct iscsi_tcp_task *tcp_task = task->dd_data;

 	/* never reached the xmit task callout */
 	if (tcp_task->dd_data)
 		kfree_skb(tcp_task->dd_data);
 	tcp_task->dd_data = NULL;

 	/* MNC - Do we need a check in case this is called but
 	 * cxgb3i_conn_alloc_pdu has never been called on the task */
 	cxgb3i_release_itt(task, task->hdr_itt);
 	iscsi_tcp_cleanup_task(task);
 }

 /*
  * We do not support ahs yet
  */
 int cxgb3i_conn_alloc_pdu(struct iscsi_task *task, u8 opcode)
 {
 	struct iscsi_tcp_task *tcp_task = task->dd_data;
 	struct sk_buff *skb;

 	task->hdr = NULL;
 	/* always allocate rooms for AHS */
 	skb = alloc_skb(sizeof(struct iscsi_hdr) + ISCSI_MAX_AHS_SIZE +
 			TX_HEADER_LEN,  GFP_ATOMIC);
 	if (!skb)
 		return -ENOMEM;

 	cxgb3i_tx_debug("task 0x%p, opcode 0x%x, skb 0x%p.\n",
 			task, opcode, skb);

 	tcp_task->dd_data = skb;
 	skb_reserve(skb, TX_HEADER_LEN);
 	task->hdr = (struct iscsi_hdr *)skb->data;
 	task->hdr_max = sizeof(struct iscsi_hdr);

 	/* data_out uses scsi_cmd's itt */
 	if (opcode != ISCSI_OP_SCSI_DATA_OUT)
 		cxgb3i_reserve_itt(task, &task->hdr->itt);

 	return 0;
 }

 int cxgb3i_conn_init_pdu(struct iscsi_task *task, unsigned int offset,
 			      unsigned int count)
 {
 	struct iscsi_tcp_task *tcp_task = task->dd_data;
 	struct sk_buff *skb = tcp_task->dd_data;
 	struct iscsi_conn *conn = task->conn;
 	struct page *pg;
 	unsigned int datalen = count;
 	int i, padlen = iscsi_padding(count);
 	skb_frag_t *frag;

 	cxgb3i_tx_debug("task 0x%p,0x%p, offset %u, count %u, skb 0x%p.\n",
 			task, task->sc, offset, count, skb);

 	skb_put(skb, task->hdr_len);
 	tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0);
 	if (!count)
 		return 0;

 	if (task->sc) {
 		struct scatterlist *sg;
 		struct scsi_data_buffer *sdb;
 		unsigned int sgoffset = offset;
 		struct page *sgpg;
 		unsigned int sglen;

 		sdb = scsi_out(task->sc);
 		sg = sdb->table.sgl;

 		for_each_sg(sdb->table.sgl, sg, sdb->table.nents, i) {
 			cxgb3i_tx_debug("sg %d, page 0x%p, len %u offset %u\n",
 					i, sg_page(sg), sg->length, sg->offset);

 			if (sgoffset < sg->length)
 				break;
 			sgoffset -= sg->length;
 		}
 		sgpg = sg_page(sg);
 		sglen = sg->length - sgoffset;

 		do {
 			int j = skb_shinfo(skb)->nr_frags;
 			unsigned int copy;

 			if (!sglen) {
 				sg = sg_next(sg);
 				sgpg = sg_page(sg);
 				sgoffset = 0;
 				sglen = sg->length;
 				++i;
 			}
 			copy = min(sglen, datalen);
 			if (j && skb_can_coalesce(skb, j, sgpg,
 						  sg->offset + sgoffset)) {
 				skb_shinfo(skb)->frags[j - 1].size += copy;
 			} else {
 				get_page(sgpg);
 				skb_fill_page_desc(skb, j, sgpg,
 						   sg->offset + sgoffset, copy);
 			}
 			sgoffset += copy;
 			sglen -= copy;
 			datalen -= copy;
 		} while (datalen);
 	} else {
 		pg = virt_to_page(task->data);

 		while (datalen) {
 			i = skb_shinfo(skb)->nr_frags;
 			frag = &skb_shinfo(skb)->frags[i];

 			get_page(pg);
 			frag->page = pg;
 			frag->page_offset = 0;
 			frag->size = min((unsigned int)PAGE_SIZE, datalen);

 			skb_shinfo(skb)->nr_frags++;
 			datalen -= frag->size;
 			pg++;
 		}
 	}

 	if (padlen) {
 		i = skb_shinfo(skb)->nr_frags;
 		frag = &skb_shinfo(skb)->frags[i];
 		frag->page = pad_page;
 		frag->page_offset = 0;
 		frag->size = padlen;
 		skb_shinfo(skb)->nr_frags++;
 	}

 	datalen = count + padlen;
 	skb->data_len += datalen;
 	skb->truesize += datalen;
 	skb->len += datalen;
 	return 0;
 }

 int cxgb3i_conn_xmit_pdu(struct iscsi_task *task)
 {
 	struct iscsi_tcp_task *tcp_task = task->dd_data;
 	struct sk_buff *skb = tcp_task->dd_data;
 	struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
 	struct cxgb3i_conn *cconn = tcp_conn->dd_data;
 	unsigned int datalen;
 	int err;

 	if (!skb)
 		return 0;

 	datalen = skb->data_len;
 	tcp_task->dd_data = NULL;
 	err = cxgb3i_c3cn_send_pdus(cconn->cep->c3cn, skb);
 	cxgb3i_tx_debug("task 0x%p, skb 0x%p, len %u/%u, rv %d.\n",
 			task, skb, skb->len, skb->data_len, err);
 	if (err > 0) {
 		int pdulen = err;

 		if (task->conn->hdrdgst_en)
 			pdulen += ISCSI_DIGEST_SIZE;
 		if (datalen && task->conn->datadgst_en)
 			pdulen += ISCSI_DIGEST_SIZE;

 		task->conn->txdata_octets += pdulen;
 		return 0;
 	}

 	if (err < 0 && err != -EAGAIN) {
 		kfree_skb(skb);
 		cxgb3i_tx_debug("itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n",
 				task->itt, skb, skb->len, skb->data_len, err);
 		iscsi_conn_printk(KERN_ERR, task->conn, "xmit err %d.\n", err);
 		iscsi_conn_failure(task->conn, ISCSI_ERR_XMIT_FAILED);
 		return err;
 	}
 	/* reset skb to send when we are called again */
 	tcp_task->dd_data = skb;
 	return -EAGAIN;
 }

 int cxgb3i_pdu_init(void)
 {
 	pad_page = alloc_page(GFP_KERNEL);
 	if (!pad_page)
 		return -ENOMEM;
 	memset(page_address(pad_page), 0, PAGE_SIZE);
 	return 0;
 }

 void cxgb3i_pdu_cleanup(void)
 {
 	if (pad_page) {
 		__free_page(pad_page);
 		pad_page = NULL;
 	}
 }

 void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn)
 {
 	struct sk_buff *skb;
 	unsigned int read = 0;
 	struct iscsi_conn *conn = c3cn->user_data;
 	int err = 0;

 	cxgb3i_rx_debug("cn 0x%p.\n", c3cn);

 	read_lock(&c3cn->callback_lock);
 	if (unlikely(!conn || conn->suspend_rx)) {
 		cxgb3i_rx_debug("conn 0x%p, id %d, suspend_rx %lu!\n",
 				conn, conn ? conn->id : 0xFF,
 				conn ? conn->suspend_rx : 0xFF);
 		read_unlock(&c3cn->callback_lock);
 		return;
 	}
 	skb = skb_peek(&c3cn->receive_queue);
 	while (!err && skb) {
 		__skb_unlink(skb, &c3cn->receive_queue);
 		read += skb_ulp_pdulen(skb);
 		err = cxgb3i_conn_read_pdu_skb(conn, skb);
 		__kfree_skb(skb);
 		skb = skb_peek(&c3cn->receive_queue);
 	}
 	read_unlock(&c3cn->callback_lock);
 	if (c3cn) {
 		c3cn->copied_seq += read;
 		cxgb3i_c3cn_rx_credits(c3cn, read);
 	}
 	conn->rxdata_octets += read;
 }

 void cxgb3i_conn_tx_open(struct s3_conn *c3cn)
 {
 	struct iscsi_conn *conn = c3cn->user_data;

 	cxgb3i_tx_debug("cn 0x%p.\n", c3cn);
 	if (conn) {
 		cxgb3i_tx_debug("cn 0x%p, cid %d.\n", c3cn, conn->id);
 		scsi_queue_work(conn->session->host, &conn->xmitwork);
 	}
 }

 void cxgb3i_conn_closing(struct s3_conn *c3cn)
 {
 	struct iscsi_conn *conn;

 	read_lock(&c3cn->callback_lock);
 	conn = c3cn->user_data;
 	if (conn && c3cn->state != C3CN_STATE_ESTABLISHED)
 		iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
 	read_unlock(&c3cn->callback_lock);
 }
	/*
	* cxgb3i_pdu.c: Chelsio S3xx iSCSI driver.
	*
	* Copyright (c) 2008 Chelsio Communications, Inc.
	* Copyright (c) 2008 Mike Christie
	* Copyright (c) 2008 Red Hat, Inc. All rights reserved.
	*
	* This program is free software; you can redistribute it and/or modify
	* it under the terms of the GNU General Public License as published by
	* the Free Software Foundation.
	*
	* Written by: Karen Xie (kxie@chelsio.com)
	*/

	#include <linux/skbuff.h>
	#include <linux/crypto.h>
	#include <scsi/scsi_cmnd.h>
	#include <scsi/scsi_host.h>

	#include "cxgb3i.h"
	#include "cxgb3i_pdu.h"

	#ifdef __DEBUG_CXGB3I_RX__
	#define cxgb3i_rx_debug cxgb3i_log_debug
	#else
	#define cxgb3i_rx_debug(fmt...)
	#endif

	#ifdef __DEBUG_CXGB3I_TX__
	#define cxgb3i_tx_debug cxgb3i_log_debug
	#else
	#define cxgb3i_tx_debug(fmt...)
	#endif

	static struct page *pad_page;

	/*
	* pdu receive, interact with libiscsi_tcp
	*/
	static inline int read_pdu_skb(struct iscsi_conn conn, struct sk_buff skb,
	unsigned int offset, int offloaded)
	{
	int status = 0;
	int bytes_read;

	bytes_read = iscsi_tcp_recv_skb(conn, skb, offset, offloaded, &status);
	switch (status) {
	case ISCSI_TCP_CONN_ERR:
	return -EIO;
	case ISCSI_TCP_SUSPENDED:
	/* no transfer - just have caller flush queue */
	return bytes_read;
	case ISCSI_TCP_SKB_DONE:
	/*
	* pdus should always fit in the skb and we should get
	* segment done notifcation.
	*/
	iscsi_conn_printk(KERN_ERR, conn, "Invalid pdu or skb.");
	return -EFAULT;
	case ISCSI_TCP_SEGMENT_DONE:
	return bytes_read;
	default:
	iscsi_conn_printk(KERN_ERR, conn, "Invalid iscsi_tcp_recv_skb "
	"status %d\n", status);
	return -EINVAL;
	}
	}

	static int cxgb3i_conn_read_pdu_skb(struct iscsi_conn *conn,
	struct sk_buff *skb)
	{
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
	bool offloaded = 0;
	unsigned int offset;
	int rc;

	cxgb3i_rx_debug("conn 0x%p, skb 0x%p, len %u, flag 0x%x.\n",
	conn, skb, skb->len, skb_ulp_mode(skb));

	if (!iscsi_tcp_recv_segment_is_hdr(tcp_conn)) {
	iscsi_conn_failure(conn, ISCSI_ERR_PROTO);
	return -EIO;
	}

	if (conn->hdrdgst_en && (skb_ulp_mode(skb) & ULP2_FLAG_HCRC_ERROR)) {
	iscsi_conn_failure(conn, ISCSI_ERR_HDR_DGST);
	return -EIO;
	}

	if (conn->datadgst_en && (skb_ulp_mode(skb) & ULP2_FLAG_DCRC_ERROR)) {
	iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
	return -EIO;
	}

	/* iscsi hdr */
	rc = read_pdu_skb(conn, skb, 0, 0);
	if (rc <= 0)
	return rc;

	if (iscsi_tcp_recv_segment_is_hdr(tcp_conn))
	return 0;

	offset = rc;
	if (conn->hdrdgst_en)
	offset += ISCSI_DIGEST_SIZE;

	/* iscsi data */
	if (skb_ulp_mode(skb) & ULP2_FLAG_DATA_DDPED) {
	cxgb3i_rx_debug("skb 0x%p, opcode 0x%x, data %u, ddp'ed, "
	"itt 0x%x.\n",
	skb,
	tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK,
	tcp_conn->in.datalen,
	ntohl(tcp_conn->in.hdr->itt));
	offloaded = 1;
	} else {
	cxgb3i_rx_debug("skb 0x%p, opcode 0x%x, data %u, NOT ddp'ed, "
	"itt 0x%x.\n",
	skb,
	tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK,
	tcp_conn->in.datalen,
	ntohl(tcp_conn->in.hdr->itt));
	offset += sizeof(struct cpl_iscsi_hdr_norss);
	}

	rc = read_pdu_skb(conn, skb, offset, offloaded);
	if (rc < 0)
	return rc;
	else
	return 0;
	}

	/*
	* pdu transmit, interact with libiscsi_tcp
	*/
	static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc)
	{
	u8 submode = 0;

	if (hcrc)
	submode \|= 1;
	if (dcrc)
	submode \|= 2;
	skb_ulp_mode(skb) = (ULP_MODE_ISCSI << 4) \| submode;
	}

	void cxgb3i_conn_cleanup_task(struct iscsi_task *task)
	{
	struct iscsi_tcp_task *tcp_task = task->dd_data;

	/* never reached the xmit task callout */
	if (tcp_task->dd_data)
	kfree_skb(tcp_task->dd_data);
	tcp_task->dd_data = NULL;

	/* MNC - Do we need a check in case this is called but
	* cxgb3i_conn_alloc_pdu has never been called on the task */
	cxgb3i_release_itt(task, task->hdr_itt);
	iscsi_tcp_cleanup_task(task);
	}

	/*
	* We do not support ahs yet
	*/
	int cxgb3i_conn_alloc_pdu(struct iscsi_task *task, u8 opcode)
	{
	struct iscsi_tcp_task *tcp_task = task->dd_data;
	struct sk_buff *skb;

	task->hdr = NULL;
	/* always allocate rooms for AHS */
	skb = alloc_skb(sizeof(struct iscsi_hdr) + ISCSI_MAX_AHS_SIZE +
	TX_HEADER_LEN, GFP_ATOMIC);
	if (!skb)
	return -ENOMEM;

	cxgb3i_tx_debug("task 0x%p, opcode 0x%x, skb 0x%p.\n",
	task, opcode, skb);

	tcp_task->dd_data = skb;
	skb_reserve(skb, TX_HEADER_LEN);
	task->hdr = (struct iscsi_hdr *)skb->data;
	task->hdr_max = sizeof(struct iscsi_hdr);

	/* data_out uses scsi_cmd's itt */
	if (opcode != ISCSI_OP_SCSI_DATA_OUT)
	cxgb3i_reserve_itt(task, &task->hdr->itt);

	return 0;
	}

	int cxgb3i_conn_init_pdu(struct iscsi_task *task, unsigned int offset,
	unsigned int count)
	{
	struct iscsi_tcp_task *tcp_task = task->dd_data;
	struct sk_buff *skb = tcp_task->dd_data;
	struct iscsi_conn *conn = task->conn;
	struct page *pg;
	unsigned int datalen = count;
	int i, padlen = iscsi_padding(count);
	skb_frag_t *frag;

	cxgb3i_tx_debug("task 0x%p,0x%p, offset %u, count %u, skb 0x%p.\n",
	task, task->sc, offset, count, skb);

	skb_put(skb, task->hdr_len);
	tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0);
	if (!count)
	return 0;

	if (task->sc) {
	struct scatterlist *sg;
	struct scsi_data_buffer *sdb;
	unsigned int sgoffset = offset;
	struct page *sgpg;
	unsigned int sglen;

	sdb = scsi_out(task->sc);
	sg = sdb->table.sgl;

	for_each_sg(sdb->table.sgl, sg, sdb->table.nents, i) {
	cxgb3i_tx_debug("sg %d, page 0x%p, len %u offset %u\n",
	i, sg_page(sg), sg->length, sg->offset);

	if (sgoffset < sg->length)
	break;
	sgoffset -= sg->length;
	}
	sgpg = sg_page(sg);
	sglen = sg->length - sgoffset;

	do {
	int j = skb_shinfo(skb)->nr_frags;
	unsigned int copy;

	if (!sglen) {
	sg = sg_next(sg);
	sgpg = sg_page(sg);
	sgoffset = 0;
	sglen = sg->length;
	++i;
	}
	copy = min(sglen, datalen);
	if (j && skb_can_coalesce(skb, j, sgpg,
	sg->offset + sgoffset)) {
	skb_shinfo(skb)->frags[j - 1].size += copy;
	} else {
	get_page(sgpg);
	skb_fill_page_desc(skb, j, sgpg,
	sg->offset + sgoffset, copy);
	}
	sgoffset += copy;
	sglen -= copy;
	datalen -= copy;
	} while (datalen);
	} else {
	pg = virt_to_page(task->data);

	while (datalen) {
	i = skb_shinfo(skb)->nr_frags;
	frag = &skb_shinfo(skb)->frags[i];

	get_page(pg);
	frag->page = pg;
	frag->page_offset = 0;
	frag->size = min((unsigned int)PAGE_SIZE, datalen);

	skb_shinfo(skb)->nr_frags++;
	datalen -= frag->size;
	pg++;
	}
	}

	if (padlen) {
	i = skb_shinfo(skb)->nr_frags;
	frag = &skb_shinfo(skb)->frags[i];
	frag->page = pad_page;
	frag->page_offset = 0;
	frag->size = padlen;
	skb_shinfo(skb)->nr_frags++;
	}

	datalen = count + padlen;
	skb->data_len += datalen;
	skb->truesize += datalen;
	skb->len += datalen;
	return 0;
	}

	int cxgb3i_conn_xmit_pdu(struct iscsi_task *task)
	{
	struct iscsi_tcp_task *tcp_task = task->dd_data;
	struct sk_buff *skb = tcp_task->dd_data;
	struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data;
	struct cxgb3i_conn *cconn = tcp_conn->dd_data;
	unsigned int datalen;
	int err;

	if (!skb)
	return 0;

	datalen = skb->data_len;
	tcp_task->dd_data = NULL;
	err = cxgb3i_c3cn_send_pdus(cconn->cep->c3cn, skb);
	cxgb3i_tx_debug("task 0x%p, skb 0x%p, len %u/%u, rv %d.\n",
	task, skb, skb->len, skb->data_len, err);
	if (err > 0) {
	int pdulen = err;

	if (task->conn->hdrdgst_en)
	pdulen += ISCSI_DIGEST_SIZE;
	if (datalen && task->conn->datadgst_en)
	pdulen += ISCSI_DIGEST_SIZE;

	task->conn->txdata_octets += pdulen;
	return 0;
	}

	if (err < 0 && err != -EAGAIN) {
	kfree_skb(skb);
	cxgb3i_tx_debug("itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n",
	task->itt, skb, skb->len, skb->data_len, err);
	iscsi_conn_printk(KERN_ERR, task->conn, "xmit err %d.\n", err);
	iscsi_conn_failure(task->conn, ISCSI_ERR_XMIT_FAILED);
	return err;
	}
	/* reset skb to send when we are called again */
	tcp_task->dd_data = skb;
	return -EAGAIN;
	}

	int cxgb3i_pdu_init(void)
	{
	pad_page = alloc_page(GFP_KERNEL);
	if (!pad_page)
	return -ENOMEM;
	memset(page_address(pad_page), 0, PAGE_SIZE);
	return 0;
	}

	void cxgb3i_pdu_cleanup(void)
	{
	if (pad_page) {
	__free_page(pad_page);
	pad_page = NULL;
	}
	}

	void cxgb3i_conn_pdu_ready(struct s3_conn *c3cn)
	{
	struct sk_buff *skb;
	unsigned int read = 0;
	struct iscsi_conn *conn = c3cn->user_data;
	int err = 0;

	cxgb3i_rx_debug("cn 0x%p.\n", c3cn);

	read_lock(&c3cn->callback_lock);
	if (unlikely(!conn \|\| conn->suspend_rx)) {
	cxgb3i_rx_debug("conn 0x%p, id %d, suspend_rx %lu!\n",
	conn, conn ? conn->id : 0xFF,
	conn ? conn->suspend_rx : 0xFF);
	read_unlock(&c3cn->callback_lock);
	return;
	}
	skb = skb_peek(&c3cn->receive_queue);
	while (!err && skb) {
	__skb_unlink(skb, &c3cn->receive_queue);
	read += skb_ulp_pdulen(skb);
	err = cxgb3i_conn_read_pdu_skb(conn, skb);
	__kfree_skb(skb);
	skb = skb_peek(&c3cn->receive_queue);
	}
	read_unlock(&c3cn->callback_lock);
	if (c3cn) {
	c3cn->copied_seq += read;
	cxgb3i_c3cn_rx_credits(c3cn, read);
	}
	conn->rxdata_octets += read;
	}

	void cxgb3i_conn_tx_open(struct s3_conn *c3cn)
	{
	struct iscsi_conn *conn = c3cn->user_data;

	cxgb3i_tx_debug("cn 0x%p.\n", c3cn);
	if (conn) {
	cxgb3i_tx_debug("cn 0x%p, cid %d.\n", c3cn, conn->id);
	scsi_queue_work(conn->session->host, &conn->xmitwork);
	}
	}

	void cxgb3i_conn_closing(struct s3_conn *c3cn)
	{
	struct iscsi_conn *conn;

	read_lock(&c3cn->callback_lock);
	conn = c3cn->user_data;
	if (conn && c3cn->state != C3CN_STATE_ESTABLISHED)
	iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
	read_unlock(&c3cn->callback_lock);
	}