set_bit(XMSTATE_BIT_CMD_HDR_XMIT, &tcp_ctask->xmstate);	}	if (test_bit(XMSTATE_BIT_CMD_HDR_XMIT, &tcp_ctask->xmstate)) {		rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->imm_count);		if (rc)			return rc;		clear_bit(XMSTATE_BIT_CMD_HDR_XMIT, &tcp_ctask->xmstate);		if (sc->sc_data_direction != DMA_TO_DEVICE)			return 0;		if (ctask->imm_count) {			set_bit(XMSTATE_BIT_IMM_DATA, &tcp_ctask->xmstate);			iscsi_set_padding(tcp_ctask, ctask->imm_count);			if (ctask->conn->datadgst_en) {				iscsi_data_digest_init(ctask->conn->dd_data,						       tcp_ctask);				tcp_ctask->immdigest = 0;			}		}		if (ctask->unsol_count) {			set_bit(XMSTATE_BIT_UNS_HDR, &tcp_ctask->xmstate);			set_bit(XMSTATE_BIT_UNS_INIT, &tcp_ctask->xmstate);		}	}	return rc;}static intiscsi_send_padding(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask){	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;	int sent = 0, rc;	if (test_bit(XMSTATE_BIT_W_PAD, &tcp_ctask->xmstate)) {		iscsi_buf_init_iov(&tcp_ctask->sendbuf, (char*)&tcp_ctask->pad,				   tcp_ctask->pad_count);		if (conn->datadgst_en)			crypto_hash_update(&tcp_conn->tx_hash,					   &tcp_ctask->sendbuf.sg,					   tcp_ctask->sendbuf.sg.length);	} else if (!test_bit(XMSTATE_BIT_W_RESEND_PAD, &tcp_ctask->xmstate))		return 0;	clear_bit(XMSTATE_BIT_W_PAD, &tcp_ctask->xmstate);	clear_bit(XMSTATE_BIT_W_RESEND_PAD, &tcp_ctask->xmstate);	debug_scsi("sending %d pad bytes for itt 0x%x\n",		   tcp_ctask->pad_count, ctask->itt);	rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, &tcp_ctask->pad_count,			   &sent);	if (rc) {		debug_scsi("padding send failed %d\n", rc);		set_bit(XMSTATE_BIT_W_RESEND_PAD, &tcp_ctask->xmstate);	}	return rc;}static intiscsi_send_digest(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,			struct iscsi_buf *buf, uint32_t *digest){	struct iscsi_tcp_cmd_task *tcp_ctask;	struct iscsi_tcp_conn *tcp_conn;	int rc, sent = 0;	if (!conn->datadgst_en)		return 0;	tcp_ctask = ctask->dd_data;	tcp_conn = conn->dd_data;	if (!test_bit(XMSTATE_BIT_W_RESEND_DATA_DIGEST, &tcp_ctask->xmstate)) {		crypto_hash_final(&tcp_conn->tx_hash, (u8*)digest);		iscsi_buf_init_iov(buf, (char*)digest, 4);	}	clear_bit(XMSTATE_BIT_W_RESEND_DATA_DIGEST, &tcp_ctask->xmstate);	rc = iscsi_sendpage(conn, buf, &tcp_ctask->digest_count, &sent);	if (!rc)		debug_scsi("sent digest 0x%x for itt 0x%x\n", *digest,			  ctask->itt);	else {		debug_scsi("sending digest 0x%x failed for itt 0x%x!\n",			  *digest, ctask->itt);		set_bit(XMSTATE_BIT_W_RESEND_DATA_DIGEST, &tcp_ctask->xmstate);	}	return rc;}static intiscsi_send_data(struct iscsi_cmd_task *ctask, struct iscsi_buf *sendbuf,		struct scatterlist **sg, int *sent, int *count,		struct iscsi_buf *digestbuf, uint32_t *digest){	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;	struct iscsi_conn *conn = ctask->conn;	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;	int rc, buf_sent, offset;	while (*count) {		buf_sent = 0;		offset = sendbuf->sent;		rc = iscsi_sendpage(conn, sendbuf, count, &buf_sent);		*sent = *sent + buf_sent;		if (buf_sent && conn->datadgst_en)			partial_sg_digest_update(&tcp_conn->tx_hash,				&sendbuf->sg, sendbuf->sg.offset + offset,				buf_sent);		if (!iscsi_buf_left(sendbuf) && *sg != tcp_ctask->bad_sg) {			iscsi_buf_init_sg(sendbuf, *sg);			*sg = *sg + 1;		}		if (rc)			return rc;	}	rc = iscsi_send_padding(conn, ctask);	if (rc)		return rc;	return iscsi_send_digest(conn, ctask, digestbuf, digest);}static intiscsi_send_unsol_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask){	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;	struct iscsi_data_task *dtask;	int rc;	set_bit(XMSTATE_BIT_UNS_DATA, &tcp_ctask->xmstate);	if (test_bit(XMSTATE_BIT_UNS_INIT, &tcp_ctask->xmstate)) {		dtask = &tcp_ctask->unsol_dtask;		iscsi_prep_unsolicit_data_pdu(ctask, &dtask->hdr);		iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)&dtask->hdr,				   sizeof(struct iscsi_hdr));		if (conn->hdrdgst_en)			iscsi_hdr_digest(conn, &tcp_ctask->headbuf,					(u8*)dtask->hdrext);		clear_bit(XMSTATE_BIT_UNS_INIT, &tcp_ctask->xmstate);		iscsi_set_padding(tcp_ctask, ctask->data_count);	}	rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->data_count);	if (rc) {		clear_bit(XMSTATE_BIT_UNS_DATA, &tcp_ctask->xmstate);		set_bit(XMSTATE_BIT_UNS_HDR, &tcp_ctask->xmstate);		return rc;	}	if (conn->datadgst_en) {		dtask = &tcp_ctask->unsol_dtask;		iscsi_data_digest_init(ctask->conn->dd_data, tcp_ctask);		dtask->digest = 0;	}	debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",		   ctask->itt, ctask->unsol_count, tcp_ctask->sent);	return 0;}static intiscsi_send_unsol_pdu(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask){	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;	int rc;	if (test_and_clear_bit(XMSTATE_BIT_UNS_HDR, &tcp_ctask->xmstate)) {		BUG_ON(!ctask->unsol_count);send_hdr:		rc = iscsi_send_unsol_hdr(conn, ctask);		if (rc)			return rc;	}	if (test_bit(XMSTATE_BIT_UNS_DATA, &tcp_ctask->xmstate)) {		struct iscsi_data_task *dtask = &tcp_ctask->unsol_dtask;		int start = tcp_ctask->sent;		rc = iscsi_send_data(ctask, &tcp_ctask->sendbuf, &tcp_ctask->sg,				     &tcp_ctask->sent, &ctask->data_count,				     &dtask->digestbuf, &dtask->digest);		ctask->unsol_count -= tcp_ctask->sent - start;		if (rc)			return rc;		clear_bit(XMSTATE_BIT_UNS_DATA, &tcp_ctask->xmstate);		/*		 * Done with the Data-Out. Next, check if we need		 * to send another unsolicited Data-Out.		 */		if (ctask->unsol_count) {			debug_scsi("sending more uns\n");			set_bit(XMSTATE_BIT_UNS_INIT, &tcp_ctask->xmstate);			goto send_hdr;		}	}	return 0;}static int iscsi_send_sol_pdu(struct iscsi_conn *conn,			      struct iscsi_cmd_task *ctask){	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;	struct iscsi_session *session = conn->session;	struct iscsi_r2t_info *r2t;	struct iscsi_data_task *dtask;	int left, rc;	if (test_bit(XMSTATE_BIT_SOL_HDR_INIT, &tcp_ctask->xmstate)) {		if (!tcp_ctask->r2t) {			spin_lock_bh(&session->lock);			__kfifo_get(tcp_ctask->r2tqueue, (void*)&tcp_ctask->r2t,				    sizeof(void*));			spin_unlock_bh(&session->lock);		}send_hdr:		r2t = tcp_ctask->r2t;		dtask = &r2t->dtask;		if (conn->hdrdgst_en)			iscsi_hdr_digest(conn, &r2t->headbuf,					(u8*)dtask->hdrext);		clear_bit(XMSTATE_BIT_SOL_HDR_INIT, &tcp_ctask->xmstate);		set_bit(XMSTATE_BIT_SOL_HDR, &tcp_ctask->xmstate);	}	if (test_bit(XMSTATE_BIT_SOL_HDR, &tcp_ctask->xmstate)) {		r2t = tcp_ctask->r2t;		dtask = &r2t->dtask;		rc = iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count);		if (rc)			return rc;		clear_bit(XMSTATE_BIT_SOL_HDR, &tcp_ctask->xmstate);		set_bit(XMSTATE_BIT_SOL_DATA, &tcp_ctask->xmstate);		if (conn->datadgst_en) {			iscsi_data_digest_init(conn->dd_data, tcp_ctask);			dtask->digest = 0;		}		iscsi_set_padding(tcp_ctask, r2t->data_count);		debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",			r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,			r2t->sent);	}	if (test_bit(XMSTATE_BIT_SOL_DATA, &tcp_ctask->xmstate)) {		r2t = tcp_ctask->r2t;		dtask = &r2t->dtask;		rc = iscsi_send_data(ctask, &r2t->sendbuf, &r2t->sg,				     &r2t->sent, &r2t->data_count,				     &dtask->digestbuf, &dtask->digest);		if (rc)			return rc;		clear_bit(XMSTATE_BIT_SOL_DATA, &tcp_ctask->xmstate);		/*		 * Done with this Data-Out. Next, check if we have		 * to send another Data-Out for this R2T.		 */		BUG_ON(r2t->data_length - r2t->sent < 0);		left = r2t->data_length - r2t->sent;		if (left) {			iscsi_solicit_data_cont(conn, ctask, r2t, left);			goto send_hdr;		}		/*		 * Done with this R2T. Check if there are more		 * outstanding R2Ts ready to be processed.		 */		spin_lock_bh(&session->lock);		tcp_ctask->r2t = NULL;		__kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,			    sizeof(void*));		if (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t,				sizeof(void*))) {			tcp_ctask->r2t = r2t;			spin_unlock_bh(&session->lock);			goto send_hdr;		}		spin_unlock_bh(&session->lock);	}	return 0;}/** * iscsi_tcp_ctask_xmit - xmit normal PDU task * @conn: iscsi connection * @ctask: iscsi command task * * Notes: *	The function can return -EAGAIN in which case caller must *	call it again later, or recover. '0' return code means successful *	xmit. *	The function is devided to logical helpers (above) for the different *	xmit stages. * *iscsi_send_cmd_hdr() *	XMSTATE_BIT_CMD_HDR_INIT - prepare Header and Data buffers Calculate *	                           Header Digest *	XMSTATE_BIT_CMD_HDR_XMIT - Transmit header in progress * *iscsi_send_padding *	XMSTATE_BIT_W_PAD        - Prepare and send pading *	XMSTATE_BIT_W_RESEND_PAD - retry send pading * *iscsi_send_digest *	XMSTATE_BIT_W_RESEND_DATA_DIGEST - Finalize and send Data Digest *	XMSTATE_BIT_W_RESEND_DATA_DIGEST - retry sending digest * *iscsi_send_unsol_hdr *	XMSTATE_BIT_UNS_INIT     - prepare un-solicit data header and digest *	XMSTATE_BIT_UNS_HDR      - send un-solicit header * *iscsi_send_unsol_pdu *	XMSTATE_BIT_UNS_DATA     - send un-solicit data in progress * *iscsi_send_sol_pdu *	XMSTATE_BIT_SOL_HDR_INIT - solicit data header and digest initialize *	XMSTATE_BIT_SOL_HDR      - send solicit header *	XMSTATE_BIT_SOL_DATA     - send solicit data * *iscsi_tcp_ctask_xmit *	XMSTATE_BIT_IMM_DATA     - xmit managment data (??) **/static intiscsi_tcp_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask){	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;	int rc = 0;	debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n",		conn->id, tcp_ctask->xmstate, ctask->itt);	rc = iscsi_send_cmd_hdr(conn, ctask);	if (rc)		return rc;	if (ctask->sc->sc_data_direction != DMA_TO_DEVICE)		return 0;	if (test_bit(XMSTATE_BIT_IMM_DATA, &tcp_ctask->xmstate)) {		rc = iscsi_send_data(ctask, &tcp_ctask->sendbuf, &tcp_ctask->sg,				     &tcp_ctask->sent, &ctask->imm_count,				     &tcp_ctask->immbuf, &tcp_ctask->immdigest);		if (rc)			return rc;		clear_bit(XMSTATE_BIT_IMM_DATA, &tcp_ctask->xmstate);	}	rc = iscsi_send_unsol_pdu(conn, ctask);	if (rc)		return rc;	rc = iscsi_send_sol_pdu(conn, ctask);	if (rc)		return rc;	return rc;}static struct iscsi_cls_conn *iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx){	struct iscsi_conn *conn;	struct iscsi_cls_conn *cls_conn;	struct iscsi_tcp_conn *tcp_conn;	cls_conn = iscsi_conn_setup(cls_session, conn_idx);	if (!cls_conn)		return NULL;	conn = cls_conn->dd_data;	/*	 * due to strange issues with iser these are not set	 * in iscsi_conn_setup	 */	conn->max_recv_dlength = ISCSI_DEF_MAX_RECV_SEG_LEN;	tcp_conn = kzalloc(sizeof(*tcp_conn), GFP_KERNEL);	if (!tcp_conn)		goto tcp_conn_alloc_fail;	conn->dd_data = tcp_conn;	tcp_conn->iscsi_conn = conn;	tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;	/* initial operational parameters */	tcp_conn->hdr_size = sizeof(struct iscsi_hdr);	tcp_conn->tx_hash.tfm = crypto_alloc_hash("crc32c", 0,						  CRYPTO_ALG_ASYNC);	tcp_conn->tx_hash.flags = 0;	if (IS_ERR(tcp_conn->tx_hash.tfm)) {		printk(KERN_ERR "Could not create connection due to crc32c "		       "loading error %ld. Make sure the crc32c module is "		       "built as a module or into the kernel\n",			PTR_ERR(tcp_conn->tx_hash.tfm));		goto free_tcp_conn;	}	tcp_conn->rx_hash.tfm = crypto_alloc_hash("crc32c", 0,						  CRYPTO_ALG_ASYNC);	tcp_conn->rx_hash.flags = 0;	if (IS_ERR(tcp_conn->rx_hash.tfm)) {		printk(KERN_ERR "Could not create connection due to crc32c "		       "loading error %ld. Make sure the crc32c module is "		       "built as a module or into the kernel\n",			PTR_ERR(tcp_conn->rx_hash.tfm));		goto free_tx_tfm;	}	return cls_conn;free_tx_tfm:	crypto_free_hash(tcp_conn->tx_hash.tfm);free_tcp_conn:	kfree(tcp_conn);tcp_conn_alloc_fail:	iscsi_conn_teardown(cls_conn);	return NULL;}static voidiscsi_tcp_release_conn(struct iscsi_conn *conn){	struct iscsi_session *session = conn->session;	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;	struct socket *sock = tcp_conn->sock;	if (!sock)		return;	sock_hold(sock->sk);	iscsi_conn_restore_callbacks(tcp_conn);	sock_put(sock->sk);	spin_lock_bh(&session->lock);	tcp_conn->sock = NULL;	conn->recv_lock = NULL;	spin_unlock_bh(&session->lock);	sockfd_put(sock);}static voidiscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn){	struct iscsi_conn *conn = cls_conn->dd_data;	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;	iscsi_tcp_release_conn(conn);	iscsi_conn_teardown(cls_conn);	if (tcp_conn->tx_hash.tfm)		crypto_free_hash(tcp_conn->tx_hash.tfm);	if (tcp_conn->rx_hash.tfm)