iscsi_tcp.c 57.8 KB
Newer Older
1 2 3 4 5
/*
 * iSCSI Initiator over TCP/IP Data-Path
 *
 * Copyright (C) 2004 Dmitry Yusupov
 * Copyright (C) 2004 Alex Aizman
6 7
 * Copyright (C) 2005 - 2006 Mike Christie
 * Copyright (C) 2006 Red Hat, Inc.  All rights reserved.
8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
 * maintained by open-iscsi@googlegroups.com
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published
 * by the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.
 *
 * See the file COPYING included with this distribution for more details.
 *
 * Credits:
 *	Christoph Hellwig
 *	FUJITA Tomonori
 *	Arne Redlich
 *	Zhenyu Wang
 */

#include <linux/types.h>
#include <linux/list.h>
#include <linux/inet.h>
#include <linux/blkdev.h>
#include <linux/crypto.h>
#include <linux/delay.h>
#include <linux/kfifo.h>
#include <linux/scatterlist.h>
37
#include <linux/mutex.h>
38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
#include <net/tcp.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_host.h>
#include <scsi/scsi.h>
#include <scsi/scsi_transport_iscsi.h>

#include "iscsi_tcp.h"

MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
	      "Alex Aizman <itn780@yahoo.com>");
MODULE_DESCRIPTION("iSCSI/TCP data-path");
MODULE_LICENSE("GPL");
/* #define DEBUG_TCP */
#define DEBUG_ASSERT

#ifdef DEBUG_TCP
54
#define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt)
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
#else
#define debug_tcp(fmt...)
#endif

#ifndef DEBUG_ASSERT
#ifdef BUG_ON
#undef BUG_ON
#endif
#define BUG_ON(expr)
#endif

static unsigned int iscsi_max_lun = 512;
module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);

static inline void
iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size)
{
72 73
	ibuf->sg.page = virt_to_page(vbuf);
	ibuf->sg.offset = offset_in_page(vbuf);
74 75
	ibuf->sg.length = size;
	ibuf->sent = 0;
76
	ibuf->use_sendmsg = 1;
77 78 79 80 81
}

static inline void
iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg)
{
82 83 84
	ibuf->sg.page = sg->page;
	ibuf->sg.offset = sg->offset;
	ibuf->sg.length = sg->length;
85 86 87
	/*
	 * Fastpath: sg element fits into single page
	 */
M
Mike Christie 已提交
88
	if (sg->length + sg->offset <= PAGE_SIZE && !PageSlab(sg->page))
89 90 91
		ibuf->use_sendmsg = 0;
	else
		ibuf->use_sendmsg = 1;
92 93 94 95 96 97 98 99 100 101 102 103 104 105
	ibuf->sent = 0;
}

static inline int
iscsi_buf_left(struct iscsi_buf *ibuf)
{
	int rc;

	rc = ibuf->sg.length - ibuf->sent;
	BUG_ON(rc < 0);
	return rc;
}

static inline void
106 107
iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf,
		 u8* crc)
108
{
109
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
110

111
	crypto_hash_digest(&tcp_conn->tx_hash, &buf->sg, buf->sg.length, crc);
112
	buf->sg.length = tcp_conn->hdr_size;
113 114 115
}

static inline int
116
iscsi_hdr_extract(struct iscsi_tcp_conn *tcp_conn)
117
{
118
	struct sk_buff *skb = tcp_conn->in.skb;
119

120
	tcp_conn->in.zero_copy_hdr = 0;
121

122 123
	if (tcp_conn->in.copy >= tcp_conn->hdr_size &&
	    tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
124 125 126 127 128
		/*
		 * Zero-copy PDU Header: using connection context
		 * to store header pointer.
		 */
		if (skb_shinfo(skb)->frag_list == NULL &&
129 130 131 132 133
		    !skb_shinfo(skb)->nr_frags) {
			tcp_conn->in.hdr = (struct iscsi_hdr *)
				((char*)skb->data + tcp_conn->in.offset);
			tcp_conn->in.zero_copy_hdr = 1;
		} else {
134 135
			/* ignoring return code since we checked
			 * in.copy before */
136 137 138
			skb_copy_bits(skb, tcp_conn->in.offset,
				&tcp_conn->hdr, tcp_conn->hdr_size);
			tcp_conn->in.hdr = &tcp_conn->hdr;
139
		}
140 141
		tcp_conn->in.offset += tcp_conn->hdr_size;
		tcp_conn->in.copy -= tcp_conn->hdr_size;
142 143 144 145 146 147 148 149 150
	} else {
		int hdr_remains;
		int copylen;

		/*
		 * PDU header scattered across SKB's,
		 * copying it... This'll happen quite rarely.
		 */

151 152
		if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER)
			tcp_conn->in.hdr_offset = 0;
153

154
		hdr_remains = tcp_conn->hdr_size - tcp_conn->in.hdr_offset;
155 156
		BUG_ON(hdr_remains <= 0);

157 158 159 160
		copylen = min(tcp_conn->in.copy, hdr_remains);
		skb_copy_bits(skb, tcp_conn->in.offset,
			(char*)&tcp_conn->hdr + tcp_conn->in.hdr_offset,
			copylen);
161 162

		debug_tcp("PDU gather offset %d bytes %d in.offset %d "
163 164
		       "in.copy %d\n", tcp_conn->in.hdr_offset, copylen,
		       tcp_conn->in.offset, tcp_conn->in.copy);
165

166 167
		tcp_conn->in.offset += copylen;
		tcp_conn->in.copy -= copylen;
168
		if (copylen < hdr_remains)  {
169 170
			tcp_conn->in_progress = IN_PROGRESS_HEADER_GATHER;
			tcp_conn->in.hdr_offset += copylen;
171 172
		        return -EAGAIN;
		}
173 174 175
		tcp_conn->in.hdr = &tcp_conn->hdr;
		tcp_conn->discontiguous_hdr_cnt++;
	        tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
176 177 178 179 180
	}

	return 0;
}

M
Mike Christie 已提交
181 182 183 184
/*
 * must be called with session lock
 */
static void
185
iscsi_tcp_cleanup_ctask(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
186
{
187
	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
188
	struct iscsi_r2t_info *r2t;
M
Mike Christie 已提交
189
	struct scsi_cmnd *sc;
190

191 192 193 194 195 196 197
	/* flush ctask's r2t queues */
	while (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
		__kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,
			    sizeof(void*));
		debug_scsi("iscsi_tcp_cleanup_ctask pending r2t dropped\n");
	}

M
Mike Christie 已提交
198 199
	sc = ctask->sc;
	if (unlikely(!sc))
200
		return;
M
Mike Christie 已提交
201

202 203
	tcp_ctask->xmstate = XMSTATE_IDLE;
	tcp_ctask->r2t = NULL;
204 205 206 207 208 209 210 211 212 213 214
}

/**
 * iscsi_data_rsp - SCSI Data-In Response processing
 * @conn: iscsi connection
 * @ctask: scsi command task
 **/
static int
iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
{
	int rc;
215 216 217
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
	struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr;
218 219 220 221 222 223 224 225 226
	struct iscsi_session *session = conn->session;
	int datasn = be32_to_cpu(rhdr->datasn);

	rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
	if (rc)
		return rc;
	/*
	 * setup Data-In byte counter (gets decremented..)
	 */
227
	ctask->data_count = tcp_conn->in.datalen;
228

229
	if (tcp_conn->in.datalen == 0)
230 231 232 233 234 235 236
		return 0;

	if (ctask->datasn != datasn)
		return ISCSI_ERR_DATASN;

	ctask->datasn++;

237 238
	tcp_ctask->data_offset = be32_to_cpu(rhdr->offset);
	if (tcp_ctask->data_offset + tcp_conn->in.datalen > ctask->total_length)
239 240 241 242 243 244
		return ISCSI_ERR_DATA_OFFSET;

	if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
		struct scsi_cmnd *sc = ctask->sc;

		conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
245
		if (rhdr->flags & ISCSI_FLAG_DATA_UNDERFLOW) {
246 247 248 249 250 251 252 253 254
			int res_count = be32_to_cpu(rhdr->residual_count);

			if (res_count > 0 &&
			    res_count <= sc->request_bufflen) {
				sc->resid = res_count;
				sc->result = (DID_OK << 16) | rhdr->cmd_status;
			} else
				sc->result = (DID_BAD_TARGET << 16) |
					rhdr->cmd_status;
255
		} else if (rhdr->flags & ISCSI_FLAG_DATA_OVERFLOW) {
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284
			sc->resid = be32_to_cpu(rhdr->residual_count);
			sc->result = (DID_OK << 16) | rhdr->cmd_status;
		} else
			sc->result = (DID_OK << 16) | rhdr->cmd_status;
	}

	conn->datain_pdus_cnt++;
	return 0;
}

/**
 * iscsi_solicit_data_init - initialize first Data-Out
 * @conn: iscsi connection
 * @ctask: scsi command task
 * @r2t: R2T info
 *
 * Notes:
 *	Initialize first Data-Out within this R2T sequence and finds
 *	proper data_offset within this SCSI command.
 *
 *	This function is called with connection lock taken.
 **/
static void
iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
			struct iscsi_r2t_info *r2t)
{
	struct iscsi_data *hdr;
	struct scsi_cmnd *sc = ctask->sc;

285
	hdr = &r2t->dtask.hdr;
286 287 288 289 290
	memset(hdr, 0, sizeof(struct iscsi_data));
	hdr->ttt = r2t->ttt;
	hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
	r2t->solicit_datasn++;
	hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
291 292
	memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
	hdr->itt = ctask->hdr->itt;
293 294 295 296 297 298 299 300 301 302 303 304 305 306 307
	hdr->exp_statsn = r2t->exp_statsn;
	hdr->offset = cpu_to_be32(r2t->data_offset);
	if (r2t->data_length > conn->max_xmit_dlength) {
		hton24(hdr->dlength, conn->max_xmit_dlength);
		r2t->data_count = conn->max_xmit_dlength;
		hdr->flags = 0;
	} else {
		hton24(hdr->dlength, r2t->data_length);
		r2t->data_count = r2t->data_length;
		hdr->flags = ISCSI_FLAG_CMD_FINAL;
	}
	conn->dataout_pdus_cnt++;

	r2t->sent = 0;

308
	iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
309
			   sizeof(struct iscsi_hdr));
310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337

	if (sc->use_sg) {
		int i, sg_count = 0;
		struct scatterlist *sg = sc->request_buffer;

		r2t->sg = NULL;
		for (i = 0; i < sc->use_sg; i++, sg += 1) {
			/* FIXME: prefetch ? */
			if (sg_count + sg->length > r2t->data_offset) {
				int page_offset;

				/* sg page found! */

				/* offset within this page */
				page_offset = r2t->data_offset - sg_count;

				/* fill in this buffer */
				iscsi_buf_init_sg(&r2t->sendbuf, sg);
				r2t->sendbuf.sg.offset += page_offset;
				r2t->sendbuf.sg.length -= page_offset;

				/* xmit logic will continue with next one */
				r2t->sg = sg + 1;
				break;
			}
			sg_count += sg->length;
		}
		BUG_ON(r2t->sg == NULL);
338 339
	} else {
		iscsi_buf_init_iov(&r2t->sendbuf,
340 341
			    (char*)sc->request_buffer + r2t->data_offset,
			    r2t->data_count);
342 343
		r2t->sg = NULL;
	}
344 345 346 347 348 349 350 351 352 353 354 355
}

/**
 * iscsi_r2t_rsp - iSCSI R2T Response processing
 * @conn: iscsi connection
 * @ctask: scsi command task
 **/
static int
iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
{
	struct iscsi_r2t_info *r2t;
	struct iscsi_session *session = conn->session;
356 357 358
	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
	struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr;
359 360 361
	int r2tsn = be32_to_cpu(rhdr->r2tsn);
	int rc;

362 363 364
	if (tcp_conn->in.datalen) {
		printk(KERN_ERR "iscsi_tcp: invalid R2t with datalen %d\n",
		       tcp_conn->in.datalen);
365
		return ISCSI_ERR_DATALEN;
366
	}
367

368
	if (tcp_ctask->exp_r2tsn && tcp_ctask->exp_r2tsn != r2tsn)
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385
		return ISCSI_ERR_R2TSN;

	rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
	if (rc)
		return rc;

	/* FIXME: use R2TSN to detect missing R2T */

	/* fill-in new R2T associated with the task */
	spin_lock(&session->lock);
	if (!ctask->sc || ctask->mtask ||
	     session->state != ISCSI_STATE_LOGGED_IN) {
		printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in "
		       "recovery...\n", ctask->itt);
		spin_unlock(&session->lock);
		return 0;
	}
386

387
	rc = __kfifo_get(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
388 389 390 391
	BUG_ON(!rc);

	r2t->exp_statsn = rhdr->statsn;
	r2t->data_length = be32_to_cpu(rhdr->data_length);
392 393
	if (r2t->data_length == 0) {
		printk(KERN_ERR "iscsi_tcp: invalid R2T with zero data len\n");
394 395 396 397
		spin_unlock(&session->lock);
		return ISCSI_ERR_DATALEN;
	}

398 399 400 401 402
	if (r2t->data_length > session->max_burst)
		debug_scsi("invalid R2T with data len %u and max burst %u."
			   "Attempting to execute request.\n",
			    r2t->data_length, session->max_burst);

403 404 405
	r2t->data_offset = be32_to_cpu(rhdr->data_offset);
	if (r2t->data_offset + r2t->data_length > ctask->total_length) {
		spin_unlock(&session->lock);
406 407 408
		printk(KERN_ERR "iscsi_tcp: invalid R2T with data len %u at "
		       "offset %u and total length %d\n", r2t->data_length,
		       r2t->data_offset, ctask->total_length);
409 410 411 412 413 414 415 416
		return ISCSI_ERR_DATALEN;
	}

	r2t->ttt = rhdr->ttt; /* no flip */
	r2t->solicit_datasn = 0;

	iscsi_solicit_data_init(conn, ctask, r2t);

417 418 419
	tcp_ctask->exp_r2tsn = r2tsn + 1;
	tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
	__kfifo_put(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*));
420
	list_move_tail(&ctask->running, &conn->xmitqueue);
421

422
	scsi_queue_work(session->host, &conn->xmitwork);
423 424 425 426 427 428 429
	conn->r2t_pdus_cnt++;
	spin_unlock(&session->lock);

	return 0;
}

static int
430
iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
431
{
432
	int rc = 0, opcode, ahslen;
433 434
	struct iscsi_hdr *hdr;
	struct iscsi_session *session = conn->session;
435 436
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
	uint32_t cdgst, rdgst = 0, itt;
437

438
	hdr = tcp_conn->in.hdr;
439 440

	/* verify PDU length */
441 442
	tcp_conn->in.datalen = ntoh24(hdr->dlength);
	if (tcp_conn->in.datalen > conn->max_recv_dlength) {
443
		printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
444
		       tcp_conn->in.datalen, conn->max_recv_dlength);
445 446
		return ISCSI_ERR_DATALEN;
	}
447
	tcp_conn->data_copied = 0;
448 449

	/* read AHS */
450 451 452 453
	ahslen = hdr->hlength << 2;
	tcp_conn->in.offset += ahslen;
	tcp_conn->in.copy -= ahslen;
	if (tcp_conn->in.copy < 0) {
454
		printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
455
		       "%d bytes\n", ahslen);
456 457 458 459
		return ISCSI_ERR_AHSLEN;
	}

	/* calculate read padding */
460 461 462 463
	tcp_conn->in.padding = tcp_conn->in.datalen & (ISCSI_PAD_LEN-1);
	if (tcp_conn->in.padding) {
		tcp_conn->in.padding = ISCSI_PAD_LEN - tcp_conn->in.padding;
		debug_scsi("read padding %d bytes\n", tcp_conn->in.padding);
464 465 466 467 468 469
	}

	if (conn->hdrdgst_en) {
		struct scatterlist sg;

		sg_init_one(&sg, (u8 *)hdr,
470
			    sizeof(struct iscsi_hdr) + ahslen);
471 472
		crypto_hash_digest(&tcp_conn->rx_hash, &sg, sg.length,
				   (u8 *)&cdgst);
473
		rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
474
				     ahslen);
475
		if (cdgst != rdgst) {
476 477
			printk(KERN_ERR "iscsi_tcp: hdrdgst error "
			       "recv 0x%x calc 0x%x\n", rdgst, cdgst);
478 479
			return ISCSI_ERR_HDR_DGST;
		}
480 481
	}

482
	opcode = hdr->opcode & ISCSI_OPCODE_MASK;
483
	/* verify itt (itt encoding: age+cid+itt) */
484 485 486 487 488 489
	rc = iscsi_verify_itt(conn, hdr, &itt);
	if (rc == ISCSI_ERR_NO_SCSI_CMD) {
		tcp_conn->in.datalen = 0; /* force drop */
		return 0;
	} else if (rc)
		return rc;
490 491

	debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
492 493
		  opcode, tcp_conn->in.offset, tcp_conn->in.copy,
		  ahslen, tcp_conn->in.datalen);
494

495 496 497 498
	switch(opcode) {
	case ISCSI_OP_SCSI_DATA_IN:
		tcp_conn->in.ctask = session->cmds[itt];
		rc = iscsi_data_rsp(conn, tcp_conn->in.ctask);
499 500
		if (rc)
			return rc;
501 502 503 504 505
		/* fall through */
	case ISCSI_OP_SCSI_CMD_RSP:
		tcp_conn->in.ctask = session->cmds[itt];
		if (tcp_conn->in.datalen)
			goto copy_hdr;
506

507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
		spin_lock(&session->lock);
		rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
		spin_unlock(&session->lock);
		break;
	case ISCSI_OP_R2T:
		tcp_conn->in.ctask = session->cmds[itt];
		if (ahslen)
			rc = ISCSI_ERR_AHSLEN;
		else if (tcp_conn->in.ctask->sc->sc_data_direction ==
								DMA_TO_DEVICE)
			rc = iscsi_r2t_rsp(conn, tcp_conn->in.ctask);
		else
			rc = ISCSI_ERR_PROTO;
		break;
	case ISCSI_OP_LOGIN_RSP:
	case ISCSI_OP_TEXT_RSP:
	case ISCSI_OP_REJECT:
	case ISCSI_OP_ASYNC_EVENT:
525 526 527 528 529 530 531 532 533 534 535 536 537 538 539
		/*
		 * It is possible that we could get a PDU with a buffer larger
		 * than 8K, but there are no targets that currently do this.
		 * For now we fail until we find a vendor that needs it
		 */
		if (DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH <
		    tcp_conn->in.datalen) {
			printk(KERN_ERR "iscsi_tcp: received buffer of len %u "
			      "but conn buffer is only %u (opcode %0x)\n",
			      tcp_conn->in.datalen,
			      DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH, opcode);
			rc = ISCSI_ERR_PROTO;
			break;
		}

540 541 542
		if (tcp_conn->in.datalen)
			goto copy_hdr;
	/* fall through */
543 544
	case ISCSI_OP_LOGOUT_RSP:
	case ISCSI_OP_NOOP_IN:
545 546 547 548 549 550 551
	case ISCSI_OP_SCSI_TMFUNC_RSP:
		rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
		break;
	default:
		rc = ISCSI_ERR_BAD_OPCODE;
		break;
	}
552 553

	return rc;
554 555 556 557 558 559 560

copy_hdr:
	/*
	 * if we did zero copy for the header but we will need multiple
	 * skbs to complete the command then we have to copy the header
	 * for later use
	 */
561
	if (tcp_conn->in.zero_copy_hdr && tcp_conn->in.copy <=
562 563 564 565 566 567 568 569 570 571
	   (tcp_conn->in.datalen + tcp_conn->in.padding +
	    (conn->datadgst_en ? 4 : 0))) {
		debug_tcp("Copying header for later use. in.copy %d in.datalen"
			  " %d\n", tcp_conn->in.copy, tcp_conn->in.datalen);
		memcpy(&tcp_conn->hdr, tcp_conn->in.hdr,
		       sizeof(struct iscsi_hdr));
		tcp_conn->in.hdr = &tcp_conn->hdr;
		tcp_conn->in.zero_copy_hdr = 0;
	}
	return 0;
572 573 574 575
}

/**
 * iscsi_ctask_copy - copy skb bits to the destanation cmd task
576
 * @conn: iscsi tcp connection
577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597
 * @ctask: scsi command task
 * @buf: buffer to copy to
 * @buf_size: size of buffer
 * @offset: offset within the buffer
 *
 * Notes:
 *	The function calls skb_copy_bits() and updates per-connection and
 *	per-cmd byte counters.
 *
 *	Read counters (in bytes):
 *
 *	conn->in.offset		offset within in progress SKB
 *	conn->in.copy		left to copy from in progress SKB
 *				including padding
 *	conn->in.copied		copied already from in progress SKB
 *	conn->data_copied	copied already from in progress buffer
 *	ctask->sent		total bytes sent up to the MidLayer
 *	ctask->data_count	left to copy from in progress Data-In
 *	buf_left		left to copy from in progress buffer
 **/
static inline int
598
iscsi_ctask_copy(struct iscsi_tcp_conn *tcp_conn, struct iscsi_cmd_task *ctask,
599 600
		void *buf, int buf_size, int offset)
{
601 602 603
	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
	int buf_left = buf_size - (tcp_conn->data_copied + offset);
	int size = min(tcp_conn->in.copy, buf_left);
604 605 606 607 608
	int rc;

	size = min(size, ctask->data_count);

	debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
609
	       size, tcp_conn->in.offset, tcp_conn->in.copied);
610 611

	BUG_ON(size <= 0);
612
	BUG_ON(tcp_ctask->sent + size > ctask->total_length);
613

614 615
	rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
			   (char*)buf + (offset + tcp_conn->data_copied), size);
616 617 618
	/* must fit into skb->len */
	BUG_ON(rc);

619 620 621 622 623
	tcp_conn->in.offset += size;
	tcp_conn->in.copy -= size;
	tcp_conn->in.copied += size;
	tcp_conn->data_copied += size;
	tcp_ctask->sent += size;
624 625
	ctask->data_count -= size;

626
	BUG_ON(tcp_conn->in.copy < 0);
627 628
	BUG_ON(ctask->data_count < 0);

629
	if (buf_size != (tcp_conn->data_copied + offset)) {
630
		if (!ctask->data_count) {
631
			BUG_ON(buf_size - tcp_conn->data_copied < 0);
632
			/* done with this PDU */
633
			return buf_size - tcp_conn->data_copied;
634 635 636 637 638
		}
		return -EAGAIN;
	}

	/* done with this buffer or with both - PDU and buffer */
639
	tcp_conn->data_copied = 0;
640 641 642 643 644
	return 0;
}

/**
 * iscsi_tcp_copy - copy skb bits to the destanation buffer
645
 * @conn: iscsi tcp connection
646 647 648 649 650 651
 *
 * Notes:
 *	The function calls skb_copy_bits() and updates per-connection
 *	byte counters.
 **/
static inline int
652
iscsi_tcp_copy(struct iscsi_conn *conn, int buf_size)
653
{
654
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
655 656
	int buf_left = buf_size - tcp_conn->data_copied;
	int size = min(tcp_conn->in.copy, buf_left);
657 658 659
	int rc;

	debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
660
	       size, tcp_conn->in.offset, tcp_conn->data_copied);
661 662
	BUG_ON(size <= 0);

663
	rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
664
			   (char*)conn->data + tcp_conn->data_copied, size);
665 666
	BUG_ON(rc);

667 668 669 670
	tcp_conn->in.offset += size;
	tcp_conn->in.copy -= size;
	tcp_conn->in.copied += size;
	tcp_conn->data_copied += size;
671

672
	if (buf_size != tcp_conn->data_copied)
673 674 675 676 677 678
		return -EAGAIN;

	return 0;
}

static inline void
679
partial_sg_digest_update(struct hash_desc *desc, struct scatterlist *sg,
680
			 int offset, int length)
681 682 683 684 685 686
{
	struct scatterlist temp;

	memcpy(&temp, sg, sizeof(struct scatterlist));
	temp.offset = offset;
	temp.length = length;
687
	crypto_hash_update(desc, &temp, length);
688 689
}

690
static void
691
iscsi_recv_digest_update(struct iscsi_tcp_conn *tcp_conn, char* buf, int len)
692 693 694 695
{
	struct scatterlist tmp;

	sg_init_one(&tmp, buf, len);
696
	crypto_hash_update(&tcp_conn->rx_hash, &tmp, len);
697 698
}

699 700
static int iscsi_scsi_data_in(struct iscsi_conn *conn)
{
701 702 703
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
	struct iscsi_cmd_task *ctask = tcp_conn->in.ctask;
	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
704
	struct scsi_cmnd *sc = ctask->sc;
705
	struct scatterlist *sg;
706 707 708 709 710 711 712 713 714
	int i, offset, rc = 0;

	BUG_ON((void*)ctask != sc->SCp.ptr);

	/*
	 * copying Data-In into the Scsi_Cmnd
	 */
	if (!sc->use_sg) {
		i = ctask->data_count;
715 716 717
		rc = iscsi_ctask_copy(tcp_conn, ctask, sc->request_buffer,
				      sc->request_bufflen,
				      tcp_ctask->data_offset);
718 719
		if (rc == -EAGAIN)
			return rc;
720
		if (conn->datadgst_en)
721 722
			iscsi_recv_digest_update(tcp_conn, sc->request_buffer,
						 i);
723 724 725 726
		rc = 0;
		goto done;
	}

727
	offset = tcp_ctask->data_offset;
728 729
	sg = sc->request_buffer;

730 731
	if (tcp_ctask->data_offset)
		for (i = 0; i < tcp_ctask->sg_count; i++)
732 733 734 735 736
			offset -= sg[i].length;
	/* we've passed through partial sg*/
	if (offset < 0)
		offset = 0;

737
	for (i = tcp_ctask->sg_count; i < sc->use_sg; i++) {
738 739 740
		char *dest;

		dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0);
741
		rc = iscsi_ctask_copy(tcp_conn, ctask, dest + sg[i].offset,
742 743 744 745 746 747 748 749
				      sg[i].length, offset);
		kunmap_atomic(dest, KM_SOFTIRQ0);
		if (rc == -EAGAIN)
			/* continue with the next SKB/PDU */
			return rc;
		if (!rc) {
			if (conn->datadgst_en) {
				if (!offset)
750
					crypto_hash_update(
751
							&tcp_conn->rx_hash,
752
							&sg[i], 1);
753
				else
754
					partial_sg_digest_update(
755
							&tcp_conn->rx_hash,
756
							&sg[i],
757 758 759 760
							sg[i].offset + offset,
							sg[i].length - offset);
			}
			offset = 0;
761
			tcp_ctask->sg_count++;
762 763 764 765 766 767 768
		}

		if (!ctask->data_count) {
			if (rc && conn->datadgst_en)
				/*
				 * data-in is complete, but buffer not...
				 */
769 770 771 772
				partial_sg_digest_update(&tcp_conn->rx_hash,
							 &sg[i],
							 sg[i].offset,
							 sg[i].length-rc);
773 774 775 776
			rc = 0;
			break;
		}

777
		if (!tcp_conn->in.copy)
778 779 780 781 782 783
			return -EAGAIN;
	}
	BUG_ON(ctask->data_count);

done:
	/* check for non-exceptional status */
784
	if (tcp_conn->in.hdr->flags & ISCSI_FLAG_DATA_STATUS) {
785 786 787
		debug_scsi("done [sc %lx res %d itt 0x%x flags 0x%x]\n",
			   (long)sc, sc->result, ctask->itt,
			   tcp_conn->in.hdr->flags);
788 789 790
		spin_lock(&conn->session->lock);
		__iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
		spin_unlock(&conn->session->lock);
791 792 793 794 795 796 797 798
	}

	return rc;
}

static int
iscsi_data_recv(struct iscsi_conn *conn)
{
799 800
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
	int rc = 0, opcode;
801

802 803
	opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK;
	switch (opcode) {
804 805 806
	case ISCSI_OP_SCSI_DATA_IN:
		rc = iscsi_scsi_data_in(conn);
		break;
807
	case ISCSI_OP_SCSI_CMD_RSP:
808 809
	case ISCSI_OP_TEXT_RSP:
	case ISCSI_OP_LOGIN_RSP:
810 811
	case ISCSI_OP_ASYNC_EVENT:
	case ISCSI_OP_REJECT:
812 813 814 815
		/*
		 * Collect data segment to the connection's data
		 * placeholder
		 */
816
		if (iscsi_tcp_copy(conn, tcp_conn->in.datalen)) {
817 818 819 820
			rc = -EAGAIN;
			goto exit;
		}

821
		rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, conn->data,
822 823
					tcp_conn->in.datalen);
		if (!rc && conn->datadgst_en && opcode != ISCSI_OP_LOGIN_RSP)
824
			iscsi_recv_digest_update(tcp_conn, conn->data,
825 826
			  			tcp_conn->in.datalen);
		break;
827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846
	default:
		BUG_ON(1);
	}
exit:
	return rc;
}

/**
 * iscsi_tcp_data_recv - TCP receive in sendfile fashion
 * @rd_desc: read descriptor
 * @skb: socket buffer
 * @offset: offset in skb
 * @len: skb->len - offset
 **/
static int
iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
		unsigned int offset, size_t len)
{
	int rc;
	struct iscsi_conn *conn = rd_desc->arg.data;
847
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
848 849 850 851 852 853 854 855
	int processed;
	char pad[ISCSI_PAD_LEN];
	struct scatterlist sg;

	/*
	 * Save current SKB and its offset in the corresponding
	 * connection context.
	 */
856 857 858 859 860 861
	tcp_conn->in.copy = skb->len - offset;
	tcp_conn->in.offset = offset;
	tcp_conn->in.skb = skb;
	tcp_conn->in.len = tcp_conn->in.copy;
	BUG_ON(tcp_conn->in.copy <= 0);
	debug_tcp("in %d bytes\n", tcp_conn->in.copy);
862 863

more:
864
	tcp_conn->in.copied = 0;
865 866 867 868 869 870 871
	rc = 0;

	if (unlikely(conn->suspend_rx)) {
		debug_tcp("conn %d Rx suspended!\n", conn->id);
		return 0;
	}

872 873 874
	if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER ||
	    tcp_conn->in_progress == IN_PROGRESS_HEADER_GATHER) {
		rc = iscsi_hdr_extract(tcp_conn);
875 876 877 878
		if (rc) {
		       if (rc == -EAGAIN)
				goto nomore;
		       else {
879
				iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
880 881 882 883 884 885 886
				return 0;
		       }
		}

		/*
		 * Verify and process incoming PDU header.
		 */
887 888
		rc = iscsi_tcp_hdr_recv(conn);
		if (!rc && tcp_conn->in.datalen) {
889
			if (conn->datadgst_en)
890
				crypto_hash_init(&tcp_conn->rx_hash);
891
			tcp_conn->in_progress = IN_PROGRESS_DATA_RECV;
892
		} else if (rc) {
893
			iscsi_conn_failure(conn, rc);
894 895 896 897
			return 0;
		}
	}

898
	if (tcp_conn->in_progress == IN_PROGRESS_DDIGEST_RECV) {
899
		uint32_t recv_digest;
900

901
		debug_tcp("extra data_recv offset %d copy %d\n",
902
			  tcp_conn->in.offset, tcp_conn->in.copy);
903 904 905 906 907 908 909 910 911
		rc = iscsi_tcp_copy(conn, sizeof(uint32_t));
		if (rc) {
			if (rc == -EAGAIN)
				goto again;
			iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
			return 0;
		}

		memcpy(&recv_digest, conn->data, sizeof(uint32_t));
912
		if (recv_digest != tcp_conn->in.datadgst) {
913 914
			debug_tcp("iscsi_tcp: data digest error!"
				  "0x%x != 0x%x\n", recv_digest,
915
				  tcp_conn->in.datadgst);
916 917 918 919 920
			iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
			return 0;
		} else {
			debug_tcp("iscsi_tcp: data digest match!"
				  "0x%x == 0x%x\n", recv_digest,
921 922
				  tcp_conn->in.datadgst);
			tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
923 924 925
		}
	}

926 927
	if (tcp_conn->in_progress == IN_PROGRESS_DATA_RECV &&
	   tcp_conn->in.copy) {
928 929

		debug_tcp("data_recv offset %d copy %d\n",
930
		       tcp_conn->in.offset, tcp_conn->in.copy);
931 932 933

		rc = iscsi_data_recv(conn);
		if (rc) {
934
			if (rc == -EAGAIN)
935
				goto again;
936
			iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
937 938
			return 0;
		}
939 940
		tcp_conn->in.copy -= tcp_conn->in.padding;
		tcp_conn->in.offset += tcp_conn->in.padding;
941
		if (conn->datadgst_en) {
942 943 944 945 946
			if (tcp_conn->in.padding) {
				debug_tcp("padding -> %d\n",
					  tcp_conn->in.padding);
				memset(pad, 0, tcp_conn->in.padding);
				sg_init_one(&sg, pad, tcp_conn->in.padding);
947
				crypto_hash_update(&tcp_conn->rx_hash,
948
						   &sg, sg.length);
949
			}
950 951
			crypto_hash_final(&tcp_conn->rx_hash,
					  (u8 *) &tcp_conn->in.datadgst);
952 953
			debug_tcp("rx digest 0x%x\n", tcp_conn->in.datadgst);
			tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
954
			tcp_conn->data_copied = 0;
955
		} else
956
			tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
957 958 959
	}

	debug_tcp("f, processed %d from out of %d padding %d\n",
960 961
	       tcp_conn->in.offset - offset, (int)len, tcp_conn->in.padding);
	BUG_ON(tcp_conn->in.offset - offset > len);
962

963
	if (tcp_conn->in.offset - offset != len) {
964
		debug_tcp("continue to process %d bytes\n",
965
		       (int)len - (tcp_conn->in.offset - offset));
966 967 968 969
		goto more;
	}

nomore:
970
	processed = tcp_conn->in.offset - offset;
971 972 973 974
	BUG_ON(processed == 0);
	return processed;

again:
975
	processed = tcp_conn->in.offset - offset;
976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992
	debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n",
	          processed, (int)len, (int)rd_desc->count);
	BUG_ON(processed == 0);
	BUG_ON(processed > len);

	conn->rxdata_octets += processed;
	return processed;
}

static void
iscsi_tcp_data_ready(struct sock *sk, int flag)
{
	struct iscsi_conn *conn = sk->sk_user_data;
	read_descriptor_t rd_desc;

	read_lock(&sk->sk_callback_lock);

993 994 995 996 997 998
	/*
	 * Use rd_desc to pass 'conn' to iscsi_tcp_data_recv.
	 * We set count to 1 because we want the network layer to
	 * hand us all the skbs that are available. iscsi_tcp_data_recv
	 * handled pdus that cross buffers or pdus that still need data.
	 */
999
	rd_desc.arg.data = conn;
1000
	rd_desc.count = 1;
1001 1002 1003 1004 1005 1006 1007 1008
	tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv);

	read_unlock(&sk->sk_callback_lock);
}

static void
iscsi_tcp_state_change(struct sock *sk)
{
1009
	struct iscsi_tcp_conn *tcp_conn;
1010 1011 1012 1013 1014 1015 1016 1017 1018
	struct iscsi_conn *conn;
	struct iscsi_session *session;
	void (*old_state_change)(struct sock *);

	read_lock(&sk->sk_callback_lock);

	conn = (struct iscsi_conn*)sk->sk_user_data;
	session = conn->session;

M
Mike Christie 已提交
1019 1020 1021
	if ((sk->sk_state == TCP_CLOSE_WAIT ||
	     sk->sk_state == TCP_CLOSE) &&
	    !atomic_read(&sk->sk_rmem_alloc)) {
1022 1023 1024 1025
		debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
		iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
	}

1026 1027
	tcp_conn = conn->dd_data;
	old_state_change = tcp_conn->old_state_change;
1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041

	read_unlock(&sk->sk_callback_lock);

	old_state_change(sk);
}

/**
 * iscsi_write_space - Called when more output buffer space is available
 * @sk: socket space is available for
 **/
static void
iscsi_write_space(struct sock *sk)
{
	struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
1042 1043 1044
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;

	tcp_conn->old_write_space(sk);
1045
	debug_tcp("iscsi_write_space: cid %d\n", conn->id);
1046
	scsi_queue_work(conn->session->host, &conn->xmitwork);
1047 1048 1049 1050 1051
}

static void
iscsi_conn_set_callbacks(struct iscsi_conn *conn)
{
1052 1053
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
	struct sock *sk = tcp_conn->sock->sk;
1054 1055 1056 1057

	/* assign new callbacks */
	write_lock_bh(&sk->sk_callback_lock);
	sk->sk_user_data = conn;
1058 1059 1060
	tcp_conn->old_data_ready = sk->sk_data_ready;
	tcp_conn->old_state_change = sk->sk_state_change;
	tcp_conn->old_write_space = sk->sk_write_space;
1061 1062 1063 1064 1065 1066 1067
	sk->sk_data_ready = iscsi_tcp_data_ready;
	sk->sk_state_change = iscsi_tcp_state_change;
	sk->sk_write_space = iscsi_write_space;
	write_unlock_bh(&sk->sk_callback_lock);
}

static void
1068
iscsi_conn_restore_callbacks(struct iscsi_tcp_conn *tcp_conn)
1069
{
1070
	struct sock *sk = tcp_conn->sock->sk;
1071 1072 1073 1074

	/* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
	write_lock_bh(&sk->sk_callback_lock);
	sk->sk_user_data    = NULL;
1075 1076 1077
	sk->sk_data_ready   = tcp_conn->old_data_ready;
	sk->sk_state_change = tcp_conn->old_state_change;
	sk->sk_write_space  = tcp_conn->old_write_space;
1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089
	sk->sk_no_check	 = 0;
	write_unlock_bh(&sk->sk_callback_lock);
}

/**
 * iscsi_send - generic send routine
 * @sk: kernel's socket
 * @buf: buffer to write from
 * @size: actual size to write
 * @flags: socket's flags
 */
static inline int
1090
iscsi_send(struct iscsi_conn *conn, struct iscsi_buf *buf, int size, int flags)
1091
{
1092 1093
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
	struct socket *sk = tcp_conn->sock;
1094
	int offset = buf->sg.offset + buf->sent, res;
1095

1096 1097 1098 1099 1100 1101 1102 1103 1104
	/*
	 * if we got use_sg=0 or are sending something we kmallocd
	 * then we did not have to do kmap (kmap returns page_address)
	 *
	 * if we got use_sg > 0, but had to drop down, we do not
	 * set clustering so this should only happen for that
	 * slab case.
	 */
	if (buf->use_sendmsg)
1105
		res = sock_no_sendpage(sk, buf->sg.page, offset, size, flags);
1106
	else
1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120
		res = tcp_conn->sendpage(sk, buf->sg.page, offset, size, flags);

	if (res >= 0) {
		conn->txdata_octets += res;
		buf->sent += res;
		return res;
	}

	tcp_conn->sendpage_failures_cnt++;
	if (res == -EAGAIN)
		res = -ENOBUFS;
	else
		iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
	return res;
1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142
}

/**
 * iscsi_sendhdr - send PDU Header via tcp_sendpage()
 * @conn: iscsi connection
 * @buf: buffer to write from
 * @datalen: lenght of data to be sent after the header
 *
 * Notes:
 *	(Tx, Fast Path)
 **/
static inline int
iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen)
{
	int flags = 0; /* MSG_DONTWAIT; */
	int res, size;

	size = buf->sg.length - buf->sent;
	BUG_ON(buf->sent + size > buf->sg.length);
	if (buf->sent + size != buf->sg.length || datalen)
		flags |= MSG_MORE;

1143
	res = iscsi_send(conn, buf, size, flags);
1144 1145 1146 1147 1148
	debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res);
	if (res >= 0) {
		if (size != res)
			return -EAGAIN;
		return 0;
1149
	}
1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174

	return res;
}

/**
 * iscsi_sendpage - send one page of iSCSI Data-Out.
 * @conn: iscsi connection
 * @buf: buffer to write from
 * @count: remaining data
 * @sent: number of bytes sent
 *
 * Notes:
 *	(Tx, Fast Path)
 **/
static inline int
iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
	       int *count, int *sent)
{
	int flags = 0; /* MSG_DONTWAIT; */
	int res, size;

	size = buf->sg.length - buf->sent;
	BUG_ON(buf->sent + size > buf->sg.length);
	if (size > *count)
		size = *count;
M
Mike Christie 已提交
1175
	if (buf->sent + size != buf->sg.length || *count != size)
1176 1177
		flags |= MSG_MORE;

1178
	res = iscsi_send(conn, buf, size, flags);
1179 1180 1181 1182 1183 1184 1185 1186
	debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n",
		  size, buf->sent, *count, *sent, res);
	if (res >= 0) {
		*count -= res;
		*sent += res;
		if (size != res)
			return -EAGAIN;
		return 0;
1187
	}
1188 1189 1190 1191 1192

	return res;
}

static inline void
1193
iscsi_data_digest_init(struct iscsi_tcp_conn *tcp_conn,
1194
		      struct iscsi_tcp_cmd_task *tcp_ctask)
1195
{
1196
	crypto_hash_init(&tcp_conn->tx_hash);
1197
	tcp_ctask->digest_count = 4;
1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220
}

/**
 * iscsi_solicit_data_cont - initialize next Data-Out
 * @conn: iscsi connection
 * @ctask: scsi command task
 * @r2t: R2T info
 * @left: bytes left to transfer
 *
 * Notes:
 *	Initialize next Data-Out within this R2T sequence and continue
 *	to process next Scatter-Gather element(if any) of this SCSI command.
 *
 *	Called under connection lock.
 **/
static void
iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
			struct iscsi_r2t_info *r2t, int left)
{
	struct iscsi_data *hdr;
	struct scsi_cmnd *sc = ctask->sc;
	int new_offset;

1221
	hdr = &r2t->dtask.hdr;
1222 1223 1224 1225 1226
	memset(hdr, 0, sizeof(struct iscsi_data));
	hdr->ttt = r2t->ttt;
	hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
	r2t->solicit_datasn++;
	hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1227 1228
	memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
	hdr->itt = ctask->hdr->itt;
1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241
	hdr->exp_statsn = r2t->exp_statsn;
	new_offset = r2t->data_offset + r2t->sent;
	hdr->offset = cpu_to_be32(new_offset);
	if (left > conn->max_xmit_dlength) {
		hton24(hdr->dlength, conn->max_xmit_dlength);
		r2t->data_count = conn->max_xmit_dlength;
	} else {
		hton24(hdr->dlength, left);
		r2t->data_count = left;
		hdr->flags = ISCSI_FLAG_CMD_FINAL;
	}
	conn->dataout_pdus_cnt++;

1242
	iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
1243
			   sizeof(struct iscsi_hdr));
1244

1245 1246 1247 1248
	if (iscsi_buf_left(&r2t->sendbuf))
		return;

	if (sc->use_sg) {
1249 1250
		iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
		r2t->sg += 1;
1251 1252
	} else {
		iscsi_buf_init_iov(&r2t->sendbuf,
1253 1254
			    (char*)sc->request_buffer + new_offset,
			    r2t->data_count);
1255 1256
		r2t->sg = NULL;
	}
1257 1258
}

1259 1260
static void iscsi_set_padding(struct iscsi_tcp_cmd_task *tcp_ctask,
			      unsigned long len)
1261
{
1262 1263 1264
	tcp_ctask->pad_count = len & (ISCSI_PAD_LEN - 1);
	if (!tcp_ctask->pad_count)
		return;
1265

1266 1267 1268
	tcp_ctask->pad_count = ISCSI_PAD_LEN - tcp_ctask->pad_count;
	debug_scsi("write padding %d bytes\n", tcp_ctask->pad_count);
	tcp_ctask->xmstate |= XMSTATE_W_PAD;
1269 1270 1271
}

/**
1272
 * iscsi_tcp_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1273 1274 1275 1276 1277
 * @conn: iscsi connection
 * @ctask: scsi command task
 * @sc: scsi command
 **/
static void
1278
iscsi_tcp_cmd_init(struct iscsi_cmd_task *ctask)
1279
{
1280 1281
	struct scsi_cmnd *sc = ctask->sc;
	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1282

1283
	BUG_ON(__kfifo_len(tcp_ctask->r2tqueue));
1284

1285 1286
	tcp_ctask->sent = 0;
	tcp_ctask->sg_count = 0;
1287 1288

	if (sc->sc_data_direction == DMA_TO_DEVICE) {
1289 1290
		tcp_ctask->xmstate = XMSTATE_W_HDR;
		tcp_ctask->exp_r2tsn = 0;
1291
		BUG_ON(ctask->total_length == 0);
1292

1293 1294 1295
		if (sc->use_sg) {
			struct scatterlist *sg = sc->request_buffer;

1296 1297
			iscsi_buf_init_sg(&tcp_ctask->sendbuf, sg);
			tcp_ctask->sg = sg + 1;
1298
			tcp_ctask->bad_sg = sg + sc->use_sg;
1299
		} else {
1300 1301 1302
			iscsi_buf_init_iov(&tcp_ctask->sendbuf,
					   sc->request_buffer,
					   sc->request_bufflen);
1303 1304
			tcp_ctask->sg = NULL;
			tcp_ctask->bad_sg = NULL;
1305
		}
1306 1307
		debug_scsi("cmd [itt 0x%x total %d imm_data %d "
			   "unsol count %d, unsol offset %d]\n",
1308
			   ctask->itt, ctask->total_length, ctask->imm_count,
1309
			   ctask->unsol_count, ctask->unsol_offset);
1310 1311
	} else
		tcp_ctask->xmstate = XMSTATE_R_HDR;
1312

1313
	iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)ctask->hdr,
1314
			    sizeof(struct iscsi_hdr));
1315 1316 1317
}

/**
1318
 * iscsi_tcp_mtask_xmit - xmit management(immediate) task
1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331
 * @conn: iscsi connection
 * @mtask: task management task
 *
 * Notes:
 *	The function can return -EAGAIN in which case caller must
 *	call it again later, or recover. '0' return code means successful
 *	xmit.
 *
 *	Management xmit state machine consists of two states:
 *		IN_PROGRESS_IMM_HEAD - PDU Header xmit in progress
 *		IN_PROGRESS_IMM_DATA - PDU Data xmit in progress
 **/
static int
1332
iscsi_tcp_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
1333
{
1334
	struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
1335
	int rc;
1336 1337

	debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n",
1338
		conn->id, tcp_mtask->xmstate, mtask->itt);
1339

1340 1341
	if (tcp_mtask->xmstate & XMSTATE_IMM_HDR) {
		tcp_mtask->xmstate &= ~XMSTATE_IMM_HDR;
1342
		if (mtask->data_count)
1343
			tcp_mtask->xmstate |= XMSTATE_IMM_DATA;
1344
		if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
M
Mike Christie 已提交
1345
		    conn->stop_stage != STOP_CONN_RECOVER &&
1346
		    conn->hdrdgst_en)
1347 1348
			iscsi_hdr_digest(conn, &tcp_mtask->headbuf,
					(u8*)tcp_mtask->hdrext);
1349 1350 1351
		rc = iscsi_sendhdr(conn, &tcp_mtask->headbuf,
				   mtask->data_count);
		if (rc) {
1352
			tcp_mtask->xmstate |= XMSTATE_IMM_HDR;
1353
			if (mtask->data_count)
1354
				tcp_mtask->xmstate &= ~XMSTATE_IMM_DATA;
1355
			return rc;
1356 1357 1358
		}
	}

1359
	if (tcp_mtask->xmstate & XMSTATE_IMM_DATA) {
1360
		BUG_ON(!mtask->data_count);
1361
		tcp_mtask->xmstate &= ~XMSTATE_IMM_DATA;
1362 1363 1364 1365
		/* FIXME: implement.
		 * Virtual buffer could be spreaded across multiple pages...
		 */
		do {
1366 1367 1368 1369 1370
			int rc;

			rc = iscsi_sendpage(conn, &tcp_mtask->sendbuf,
					&mtask->data_count, &tcp_mtask->sent);
			if (rc) {
1371
				tcp_mtask->xmstate |= XMSTATE_IMM_DATA;
1372
				return rc;
1373 1374 1375 1376
			}
		} while (mtask->data_count);
	}

1377 1378 1379 1380 1381 1382 1383 1384 1385 1386
	BUG_ON(tcp_mtask->xmstate != XMSTATE_IDLE);
	if (mtask->hdr->itt == cpu_to_be32(ISCSI_RESERVED_TAG)) {
		struct iscsi_session *session = conn->session;

		spin_lock_bh(&session->lock);
		list_del(&conn->mtask->running);
		__kfifo_put(session->mgmtpool.queue, (void*)&conn->mtask,
			    sizeof(void*));
		spin_unlock_bh(&session->lock);
	}
1387 1388 1389 1390
	return 0;
}

static inline int
1391 1392
iscsi_send_read_hdr(struct iscsi_conn *conn,
		    struct iscsi_tcp_cmd_task *tcp_ctask)
1393
{
1394 1395
	int rc;

1396
	tcp_ctask->xmstate &= ~XMSTATE_R_HDR;
1397
	if (conn->hdrdgst_en)
1398 1399
		iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
				 (u8*)tcp_ctask->hdrext);
1400 1401
	rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, 0);
	if (!rc) {
1402
		BUG_ON(tcp_ctask->xmstate != XMSTATE_IDLE);
1403 1404
		return 0; /* wait for Data-In */
	}
1405
	tcp_ctask->xmstate |= XMSTATE_R_HDR;
1406
	return rc;
1407 1408 1409
}

static inline int
1410
iscsi_send_write_hdr(struct iscsi_conn *conn,
1411
		     struct iscsi_cmd_task *ctask)
1412
{
1413
	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1414
	int rc;
1415 1416

	tcp_ctask->xmstate &= ~XMSTATE_W_HDR;
1417
	if (conn->hdrdgst_en)
1418 1419
		iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
				 (u8*)tcp_ctask->hdrext);
1420
	rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->imm_count);
1421
	if (rc) {
1422
		tcp_ctask->xmstate |= XMSTATE_W_HDR;
1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438
		return rc;
	}

	if (ctask->imm_count) {
		tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
		iscsi_set_padding(tcp_ctask, ctask->imm_count);

		if (ctask->conn->datadgst_en) {
			iscsi_data_digest_init(ctask->conn->dd_data, tcp_ctask);
			tcp_ctask->immdigest = 0;
		}
	}

	if (ctask->unsol_count)
		tcp_ctask->xmstate |= XMSTATE_UNS_HDR | XMSTATE_UNS_INIT;
	return 0;
1439 1440
}

1441 1442
static int
iscsi_send_padding(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1443
{
1444
	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1445 1446
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
	int sent = 0, rc;
1447

1448 1449 1450 1451
	if (tcp_ctask->xmstate & XMSTATE_W_PAD) {
		iscsi_buf_init_iov(&tcp_ctask->sendbuf, (char*)&tcp_ctask->pad,
				   tcp_ctask->pad_count);
		if (conn->datadgst_en)
1452 1453 1454
			crypto_hash_update(&tcp_conn->tx_hash,
					   &tcp_ctask->sendbuf.sg,
					   tcp_ctask->sendbuf.sg.length);
1455 1456 1457 1458 1459 1460 1461 1462 1463
	} else if (!(tcp_ctask->xmstate & XMSTATE_W_RESEND_PAD))
		return 0;

	tcp_ctask->xmstate &= ~XMSTATE_W_PAD;
	tcp_ctask->xmstate &= ~XMSTATE_W_RESEND_PAD;
	debug_scsi("sending %d pad bytes for itt 0x%x\n",
		   tcp_ctask->pad_count, ctask->itt);
	rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, &tcp_ctask->pad_count,
			   &sent);
1464
	if (rc) {
1465 1466
		debug_scsi("padding send failed %d\n", rc);
		tcp_ctask->xmstate |= XMSTATE_W_RESEND_PAD;
1467
	}
1468
	return rc;
1469 1470
}

1471 1472 1473
static int
iscsi_send_digest(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
			struct iscsi_buf *buf, uint32_t *digest)
1474
{
1475 1476 1477
	struct iscsi_tcp_cmd_task *tcp_ctask;
	struct iscsi_tcp_conn *tcp_conn;
	int rc, sent = 0;
1478

1479 1480
	if (!conn->datadgst_en)
		return 0;
1481

1482 1483
	tcp_ctask = ctask->dd_data;
	tcp_conn = conn->dd_data;
1484

1485
	if (!(tcp_ctask->xmstate & XMSTATE_W_RESEND_DATA_DIGEST)) {
1486
		crypto_hash_final(&tcp_conn->tx_hash, (u8*)digest);
1487 1488 1489
		iscsi_buf_init_iov(buf, (char*)digest, 4);
	}
	tcp_ctask->xmstate &= ~XMSTATE_W_RESEND_DATA_DIGEST;
1490

1491 1492 1493 1494 1495 1496 1497 1498
	rc = iscsi_sendpage(conn, buf, &tcp_ctask->digest_count, &sent);
	if (!rc)
		debug_scsi("sent digest 0x%x for itt 0x%x\n", *digest,
			  ctask->itt);
	else {
		debug_scsi("sending digest 0x%x failed for itt 0x%x!\n",
			  *digest, ctask->itt);
		tcp_ctask->xmstate |= XMSTATE_W_RESEND_DATA_DIGEST;
1499
	}
1500 1501
	return rc;
}
1502

1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519
static int
iscsi_send_data(struct iscsi_cmd_task *ctask, struct iscsi_buf *sendbuf,
		struct scatterlist **sg, int *sent, int *count,
		struct iscsi_buf *digestbuf, uint32_t *digest)
{
	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
	struct iscsi_conn *conn = ctask->conn;
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
	int rc, buf_sent, offset;

	while (*count) {
		buf_sent = 0;
		offset = sendbuf->sent;

		rc = iscsi_sendpage(conn, sendbuf, count, &buf_sent);
		*sent = *sent + buf_sent;
		if (buf_sent && conn->datadgst_en)
1520
			partial_sg_digest_update(&tcp_conn->tx_hash,
1521 1522 1523 1524 1525
				&sendbuf->sg, sendbuf->sg.offset + offset,
				buf_sent);
		if (!iscsi_buf_left(sendbuf) && *sg != tcp_ctask->bad_sg) {
			iscsi_buf_init_sg(sendbuf, *sg);
			*sg = *sg + 1;
1526
		}
1527 1528 1529

		if (rc)
			return rc;
1530 1531
	}

1532 1533 1534 1535 1536
	rc = iscsi_send_padding(conn, ctask);
	if (rc)
		return rc;

	return iscsi_send_digest(conn, ctask, digestbuf, digest);
1537 1538
}

1539 1540
static int
iscsi_send_unsol_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1541
{
1542
	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1543
	struct iscsi_data_task *dtask;
1544
	int rc;
1545

1546 1547
	tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
	if (tcp_ctask->xmstate & XMSTATE_UNS_INIT) {
1548 1549
		dtask = &tcp_ctask->unsol_dtask;

1550 1551 1552
		iscsi_prep_unsolicit_data_pdu(ctask, &dtask->hdr);
		iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)&dtask->hdr,
				   sizeof(struct iscsi_hdr));
1553
		if (conn->hdrdgst_en)
1554
			iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
1555
					(u8*)dtask->hdrext);
1556

1557
		tcp_ctask->xmstate &= ~XMSTATE_UNS_INIT;
1558
		iscsi_set_padding(tcp_ctask, ctask->data_count);
1559
	}
1560 1561 1562

	rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->data_count);
	if (rc) {
1563 1564
		tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
		tcp_ctask->xmstate |= XMSTATE_UNS_HDR;
1565
		return rc;
1566 1567
	}

1568 1569 1570 1571 1572 1573
	if (conn->datadgst_en) {
		dtask = &tcp_ctask->unsol_dtask;
		iscsi_data_digest_init(ctask->conn->dd_data, tcp_ctask);
		dtask->digest = 0;
	}

1574
	debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
1575
		   ctask->itt, ctask->unsol_count, tcp_ctask->sent);
1576 1577 1578
	return 0;
}

1579 1580
static int
iscsi_send_unsol_pdu(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1581
{
1582
	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1583
	int rc;
1584

1585 1586 1587 1588 1589 1590 1591
	if (tcp_ctask->xmstate & XMSTATE_UNS_HDR) {
		BUG_ON(!ctask->unsol_count);
		tcp_ctask->xmstate &= ~XMSTATE_UNS_HDR;
send_hdr:
		rc = iscsi_send_unsol_hdr(conn, ctask);
		if (rc)
			return rc;
1592 1593
	}

1594 1595
	if (tcp_ctask->xmstate & XMSTATE_UNS_DATA) {
		struct iscsi_data_task *dtask = &tcp_ctask->unsol_dtask;
1596
		int start = tcp_ctask->sent;
1597

1598 1599 1600
		rc = iscsi_send_data(ctask, &tcp_ctask->sendbuf, &tcp_ctask->sg,
				     &tcp_ctask->sent, &ctask->data_count,
				     &dtask->digestbuf, &dtask->digest);
1601
		ctask->unsol_count -= tcp_ctask->sent - start;
1602 1603 1604
		if (rc)
			return rc;
		tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
1605
		/*
1606 1607
		 * Done with the Data-Out. Next, check if we need
		 * to send another unsolicited Data-Out.
1608
		 */
1609 1610 1611 1612
		if (ctask->unsol_count) {
			debug_scsi("sending more uns\n");
			tcp_ctask->xmstate |= XMSTATE_UNS_INIT;
			goto send_hdr;
1613 1614 1615 1616 1617
		}
	}
	return 0;
}

1618 1619
static int iscsi_send_sol_pdu(struct iscsi_conn *conn,
			      struct iscsi_cmd_task *ctask)
1620
{
1621
	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1622 1623 1624
	struct iscsi_session *session = conn->session;
	struct iscsi_r2t_info *r2t;
	struct iscsi_data_task *dtask;
1625
	int left, rc;
1626

1627 1628
	if (tcp_ctask->xmstate & XMSTATE_SOL_HDR) {
		tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
1629
		tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
1630 1631 1632 1633 1634 1635
		if (!tcp_ctask->r2t)
			__kfifo_get(tcp_ctask->r2tqueue, (void*)&tcp_ctask->r2t,
				    sizeof(void*));
send_hdr:
		r2t = tcp_ctask->r2t;
		dtask = &r2t->dtask;
1636

1637 1638 1639 1640
		if (conn->hdrdgst_en)
			iscsi_hdr_digest(conn, &r2t->headbuf,
					(u8*)dtask->hdrext);
		rc = iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count);
1641
		if (rc) {
1642 1643
			tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
			tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
1644
			return rc;
1645 1646 1647
		}

		if (conn->datadgst_en) {
1648 1649
			iscsi_data_digest_init(conn->dd_data, tcp_ctask);
			dtask->digest = 0;
1650 1651
		}

1652 1653 1654 1655
		iscsi_set_padding(tcp_ctask, r2t->data_count);
		debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
			r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
			r2t->sent);
1656 1657
	}

1658 1659 1660
	if (tcp_ctask->xmstate & XMSTATE_SOL_DATA) {
		r2t = tcp_ctask->r2t;
		dtask = &r2t->dtask;
1661

1662 1663 1664 1665 1666 1667
		rc = iscsi_send_data(ctask, &r2t->sendbuf, &r2t->sg,
				     &r2t->sent, &r2t->data_count,
				     &dtask->digestbuf, &dtask->digest);
		if (rc)
			return rc;
		tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
1668

1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680
		/*
		 * Done with this Data-Out. Next, check if we have
		 * to send another Data-Out for this R2T.
		 */
		BUG_ON(r2t->data_length - r2t->sent < 0);
		left = r2t->data_length - r2t->sent;
		if (left) {
			iscsi_solicit_data_cont(conn, ctask, r2t, left);
			tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
			tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
			goto send_hdr;
		}
1681

1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696
		/*
		 * Done with this R2T. Check if there are more
		 * outstanding R2Ts ready to be processed.
		 */
		spin_lock_bh(&session->lock);
		tcp_ctask->r2t = NULL;
		__kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,
			    sizeof(void*));
		if (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t,
				sizeof(void*))) {
			tcp_ctask->r2t = r2t;
			tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
			tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
			spin_unlock_bh(&session->lock);
			goto send_hdr;
1697
		}
1698
		spin_unlock_bh(&session->lock);
1699 1700 1701 1702 1703
	}
	return 0;
}

static int
1704
iscsi_tcp_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1705
{
1706
	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1707 1708 1709
	int rc = 0;

	debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n",
1710
		conn->id, tcp_ctask->xmstate, ctask->itt);
1711 1712 1713 1714 1715 1716 1717

	/*
	 * serialize with TMF AbortTask
	 */
	if (ctask->mtask)
		return rc;

1718
	if (tcp_ctask->xmstate & XMSTATE_R_HDR)
1719
		return iscsi_send_read_hdr(conn, tcp_ctask);
1720

1721
	if (tcp_ctask->xmstate & XMSTATE_W_HDR) {
1722
		rc = iscsi_send_write_hdr(conn, ctask);
1723 1724 1725 1726
		if (rc)
			return rc;
	}

1727
	if (tcp_ctask->xmstate & XMSTATE_IMM_DATA) {
1728 1729 1730
		rc = iscsi_send_data(ctask, &tcp_ctask->sendbuf, &tcp_ctask->sg,
				     &tcp_ctask->sent, &ctask->imm_count,
				     &tcp_ctask->immbuf, &tcp_ctask->immdigest);
1731 1732
		if (rc)
			return rc;
1733
		tcp_ctask->xmstate &= ~XMSTATE_IMM_DATA;
1734 1735
	}

1736 1737 1738
	rc = iscsi_send_unsol_pdu(conn, ctask);
	if (rc)
		return rc;
1739

1740 1741 1742
	rc = iscsi_send_sol_pdu(conn, ctask);
	if (rc)
		return rc;
1743 1744 1745 1746

	return rc;
}

1747 1748
static struct iscsi_cls_conn *
iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
1749
{
1750 1751 1752
	struct iscsi_conn *conn;
	struct iscsi_cls_conn *cls_conn;
	struct iscsi_tcp_conn *tcp_conn;
1753

1754 1755 1756 1757
	cls_conn = iscsi_conn_setup(cls_session, conn_idx);
	if (!cls_conn)
		return NULL;
	conn = cls_conn->dd_data;
1758
	/*
1759 1760
	 * due to strange issues with iser these are not set
	 * in iscsi_conn_setup
1761
	 */
1762
	conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
1763

1764 1765 1766
	tcp_conn = kzalloc(sizeof(*tcp_conn), GFP_KERNEL);
	if (!tcp_conn)
		goto tcp_conn_alloc_fail;
1767

1768 1769 1770 1771 1772
	conn->dd_data = tcp_conn;
	tcp_conn->iscsi_conn = conn;
	tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
	/* initial operational parameters */
	tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
1773

1774 1775 1776 1777
	tcp_conn->tx_hash.tfm = crypto_alloc_hash("crc32c", 0,
						  CRYPTO_ALG_ASYNC);
	tcp_conn->tx_hash.flags = 0;
	if (!tcp_conn->tx_hash.tfm)
1778 1779
		goto free_tcp_conn;

1780 1781 1782 1783
	tcp_conn->rx_hash.tfm = crypto_alloc_hash("crc32c", 0,
						  CRYPTO_ALG_ASYNC);
	tcp_conn->rx_hash.flags = 0;
	if (!tcp_conn->rx_hash.tfm)
1784 1785
		goto free_tx_tfm;

1786
	return cls_conn;
1787

1788
free_tx_tfm:
1789
	crypto_free_hash(tcp_conn->tx_hash.tfm);
1790 1791
free_tcp_conn:
	kfree(tcp_conn);
1792 1793 1794
tcp_conn_alloc_fail:
	iscsi_conn_teardown(cls_conn);
	return NULL;
1795 1796
}

1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813
static void
iscsi_tcp_release_conn(struct iscsi_conn *conn)
{
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;

	if (!tcp_conn->sock)
		return;

	sock_hold(tcp_conn->sock->sk);
	iscsi_conn_restore_callbacks(tcp_conn);
	sock_put(tcp_conn->sock->sk);

	sock_release(tcp_conn->sock);
	tcp_conn->sock = NULL;
	conn->recv_lock = NULL;
}

1814
static void
1815
iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
1816
{
1817 1818
	struct iscsi_conn *conn = cls_conn->dd_data;
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1819

1820
	iscsi_tcp_release_conn(conn);
1821
	iscsi_conn_teardown(cls_conn);
1822

P
Pete Wyckoff 已提交
1823 1824 1825 1826
	if (tcp_conn->tx_hash.tfm)
		crypto_free_hash(tcp_conn->tx_hash.tfm);
	if (tcp_conn->rx_hash.tfm)
		crypto_free_hash(tcp_conn->rx_hash.tfm);
1827

1828 1829
	kfree(tcp_conn);
}
1830

1831 1832 1833 1834
static void
iscsi_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
{
	struct iscsi_conn *conn = cls_conn->dd_data;
1835
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1836 1837 1838

	iscsi_conn_stop(cls_conn, flag);
	iscsi_tcp_release_conn(conn);
1839
	tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
1840 1841
}

1842 1843
static int
iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
1844
		    struct iscsi_cls_conn *cls_conn, uint64_t transport_eph,
1845 1846 1847 1848 1849 1850 1851
		    int is_leading)
{
	struct iscsi_conn *conn = cls_conn->dd_data;
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
	struct sock *sk;
	struct socket *sock;
	int err;
1852

1853
	/* lookup for existing socket */
1854
	sock = sockfd_lookup((int)transport_eph, &err);
1855 1856 1857
	if (!sock) {
		printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err);
		return -EEXIST;
1858 1859
	}

1860 1861 1862
	err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
	if (err)
		return err;
1863

1864 1865
	/* bind iSCSI connection and socket */
	tcp_conn->sock = sock;
1866

1867 1868 1869 1870 1871
	/* setup Socket parameters */
	sk = sock->sk;
	sk->sk_reuse = 1;
	sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
	sk->sk_allocation = GFP_ATOMIC;
1872

1873
	/* FIXME: disable Nagle's algorithm */
1874

1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885
	/*
	 * Intercept TCP callbacks for sendfile like receive
	 * processing.
	 */
	conn->recv_lock = &sk->sk_callback_lock;
	iscsi_conn_set_callbacks(conn);
	tcp_conn->sendpage = tcp_conn->sock->ops->sendpage;
	/*
	 * set receive state machine into initial state
	 */
	tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1886 1887 1888 1889

	return 0;
}

1890
/* called with host lock */
M
Mike Christie 已提交
1891
static void
1892 1893
iscsi_tcp_mgmt_init(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask,
		    char *data, uint32_t data_size)
1894
{
1895
	struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;
1896

1897 1898
	iscsi_buf_init_iov(&tcp_mtask->headbuf, (char*)mtask->hdr,
			   sizeof(struct iscsi_hdr));
1899
	tcp_mtask->xmstate = XMSTATE_IMM_HDR;
1900
	tcp_mtask->sent = 0;
1901

1902 1903
	if (mtask->data_count)
		iscsi_buf_init_iov(&tcp_mtask->sendbuf, (char*)mtask->data,
1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917
				    mtask->data_count);
}

static int
iscsi_r2tpool_alloc(struct iscsi_session *session)
{
	int i;
	int cmd_i;

	/*
	 * initialize per-task: R2T pool and xmit queue
	 */
	for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
	        struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
1918
		struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1919 1920 1921 1922 1923 1924 1925 1926

		/*
		 * pre-allocated x4 as much r2ts to handle race when
		 * target acks DataOut faster than we data_xmit() queues
		 * could replenish r2tqueue.
		 */

		/* R2T pool */
1927 1928 1929
		if (iscsi_pool_init(&tcp_ctask->r2tpool, session->max_r2t * 4,
				    (void***)&tcp_ctask->r2ts,
				    sizeof(struct iscsi_r2t_info))) {
1930 1931 1932 1933
			goto r2t_alloc_fail;
		}

		/* R2T xmit queue */
1934
		tcp_ctask->r2tqueue = kfifo_alloc(
1935
		      session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
1936 1937 1938
		if (tcp_ctask->r2tqueue == ERR_PTR(-ENOMEM)) {
			iscsi_pool_free(&tcp_ctask->r2tpool,
					(void**)tcp_ctask->r2ts);
1939 1940 1941 1942 1943 1944 1945 1946
			goto r2t_alloc_fail;
		}
	}

	return 0;

r2t_alloc_fail:
	for (i = 0; i < cmd_i; i++) {
1947 1948 1949 1950 1951 1952
		struct iscsi_cmd_task *ctask = session->cmds[i];
		struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;

		kfifo_free(tcp_ctask->r2tqueue);
		iscsi_pool_free(&tcp_ctask->r2tpool,
				(void**)tcp_ctask->r2ts);
1953 1954 1955 1956 1957 1958 1959 1960 1961 1962
	}
	return -ENOMEM;
}

static void
iscsi_r2tpool_free(struct iscsi_session *session)
{
	int i;

	for (i = 0; i < session->cmds_max; i++) {
1963 1964
		struct iscsi_cmd_task *ctask = session->cmds[i];
		struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
1965

1966 1967 1968
		kfifo_free(tcp_ctask->r2tqueue);
		iscsi_pool_free(&tcp_ctask->r2tpool,
				(void**)tcp_ctask->r2ts);
1969 1970 1971 1972
	}
}

static int
1973
iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
1974
		     char *buf, int buflen)
1975
{
1976
	struct iscsi_conn *conn = cls_conn->dd_data;
1977
	struct iscsi_session *session = conn->session;
1978
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1979
	int value;
1980 1981 1982

	switch(param) {
	case ISCSI_PARAM_HDRDGST_EN:
1983
		iscsi_set_param(cls_conn, param, buf, buflen);
1984
		tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
1985
		if (conn->hdrdgst_en)
1986
			tcp_conn->hdr_size += sizeof(__u32);
1987 1988
		break;
	case ISCSI_PARAM_DATADGST_EN:
1989
		iscsi_set_param(cls_conn, param, buf, buflen);
1990 1991
		tcp_conn->sendpage = conn->datadgst_en ?
			sock_no_sendpage : tcp_conn->sock->ops->sendpage;
1992 1993
		break;
	case ISCSI_PARAM_MAX_R2T:
1994
		sscanf(buf, "%d", &value);
1995 1996 1997
		if (session->max_r2t == roundup_pow_of_two(value))
			break;
		iscsi_r2tpool_free(session);
1998
		iscsi_set_param(cls_conn, param, buf, buflen);
1999 2000 2001 2002 2003 2004
		if (session->max_r2t & (session->max_r2t - 1))
			session->max_r2t = roundup_pow_of_two(session->max_r2t);
		if (iscsi_r2tpool_alloc(session))
			return -ENOMEM;
		break;
	default:
2005
		return iscsi_set_param(cls_conn, param, buf, buflen);
2006 2007 2008 2009 2010 2011
	}

	return 0;
}

static int
2012 2013
iscsi_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn,
			 enum iscsi_param param, char *buf)
2014
{
2015
	struct iscsi_conn *conn = cls_conn->dd_data;
2016
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2017
	struct inet_sock *inet;
2018 2019 2020
	struct ipv6_pinfo *np;
	struct sock *sk;
	int len;
2021 2022

	switch(param) {
2023 2024
	case ISCSI_PARAM_CONN_PORT:
		mutex_lock(&conn->xmitmutex);
2025
		if (!tcp_conn->sock) {
2026 2027 2028 2029
			mutex_unlock(&conn->xmitmutex);
			return -EINVAL;
		}

2030
		inet = inet_sk(tcp_conn->sock->sk);
2031
		len = sprintf(buf, "%hu\n", be16_to_cpu(inet->dport));
2032
		mutex_unlock(&conn->xmitmutex);
2033
		break;
2034 2035
	case ISCSI_PARAM_CONN_ADDRESS:
		mutex_lock(&conn->xmitmutex);
2036
		if (!tcp_conn->sock) {
2037 2038 2039 2040
			mutex_unlock(&conn->xmitmutex);
			return -EINVAL;
		}

2041
		sk = tcp_conn->sock->sk;
2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054
		if (sk->sk_family == PF_INET) {
			inet = inet_sk(sk);
			len = sprintf(buf, "%u.%u.%u.%u\n",
				      NIPQUAD(inet->daddr));
		} else {
			np = inet6_sk(sk);
			len = sprintf(buf,
				"%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n",
				NIP6(np->daddr));
		}
		mutex_unlock(&conn->xmitmutex);
		break;
	default:
2055
		return iscsi_conn_get_param(cls_conn, param, buf);
2056 2057 2058 2059 2060
	}

	return len;
}

2061
static void
2062
iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats)
2063
{
2064
	struct iscsi_conn *conn = cls_conn->dd_data;
2065
	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077

	stats->txdata_octets = conn->txdata_octets;
	stats->rxdata_octets = conn->rxdata_octets;
	stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
	stats->dataout_pdus = conn->dataout_pdus_cnt;
	stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
	stats->datain_pdus = conn->datain_pdus_cnt;
	stats->r2t_pdus = conn->r2t_pdus_cnt;
	stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
	stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
	stats->custom_length = 3;
	strcpy(stats->custom[0].desc, "tx_sendpage_failures");
2078
	stats->custom[0].value = tcp_conn->sendpage_failures_cnt;
2079
	strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
2080
	stats->custom[1].value = tcp_conn->discontiguous_hdr_cnt;
2081 2082 2083 2084
	strcpy(stats->custom[2].desc, "eh_abort_cnt");
	stats->custom[2].value = conn->eh_abort_cnt;
}

2085 2086 2087 2088
static struct iscsi_cls_session *
iscsi_tcp_session_create(struct iscsi_transport *iscsit,
			 struct scsi_transport_template *scsit,
			 uint32_t initial_cmdsn, uint32_t *hostno)
2089
{
2090 2091 2092 2093
	struct iscsi_cls_session *cls_session;
	struct iscsi_session *session;
	uint32_t hn;
	int cmd_i;
2094

2095 2096 2097 2098 2099 2100 2101
	cls_session = iscsi_session_setup(iscsit, scsit,
					 sizeof(struct iscsi_tcp_cmd_task),
					 sizeof(struct iscsi_tcp_mgmt_task),
					 initial_cmdsn, &hn);
	if (!cls_session)
		return NULL;
	*hostno = hn;
2102

2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131
	session = class_to_transport_session(cls_session);
	for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
		struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
		struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;

		ctask->hdr = &tcp_ctask->hdr;
	}

	for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) {
		struct iscsi_mgmt_task *mtask = session->mgmt_cmds[cmd_i];
		struct iscsi_tcp_mgmt_task *tcp_mtask = mtask->dd_data;

		mtask->hdr = &tcp_mtask->hdr;
	}

	if (iscsi_r2tpool_alloc(class_to_transport_session(cls_session)))
		goto r2tpool_alloc_fail;

	return cls_session;

r2tpool_alloc_fail:
	iscsi_session_teardown(cls_session);
	return NULL;
}

static void iscsi_tcp_session_destroy(struct iscsi_cls_session *cls_session)
{
	iscsi_r2tpool_free(class_to_transport_session(cls_session));
	iscsi_session_teardown(cls_session);
2132 2133
}

2134
static struct scsi_host_template iscsi_sht = {
2135
	.name			= "iSCSI Initiator over TCP/IP",
2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147
	.queuecommand           = iscsi_queuecommand,
	.change_queue_depth	= iscsi_change_queue_depth,
	.can_queue		= ISCSI_XMIT_CMDS_MAX - 1,
	.sg_tablesize		= ISCSI_SG_TABLESIZE,
	.cmd_per_lun		= ISCSI_DEF_CMD_PER_LUN,
	.eh_abort_handler       = iscsi_eh_abort,
	.eh_host_reset_handler	= iscsi_eh_host_reset,
	.use_clustering         = DISABLE_CLUSTERING,
	.proc_name		= "iscsi_tcp",
	.this_id		= -1,
};

2148 2149 2150 2151 2152
static struct iscsi_transport iscsi_tcp_transport = {
	.owner			= THIS_MODULE,
	.name			= "tcp",
	.caps			= CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
				  | CAP_DATADGST,
2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165
	.param_mask		= ISCSI_MAX_RECV_DLENGTH |
				  ISCSI_MAX_XMIT_DLENGTH |
				  ISCSI_HDRDGST_EN |
				  ISCSI_DATADGST_EN |
				  ISCSI_INITIAL_R2T_EN |
				  ISCSI_MAX_R2T |
				  ISCSI_IMM_DATA_EN |
				  ISCSI_FIRST_BURST |
				  ISCSI_MAX_BURST |
				  ISCSI_PDU_INORDER_EN |
				  ISCSI_DATASEQ_INORDER_EN |
				  ISCSI_ERL |
				  ISCSI_CONN_PORT |
2166
				  ISCSI_CONN_ADDRESS |
2167 2168 2169 2170 2171
				  ISCSI_EXP_STATSN |
				  ISCSI_PERSISTENT_PORT |
				  ISCSI_PERSISTENT_ADDRESS |
				  ISCSI_TARGET_NAME |
				  ISCSI_TPGT,
2172
	.host_template		= &iscsi_sht,
2173
	.conndata_size		= sizeof(struct iscsi_conn),
2174 2175
	.max_conn		= 1,
	.max_cmd_len		= ISCSI_TCP_MAX_CMD_LEN,
2176 2177 2178 2179 2180 2181 2182
	/* session management */
	.create_session		= iscsi_tcp_session_create,
	.destroy_session	= iscsi_tcp_session_destroy,
	/* connection management */
	.create_conn		= iscsi_tcp_conn_create,
	.bind_conn		= iscsi_tcp_conn_bind,
	.destroy_conn		= iscsi_tcp_conn_destroy,
2183
	.set_param		= iscsi_conn_set_param,
2184
	.get_conn_param		= iscsi_tcp_conn_get_param,
2185
	.get_session_param	= iscsi_session_get_param,
2186
	.start_conn		= iscsi_conn_start,
2187
	.stop_conn		= iscsi_tcp_conn_stop,
2188
	/* IO */
2189 2190
	.send_pdu		= iscsi_conn_send_pdu,
	.get_stats		= iscsi_conn_get_stats,
2191 2192 2193 2194 2195 2196
	.init_cmd_task		= iscsi_tcp_cmd_init,
	.init_mgmt_task		= iscsi_tcp_mgmt_init,
	.xmit_cmd_task		= iscsi_tcp_ctask_xmit,
	.xmit_mgmt_task		= iscsi_tcp_mtask_xmit,
	.cleanup_cmd_task	= iscsi_tcp_cleanup_ctask,
	/* recovery */
M
Mike Christie 已提交
2197
	.session_recovery_timedout = iscsi_session_recovery_timedout,
2198 2199 2200 2201 2202 2203
};

static int __init
iscsi_tcp_init(void)
{
	if (iscsi_max_lun < 1) {
O
Or Gerlitz 已提交
2204 2205
		printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n",
		       iscsi_max_lun);
2206 2207 2208 2209
		return -EINVAL;
	}
	iscsi_tcp_transport.max_lun = iscsi_max_lun;

2210
	if (!iscsi_register_transport(&iscsi_tcp_transport))
2211
		return -ENODEV;
2212

2213
	return 0;
2214 2215 2216 2217 2218 2219 2220 2221 2222 2223
}

static void __exit
iscsi_tcp_exit(void)
{
	iscsi_unregister_transport(&iscsi_tcp_transport);
}

module_init(iscsi_tcp_init);
module_exit(iscsi_tcp_exit);