target_core_transport.c 131.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43
/*******************************************************************************
 * Filename:  target_core_transport.c
 *
 * This file contains the Generic Target Engine Core.
 *
 * Copyright (c) 2002, 2003, 2004, 2005 PyX Technologies, Inc.
 * Copyright (c) 2005, 2006, 2007 SBE, Inc.
 * Copyright (c) 2007-2010 Rising Tide Systems
 * Copyright (c) 2008-2010 Linux-iSCSI.org
 *
 * Nicholas A. Bellinger <nab@kernel.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 ******************************************************************************/

#include <linux/net.h>
#include <linux/delay.h>
#include <linux/string.h>
#include <linux/timer.h>
#include <linux/slab.h>
#include <linux/blkdev.h>
#include <linux/spinlock.h>
#include <linux/kthread.h>
#include <linux/in.h>
#include <linux/cdrom.h>
#include <asm/unaligned.h>
#include <net/sock.h>
#include <net/tcp.h>
#include <scsi/scsi.h>
#include <scsi/scsi_cmnd.h>
44
#include <scsi/scsi_tcq.h>
45 46 47 48 49 50 51 52 53 54 55 56 57 58

#include <target/target_core_base.h>
#include <target/target_core_device.h>
#include <target/target_core_tmr.h>
#include <target/target_core_tpg.h>
#include <target/target_core_transport.h>
#include <target/target_core_fabric_ops.h>
#include <target/target_core_configfs.h>

#include "target_core_alua.h"
#include "target_core_hba.h"
#include "target_core_pr.h"
#include "target_core_ua.h"

59
static int sub_api_initialized;
60

61
static struct workqueue_struct *target_completion_wq;
62 63 64 65 66 67 68 69 70 71 72
static struct kmem_cache *se_cmd_cache;
static struct kmem_cache *se_sess_cache;
struct kmem_cache *se_tmr_req_cache;
struct kmem_cache *se_ua_cache;
struct kmem_cache *t10_pr_reg_cache;
struct kmem_cache *t10_alua_lu_gp_cache;
struct kmem_cache *t10_alua_lu_gp_mem_cache;
struct kmem_cache *t10_alua_tg_pt_gp_cache;
struct kmem_cache *t10_alua_tg_pt_gp_mem_cache;

static int transport_generic_write_pending(struct se_cmd *);
73
static int transport_processing_thread(void *param);
74 75
static int __transport_execute_tasks(struct se_device *dev);
static void transport_complete_task_attr(struct se_cmd *cmd);
76
static void transport_handle_queue_full(struct se_cmd *cmd,
77
		struct se_device *dev);
78 79
static void transport_direct_request_timeout(struct se_cmd *cmd);
static void transport_free_dev_tasks(struct se_cmd *cmd);
80
static u32 transport_allocate_tasks(struct se_cmd *cmd,
81
		unsigned long long starting_lba,
82
		enum dma_data_direction data_direction,
83
		struct scatterlist *sgl, unsigned int nents);
84
static int transport_generic_get_mem(struct se_cmd *cmd);
85
static void transport_put_cmd(struct se_cmd *cmd);
86
static void transport_remove_cmd_from_queue(struct se_cmd *cmd);
87
static int transport_set_sense_codes(struct se_cmd *cmd, u8 asc, u8 ascq);
88 89
static void transport_generic_request_failure(struct se_cmd *, int, int);
static void target_complete_ok_work(struct work_struct *work);
90

91
int init_se_kmem_caches(void)
92 93 94
{
	se_cmd_cache = kmem_cache_create("se_cmd_cache",
			sizeof(struct se_cmd), __alignof__(struct se_cmd), 0, NULL);
95 96
	if (!se_cmd_cache) {
		pr_err("kmem_cache_create for struct se_cmd failed\n");
97 98 99 100 101
		goto out;
	}
	se_tmr_req_cache = kmem_cache_create("se_tmr_cache",
			sizeof(struct se_tmr_req), __alignof__(struct se_tmr_req),
			0, NULL);
102 103
	if (!se_tmr_req_cache) {
		pr_err("kmem_cache_create() for struct se_tmr_req"
104
				" failed\n");
105
		goto out_free_cmd_cache;
106 107 108 109
	}
	se_sess_cache = kmem_cache_create("se_sess_cache",
			sizeof(struct se_session), __alignof__(struct se_session),
			0, NULL);
110 111
	if (!se_sess_cache) {
		pr_err("kmem_cache_create() for struct se_session"
112
				" failed\n");
113
		goto out_free_tmr_req_cache;
114 115 116 117
	}
	se_ua_cache = kmem_cache_create("se_ua_cache",
			sizeof(struct se_ua), __alignof__(struct se_ua),
			0, NULL);
118 119
	if (!se_ua_cache) {
		pr_err("kmem_cache_create() for struct se_ua failed\n");
120
		goto out_free_sess_cache;
121 122 123 124
	}
	t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache",
			sizeof(struct t10_pr_registration),
			__alignof__(struct t10_pr_registration), 0, NULL);
125 126
	if (!t10_pr_reg_cache) {
		pr_err("kmem_cache_create() for struct t10_pr_registration"
127
				" failed\n");
128
		goto out_free_ua_cache;
129 130 131 132
	}
	t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache",
			sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp),
			0, NULL);
133 134
	if (!t10_alua_lu_gp_cache) {
		pr_err("kmem_cache_create() for t10_alua_lu_gp_cache"
135
				" failed\n");
136
		goto out_free_pr_reg_cache;
137 138 139 140
	}
	t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache",
			sizeof(struct t10_alua_lu_gp_member),
			__alignof__(struct t10_alua_lu_gp_member), 0, NULL);
141 142
	if (!t10_alua_lu_gp_mem_cache) {
		pr_err("kmem_cache_create() for t10_alua_lu_gp_mem_"
143
				"cache failed\n");
144
		goto out_free_lu_gp_cache;
145 146 147 148
	}
	t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache",
			sizeof(struct t10_alua_tg_pt_gp),
			__alignof__(struct t10_alua_tg_pt_gp), 0, NULL);
149 150
	if (!t10_alua_tg_pt_gp_cache) {
		pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_"
151
				"cache failed\n");
152
		goto out_free_lu_gp_mem_cache;
153 154 155 156 157 158
	}
	t10_alua_tg_pt_gp_mem_cache = kmem_cache_create(
			"t10_alua_tg_pt_gp_mem_cache",
			sizeof(struct t10_alua_tg_pt_gp_member),
			__alignof__(struct t10_alua_tg_pt_gp_member),
			0, NULL);
159 160
	if (!t10_alua_tg_pt_gp_mem_cache) {
		pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_"
161
				"mem_t failed\n");
162
		goto out_free_tg_pt_gp_cache;
163 164
	}

165 166 167 168 169
	target_completion_wq = alloc_workqueue("target_completion",
					       WQ_MEM_RECLAIM, 0);
	if (!target_completion_wq)
		goto out_free_tg_pt_gp_mem_cache;

170
	return 0;
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189

out_free_tg_pt_gp_mem_cache:
	kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache);
out_free_tg_pt_gp_cache:
	kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
out_free_lu_gp_mem_cache:
	kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
out_free_lu_gp_cache:
	kmem_cache_destroy(t10_alua_lu_gp_cache);
out_free_pr_reg_cache:
	kmem_cache_destroy(t10_pr_reg_cache);
out_free_ua_cache:
	kmem_cache_destroy(se_ua_cache);
out_free_sess_cache:
	kmem_cache_destroy(se_sess_cache);
out_free_tmr_req_cache:
	kmem_cache_destroy(se_tmr_req_cache);
out_free_cmd_cache:
	kmem_cache_destroy(se_cmd_cache);
190
out:
191
	return -ENOMEM;
192 193
}

194
void release_se_kmem_caches(void)
195
{
196
	destroy_workqueue(target_completion_wq);
197 198 199 200 201 202 203 204 205 206 207
	kmem_cache_destroy(se_cmd_cache);
	kmem_cache_destroy(se_tmr_req_cache);
	kmem_cache_destroy(se_sess_cache);
	kmem_cache_destroy(se_ua_cache);
	kmem_cache_destroy(t10_pr_reg_cache);
	kmem_cache_destroy(t10_alua_lu_gp_cache);
	kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
	kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
	kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache);
}

208 209 210
/* This code ensures unique mib indexes are handed out. */
static DEFINE_SPINLOCK(scsi_mib_index_lock);
static u32 scsi_mib_index[SCSI_INDEX_TYPE_MAX];
211 212 213 214 215 216 217 218

/*
 * Allocate a new row index for the entry type specified
 */
u32 scsi_get_new_index(scsi_index_t type)
{
	u32 new_index;

219
	BUG_ON((type < 0) || (type >= SCSI_INDEX_TYPE_MAX));
220

221 222 223
	spin_lock(&scsi_mib_index_lock);
	new_index = ++scsi_mib_index[type];
	spin_unlock(&scsi_mib_index_lock);
224 225 226 227

	return new_index;
}

228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
void transport_init_queue_obj(struct se_queue_obj *qobj)
{
	atomic_set(&qobj->queue_cnt, 0);
	INIT_LIST_HEAD(&qobj->qobj_list);
	init_waitqueue_head(&qobj->thread_wq);
	spin_lock_init(&qobj->cmd_queue_lock);
}
EXPORT_SYMBOL(transport_init_queue_obj);

static int transport_subsystem_reqmods(void)
{
	int ret;

	ret = request_module("target_core_iblock");
	if (ret != 0)
243
		pr_err("Unable to load target_core_iblock\n");
244 245 246

	ret = request_module("target_core_file");
	if (ret != 0)
247
		pr_err("Unable to load target_core_file\n");
248 249 250

	ret = request_module("target_core_pscsi");
	if (ret != 0)
251
		pr_err("Unable to load target_core_pscsi\n");
252 253 254

	ret = request_module("target_core_stgt");
	if (ret != 0)
255
		pr_err("Unable to load target_core_stgt\n");
256 257 258 259 260 261

	return 0;
}

int transport_subsystem_check_init(void)
{
262 263 264
	int ret;

	if (sub_api_initialized)
265 266 267 268
		return 0;
	/*
	 * Request the loading of known TCM subsystem plugins..
	 */
269 270 271
	ret = transport_subsystem_reqmods();
	if (ret < 0)
		return ret;
272

273
	sub_api_initialized = 1;
274 275 276 277 278 279 280 281
	return 0;
}

struct se_session *transport_init_session(void)
{
	struct se_session *se_sess;

	se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL);
282 283
	if (!se_sess) {
		pr_err("Unable to allocate struct se_session from"
284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
				" se_sess_cache\n");
		return ERR_PTR(-ENOMEM);
	}
	INIT_LIST_HEAD(&se_sess->sess_list);
	INIT_LIST_HEAD(&se_sess->sess_acl_list);

	return se_sess;
}
EXPORT_SYMBOL(transport_init_session);

/*
 * Called with spin_lock_bh(&struct se_portal_group->session_lock called.
 */
void __transport_register_session(
	struct se_portal_group *se_tpg,
	struct se_node_acl *se_nacl,
	struct se_session *se_sess,
	void *fabric_sess_ptr)
{
	unsigned char buf[PR_REG_ISID_LEN];

	se_sess->se_tpg = se_tpg;
	se_sess->fabric_sess_ptr = fabric_sess_ptr;
	/*
	 * Used by struct se_node_acl's under ConfigFS to locate active se_session-t
	 *
	 * Only set for struct se_session's that will actually be moving I/O.
	 * eg: *NOT* discovery sessions.
	 */
	if (se_nacl) {
		/*
		 * If the fabric module supports an ISID based TransportID,
		 * save this value in binary from the fabric I_T Nexus now.
		 */
318
		if (se_tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) {
319
			memset(&buf[0], 0, PR_REG_ISID_LEN);
320
			se_tpg->se_tpg_tfo->sess_get_initiator_sid(se_sess,
321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336
					&buf[0], PR_REG_ISID_LEN);
			se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]);
		}
		spin_lock_irq(&se_nacl->nacl_sess_lock);
		/*
		 * The se_nacl->nacl_sess pointer will be set to the
		 * last active I_T Nexus for each struct se_node_acl.
		 */
		se_nacl->nacl_sess = se_sess;

		list_add_tail(&se_sess->sess_acl_list,
			      &se_nacl->acl_sess_list);
		spin_unlock_irq(&se_nacl->nacl_sess_lock);
	}
	list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list);

337
	pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n",
338
		se_tpg->se_tpg_tfo->get_fabric_name(), se_sess->fabric_sess_ptr);
339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
}
EXPORT_SYMBOL(__transport_register_session);

void transport_register_session(
	struct se_portal_group *se_tpg,
	struct se_node_acl *se_nacl,
	struct se_session *se_sess,
	void *fabric_sess_ptr)
{
	spin_lock_bh(&se_tpg->session_lock);
	__transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr);
	spin_unlock_bh(&se_tpg->session_lock);
}
EXPORT_SYMBOL(transport_register_session);

void transport_deregister_session_configfs(struct se_session *se_sess)
{
	struct se_node_acl *se_nacl;
357
	unsigned long flags;
358 359 360 361
	/*
	 * Used by struct se_node_acl's under ConfigFS to locate active struct se_session
	 */
	se_nacl = se_sess->se_node_acl;
362
	if (se_nacl) {
363
		spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
364 365 366 367 368 369 370 371 372 373 374 375 376
		list_del(&se_sess->sess_acl_list);
		/*
		 * If the session list is empty, then clear the pointer.
		 * Otherwise, set the struct se_session pointer from the tail
		 * element of the per struct se_node_acl active session list.
		 */
		if (list_empty(&se_nacl->acl_sess_list))
			se_nacl->nacl_sess = NULL;
		else {
			se_nacl->nacl_sess = container_of(
					se_nacl->acl_sess_list.prev,
					struct se_session, sess_acl_list);
		}
377
		spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
378 379 380 381 382 383 384 385 386 387 388 389 390 391
	}
}
EXPORT_SYMBOL(transport_deregister_session_configfs);

void transport_free_session(struct se_session *se_sess)
{
	kmem_cache_free(se_sess_cache, se_sess);
}
EXPORT_SYMBOL(transport_free_session);

void transport_deregister_session(struct se_session *se_sess)
{
	struct se_portal_group *se_tpg = se_sess->se_tpg;
	struct se_node_acl *se_nacl;
392
	unsigned long flags;
393

394
	if (!se_tpg) {
395 396 397 398
		transport_free_session(se_sess);
		return;
	}

399
	spin_lock_irqsave(&se_tpg->session_lock, flags);
400 401 402
	list_del(&se_sess->sess_list);
	se_sess->se_tpg = NULL;
	se_sess->fabric_sess_ptr = NULL;
403
	spin_unlock_irqrestore(&se_tpg->session_lock, flags);
404 405 406 407 408 409

	/*
	 * Determine if we need to do extra work for this initiator node's
	 * struct se_node_acl if it had been previously dynamically generated.
	 */
	se_nacl = se_sess->se_node_acl;
410
	if (se_nacl) {
411
		spin_lock_irqsave(&se_tpg->acl_node_lock, flags);
412
		if (se_nacl->dynamic_node_acl) {
413 414
			if (!se_tpg->se_tpg_tfo->tpg_check_demo_mode_cache(
					se_tpg)) {
415 416
				list_del(&se_nacl->acl_list);
				se_tpg->num_node_acls--;
417
				spin_unlock_irqrestore(&se_tpg->acl_node_lock, flags);
418 419 420

				core_tpg_wait_for_nacl_pr_ref(se_nacl);
				core_free_device_list_for_node(se_nacl, se_tpg);
421
				se_tpg->se_tpg_tfo->tpg_release_fabric_acl(se_tpg,
422
						se_nacl);
423
				spin_lock_irqsave(&se_tpg->acl_node_lock, flags);
424 425
			}
		}
426
		spin_unlock_irqrestore(&se_tpg->acl_node_lock, flags);
427 428 429 430
	}

	transport_free_session(se_sess);

431
	pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n",
432
		se_tpg->se_tpg_tfo->get_fabric_name());
433 434 435 436
}
EXPORT_SYMBOL(transport_deregister_session);

/*
437
 * Called with cmd->t_state_lock held.
438 439 440
 */
static void transport_all_task_dev_remove_state(struct se_cmd *cmd)
{
441
	struct se_device *dev = cmd->se_dev;
442 443 444
	struct se_task *task;
	unsigned long flags;

445 446
	if (!dev)
		return;
447

448
	list_for_each_entry(task, &cmd->t_task_list, t_list) {
449
		if (task->task_flags & TF_ACTIVE)
450 451
			continue;

452
		if (!atomic_read(&task->task_state_active))
453 454 455 456
			continue;

		spin_lock_irqsave(&dev->execute_task_lock, flags);
		list_del(&task->t_state_list);
457 458
		pr_debug("Removed ITT: 0x%08x dev: %p task[%p]\n",
			cmd->se_tfo->get_task_tag(cmd), dev, task);
459 460 461
		spin_unlock_irqrestore(&dev->execute_task_lock, flags);

		atomic_set(&task->task_state_active, 0);
462
		atomic_dec(&cmd->t_task_cdbs_ex_left);
463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480
	}
}

/*	transport_cmd_check_stop():
 *
 *	'transport_off = 1' determines if t_transport_active should be cleared.
 *	'transport_off = 2' determines if task_dev_state should be removed.
 *
 *	A non-zero u8 t_state sets cmd->t_state.
 *	Returns 1 when command is stopped, else 0.
 */
static int transport_cmd_check_stop(
	struct se_cmd *cmd,
	int transport_off,
	u8 t_state)
{
	unsigned long flags;

481
	spin_lock_irqsave(&cmd->t_state_lock, flags);
482 483 484 485
	/*
	 * Determine if IOCTL context caller in requesting the stopping of this
	 * command for LUN shutdown purposes.
	 */
486
	if (atomic_read(&cmd->transport_lun_stop)) {
487
		pr_debug("%s:%d atomic_read(&cmd->transport_lun_stop)"
488
			" == TRUE for ITT: 0x%08x\n", __func__, __LINE__,
489
			cmd->se_tfo->get_task_tag(cmd));
490

491
		atomic_set(&cmd->t_transport_active, 0);
492 493
		if (transport_off == 2)
			transport_all_task_dev_remove_state(cmd);
494
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
495

496
		complete(&cmd->transport_lun_stop_comp);
497 498 499 500
		return 1;
	}
	/*
	 * Determine if frontend context caller is requesting the stopping of
501
	 * this command for frontend exceptions.
502
	 */
503
	if (atomic_read(&cmd->t_transport_stop)) {
504
		pr_debug("%s:%d atomic_read(&cmd->t_transport_stop) =="
505
			" TRUE for ITT: 0x%08x\n", __func__, __LINE__,
506
			cmd->se_tfo->get_task_tag(cmd));
507 508 509 510 511 512 513 514 515 516

		if (transport_off == 2)
			transport_all_task_dev_remove_state(cmd);

		/*
		 * Clear struct se_cmd->se_lun before the transport_off == 2 handoff
		 * to FE.
		 */
		if (transport_off == 2)
			cmd->se_lun = NULL;
517
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
518

519
		complete(&cmd->t_transport_stop_comp);
520 521 522
		return 1;
	}
	if (transport_off) {
523
		atomic_set(&cmd->t_transport_active, 0);
524 525 526 527 528 529 530 531 532
		if (transport_off == 2) {
			transport_all_task_dev_remove_state(cmd);
			/*
			 * Clear struct se_cmd->se_lun before the transport_off == 2
			 * handoff to fabric module.
			 */
			cmd->se_lun = NULL;
			/*
			 * Some fabric modules like tcm_loop can release
L
Lucas De Marchi 已提交
533
			 * their internally allocated I/O reference now and
534 535
			 * struct se_cmd now.
			 */
536
			if (cmd->se_tfo->check_stop_free != NULL) {
537
				spin_unlock_irqrestore(
538
					&cmd->t_state_lock, flags);
539

540
				cmd->se_tfo->check_stop_free(cmd);
541 542 543
				return 1;
			}
		}
544
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
545 546 547 548

		return 0;
	} else if (t_state)
		cmd->t_state = t_state;
549
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
550 551 552 553 554 555 556 557 558 559 560

	return 0;
}

static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd)
{
	return transport_cmd_check_stop(cmd, 2, 0);
}

static void transport_lun_remove_cmd(struct se_cmd *cmd)
{
561
	struct se_lun *lun = cmd->se_lun;
562 563 564 565 566
	unsigned long flags;

	if (!lun)
		return;

567
	spin_lock_irqsave(&cmd->t_state_lock, flags);
568
	if (!atomic_read(&cmd->transport_dev_active)) {
569
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
570 571
		goto check_lun;
	}
572
	atomic_set(&cmd->transport_dev_active, 0);
573
	transport_all_task_dev_remove_state(cmd);
574
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
575 576 577 578


check_lun:
	spin_lock_irqsave(&lun->lun_cmd_lock, flags);
579
	if (atomic_read(&cmd->transport_lun_active)) {
580
		list_del(&cmd->se_lun_node);
581
		atomic_set(&cmd->transport_lun_active, 0);
582
#if 0
583
		pr_debug("Removed ITT: 0x%08x from LUN LIST[%d]\n"
584
			cmd->se_tfo->get_task_tag(cmd), lun->unpacked_lun);
585 586 587 588 589 590 591
#endif
	}
	spin_unlock_irqrestore(&lun->lun_cmd_lock, flags);
}

void transport_cmd_finish_abort(struct se_cmd *cmd, int remove)
{
592 593
	if (!cmd->se_tmr_req)
		transport_lun_remove_cmd(cmd);
594 595 596

	if (transport_cmd_check_stop_to_fabric(cmd))
		return;
597
	if (remove) {
598
		transport_remove_cmd_from_queue(cmd);
599
		transport_put_cmd(cmd);
600
	}
601 602
}

603 604
static void transport_add_cmd_to_queue(struct se_cmd *cmd, int t_state,
		bool at_head)
605 606
{
	struct se_device *dev = cmd->se_dev;
607
	struct se_queue_obj *qobj = &dev->dev_queue_obj;
608 609 610
	unsigned long flags;

	if (t_state) {
611
		spin_lock_irqsave(&cmd->t_state_lock, flags);
612
		cmd->t_state = t_state;
613 614
		atomic_set(&cmd->t_transport_active, 1);
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
615 616 617
	}

	spin_lock_irqsave(&qobj->cmd_queue_lock, flags);
618 619 620 621 622 623 624

	/* If the cmd is already on the list, remove it before we add it */
	if (!list_empty(&cmd->se_queue_node))
		list_del(&cmd->se_queue_node);
	else
		atomic_inc(&qobj->queue_cnt);

625
	if (at_head)
626
		list_add(&cmd->se_queue_node, &qobj->qobj_list);
627
	else
628
		list_add_tail(&cmd->se_queue_node, &qobj->qobj_list);
629
	atomic_set(&cmd->t_transport_queue_active, 1);
630 631 632 633 634
	spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);

	wake_up_interruptible(&qobj->thread_wq);
}

635 636
static struct se_cmd *
transport_get_cmd_from_queue(struct se_queue_obj *qobj)
637
{
638
	struct se_cmd *cmd;
639 640 641 642 643 644 645
	unsigned long flags;

	spin_lock_irqsave(&qobj->cmd_queue_lock, flags);
	if (list_empty(&qobj->qobj_list)) {
		spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
		return NULL;
	}
646
	cmd = list_first_entry(&qobj->qobj_list, struct se_cmd, se_queue_node);
647

648
	atomic_set(&cmd->t_transport_queue_active, 0);
649

650
	list_del_init(&cmd->se_queue_node);
651 652 653
	atomic_dec(&qobj->queue_cnt);
	spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);

654
	return cmd;
655 656
}

657
static void transport_remove_cmd_from_queue(struct se_cmd *cmd)
658
{
659
	struct se_queue_obj *qobj = &cmd->se_dev->dev_queue_obj;
660 661 662
	unsigned long flags;

	spin_lock_irqsave(&qobj->cmd_queue_lock, flags);
663
	if (!atomic_read(&cmd->t_transport_queue_active)) {
664 665 666
		spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
		return;
	}
667 668 669
	atomic_set(&cmd->t_transport_queue_active, 0);
	atomic_dec(&qobj->queue_cnt);
	list_del_init(&cmd->se_queue_node);
670 671
	spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);

672
	if (atomic_read(&cmd->t_transport_queue_active)) {
673
		pr_err("ITT: 0x%08x t_transport_queue_active: %d\n",
674
			cmd->se_tfo->get_task_tag(cmd),
675
			atomic_read(&cmd->t_transport_queue_active));
676 677 678 679 680 681 682 683 684
	}
}

/*
 * Completion function used by TCM subsystem plugins (such as FILEIO)
 * for queueing up response from struct se_subsystem_api->do_task()
 */
void transport_complete_sync_cache(struct se_cmd *cmd, int good)
{
685
	struct se_task *task = list_entry(cmd->t_task_list.next,
686 687 688 689 690 691 692 693
				struct se_task, t_list);

	if (good) {
		cmd->scsi_status = SAM_STAT_GOOD;
		task->task_scsi_status = GOOD;
	} else {
		task->task_scsi_status = SAM_STAT_CHECK_CONDITION;
		task->task_error_status = PYX_TRANSPORT_ILLEGAL_REQUEST;
694
		task->task_se_cmd->transport_error_status =
695 696 697 698 699 700 701
					PYX_TRANSPORT_ILLEGAL_REQUEST;
	}

	transport_complete_task(task, good);
}
EXPORT_SYMBOL(transport_complete_sync_cache);

702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728
static void target_complete_timeout_work(struct work_struct *work)
{
	struct se_cmd *cmd = container_of(work, struct se_cmd, work);
	unsigned long flags;

	/*
	 * Reset cmd->t_se_count to allow transport_put_cmd()
	 * to allow last call to free memory resources.
	 */
	spin_lock_irqsave(&cmd->t_state_lock, flags);
	if (atomic_read(&cmd->t_transport_timeout) > 1) {
		int tmp = (atomic_read(&cmd->t_transport_timeout) - 1);

		atomic_sub(tmp, &cmd->t_se_count);
	}
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);

	transport_put_cmd(cmd);
}

static void target_complete_failure_work(struct work_struct *work)
{
	struct se_cmd *cmd = container_of(work, struct se_cmd, work);

	transport_generic_request_failure(cmd, 1, 1);
}

729 730 731 732 733 734 735
/*	transport_complete_task():
 *
 *	Called from interrupt and non interrupt context depending
 *	on the transport plugin.
 */
void transport_complete_task(struct se_task *task, int success)
{
736
	struct se_cmd *cmd = task->task_se_cmd;
737
	struct se_device *dev = cmd->se_dev;
738 739
	unsigned long flags;
#if 0
740
	pr_debug("task: %p CDB: 0x%02x obj_ptr: %p\n", task,
741
			cmd->t_task_cdb[0], dev);
742
#endif
743
	if (dev)
744 745
		atomic_inc(&dev->depth_left);

746 747
	del_timer(&task->task_timer);

748
	spin_lock_irqsave(&cmd->t_state_lock, flags);
749
	task->task_flags &= ~TF_ACTIVE;
750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767

	/*
	 * See if any sense data exists, if so set the TASK_SENSE flag.
	 * Also check for any other post completion work that needs to be
	 * done by the plugins.
	 */
	if (dev && dev->transport->transport_complete) {
		if (dev->transport->transport_complete(task) != 0) {
			cmd->se_cmd_flags |= SCF_TRANSPORT_TASK_SENSE;
			task->task_sense = 1;
			success = 1;
		}
	}

	/*
	 * See if we are waiting for outstanding struct se_task
	 * to complete for an exception condition
	 */
768
	if (task->task_flags & TF_REQUEST_STOP) {
769
		/*
770
		 * Decrement cmd->t_se_count if this task had
771 772
		 * previously thrown its timeout exception handler.
		 */
773
		if (task->task_flags & TF_TIMEOUT) {
774
			atomic_dec(&cmd->t_se_count);
775
			task->task_flags &= ~TF_TIMEOUT;
776
		}
777
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
778 779 780 781 782 783 784 785 786

		complete(&task->task_stop_comp);
		return;
	}
	/*
	 * If the task's timeout handler has fired, use the t_task_cdbs_timeout
	 * left counter to determine when the struct se_cmd is ready to be queued to
	 * the processing thread.
	 */
787
	if (task->task_flags & TF_TIMEOUT) {
788 789
		if (!atomic_dec_and_test(&cmd->t_task_cdbs_timeout_left)) {
			spin_unlock_irqrestore(&cmd->t_state_lock, flags);
790 791
			return;
		}
792 793
		INIT_WORK(&cmd->work, target_complete_timeout_work);
		goto out_queue;
794
	}
795
	atomic_dec(&cmd->t_task_cdbs_timeout_left);
796 797 798 799 800 801

	/*
	 * Decrement the outstanding t_task_cdbs_left count.  The last
	 * struct se_task from struct se_cmd will complete itself into the
	 * device queue depending upon int success.
	 */
802
	if (!atomic_dec_and_test(&cmd->t_task_cdbs_left)) {
803
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
804 805 806
		return;
	}

807
	if (!success || cmd->t_tasks_failed) {
808 809 810 811 812 813
		if (!task->task_error_status) {
			task->task_error_status =
				PYX_TRANSPORT_UNKNOWN_SAM_OPCODE;
			cmd->transport_error_status =
				PYX_TRANSPORT_UNKNOWN_SAM_OPCODE;
		}
814
		INIT_WORK(&cmd->work, target_complete_failure_work);
815
	} else {
816
		atomic_set(&cmd->t_transport_complete, 1);
817
		INIT_WORK(&cmd->work, target_complete_ok_work);
818
	}
819 820 821 822

out_queue:
	cmd->t_state = TRANSPORT_COMPLETE;
	atomic_set(&cmd->t_transport_active, 1);
823
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
824

825
	queue_work(target_completion_wq, &cmd->work);
826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854
}
EXPORT_SYMBOL(transport_complete_task);

/*
 * Called by transport_add_tasks_from_cmd() once a struct se_cmd's
 * struct se_task list are ready to be added to the active execution list
 * struct se_device

 * Called with se_dev_t->execute_task_lock called.
 */
static inline int transport_add_task_check_sam_attr(
	struct se_task *task,
	struct se_task *task_prev,
	struct se_device *dev)
{
	/*
	 * No SAM Task attribute emulation enabled, add to tail of
	 * execution queue
	 */
	if (dev->dev_task_attr_type != SAM_TASK_ATTR_EMULATED) {
		list_add_tail(&task->t_execute_list, &dev->execute_task_list);
		return 0;
	}
	/*
	 * HEAD_OF_QUEUE attribute for received CDB, which means
	 * the first task that is associated with a struct se_cmd goes to
	 * head of the struct se_device->execute_task_list, and task_prev
	 * after that for each subsequent task
	 */
855
	if (task->task_se_cmd->sam_task_attr == MSG_HEAD_TAG) {
856 857 858 859 860
		list_add(&task->t_execute_list,
				(task_prev != NULL) ?
				&task_prev->t_execute_list :
				&dev->execute_task_list);

861
		pr_debug("Set HEAD_OF_QUEUE for task CDB: 0x%02x"
862
				" in execution queue\n",
863
				task->task_se_cmd->t_task_cdb[0]);
864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904
		return 1;
	}
	/*
	 * For ORDERED, SIMPLE or UNTAGGED attribute tasks once they have been
	 * transitioned from Dermant -> Active state, and are added to the end
	 * of the struct se_device->execute_task_list
	 */
	list_add_tail(&task->t_execute_list, &dev->execute_task_list);
	return 0;
}

/*	__transport_add_task_to_execute_queue():
 *
 *	Called with se_dev_t->execute_task_lock called.
 */
static void __transport_add_task_to_execute_queue(
	struct se_task *task,
	struct se_task *task_prev,
	struct se_device *dev)
{
	int head_of_queue;

	head_of_queue = transport_add_task_check_sam_attr(task, task_prev, dev);
	atomic_inc(&dev->execute_tasks);

	if (atomic_read(&task->task_state_active))
		return;
	/*
	 * Determine if this task needs to go to HEAD_OF_QUEUE for the
	 * state list as well.  Running with SAM Task Attribute emulation
	 * will always return head_of_queue == 0 here
	 */
	if (head_of_queue)
		list_add(&task->t_state_list, (task_prev) ?
				&task_prev->t_state_list :
				&dev->state_task_list);
	else
		list_add_tail(&task->t_state_list, &dev->state_task_list);

	atomic_set(&task->task_state_active, 1);

905
	pr_debug("Added ITT: 0x%08x task[%p] to dev: %p\n",
906
		task->task_se_cmd->se_tfo->get_task_tag(task->task_se_cmd),
907 908 909 910 911
		task, dev);
}

static void transport_add_tasks_to_state_queue(struct se_cmd *cmd)
{
912
	struct se_device *dev = cmd->se_dev;
913 914 915
	struct se_task *task;
	unsigned long flags;

916 917
	spin_lock_irqsave(&cmd->t_state_lock, flags);
	list_for_each_entry(task, &cmd->t_task_list, t_list) {
918 919 920 921 922 923 924
		if (atomic_read(&task->task_state_active))
			continue;

		spin_lock(&dev->execute_task_lock);
		list_add_tail(&task->t_state_list, &dev->state_task_list);
		atomic_set(&task->task_state_active, 1);

925 926
		pr_debug("Added ITT: 0x%08x task[%p] to dev: %p\n",
			task->task_se_cmd->se_tfo->get_task_tag(
927 928 929 930
			task->task_se_cmd), task, dev);

		spin_unlock(&dev->execute_task_lock);
	}
931
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
932 933 934 935
}

static void transport_add_tasks_from_cmd(struct se_cmd *cmd)
{
936
	struct se_device *dev = cmd->se_dev;
937 938 939 940
	struct se_task *task, *task_prev = NULL;
	unsigned long flags;

	spin_lock_irqsave(&dev->execute_task_lock, flags);
941
	list_for_each_entry(task, &cmd->t_task_list, t_list) {
942
		if (!list_empty(&task->t_execute_list))
943 944 945 946 947 948 949 950 951 952 953
			continue;
		/*
		 * __transport_add_task_to_execute_queue() handles the
		 * SAM Task Attribute emulation if enabled
		 */
		__transport_add_task_to_execute_queue(task, task_prev, dev);
		task_prev = task;
	}
	spin_unlock_irqrestore(&dev->execute_task_lock, flags);
}

954 955 956 957 958 959 960
void __transport_remove_task_from_execute_queue(struct se_task *task,
		struct se_device *dev)
{
	list_del_init(&task->t_execute_list);
	atomic_dec(&dev->execute_tasks);
}

961
void transport_remove_task_from_execute_queue(
962 963 964 965 966
	struct se_task *task,
	struct se_device *dev)
{
	unsigned long flags;

967
	if (WARN_ON(list_empty(&task->t_execute_list)))
968 969
		return;

970
	spin_lock_irqsave(&dev->execute_task_lock, flags);
971
	__transport_remove_task_from_execute_queue(task, dev);
972 973 974
	spin_unlock_irqrestore(&dev->execute_task_lock, flags);
}

975 976 977 978 979 980 981 982
/*
 * Handle QUEUE_FULL / -EAGAIN status
 */

static void target_qf_do_work(struct work_struct *work)
{
	struct se_device *dev = container_of(work, struct se_device,
					qf_work_queue);
983
	LIST_HEAD(qf_cmd_list);
984 985 986
	struct se_cmd *cmd, *cmd_tmp;

	spin_lock_irq(&dev->qf_cmd_lock);
987 988
	list_splice_init(&dev->qf_cmd_list, &qf_cmd_list);
	spin_unlock_irq(&dev->qf_cmd_lock);
989

990
	list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) {
991 992 993 994
		list_del(&cmd->se_qf_node);
		atomic_dec(&dev->dev_qf_count);
		smp_mb__after_atomic_dec();

995
		pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue"
996
			" context: %s\n", cmd->se_tfo->get_fabric_name(), cmd,
997
			(cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" :
998 999
			(cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING"
			: "UNKNOWN");
1000 1001

		transport_add_cmd_to_queue(cmd, cmd->t_state, true);
1002 1003 1004
	}
}

1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051
unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd)
{
	switch (cmd->data_direction) {
	case DMA_NONE:
		return "NONE";
	case DMA_FROM_DEVICE:
		return "READ";
	case DMA_TO_DEVICE:
		return "WRITE";
	case DMA_BIDIRECTIONAL:
		return "BIDI";
	default:
		break;
	}

	return "UNKNOWN";
}

void transport_dump_dev_state(
	struct se_device *dev,
	char *b,
	int *bl)
{
	*bl += sprintf(b + *bl, "Status: ");
	switch (dev->dev_status) {
	case TRANSPORT_DEVICE_ACTIVATED:
		*bl += sprintf(b + *bl, "ACTIVATED");
		break;
	case TRANSPORT_DEVICE_DEACTIVATED:
		*bl += sprintf(b + *bl, "DEACTIVATED");
		break;
	case TRANSPORT_DEVICE_SHUTDOWN:
		*bl += sprintf(b + *bl, "SHUTDOWN");
		break;
	case TRANSPORT_DEVICE_OFFLINE_ACTIVATED:
	case TRANSPORT_DEVICE_OFFLINE_DEACTIVATED:
		*bl += sprintf(b + *bl, "OFFLINE");
		break;
	default:
		*bl += sprintf(b + *bl, "UNKNOWN=%d", dev->dev_status);
		break;
	}

	*bl += sprintf(b + *bl, "  Execute/Left/Max Queue Depth: %d/%d/%d",
		atomic_read(&dev->execute_tasks), atomic_read(&dev->depth_left),
		dev->queue_depth);
	*bl += sprintf(b + *bl, "  SectorSize: %u  MaxSectors: %u\n",
1052
		dev->se_sub_dev->se_dev_attrib.block_size, dev->se_sub_dev->se_dev_attrib.max_sectors);
1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
	*bl += sprintf(b + *bl, "        ");
}

void transport_dump_vpd_proto_id(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
	int len;

	memset(buf, 0, VPD_TMP_BUF_SIZE);
	len = sprintf(buf, "T10 VPD Protocol Identifier: ");

	switch (vpd->protocol_identifier) {
	case 0x00:
		sprintf(buf+len, "Fibre Channel\n");
		break;
	case 0x10:
		sprintf(buf+len, "Parallel SCSI\n");
		break;
	case 0x20:
		sprintf(buf+len, "SSA\n");
		break;
	case 0x30:
		sprintf(buf+len, "IEEE 1394\n");
		break;
	case 0x40:
		sprintf(buf+len, "SCSI Remote Direct Memory Access"
				" Protocol\n");
		break;
	case 0x50:
		sprintf(buf+len, "Internet SCSI (iSCSI)\n");
		break;
	case 0x60:
		sprintf(buf+len, "SAS Serial SCSI Protocol\n");
		break;
	case 0x70:
		sprintf(buf+len, "Automation/Drive Interface Transport"
				" Protocol\n");
		break;
	case 0x80:
		sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n");
		break;
	default:
		sprintf(buf+len, "Unknown 0x%02x\n",
				vpd->protocol_identifier);
		break;
	}

	if (p_buf)
		strncpy(p_buf, buf, p_buf_len);
	else
1106
		pr_debug("%s", buf);
1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130
}

void
transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83)
{
	/*
	 * Check if the Protocol Identifier Valid (PIV) bit is set..
	 *
	 * from spc3r23.pdf section 7.5.1
	 */
	 if (page_83[1] & 0x80) {
		vpd->protocol_identifier = (page_83[0] & 0xf0);
		vpd->protocol_identifier_set = 1;
		transport_dump_vpd_proto_id(vpd, NULL, 0);
	}
}
EXPORT_SYMBOL(transport_set_vpd_proto_id);

int transport_dump_vpd_assoc(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
1131 1132
	int ret = 0;
	int len;
1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148

	memset(buf, 0, VPD_TMP_BUF_SIZE);
	len = sprintf(buf, "T10 VPD Identifier Association: ");

	switch (vpd->association) {
	case 0x00:
		sprintf(buf+len, "addressed logical unit\n");
		break;
	case 0x10:
		sprintf(buf+len, "target port\n");
		break;
	case 0x20:
		sprintf(buf+len, "SCSI target device\n");
		break;
	default:
		sprintf(buf+len, "Unknown 0x%02x\n", vpd->association);
1149
		ret = -EINVAL;
1150 1151 1152 1153 1154 1155
		break;
	}

	if (p_buf)
		strncpy(p_buf, buf, p_buf_len);
	else
1156
		pr_debug("%s", buf);
1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178

	return ret;
}

int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83)
{
	/*
	 * The VPD identification association..
	 *
	 * from spc3r23.pdf Section 7.6.3.1 Table 297
	 */
	vpd->association = (page_83[1] & 0x30);
	return transport_dump_vpd_assoc(vpd, NULL, 0);
}
EXPORT_SYMBOL(transport_set_vpd_assoc);

int transport_dump_vpd_ident_type(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
1179 1180
	int ret = 0;
	int len;
1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206

	memset(buf, 0, VPD_TMP_BUF_SIZE);
	len = sprintf(buf, "T10 VPD Identifier Type: ");

	switch (vpd->device_identifier_type) {
	case 0x00:
		sprintf(buf+len, "Vendor specific\n");
		break;
	case 0x01:
		sprintf(buf+len, "T10 Vendor ID based\n");
		break;
	case 0x02:
		sprintf(buf+len, "EUI-64 based\n");
		break;
	case 0x03:
		sprintf(buf+len, "NAA\n");
		break;
	case 0x04:
		sprintf(buf+len, "Relative target port identifier\n");
		break;
	case 0x08:
		sprintf(buf+len, "SCSI name string\n");
		break;
	default:
		sprintf(buf+len, "Unsupported: 0x%02x\n",
				vpd->device_identifier_type);
1207
		ret = -EINVAL;
1208 1209 1210
		break;
	}

1211 1212 1213
	if (p_buf) {
		if (p_buf_len < strlen(buf)+1)
			return -EINVAL;
1214
		strncpy(p_buf, buf, p_buf_len);
1215
	} else {
1216
		pr_debug("%s", buf);
1217
	}
1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259

	return ret;
}

int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83)
{
	/*
	 * The VPD identifier type..
	 *
	 * from spc3r23.pdf Section 7.6.3.1 Table 298
	 */
	vpd->device_identifier_type = (page_83[1] & 0x0f);
	return transport_dump_vpd_ident_type(vpd, NULL, 0);
}
EXPORT_SYMBOL(transport_set_vpd_ident_type);

int transport_dump_vpd_ident(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
	int ret = 0;

	memset(buf, 0, VPD_TMP_BUF_SIZE);

	switch (vpd->device_identifier_code_set) {
	case 0x01: /* Binary */
		sprintf(buf, "T10 VPD Binary Device Identifier: %s\n",
			&vpd->device_identifier[0]);
		break;
	case 0x02: /* ASCII */
		sprintf(buf, "T10 VPD ASCII Device Identifier: %s\n",
			&vpd->device_identifier[0]);
		break;
	case 0x03: /* UTF-8 */
		sprintf(buf, "T10 VPD UTF-8 Device Identifier: %s\n",
			&vpd->device_identifier[0]);
		break;
	default:
		sprintf(buf, "T10 VPD Device Identifier encoding unsupported:"
			" 0x%02x", vpd->device_identifier_code_set);
1260
		ret = -EINVAL;
1261 1262 1263 1264 1265 1266
		break;
	}

	if (p_buf)
		strncpy(p_buf, buf, p_buf_len);
	else
1267
		pr_debug("%s", buf);
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317

	return ret;
}

int
transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83)
{
	static const char hex_str[] = "0123456789abcdef";
	int j = 0, i = 4; /* offset to start of the identifer */

	/*
	 * The VPD Code Set (encoding)
	 *
	 * from spc3r23.pdf Section 7.6.3.1 Table 296
	 */
	vpd->device_identifier_code_set = (page_83[0] & 0x0f);
	switch (vpd->device_identifier_code_set) {
	case 0x01: /* Binary */
		vpd->device_identifier[j++] =
				hex_str[vpd->device_identifier_type];
		while (i < (4 + page_83[3])) {
			vpd->device_identifier[j++] =
				hex_str[(page_83[i] & 0xf0) >> 4];
			vpd->device_identifier[j++] =
				hex_str[page_83[i] & 0x0f];
			i++;
		}
		break;
	case 0x02: /* ASCII */
	case 0x03: /* UTF-8 */
		while (i < (4 + page_83[3]))
			vpd->device_identifier[j++] = page_83[i++];
		break;
	default:
		break;
	}

	return transport_dump_vpd_ident(vpd, NULL, 0);
}
EXPORT_SYMBOL(transport_set_vpd_ident);

static void core_setup_task_attr_emulation(struct se_device *dev)
{
	/*
	 * If this device is from Target_Core_Mod/pSCSI, disable the
	 * SAM Task Attribute emulation.
	 *
	 * This is currently not available in upsream Linux/SCSI Target
	 * mode code, and is assumed to be disabled while using TCM/pSCSI.
	 */
1318
	if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV) {
1319 1320 1321 1322 1323
		dev->dev_task_attr_type = SAM_TASK_ATTR_PASSTHROUGH;
		return;
	}

	dev->dev_task_attr_type = SAM_TASK_ATTR_EMULATED;
1324
	pr_debug("%s: Using SAM_TASK_ATTR_EMULATED for SPC: 0x%02x"
1325 1326
		" device\n", dev->transport->name,
		dev->transport->get_device_rev(dev));
1327 1328 1329 1330
}

static void scsi_dump_inquiry(struct se_device *dev)
{
1331
	struct t10_wwn *wwn = &dev->se_sub_dev->t10_wwn;
1332 1333 1334 1335
	int i, device_type;
	/*
	 * Print Linux/SCSI style INQUIRY formatting to the kernel ring buffer
	 */
1336
	pr_debug("  Vendor: ");
1337 1338
	for (i = 0; i < 8; i++)
		if (wwn->vendor[i] >= 0x20)
1339
			pr_debug("%c", wwn->vendor[i]);
1340
		else
1341
			pr_debug(" ");
1342

1343
	pr_debug("  Model: ");
1344 1345
	for (i = 0; i < 16; i++)
		if (wwn->model[i] >= 0x20)
1346
			pr_debug("%c", wwn->model[i]);
1347
		else
1348
			pr_debug(" ");
1349

1350
	pr_debug("  Revision: ");
1351 1352
	for (i = 0; i < 4; i++)
		if (wwn->revision[i] >= 0x20)
1353
			pr_debug("%c", wwn->revision[i]);
1354
		else
1355
			pr_debug(" ");
1356

1357
	pr_debug("\n");
1358

1359
	device_type = dev->transport->get_device_type(dev);
1360 1361
	pr_debug("  Type:   %s ", scsi_device_type(device_type));
	pr_debug("                 ANSI SCSI revision: %02x\n",
1362
				dev->transport->get_device_rev(dev));
1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374
}

struct se_device *transport_add_device_to_core_hba(
	struct se_hba *hba,
	struct se_subsystem_api *transport,
	struct se_subsystem_dev *se_dev,
	u32 device_flags,
	void *transport_dev,
	struct se_dev_limits *dev_limits,
	const char *inquiry_prod,
	const char *inquiry_rev)
{
1375
	int force_pt;
1376 1377 1378
	struct se_device  *dev;

	dev = kzalloc(sizeof(struct se_device), GFP_KERNEL);
1379 1380
	if (!dev) {
		pr_err("Unable to allocate memory for se_dev_t\n");
1381 1382 1383
		return NULL;
	}

1384
	transport_init_queue_obj(&dev->dev_queue_obj);
1385 1386
	dev->dev_flags		= device_flags;
	dev->dev_status		|= TRANSPORT_DEVICE_DEACTIVATED;
1387
	dev->dev_ptr		= transport_dev;
1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398
	dev->se_hba		= hba;
	dev->se_sub_dev		= se_dev;
	dev->transport		= transport;
	atomic_set(&dev->active_cmds, 0);
	INIT_LIST_HEAD(&dev->dev_list);
	INIT_LIST_HEAD(&dev->dev_sep_list);
	INIT_LIST_HEAD(&dev->dev_tmr_list);
	INIT_LIST_HEAD(&dev->execute_task_list);
	INIT_LIST_HEAD(&dev->delayed_cmd_list);
	INIT_LIST_HEAD(&dev->ordered_cmd_list);
	INIT_LIST_HEAD(&dev->state_task_list);
1399
	INIT_LIST_HEAD(&dev->qf_cmd_list);
1400 1401 1402 1403 1404 1405 1406 1407 1408 1409
	spin_lock_init(&dev->execute_task_lock);
	spin_lock_init(&dev->delayed_cmd_lock);
	spin_lock_init(&dev->ordered_cmd_lock);
	spin_lock_init(&dev->state_task_lock);
	spin_lock_init(&dev->dev_alua_lock);
	spin_lock_init(&dev->dev_reservation_lock);
	spin_lock_init(&dev->dev_status_lock);
	spin_lock_init(&dev->dev_status_thr_lock);
	spin_lock_init(&dev->se_port_lock);
	spin_lock_init(&dev->se_tmr_lock);
1410
	spin_lock_init(&dev->qf_cmd_lock);
1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447

	dev->queue_depth	= dev_limits->queue_depth;
	atomic_set(&dev->depth_left, dev->queue_depth);
	atomic_set(&dev->dev_ordered_id, 0);

	se_dev_set_default_attribs(dev, dev_limits);

	dev->dev_index = scsi_get_new_index(SCSI_DEVICE_INDEX);
	dev->creation_time = get_jiffies_64();
	spin_lock_init(&dev->stats_lock);

	spin_lock(&hba->device_lock);
	list_add_tail(&dev->dev_list, &hba->hba_dev_list);
	hba->dev_count++;
	spin_unlock(&hba->device_lock);
	/*
	 * Setup the SAM Task Attribute emulation for struct se_device
	 */
	core_setup_task_attr_emulation(dev);
	/*
	 * Force PR and ALUA passthrough emulation with internal object use.
	 */
	force_pt = (hba->hba_flags & HBA_FLAGS_INTERNAL_USE);
	/*
	 * Setup the Reservations infrastructure for struct se_device
	 */
	core_setup_reservations(dev, force_pt);
	/*
	 * Setup the Asymmetric Logical Unit Assignment for struct se_device
	 */
	if (core_setup_alua(dev, force_pt) < 0)
		goto out;

	/*
	 * Startup the struct se_device processing thread
	 */
	dev->process_thread = kthread_run(transport_processing_thread, dev,
1448
					  "LIO_%s", dev->transport->name);
1449
	if (IS_ERR(dev->process_thread)) {
1450
		pr_err("Unable to create kthread: LIO_%s\n",
1451
			dev->transport->name);
1452 1453
		goto out;
	}
1454 1455 1456 1457
	/*
	 * Setup work_queue for QUEUE_FULL
	 */
	INIT_WORK(&dev->qf_work_queue, target_qf_do_work);
1458 1459 1460 1461 1462 1463 1464 1465
	/*
	 * Preload the initial INQUIRY const values if we are doing
	 * anything virtual (IBLOCK, FILEIO, RAMDISK), but not for TCM/pSCSI
	 * passthrough because this is being provided by the backend LLD.
	 * This is required so that transport_get_inquiry() copies these
	 * originals once back into DEV_T10_WWN(dev) for the virtual device
	 * setup.
	 */
1466
	if (dev->transport->transport_type != TRANSPORT_PLUGIN_PHBA_PDEV) {
1467
		if (!inquiry_prod || !inquiry_rev) {
1468
			pr_err("All non TCM/pSCSI plugins require"
1469 1470 1471 1472
				" INQUIRY consts\n");
			goto out;
		}

1473 1474 1475
		strncpy(&dev->se_sub_dev->t10_wwn.vendor[0], "LIO-ORG", 8);
		strncpy(&dev->se_sub_dev->t10_wwn.model[0], inquiry_prod, 16);
		strncpy(&dev->se_sub_dev->t10_wwn.revision[0], inquiry_rev, 4);
1476 1477 1478
	}
	scsi_dump_inquiry(dev);

1479
	return dev;
1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527
out:
	kthread_stop(dev->process_thread);

	spin_lock(&hba->device_lock);
	list_del(&dev->dev_list);
	hba->dev_count--;
	spin_unlock(&hba->device_lock);

	se_release_vpd_for_dev(dev);

	kfree(dev);

	return NULL;
}
EXPORT_SYMBOL(transport_add_device_to_core_hba);

/*	transport_generic_prepare_cdb():
 *
 *	Since the Initiator sees iSCSI devices as LUNs,  the SCSI CDB will
 *	contain the iSCSI LUN in bits 7-5 of byte 1 as per SAM-2.
 *	The point of this is since we are mapping iSCSI LUNs to
 *	SCSI Target IDs having a non-zero LUN in the CDB will throw the
 *	devices and HBAs for a loop.
 */
static inline void transport_generic_prepare_cdb(
	unsigned char *cdb)
{
	switch (cdb[0]) {
	case READ_10: /* SBC - RDProtect */
	case READ_12: /* SBC - RDProtect */
	case READ_16: /* SBC - RDProtect */
	case SEND_DIAGNOSTIC: /* SPC - SELF-TEST Code */
	case VERIFY: /* SBC - VRProtect */
	case VERIFY_16: /* SBC - VRProtect */
	case WRITE_VERIFY: /* SBC - VRProtect */
	case WRITE_VERIFY_12: /* SBC - VRProtect */
		break;
	default:
		cdb[1] &= 0x1f; /* clear logical unit number */
		break;
	}
}

static struct se_task *
transport_generic_get_task(struct se_cmd *cmd,
		enum dma_data_direction data_direction)
{
	struct se_task *task;
1528
	struct se_device *dev = cmd->se_dev;
1529

1530
	task = dev->transport->alloc_task(cmd->t_task_cdb);
1531
	if (!task) {
1532
		pr_err("Unable to allocate struct se_task\n");
1533 1534 1535 1536 1537 1538
		return NULL;
	}

	INIT_LIST_HEAD(&task->t_list);
	INIT_LIST_HEAD(&task->t_execute_list);
	INIT_LIST_HEAD(&task->t_state_list);
1539
	init_timer(&task->task_timer);
1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561
	init_completion(&task->task_stop_comp);
	task->task_se_cmd = cmd;
	task->task_data_direction = data_direction;

	return task;
}

static int transport_generic_cmd_sequencer(struct se_cmd *, unsigned char *);

/*
 * Used by fabric modules containing a local struct se_cmd within their
 * fabric dependent per I/O descriptor.
 */
void transport_init_se_cmd(
	struct se_cmd *cmd,
	struct target_core_fabric_ops *tfo,
	struct se_session *se_sess,
	u32 data_length,
	int data_direction,
	int task_attr,
	unsigned char *sense_buffer)
{
1562 1563 1564
	INIT_LIST_HEAD(&cmd->se_lun_node);
	INIT_LIST_HEAD(&cmd->se_delayed_node);
	INIT_LIST_HEAD(&cmd->se_ordered_node);
1565
	INIT_LIST_HEAD(&cmd->se_qf_node);
1566
	INIT_LIST_HEAD(&cmd->se_queue_node);
1567

1568 1569 1570 1571 1572 1573
	INIT_LIST_HEAD(&cmd->t_task_list);
	init_completion(&cmd->transport_lun_fe_stop_comp);
	init_completion(&cmd->transport_lun_stop_comp);
	init_completion(&cmd->t_transport_stop_comp);
	spin_lock_init(&cmd->t_state_lock);
	atomic_set(&cmd->transport_dev_active, 1);
1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589

	cmd->se_tfo = tfo;
	cmd->se_sess = se_sess;
	cmd->data_length = data_length;
	cmd->data_direction = data_direction;
	cmd->sam_task_attr = task_attr;
	cmd->sense_buffer = sense_buffer;
}
EXPORT_SYMBOL(transport_init_se_cmd);

static int transport_check_alloc_task_attr(struct se_cmd *cmd)
{
	/*
	 * Check if SAM Task Attribute emulation is enabled for this
	 * struct se_device storage object
	 */
1590
	if (cmd->se_dev->dev_task_attr_type != SAM_TASK_ATTR_EMULATED)
1591 1592
		return 0;

1593
	if (cmd->sam_task_attr == MSG_ACA_TAG) {
1594
		pr_debug("SAM Task Attribute ACA"
1595
			" emulation is not supported\n");
1596
		return -EINVAL;
1597 1598 1599 1600 1601
	}
	/*
	 * Used to determine when ORDERED commands should go from
	 * Dormant to Active status.
	 */
1602
	cmd->se_ordered_id = atomic_inc_return(&cmd->se_dev->dev_ordered_id);
1603
	smp_mb__after_atomic_inc();
1604
	pr_debug("Allocated se_ordered_id: %u for Task Attr: 0x%02x on %s\n",
1605
			cmd->se_ordered_id, cmd->sam_task_attr,
1606
			cmd->se_dev->transport->name);
1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625
	return 0;
}

/*	transport_generic_allocate_tasks():
 *
 *	Called from fabric RX Thread.
 */
int transport_generic_allocate_tasks(
	struct se_cmd *cmd,
	unsigned char *cdb)
{
	int ret;

	transport_generic_prepare_cdb(cdb);
	/*
	 * Ensure that the received CDB is less than the max (252 + 8) bytes
	 * for VARIABLE_LENGTH_CMD
	 */
	if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) {
1626
		pr_err("Received SCSI CDB with command_size: %d that"
1627 1628
			" exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
			scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE);
1629
		return -EINVAL;
1630 1631 1632 1633 1634 1635
	}
	/*
	 * If the received CDB is larger than TCM_MAX_COMMAND_SIZE,
	 * allocate the additional extended CDB buffer now..  Otherwise
	 * setup the pointer from __t_task_cdb to t_task_cdb.
	 */
1636 1637
	if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) {
		cmd->t_task_cdb = kzalloc(scsi_command_size(cdb),
1638
						GFP_KERNEL);
1639 1640
		if (!cmd->t_task_cdb) {
			pr_err("Unable to allocate cmd->t_task_cdb"
1641
				" %u > sizeof(cmd->__t_task_cdb): %lu ops\n",
1642
				scsi_command_size(cdb),
1643
				(unsigned long)sizeof(cmd->__t_task_cdb));
1644
			return -ENOMEM;
1645 1646
		}
	} else
1647
		cmd->t_task_cdb = &cmd->__t_task_cdb[0];
1648
	/*
1649
	 * Copy the original CDB into cmd->
1650
	 */
1651
	memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb));
1652 1653 1654
	/*
	 * Setup the received CDB based on SCSI defined opcodes and
	 * perform unit attention, persistent reservations and ALUA
1655
	 * checks for virtual device backends.  The cmd->t_task_cdb
1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666
	 * pointer is expected to be setup before we reach this point.
	 */
	ret = transport_generic_cmd_sequencer(cmd, cdb);
	if (ret < 0)
		return ret;
	/*
	 * Check for SAM Task Attribute Emulation
	 */
	if (transport_check_alloc_task_attr(cmd) < 0) {
		cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
		cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
1667
		return -EINVAL;
1668 1669 1670 1671 1672 1673 1674 1675 1676
	}
	spin_lock(&cmd->se_lun->lun_sep_lock);
	if (cmd->se_lun->lun_sep)
		cmd->se_lun->lun_sep->sep_stats.cmd_pdus++;
	spin_unlock(&cmd->se_lun->lun_sep_lock);
	return 0;
}
EXPORT_SYMBOL(transport_generic_allocate_tasks);

1677 1678 1679 1680 1681 1682 1683
/*
 * Used by fabric module frontends to queue tasks directly.
 * Many only be used from process context only
 */
int transport_handle_cdb_direct(
	struct se_cmd *cmd)
{
1684 1685
	int ret;

1686 1687
	if (!cmd->se_lun) {
		dump_stack();
1688
		pr_err("cmd->se_lun is NULL\n");
1689 1690 1691 1692
		return -EINVAL;
	}
	if (in_interrupt()) {
		dump_stack();
1693
		pr_err("transport_generic_handle_cdb cannot be called"
1694 1695 1696
				" from interrupt context\n");
		return -EINVAL;
	}
1697 1698 1699 1700
	/*
	 * Set TRANSPORT_NEW_CMD state and cmd->t_transport_active=1 following
	 * transport_generic_handle_cdb*() -> transport_add_cmd_to_queue()
	 * in existing usage to ensure that outstanding descriptors are handled
1701
	 * correctly during shutdown via transport_wait_for_tasks()
1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717
	 *
	 * Also, we don't take cmd->t_state_lock here as we only expect
	 * this to be called for initial descriptor submission.
	 */
	cmd->t_state = TRANSPORT_NEW_CMD;
	atomic_set(&cmd->t_transport_active, 1);
	/*
	 * transport_generic_new_cmd() is already handling QUEUE_FULL,
	 * so follow TRANSPORT_NEW_CMD processing thread context usage
	 * and call transport_generic_request_failure() if necessary..
	 */
	ret = transport_generic_new_cmd(cmd);
	if (ret == -EAGAIN)
		return 0;
	else if (ret < 0) {
		cmd->transport_error_status = ret;
1718
		transport_generic_request_failure(cmd, 0,
1719 1720 1721
				(cmd->data_direction != DMA_TO_DEVICE));
	}
	return 0;
1722 1723 1724
}
EXPORT_SYMBOL(transport_handle_cdb_direct);

1725 1726 1727 1728 1729 1730 1731 1732
/*
 * Used by fabric module frontends defining a TFO->new_cmd_map() caller
 * to  queue up a newly setup se_cmd w/ TRANSPORT_NEW_CMD_MAP in order to
 * complete setup in TCM process context w/ TFO->new_cmd_map().
 */
int transport_generic_handle_cdb_map(
	struct se_cmd *cmd)
{
1733
	if (!cmd->se_lun) {
1734
		dump_stack();
1735
		pr_err("cmd->se_lun is NULL\n");
1736
		return -EINVAL;
1737 1738
	}

1739
	transport_add_cmd_to_queue(cmd, TRANSPORT_NEW_CMD_MAP, false);
1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757
	return 0;
}
EXPORT_SYMBOL(transport_generic_handle_cdb_map);

/*	transport_generic_handle_data():
 *
 *
 */
int transport_generic_handle_data(
	struct se_cmd *cmd)
{
	/*
	 * For the software fabric case, then we assume the nexus is being
	 * failed/shutdown when signals are pending from the kthread context
	 * caller, so we return a failure.  For the HW target mode case running
	 * in interrupt code, the signal_pending() check is skipped.
	 */
	if (!in_interrupt() && signal_pending(current))
1758
		return -EPERM;
1759 1760 1761 1762
	/*
	 * If the received CDB has aleady been ABORTED by the generic
	 * target engine, we now call transport_check_aborted_status()
	 * to queue any delated TASK_ABORTED status for the received CDB to the
L
Lucas De Marchi 已提交
1763
	 * fabric module as we are expecting no further incoming DATA OUT
1764 1765 1766 1767 1768
	 * sequences at this point.
	 */
	if (transport_check_aborted_status(cmd, 1) != 0)
		return 0;

1769
	transport_add_cmd_to_queue(cmd, TRANSPORT_PROCESS_WRITE, false);
1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780
	return 0;
}
EXPORT_SYMBOL(transport_generic_handle_data);

/*	transport_generic_handle_tmr():
 *
 *
 */
int transport_generic_handle_tmr(
	struct se_cmd *cmd)
{
1781
	transport_add_cmd_to_queue(cmd, TRANSPORT_PROCESS_TMR, false);
1782 1783 1784 1785
	return 0;
}
EXPORT_SYMBOL(transport_generic_handle_tmr);

1786 1787 1788
void transport_generic_free_cmd_intr(
	struct se_cmd *cmd)
{
1789
	transport_add_cmd_to_queue(cmd, TRANSPORT_FREE_CMD_INTR, false);
1790 1791 1792
}
EXPORT_SYMBOL(transport_generic_free_cmd_intr);

1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806
/*
 * If the task is active, request it to be stopped and sleep until it
 * has completed.
 */
bool target_stop_task(struct se_task *task, unsigned long *flags)
{
	struct se_cmd *cmd = task->task_se_cmd;
	bool was_active = false;

	if (task->task_flags & TF_ACTIVE) {
		task->task_flags |= TF_REQUEST_STOP;
		spin_unlock_irqrestore(&cmd->t_state_lock, *flags);

		pr_debug("Task %p waiting to complete\n", task);
1807
		del_timer_sync(&task->task_timer);
1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819
		wait_for_completion(&task->task_stop_comp);
		pr_debug("Task %p stopped successfully\n", task);

		spin_lock_irqsave(&cmd->t_state_lock, *flags);
		atomic_dec(&cmd->t_task_cdbs_left);
		task->task_flags &= ~(TF_ACTIVE | TF_REQUEST_STOP);
		was_active = true;
	}

	return was_active;
}

1820 1821 1822 1823 1824 1825
static int transport_stop_tasks_for_cmd(struct se_cmd *cmd)
{
	struct se_task *task, *task_tmp;
	unsigned long flags;
	int ret = 0;

1826
	pr_debug("ITT[0x%08x] - Stopping tasks\n",
1827
		cmd->se_tfo->get_task_tag(cmd));
1828 1829 1830 1831

	/*
	 * No tasks remain in the execution queue
	 */
1832
	spin_lock_irqsave(&cmd->t_state_lock, flags);
1833
	list_for_each_entry_safe(task, task_tmp,
1834
				&cmd->t_task_list, t_list) {
1835
		pr_debug("Processing task %p\n", task);
1836 1837 1838 1839
		/*
		 * If the struct se_task has not been sent and is not active,
		 * remove the struct se_task from the execution queue.
		 */
1840
		if (!(task->task_flags & (TF_ACTIVE | TF_SENT))) {
1841
			spin_unlock_irqrestore(&cmd->t_state_lock,
1842 1843
					flags);
			transport_remove_task_from_execute_queue(task,
1844
					cmd->se_dev);
1845

1846
			pr_debug("Task %p removed from execute queue\n", task);
1847
			spin_lock_irqsave(&cmd->t_state_lock, flags);
1848 1849 1850
			continue;
		}

1851
		if (!target_stop_task(task, &flags)) {
1852
			pr_debug("Task %p - did nothing\n", task);
1853 1854 1855
			ret++;
		}
	}
1856
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868

	return ret;
}

/*
 * Handle SAM-esque emulation for generic transport request failures.
 */
static void transport_generic_request_failure(
	struct se_cmd *cmd,
	int complete,
	int sc)
{
1869 1870
	int ret = 0;

1871
	pr_debug("-----[ Storage Engine Exception for cmd: %p ITT: 0x%08x"
1872
		" CDB: 0x%02x\n", cmd, cmd->se_tfo->get_task_tag(cmd),
1873
		cmd->t_task_cdb[0]);
1874
	pr_debug("-----[ i_state: %d t_state: %d transport_error_status: %d\n",
1875
		cmd->se_tfo->get_cmd_state(cmd),
1876
		cmd->t_state,
1877
		cmd->transport_error_status);
1878
	pr_debug("-----[ t_tasks: %d t_task_cdbs_left: %d"
1879 1880
		" t_task_cdbs_sent: %d t_task_cdbs_ex_left: %d --"
		" t_transport_active: %d t_transport_stop: %d"
1881
		" t_transport_sent: %d\n", cmd->t_task_list_num,
1882 1883 1884 1885 1886 1887
		atomic_read(&cmd->t_task_cdbs_left),
		atomic_read(&cmd->t_task_cdbs_sent),
		atomic_read(&cmd->t_task_cdbs_ex_left),
		atomic_read(&cmd->t_transport_active),
		atomic_read(&cmd->t_transport_stop),
		atomic_read(&cmd->t_transport_sent));
1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920

	/*
	 * For SAM Task Attribute emulation for failed struct se_cmd
	 */
	if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
		transport_complete_task_attr(cmd);

	if (complete) {
		transport_direct_request_timeout(cmd);
		cmd->transport_error_status = PYX_TRANSPORT_LU_COMM_FAILURE;
	}

	switch (cmd->transport_error_status) {
	case PYX_TRANSPORT_UNKNOWN_SAM_OPCODE:
		cmd->scsi_sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
		break;
	case PYX_TRANSPORT_REQ_TOO_MANY_SECTORS:
		cmd->scsi_sense_reason = TCM_SECTOR_COUNT_TOO_MANY;
		break;
	case PYX_TRANSPORT_INVALID_CDB_FIELD:
		cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
		break;
	case PYX_TRANSPORT_INVALID_PARAMETER_LIST:
		cmd->scsi_sense_reason = TCM_INVALID_PARAMETER_LIST;
		break;
	case PYX_TRANSPORT_OUT_OF_MEMORY_RESOURCES:
		if (!sc)
			transport_new_cmd_failure(cmd);
		/*
		 * Currently for PYX_TRANSPORT_OUT_OF_MEMORY_RESOURCES,
		 * we force this session to fall back to session
		 * recovery.
		 */
1921 1922
		cmd->se_tfo->fall_back_to_erl0(cmd->se_sess);
		cmd->se_tfo->stop_session(cmd->se_sess, 0, 0);
1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949

		goto check_stop;
	case PYX_TRANSPORT_LU_COMM_FAILURE:
	case PYX_TRANSPORT_ILLEGAL_REQUEST:
		cmd->scsi_sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
		break;
	case PYX_TRANSPORT_UNKNOWN_MODE_PAGE:
		cmd->scsi_sense_reason = TCM_UNKNOWN_MODE_PAGE;
		break;
	case PYX_TRANSPORT_WRITE_PROTECTED:
		cmd->scsi_sense_reason = TCM_WRITE_PROTECTED;
		break;
	case PYX_TRANSPORT_RESERVATION_CONFLICT:
		/*
		 * No SENSE Data payload for this case, set SCSI Status
		 * and queue the response to $FABRIC_MOD.
		 *
		 * Uses linux/include/scsi/scsi.h SAM status codes defs
		 */
		cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
		/*
		 * For UA Interlock Code 11b, a RESERVATION CONFLICT will
		 * establish a UNIT ATTENTION with PREVIOUS RESERVATION
		 * CONFLICT STATUS.
		 *
		 * See spc4r17, section 7.4.6 Control Mode Page, Table 349
		 */
1950 1951 1952
		if (cmd->se_sess &&
		    cmd->se_dev->se_sub_dev->se_dev_attrib.emulate_ua_intlck_ctrl == 2)
			core_scsi3_ua_allocate(cmd->se_sess->se_node_acl,
1953 1954 1955
				cmd->orig_fe_lun, 0x2C,
				ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);

1956 1957 1958
		ret = cmd->se_tfo->queue_status(cmd);
		if (ret == -EAGAIN)
			goto queue_full;
1959 1960 1961 1962 1963 1964 1965
		goto check_stop;
	case PYX_TRANSPORT_USE_SENSE_REASON:
		/*
		 * struct se_cmd->scsi_sense_reason already set
		 */
		break;
	default:
1966
		pr_err("Unknown transport error for CDB 0x%02x: %d\n",
1967
			cmd->t_task_cdb[0],
1968 1969 1970 1971
			cmd->transport_error_status);
		cmd->scsi_sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
		break;
	}
1972 1973 1974 1975 1976 1977 1978 1979
	/*
	 * If a fabric does not define a cmd->se_tfo->new_cmd_map caller,
	 * make the call to transport_send_check_condition_and_sense()
	 * directly.  Otherwise expect the fabric to make the call to
	 * transport_send_check_condition_and_sense() after handling
	 * possible unsoliticied write data payloads.
	 */
	if (!sc && !cmd->se_tfo->new_cmd_map)
1980
		transport_new_cmd_failure(cmd);
1981 1982 1983 1984 1985 1986 1987
	else {
		ret = transport_send_check_condition_and_sense(cmd,
				cmd->scsi_sense_reason, 0);
		if (ret == -EAGAIN)
			goto queue_full;
	}

1988 1989
check_stop:
	transport_lun_remove_cmd(cmd);
1990
	if (!transport_cmd_check_stop_to_fabric(cmd))
1991
		;
1992 1993 1994
	return;

queue_full:
1995 1996
	cmd->t_state = TRANSPORT_COMPLETE_QF_OK;
	transport_handle_queue_full(cmd, cmd->se_dev);
1997 1998 1999 2000 2001 2002
}

static void transport_direct_request_timeout(struct se_cmd *cmd)
{
	unsigned long flags;

2003
	spin_lock_irqsave(&cmd->t_state_lock, flags);
2004
	if (!atomic_read(&cmd->t_transport_timeout)) {
2005
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2006 2007
		return;
	}
2008 2009
	if (atomic_read(&cmd->t_task_cdbs_timeout_left)) {
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2010 2011 2012
		return;
	}

2013 2014 2015
	atomic_sub(atomic_read(&cmd->t_transport_timeout),
		   &cmd->t_se_count);
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054
}

static inline u32 transport_lba_21(unsigned char *cdb)
{
	return ((cdb[1] & 0x1f) << 16) | (cdb[2] << 8) | cdb[3];
}

static inline u32 transport_lba_32(unsigned char *cdb)
{
	return (cdb[2] << 24) | (cdb[3] << 16) | (cdb[4] << 8) | cdb[5];
}

static inline unsigned long long transport_lba_64(unsigned char *cdb)
{
	unsigned int __v1, __v2;

	__v1 = (cdb[2] << 24) | (cdb[3] << 16) | (cdb[4] << 8) | cdb[5];
	__v2 = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9];

	return ((unsigned long long)__v2) | (unsigned long long)__v1 << 32;
}

/*
 * For VARIABLE_LENGTH_CDB w/ 32 byte extended CDBs
 */
static inline unsigned long long transport_lba_64_ext(unsigned char *cdb)
{
	unsigned int __v1, __v2;

	__v1 = (cdb[12] << 24) | (cdb[13] << 16) | (cdb[14] << 8) | cdb[15];
	__v2 = (cdb[16] << 24) | (cdb[17] << 16) | (cdb[18] << 8) | cdb[19];

	return ((unsigned long long)__v2) | (unsigned long long)__v1 << 32;
}

static void transport_set_supported_SAM_opcode(struct se_cmd *se_cmd)
{
	unsigned long flags;

2055
	spin_lock_irqsave(&se_cmd->t_state_lock, flags);
2056
	se_cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE;
2057
	spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
2058 2059 2060 2061 2062 2063 2064 2065
}

/*
 * Called from interrupt context.
 */
static void transport_task_timeout_handler(unsigned long data)
{
	struct se_task *task = (struct se_task *)data;
2066
	struct se_cmd *cmd = task->task_se_cmd;
2067 2068
	unsigned long flags;

2069
	pr_debug("transport task timeout fired! task: %p cmd: %p\n", task, cmd);
2070

2071
	spin_lock_irqsave(&cmd->t_state_lock, flags);
2072 2073 2074 2075

	/*
	 * Determine if transport_complete_task() has already been called.
	 */
2076 2077 2078
	if (!(task->task_flags & TF_ACTIVE)) {
		pr_debug("transport task: %p cmd: %p timeout !TF_ACTIVE\n",
			 task, cmd);
2079
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2080 2081 2082
		return;
	}

2083 2084 2085
	atomic_inc(&cmd->t_se_count);
	atomic_inc(&cmd->t_transport_timeout);
	cmd->t_tasks_failed = 1;
2086

2087
	task->task_flags |= TF_TIMEOUT;
2088 2089 2090
	task->task_error_status = PYX_TRANSPORT_TASK_TIMEOUT;
	task->task_scsi_status = 1;

2091 2092
	if (task->task_flags & TF_REQUEST_STOP) {
		pr_debug("transport task: %p cmd: %p timeout TF_REQUEST_STOP"
2093
				" == 1\n", task, cmd);
2094
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2095 2096 2097 2098
		complete(&task->task_stop_comp);
		return;
	}

2099 2100
	if (!atomic_dec_and_test(&cmd->t_task_cdbs_left)) {
		pr_debug("transport task: %p cmd: %p timeout non zero"
2101
				" t_task_cdbs_left\n", task, cmd);
2102
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2103 2104
		return;
	}
2105
	pr_debug("transport task: %p cmd: %p timeout ZERO t_task_cdbs_left\n",
2106 2107
			task, cmd);

2108 2109 2110
	INIT_WORK(&cmd->work, target_complete_failure_work);
	cmd->t_state = TRANSPORT_COMPLETE;
	atomic_set(&cmd->t_transport_active, 1);
2111
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2112

2113
	queue_work(target_completion_wq, &cmd->work);
2114 2115 2116 2117
}

static void transport_start_task_timer(struct se_task *task)
{
2118
	struct se_device *dev = task->task_se_cmd->se_dev;
2119 2120 2121 2122 2123
	int timeout;

	/*
	 * If the task_timeout is disabled, exit now.
	 */
2124
	timeout = dev->se_sub_dev->se_dev_attrib.task_timeout;
2125
	if (!timeout)
2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141
		return;

	task->task_timer.expires = (get_jiffies_64() + timeout * HZ);
	task->task_timer.data = (unsigned long) task;
	task->task_timer.function = transport_task_timeout_handler;
	add_timer(&task->task_timer);
}

static inline int transport_tcq_window_closed(struct se_device *dev)
{
	if (dev->dev_tcq_window_closed++ <
			PYX_TRANSPORT_WINDOW_CLOSED_THRESHOLD) {
		msleep(PYX_TRANSPORT_WINDOW_CLOSED_WAIT_SHORT);
	} else
		msleep(PYX_TRANSPORT_WINDOW_CLOSED_WAIT_LONG);

2142
	wake_up_interruptible(&dev->dev_queue_obj.thread_wq);
2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154
	return 0;
}

/*
 * Called from Fabric Module context from transport_execute_tasks()
 *
 * The return of this function determins if the tasks from struct se_cmd
 * get added to the execution queue in transport_execute_tasks(),
 * or are added to the delayed or ordered lists here.
 */
static inline int transport_execute_task_attr(struct se_cmd *cmd)
{
2155
	if (cmd->se_dev->dev_task_attr_type != SAM_TASK_ATTR_EMULATED)
2156 2157
		return 1;
	/*
L
Lucas De Marchi 已提交
2158
	 * Check for the existence of HEAD_OF_QUEUE, and if true return 1
2159 2160
	 * to allow the passed struct se_cmd list of tasks to the front of the list.
	 */
2161
	 if (cmd->sam_task_attr == MSG_HEAD_TAG) {
2162
		atomic_inc(&cmd->se_dev->dev_hoq_count);
2163
		smp_mb__after_atomic_inc();
2164
		pr_debug("Added HEAD_OF_QUEUE for CDB:"
2165
			" 0x%02x, se_ordered_id: %u\n",
2166
			cmd->t_task_cdb[0],
2167 2168
			cmd->se_ordered_id);
		return 1;
2169
	} else if (cmd->sam_task_attr == MSG_ORDERED_TAG) {
2170 2171 2172 2173
		spin_lock(&cmd->se_dev->ordered_cmd_lock);
		list_add_tail(&cmd->se_ordered_node,
				&cmd->se_dev->ordered_cmd_list);
		spin_unlock(&cmd->se_dev->ordered_cmd_lock);
2174

2175
		atomic_inc(&cmd->se_dev->dev_ordered_sync);
2176 2177
		smp_mb__after_atomic_inc();

2178
		pr_debug("Added ORDERED for CDB: 0x%02x to ordered"
2179
				" list, se_ordered_id: %u\n",
2180
				cmd->t_task_cdb[0],
2181 2182 2183 2184 2185 2186
				cmd->se_ordered_id);
		/*
		 * Add ORDERED command to tail of execution queue if
		 * no other older commands exist that need to be
		 * completed first.
		 */
2187
		if (!atomic_read(&cmd->se_dev->simple_cmds))
2188 2189 2190 2191 2192
			return 1;
	} else {
		/*
		 * For SIMPLE and UNTAGGED Task Attribute commands
		 */
2193
		atomic_inc(&cmd->se_dev->simple_cmds);
2194 2195 2196 2197 2198 2199 2200
		smp_mb__after_atomic_inc();
	}
	/*
	 * Otherwise if one or more outstanding ORDERED task attribute exist,
	 * add the dormant task(s) built for the passed struct se_cmd to the
	 * execution queue and become in Active state for this struct se_device.
	 */
2201
	if (atomic_read(&cmd->se_dev->dev_ordered_sync) != 0) {
2202 2203
		/*
		 * Otherwise, add cmd w/ tasks to delayed cmd queue that
L
Lucas De Marchi 已提交
2204
		 * will be drained upon completion of HEAD_OF_QUEUE task.
2205
		 */
2206
		spin_lock(&cmd->se_dev->delayed_cmd_lock);
2207
		cmd->se_cmd_flags |= SCF_DELAYED_CMD_FROM_SAM_ATTR;
2208 2209 2210
		list_add_tail(&cmd->se_delayed_node,
				&cmd->se_dev->delayed_cmd_list);
		spin_unlock(&cmd->se_dev->delayed_cmd_lock);
2211

2212
		pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to"
2213
			" delayed CMD list, se_ordered_id: %u\n",
2214
			cmd->t_task_cdb[0], cmd->sam_task_attr,
2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235
			cmd->se_ordered_id);
		/*
		 * Return zero to let transport_execute_tasks() know
		 * not to add the delayed tasks to the execution list.
		 */
		return 0;
	}
	/*
	 * Otherwise, no ORDERED task attributes exist..
	 */
	return 1;
}

/*
 * Called from fabric module context in transport_generic_new_cmd() and
 * transport_generic_process_write()
 */
static int transport_execute_tasks(struct se_cmd *cmd)
{
	int add_tasks;

2236 2237
	if (se_dev_check_online(cmd->se_orig_obj_ptr) != 0) {
		cmd->transport_error_status = PYX_TRANSPORT_LU_COMM_FAILURE;
2238
		transport_generic_request_failure(cmd, 0, 1);
2239
		return 0;
2240
	}
2241

2242 2243
	/*
	 * Call transport_cmd_check_stop() to see if a fabric exception
L
Lucas De Marchi 已提交
2244
	 * has occurred that prevents execution.
2245
	 */
2246
	if (!transport_cmd_check_stop(cmd, 0, TRANSPORT_PROCESSING)) {
2247 2248 2249 2250 2251
		/*
		 * Check for SAM Task Attribute emulation and HEAD_OF_QUEUE
		 * attribute for the tasks of the received struct se_cmd CDB
		 */
		add_tasks = transport_execute_task_attr(cmd);
2252
		if (!add_tasks)
2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266
			goto execute_tasks;
		/*
		 * This calls transport_add_tasks_from_cmd() to handle
		 * HEAD_OF_QUEUE ordering for SAM Task Attribute emulation
		 * (if enabled) in __transport_add_task_to_execute_queue() and
		 * transport_add_task_check_sam_attr().
		 */
		transport_add_tasks_from_cmd(cmd);
	}
	/*
	 * Kick the execution queue for the cmd associated struct se_device
	 * storage object.
	 */
execute_tasks:
2267
	__transport_execute_tasks(cmd->se_dev);
2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280
	return 0;
}

/*
 * Called to check struct se_device tcq depth window, and once open pull struct se_task
 * from struct se_device->execute_task_list and
 *
 * Called from transport_processing_thread()
 */
static int __transport_execute_tasks(struct se_device *dev)
{
	int error;
	struct se_cmd *cmd = NULL;
2281
	struct se_task *task = NULL;
2282 2283 2284 2285
	unsigned long flags;

	/*
	 * Check if there is enough room in the device and HBA queue to send
2286
	 * struct se_tasks to the selected transport.
2287 2288
	 */
check_depth:
2289
	if (!atomic_read(&dev->depth_left))
2290 2291
		return transport_tcq_window_closed(dev);

2292
	dev->dev_tcq_window_closed = 0;
2293

2294 2295 2296
	spin_lock_irq(&dev->execute_task_lock);
	if (list_empty(&dev->execute_task_list)) {
		spin_unlock_irq(&dev->execute_task_lock);
2297 2298
		return 0;
	}
2299 2300
	task = list_first_entry(&dev->execute_task_list,
				struct se_task, t_execute_list);
2301
	__transport_remove_task_from_execute_queue(task, dev);
2302
	spin_unlock_irq(&dev->execute_task_lock);
2303 2304 2305

	atomic_dec(&dev->depth_left);

2306
	cmd = task->task_se_cmd;
2307

2308
	spin_lock_irqsave(&cmd->t_state_lock, flags);
2309
	task->task_flags |= (TF_ACTIVE | TF_SENT);
2310
	atomic_inc(&cmd->t_task_cdbs_sent);
2311

2312 2313
	if (atomic_read(&cmd->t_task_cdbs_sent) ==
	    cmd->t_task_list_num)
2314 2315 2316
		atomic_set(&cmd->transport_sent, 1);

	transport_start_task_timer(task);
2317
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2318 2319
	/*
	 * The struct se_cmd->transport_emulate_cdb() function pointer is used
2320
	 * to grab REPORT_LUNS and other CDBs we want to handle before they hit the
2321 2322 2323 2324 2325 2326
	 * struct se_subsystem_api->do_task() caller below.
	 */
	if (cmd->transport_emulate_cdb) {
		error = cmd->transport_emulate_cdb(cmd);
		if (error != 0) {
			cmd->transport_error_status = error;
2327 2328 2329
			spin_lock_irqsave(&cmd->t_state_lock, flags);
			task->task_flags &= ~TF_ACTIVE;
			spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2330
			del_timer_sync(&task->task_timer);
2331 2332
			atomic_set(&cmd->transport_sent, 0);
			transport_stop_tasks_for_cmd(cmd);
2333 2334
			atomic_inc(&dev->depth_left);
			transport_generic_request_failure(cmd, 0, 1);
2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358
			goto check_depth;
		}
		/*
		 * Handle the successful completion for transport_emulate_cdb()
		 * for synchronous operation, following SCF_EMULATE_CDB_ASYNC
		 * Otherwise the caller is expected to complete the task with
		 * proper status.
		 */
		if (!(cmd->se_cmd_flags & SCF_EMULATE_CDB_ASYNC)) {
			cmd->scsi_status = SAM_STAT_GOOD;
			task->task_scsi_status = GOOD;
			transport_complete_task(task, 1);
		}
	} else {
		/*
		 * Currently for all virtual TCM plugins including IBLOCK, FILEIO and
		 * RAMDISK we use the internal transport_emulate_control_cdb() logic
		 * with struct se_subsystem_api callers for the primary SPC-3 TYPE_DISK
		 * LUN emulation code.
		 *
		 * For TCM/pSCSI and all other SCF_SCSI_DATA_SG_IO_CDB I/O tasks we
		 * call ->do_task() directly and let the underlying TCM subsystem plugin
		 * code handle the CDB emulation.
		 */
2359 2360
		if ((dev->transport->transport_type != TRANSPORT_PLUGIN_PHBA_PDEV) &&
		    (!(task->task_se_cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB)))
2361 2362
			error = transport_emulate_control_cdb(task);
		else
2363
			error = dev->transport->do_task(task);
2364 2365 2366

		if (error != 0) {
			cmd->transport_error_status = error;
2367 2368 2369
			spin_lock_irqsave(&cmd->t_state_lock, flags);
			task->task_flags &= ~TF_ACTIVE;
			spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2370
			del_timer_sync(&task->task_timer);
2371 2372
			atomic_set(&cmd->transport_sent, 0);
			transport_stop_tasks_for_cmd(cmd);
2373 2374
			atomic_inc(&dev->depth_left);
			transport_generic_request_failure(cmd, 0, 1);
2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389
		}
	}

	goto check_depth;

	return 0;
}

void transport_new_cmd_failure(struct se_cmd *se_cmd)
{
	unsigned long flags;
	/*
	 * Any unsolicited data will get dumped for failed command inside of
	 * the fabric plugin
	 */
2390
	spin_lock_irqsave(&se_cmd->t_state_lock, flags);
2391 2392
	se_cmd->se_cmd_flags |= SCF_SE_CMD_FAILED;
	se_cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
2393
	spin_unlock_irqrestore(&se_cmd->t_state_lock, flags);
2394 2395 2396 2397 2398 2399 2400
}

static inline u32 transport_get_sectors_6(
	unsigned char *cdb,
	struct se_cmd *cmd,
	int *ret)
{
2401
	struct se_device *dev = cmd->se_dev;
2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412

	/*
	 * Assume TYPE_DISK for non struct se_device objects.
	 * Use 8-bit sector value.
	 */
	if (!dev)
		goto type_disk;

	/*
	 * Use 24-bit allocation length for TYPE_TAPE.
	 */
2413
	if (dev->transport->get_device_type(dev) == TYPE_TAPE)
2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428
		return (u32)(cdb[2] << 16) + (cdb[3] << 8) + cdb[4];

	/*
	 * Everything else assume TYPE_DISK Sector CDB location.
	 * Use 8-bit sector value.
	 */
type_disk:
	return (u32)cdb[4];
}

static inline u32 transport_get_sectors_10(
	unsigned char *cdb,
	struct se_cmd *cmd,
	int *ret)
{
2429
	struct se_device *dev = cmd->se_dev;
2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440

	/*
	 * Assume TYPE_DISK for non struct se_device objects.
	 * Use 16-bit sector value.
	 */
	if (!dev)
		goto type_disk;

	/*
	 * XXX_10 is not defined in SSC, throw an exception
	 */
2441 2442
	if (dev->transport->get_device_type(dev) == TYPE_TAPE) {
		*ret = -EINVAL;
2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458
		return 0;
	}

	/*
	 * Everything else assume TYPE_DISK Sector CDB location.
	 * Use 16-bit sector value.
	 */
type_disk:
	return (u32)(cdb[7] << 8) + cdb[8];
}

static inline u32 transport_get_sectors_12(
	unsigned char *cdb,
	struct se_cmd *cmd,
	int *ret)
{
2459
	struct se_device *dev = cmd->se_dev;
2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470

	/*
	 * Assume TYPE_DISK for non struct se_device objects.
	 * Use 32-bit sector value.
	 */
	if (!dev)
		goto type_disk;

	/*
	 * XXX_12 is not defined in SSC, throw an exception
	 */
2471 2472
	if (dev->transport->get_device_type(dev) == TYPE_TAPE) {
		*ret = -EINVAL;
2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488
		return 0;
	}

	/*
	 * Everything else assume TYPE_DISK Sector CDB location.
	 * Use 32-bit sector value.
	 */
type_disk:
	return (u32)(cdb[6] << 24) + (cdb[7] << 16) + (cdb[8] << 8) + cdb[9];
}

static inline u32 transport_get_sectors_16(
	unsigned char *cdb,
	struct se_cmd *cmd,
	int *ret)
{
2489
	struct se_device *dev = cmd->se_dev;
2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500

	/*
	 * Assume TYPE_DISK for non struct se_device objects.
	 * Use 32-bit sector value.
	 */
	if (!dev)
		goto type_disk;

	/*
	 * Use 24-bit allocation length for TYPE_TAPE.
	 */
2501
	if (dev->transport->get_device_type(dev) == TYPE_TAPE)
2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530
		return (u32)(cdb[12] << 16) + (cdb[13] << 8) + cdb[14];

type_disk:
	return (u32)(cdb[10] << 24) + (cdb[11] << 16) +
		    (cdb[12] << 8) + cdb[13];
}

/*
 * Used for VARIABLE_LENGTH_CDB WRITE_32 and READ_32 variants
 */
static inline u32 transport_get_sectors_32(
	unsigned char *cdb,
	struct se_cmd *cmd,
	int *ret)
{
	/*
	 * Assume TYPE_DISK for non struct se_device objects.
	 * Use 32-bit sector value.
	 */
	return (u32)(cdb[28] << 24) + (cdb[29] << 16) +
		    (cdb[30] << 8) + cdb[31];

}

static inline u32 transport_get_size(
	u32 sectors,
	unsigned char *cdb,
	struct se_cmd *cmd)
{
2531
	struct se_device *dev = cmd->se_dev;
2532

2533
	if (dev->transport->get_device_type(dev) == TYPE_TAPE) {
2534
		if (cdb[1] & 1) { /* sectors */
2535
			return dev->se_sub_dev->se_dev_attrib.block_size * sectors;
2536 2537 2538 2539
		} else /* bytes */
			return sectors;
	}
#if 0
2540
	pr_debug("Returning block_size: %u, sectors: %u == %u for"
2541 2542 2543
			" %s object\n", dev->se_sub_dev->se_dev_attrib.block_size, sectors,
			dev->se_sub_dev->se_dev_attrib.block_size * sectors,
			dev->transport->name);
2544
#endif
2545
	return dev->se_sub_dev->se_dev_attrib.block_size * sectors;
2546 2547 2548 2549 2550
}

static void transport_xor_callback(struct se_cmd *cmd)
{
	unsigned char *buf, *addr;
2551
	struct scatterlist *sg;
2552 2553
	unsigned int offset;
	int i;
2554
	int count;
2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566
	/*
	 * From sbc3r22.pdf section 5.48 XDWRITEREAD (10) command
	 *
	 * 1) read the specified logical block(s);
	 * 2) transfer logical blocks from the data-out buffer;
	 * 3) XOR the logical blocks transferred from the data-out buffer with
	 *    the logical blocks read, storing the resulting XOR data in a buffer;
	 * 4) if the DISABLE WRITE bit is set to zero, then write the logical
	 *    blocks transferred from the data-out buffer; and
	 * 5) transfer the resulting XOR data to the data-in buffer.
	 */
	buf = kmalloc(cmd->data_length, GFP_KERNEL);
2567 2568
	if (!buf) {
		pr_err("Unable to allocate xor_callback buf\n");
2569 2570 2571
		return;
	}
	/*
2572
	 * Copy the scatterlist WRITE buffer located at cmd->t_data_sg
2573 2574
	 * into the locally allocated *buf
	 */
2575 2576 2577 2578 2579
	sg_copy_to_buffer(cmd->t_data_sg,
			  cmd->t_data_nents,
			  buf,
			  cmd->data_length);

2580 2581
	/*
	 * Now perform the XOR against the BIDI read memory located at
2582
	 * cmd->t_mem_bidi_list
2583 2584 2585
	 */

	offset = 0;
2586 2587 2588
	for_each_sg(cmd->t_bidi_data_sg, sg, cmd->t_bidi_data_nents, count) {
		addr = kmap_atomic(sg_page(sg), KM_USER0);
		if (!addr)
2589 2590
			goto out;

2591 2592
		for (i = 0; i < sg->length; i++)
			*(addr + sg->offset + i) ^= *(buf + offset + i);
2593

2594
		offset += sg->length;
2595 2596
		kunmap_atomic(addr, KM_USER0);
	}
2597

2598 2599 2600 2601 2602 2603 2604 2605 2606 2607
out:
	kfree(buf);
}

/*
 * Used to obtain Sense Data from underlying Linux/SCSI struct scsi_cmnd
 */
static int transport_get_sense_data(struct se_cmd *cmd)
{
	unsigned char *buffer = cmd->sense_buffer, *sense_buffer = NULL;
2608
	struct se_device *dev = cmd->se_dev;
2609 2610 2611 2612
	struct se_task *task = NULL, *task_tmp;
	unsigned long flags;
	u32 offset = 0;

2613 2614
	WARN_ON(!cmd->se_lun);

2615 2616 2617
	if (!dev)
		return 0;

2618
	spin_lock_irqsave(&cmd->t_state_lock, flags);
2619
	if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
2620
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2621 2622 2623 2624
		return 0;
	}

	list_for_each_entry_safe(task, task_tmp,
2625
				&cmd->t_task_list, t_list) {
2626 2627 2628
		if (!task->task_sense)
			continue;

2629
		if (!dev->transport->get_sense_buffer) {
2630
			pr_err("dev->transport->get_sense_buffer"
2631 2632 2633 2634
					" is NULL\n");
			continue;
		}

2635
		sense_buffer = dev->transport->get_sense_buffer(task);
2636
		if (!sense_buffer) {
2637
			pr_err("ITT[0x%08x]_TASK[%p]: Unable to locate"
2638
				" sense buffer for task with sense\n",
2639
				cmd->se_tfo->get_task_tag(cmd), task);
2640 2641
			continue;
		}
2642
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2643

2644
		offset = cmd->se_tfo->set_fabric_sense_len(cmd,
2645 2646
				TRANSPORT_SENSE_BUFFER);

2647
		memcpy(&buffer[offset], sense_buffer,
2648 2649 2650 2651 2652 2653
				TRANSPORT_SENSE_BUFFER);
		cmd->scsi_status = task->task_scsi_status;
		/* Automatically padded */
		cmd->scsi_sense_length =
				(TRANSPORT_SENSE_BUFFER + offset);

2654
		pr_debug("HBA_[%u]_PLUG[%s]: Set SAM STATUS: 0x%02x"
2655
				" and sense\n",
2656
			dev->se_hba->hba_id, dev->transport->name,
2657 2658 2659
				cmd->scsi_status);
		return 0;
	}
2660
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677

	return -1;
}

static int
transport_handle_reservation_conflict(struct se_cmd *cmd)
{
	cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
	cmd->se_cmd_flags |= SCF_SCSI_RESERVATION_CONFLICT;
	cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
	/*
	 * For UA Interlock Code 11b, a RESERVATION CONFLICT will
	 * establish a UNIT ATTENTION with PREVIOUS RESERVATION
	 * CONFLICT STATUS.
	 *
	 * See spc4r17, section 7.4.6 Control Mode Page, Table 349
	 */
2678 2679 2680
	if (cmd->se_sess &&
	    cmd->se_dev->se_sub_dev->se_dev_attrib.emulate_ua_intlck_ctrl == 2)
		core_scsi3_ua_allocate(cmd->se_sess->se_node_acl,
2681 2682
			cmd->orig_fe_lun, 0x2C,
			ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);
2683
	return -EINVAL;
2684 2685
}

2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700
static inline long long transport_dev_end_lba(struct se_device *dev)
{
	return dev->transport->get_blocks(dev) + 1;
}

static int transport_cmd_get_valid_sectors(struct se_cmd *cmd)
{
	struct se_device *dev = cmd->se_dev;
	u32 sectors;

	if (dev->transport->get_device_type(dev) != TYPE_DISK)
		return 0;

	sectors = (cmd->data_length / dev->se_sub_dev->se_dev_attrib.block_size);

2701 2702
	if ((cmd->t_task_lba + sectors) > transport_dev_end_lba(dev)) {
		pr_err("LBA: %llu Sectors: %u exceeds"
2703 2704 2705
			" transport_dev_end_lba(): %llu\n",
			cmd->t_task_lba, sectors,
			transport_dev_end_lba(dev));
2706
		return -EINVAL;
2707 2708
	}

2709
	return 0;
2710 2711
}

2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743
static int target_check_write_same_discard(unsigned char *flags, struct se_device *dev)
{
	/*
	 * Determine if the received WRITE_SAME is used to for direct
	 * passthrough into Linux/SCSI with struct request via TCM/pSCSI
	 * or we are signaling the use of internal WRITE_SAME + UNMAP=1
	 * emulation for -> Linux/BLOCK disbard with TCM/IBLOCK code.
	 */
	int passthrough = (dev->transport->transport_type ==
				TRANSPORT_PLUGIN_PHBA_PDEV);

	if (!passthrough) {
		if ((flags[0] & 0x04) || (flags[0] & 0x02)) {
			pr_err("WRITE_SAME PBDATA and LBDATA"
				" bits not supported for Block Discard"
				" Emulation\n");
			return -ENOSYS;
		}
		/*
		 * Currently for the emulated case we only accept
		 * tpws with the UNMAP=1 bit set.
		 */
		if (!(flags[0] & 0x08)) {
			pr_err("WRITE_SAME w/o UNMAP bit not"
				" supported for Block Discard Emulation\n");
			return -ENOSYS;
		}
	}

	return 0;
}

2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757
/*	transport_generic_cmd_sequencer():
 *
 *	Generic Command Sequencer that should work for most DAS transport
 *	drivers.
 *
 *	Called from transport_generic_allocate_tasks() in the $FABRIC_MOD
 *	RX Thread.
 *
 *	FIXME: Need to support other SCSI OPCODES where as well.
 */
static int transport_generic_cmd_sequencer(
	struct se_cmd *cmd,
	unsigned char *cdb)
{
2758
	struct se_device *dev = cmd->se_dev;
2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769
	struct se_subsystem_dev *su_dev = dev->se_sub_dev;
	int ret = 0, sector_ret = 0, passthrough;
	u32 sectors = 0, size = 0, pr_reg_type = 0;
	u16 service_action;
	u8 alua_ascq = 0;
	/*
	 * Check for an existing UNIT ATTENTION condition
	 */
	if (core_scsi3_ua_check(cmd, cdb) < 0) {
		cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
		cmd->scsi_sense_reason = TCM_CHECK_CONDITION_UNIT_ATTENTION;
2770
		return -EINVAL;
2771 2772 2773 2774
	}
	/*
	 * Check status of Asymmetric Logical Unit Assignment port
	 */
2775
	ret = su_dev->t10_alua.alua_state_check(cmd, cdb, &alua_ascq);
2776 2777
	if (ret != 0) {
		/*
L
Lucas De Marchi 已提交
2778
		 * Set SCSI additional sense code (ASC) to 'LUN Not Accessible';
2779 2780 2781 2782 2783
		 * The ALUA additional sense code qualifier (ASCQ) is determined
		 * by the ALUA primary or secondary access state..
		 */
		if (ret > 0) {
#if 0
2784
			pr_debug("[%s]: ALUA TG Port not available,"
2785
				" SenseKey: NOT_READY, ASC/ASCQ: 0x04/0x%02x\n",
2786
				cmd->se_tfo->get_fabric_name(), alua_ascq);
2787 2788 2789 2790
#endif
			transport_set_sense_codes(cmd, 0x04, alua_ascq);
			cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
			cmd->scsi_sense_reason = TCM_CHECK_CONDITION_NOT_READY;
2791
			return -EINVAL;
2792 2793 2794 2795 2796 2797
		}
		goto out_invalid_cdb_field;
	}
	/*
	 * Check status for SPC-3 Persistent Reservations
	 */
2798 2799
	if (su_dev->t10_pr.pr_ops.t10_reservation_check(cmd, &pr_reg_type) != 0) {
		if (su_dev->t10_pr.pr_ops.t10_seq_non_holder(
2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814
					cmd, cdb, pr_reg_type) != 0)
			return transport_handle_reservation_conflict(cmd);
		/*
		 * This means the CDB is allowed for the SCSI Initiator port
		 * when said port is *NOT* holding the legacy SPC-2 or
		 * SPC-3 Persistent Reservation.
		 */
	}

	switch (cdb[0]) {
	case READ_6:
		sectors = transport_get_sectors_6(cdb, cmd, &sector_ret);
		if (sector_ret)
			goto out_unsupported_cdb;
		size = transport_get_size(sectors, cdb, cmd);
2815
		cmd->t_task_lba = transport_lba_21(cdb);
2816 2817 2818 2819 2820 2821 2822
		cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
		break;
	case READ_10:
		sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
		if (sector_ret)
			goto out_unsupported_cdb;
		size = transport_get_size(sectors, cdb, cmd);
2823
		cmd->t_task_lba = transport_lba_32(cdb);
2824 2825 2826 2827 2828 2829 2830
		cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
		break;
	case READ_12:
		sectors = transport_get_sectors_12(cdb, cmd, &sector_ret);
		if (sector_ret)
			goto out_unsupported_cdb;
		size = transport_get_size(sectors, cdb, cmd);
2831
		cmd->t_task_lba = transport_lba_32(cdb);
2832 2833 2834 2835 2836 2837 2838
		cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
		break;
	case READ_16:
		sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
		if (sector_ret)
			goto out_unsupported_cdb;
		size = transport_get_size(sectors, cdb, cmd);
2839
		cmd->t_task_lba = transport_lba_64(cdb);
2840 2841 2842 2843 2844 2845 2846
		cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
		break;
	case WRITE_6:
		sectors = transport_get_sectors_6(cdb, cmd, &sector_ret);
		if (sector_ret)
			goto out_unsupported_cdb;
		size = transport_get_size(sectors, cdb, cmd);
2847
		cmd->t_task_lba = transport_lba_21(cdb);
2848 2849 2850 2851 2852 2853 2854
		cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
		break;
	case WRITE_10:
		sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
		if (sector_ret)
			goto out_unsupported_cdb;
		size = transport_get_size(sectors, cdb, cmd);
2855 2856
		cmd->t_task_lba = transport_lba_32(cdb);
		cmd->t_tasks_fua = (cdb[1] & 0x8);
2857 2858 2859 2860 2861 2862 2863
		cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
		break;
	case WRITE_12:
		sectors = transport_get_sectors_12(cdb, cmd, &sector_ret);
		if (sector_ret)
			goto out_unsupported_cdb;
		size = transport_get_size(sectors, cdb, cmd);
2864 2865
		cmd->t_task_lba = transport_lba_32(cdb);
		cmd->t_tasks_fua = (cdb[1] & 0x8);
2866 2867 2868 2869 2870 2871 2872
		cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
		break;
	case WRITE_16:
		sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
		if (sector_ret)
			goto out_unsupported_cdb;
		size = transport_get_size(sectors, cdb, cmd);
2873 2874
		cmd->t_task_lba = transport_lba_64(cdb);
		cmd->t_tasks_fua = (cdb[1] & 0x8);
2875 2876 2877 2878
		cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
		break;
	case XDWRITEREAD_10:
		if ((cmd->data_direction != DMA_TO_DEVICE) ||
2879
		    !(cmd->t_tasks_bidi))
2880 2881 2882 2883 2884
			goto out_invalid_cdb_field;
		sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
		if (sector_ret)
			goto out_unsupported_cdb;
		size = transport_get_size(sectors, cdb, cmd);
2885
		cmd->t_task_lba = transport_lba_32(cdb);
2886
		cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
2887
		passthrough = (dev->transport->transport_type ==
2888 2889 2890 2891 2892 2893 2894
				TRANSPORT_PLUGIN_PHBA_PDEV);
		/*
		 * Skip the remaining assignments for TCM/PSCSI passthrough
		 */
		if (passthrough)
			break;
		/*
2895
		 * Setup BIDI XOR callback to be run after I/O completion.
2896 2897
		 */
		cmd->transport_complete_callback = &transport_xor_callback;
2898
		cmd->t_tasks_fua = (cdb[1] & 0x8);
2899 2900 2901 2902 2903 2904 2905
		break;
	case VARIABLE_LENGTH_CMD:
		service_action = get_unaligned_be16(&cdb[8]);
		/*
		 * Determine if this is TCM/PSCSI device and we should disable
		 * internal emulation for this CDB.
		 */
2906
		passthrough = (dev->transport->transport_type ==
2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918
					TRANSPORT_PLUGIN_PHBA_PDEV);

		switch (service_action) {
		case XDWRITEREAD_32:
			sectors = transport_get_sectors_32(cdb, cmd, &sector_ret);
			if (sector_ret)
				goto out_unsupported_cdb;
			size = transport_get_size(sectors, cdb, cmd);
			/*
			 * Use WRITE_32 and READ_32 opcodes for the emulated
			 * XDWRITE_READ_32 logic.
			 */
2919
			cmd->t_task_lba = transport_lba_64_ext(cdb);
2920 2921 2922 2923 2924 2925 2926 2927 2928
			cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;

			/*
			 * Skip the remaining assignments for TCM/PSCSI passthrough
			 */
			if (passthrough)
				break;

			/*
2929 2930
			 * Setup BIDI XOR callback to be run during after I/O
			 * completion.
2931 2932
			 */
			cmd->transport_complete_callback = &transport_xor_callback;
2933
			cmd->t_tasks_fua = (cdb[10] & 0x8);
2934 2935 2936 2937 2938
			break;
		case WRITE_SAME_32:
			sectors = transport_get_sectors_32(cdb, cmd, &sector_ret);
			if (sector_ret)
				goto out_unsupported_cdb;
2939

2940
			if (sectors)
2941
				size = transport_get_size(1, cdb, cmd);
2942 2943 2944 2945 2946
			else {
				pr_err("WSNZ=1, WRITE_SAME w/sectors=0 not"
				       " supported\n");
				goto out_invalid_cdb_field;
			}
2947

2948
			cmd->t_task_lba = get_unaligned_be64(&cdb[12]);
2949 2950
			cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;

2951
			if (target_check_write_same_discard(&cdb[10], dev) < 0)
2952
				goto out_invalid_cdb_field;
2953

2954 2955
			break;
		default:
2956
			pr_err("VARIABLE_LENGTH_CMD service action"
2957 2958 2959 2960
				" 0x%04x not supported\n", service_action);
			goto out_unsupported_cdb;
		}
		break;
2961
	case MAINTENANCE_IN:
2962
		if (dev->transport->get_device_type(dev) != TYPE_ROM) {
2963 2964 2965 2966 2967 2968
			/* MAINTENANCE_IN from SCC-2 */
			/*
			 * Check for emulated MI_REPORT_TARGET_PGS.
			 */
			if (cdb[1] == MI_REPORT_TARGET_PGS) {
				cmd->transport_emulate_cdb =
2969
				(su_dev->t10_alua.alua_type ==
2970
				 SPC3_ALUA_EMULATED) ?
2971
				core_emulate_report_target_port_groups :
2972 2973 2974 2975 2976 2977 2978 2979
				NULL;
			}
			size = (cdb[6] << 24) | (cdb[7] << 16) |
			       (cdb[8] << 8) | cdb[9];
		} else {
			/* GPCMD_SEND_KEY from multi media commands */
			size = (cdb[8] << 8) + cdb[9];
		}
2980
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991
		break;
	case MODE_SELECT:
		size = cdb[4];
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
		break;
	case MODE_SELECT_10:
		size = (cdb[7] << 8) + cdb[8];
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
		break;
	case MODE_SENSE:
		size = cdb[4];
2992
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
2993 2994 2995 2996 2997 2998 2999
		break;
	case MODE_SENSE_10:
	case GPCMD_READ_BUFFER_CAPACITY:
	case GPCMD_SEND_OPC:
	case LOG_SELECT:
	case LOG_SENSE:
		size = (cdb[7] << 8) + cdb[8];
3000
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3001 3002 3003
		break;
	case READ_BLOCK_LIMITS:
		size = READ_BLOCK_LEN;
3004
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015
		break;
	case GPCMD_GET_CONFIGURATION:
	case GPCMD_READ_FORMAT_CAPACITIES:
	case GPCMD_READ_DISC_INFO:
	case GPCMD_READ_TRACK_RZONE_INFO:
		size = (cdb[7] << 8) + cdb[8];
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
		break;
	case PERSISTENT_RESERVE_IN:
	case PERSISTENT_RESERVE_OUT:
		cmd->transport_emulate_cdb =
3016
			(su_dev->t10_pr.res_type ==
3017
			 SPC3_PERSISTENT_RESERVATIONS) ?
3018
			core_scsi3_emulate_pr : NULL;
3019
		size = (cdb[7] << 8) + cdb[8];
3020
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3021 3022 3023 3024 3025 3026 3027 3028
		break;
	case GPCMD_MECHANISM_STATUS:
	case GPCMD_READ_DVD_STRUCTURE:
		size = (cdb[8] << 8) + cdb[9];
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
		break;
	case READ_POSITION:
		size = READ_POSITION_LEN;
3029
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3030
		break;
3031
	case MAINTENANCE_OUT:
3032
		if (dev->transport->get_device_type(dev) != TYPE_ROM) {
3033 3034 3035 3036 3037 3038
			/* MAINTENANCE_OUT from SCC-2
			 *
			 * Check for emulated MO_SET_TARGET_PGS.
			 */
			if (cdb[1] == MO_SET_TARGET_PGS) {
				cmd->transport_emulate_cdb =
3039
				(su_dev->t10_alua.alua_type ==
3040
					SPC3_ALUA_EMULATED) ?
3041
				core_emulate_set_target_port_groups :
3042 3043 3044 3045 3046 3047 3048 3049 3050
				NULL;
			}

			size = (cdb[6] << 24) | (cdb[7] << 16) |
			       (cdb[8] << 8) | cdb[9];
		} else  {
			/* GPCMD_REPORT_KEY from multi media commands */
			size = (cdb[8] << 8) + cdb[9];
		}
3051
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3052 3053 3054 3055 3056 3057 3058
		break;
	case INQUIRY:
		size = (cdb[3] << 8) + cdb[4];
		/*
		 * Do implict HEAD_OF_QUEUE processing for INQUIRY.
		 * See spc4r17 section 5.3
		 */
3059
		if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
3060
			cmd->sam_task_attr = MSG_HEAD_TAG;
3061
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3062 3063 3064
		break;
	case READ_BUFFER:
		size = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8];
3065
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3066 3067 3068
		break;
	case READ_CAPACITY:
		size = READ_CAP_LEN;
3069
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3070 3071 3072 3073 3074
		break;
	case READ_MEDIA_SERIAL_NUMBER:
	case SECURITY_PROTOCOL_IN:
	case SECURITY_PROTOCOL_OUT:
		size = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9];
3075
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3076 3077 3078 3079 3080 3081 3082 3083 3084 3085
		break;
	case SERVICE_ACTION_IN:
	case ACCESS_CONTROL_IN:
	case ACCESS_CONTROL_OUT:
	case EXTENDED_COPY:
	case READ_ATTRIBUTE:
	case RECEIVE_COPY_RESULTS:
	case WRITE_ATTRIBUTE:
		size = (cdb[10] << 24) | (cdb[11] << 16) |
		       (cdb[12] << 8) | cdb[13];
3086
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3087 3088 3089 3090
		break;
	case RECEIVE_DIAGNOSTIC:
	case SEND_DIAGNOSTIC:
		size = (cdb[3] << 8) | cdb[4];
3091
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3092 3093 3094 3095 3096 3097
		break;
/* #warning FIXME: Figure out correct GPCMD_READ_CD blocksize. */
#if 0
	case GPCMD_READ_CD:
		sectors = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8];
		size = (2336 * sectors);
3098
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3099 3100 3101 3102
		break;
#endif
	case READ_TOC:
		size = cdb[8];
3103
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3104 3105 3106
		break;
	case REQUEST_SENSE:
		size = cdb[4];
3107
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3108 3109 3110
		break;
	case READ_ELEMENT_STATUS:
		size = 65536 * cdb[7] + 256 * cdb[8] + cdb[9];
3111
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3112 3113 3114
		break;
	case WRITE_BUFFER:
		size = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8];
3115
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135
		break;
	case RESERVE:
	case RESERVE_10:
		/*
		 * The SPC-2 RESERVE does not contain a size in the SCSI CDB.
		 * Assume the passthrough or $FABRIC_MOD will tell us about it.
		 */
		if (cdb[0] == RESERVE_10)
			size = (cdb[7] << 8) | cdb[8];
		else
			size = cmd->data_length;

		/*
		 * Setup the legacy emulated handler for SPC-2 and
		 * >= SPC-3 compatible reservation handling (CRH=1)
		 * Otherwise, we assume the underlying SCSI logic is
		 * is running in SPC_PASSTHROUGH, and wants reservations
		 * emulation disabled.
		 */
		cmd->transport_emulate_cdb =
3136
				(su_dev->t10_pr.res_type !=
3137
				 SPC_PASSTHROUGH) ?
3138
				core_scsi2_emulate_crh : NULL;
3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152
		cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
		break;
	case RELEASE:
	case RELEASE_10:
		/*
		 * The SPC-2 RELEASE does not contain a size in the SCSI CDB.
		 * Assume the passthrough or $FABRIC_MOD will tell us about it.
		*/
		if (cdb[0] == RELEASE_10)
			size = (cdb[7] << 8) | cdb[8];
		else
			size = cmd->data_length;

		cmd->transport_emulate_cdb =
3153
				(su_dev->t10_pr.res_type !=
3154
				 SPC_PASSTHROUGH) ?
3155
				core_scsi2_emulate_crh : NULL;
3156 3157 3158 3159 3160 3161 3162 3163 3164
		cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
		break;
	case SYNCHRONIZE_CACHE:
	case 0x91: /* SYNCHRONIZE_CACHE_16: */
		/*
		 * Extract LBA and range to be flushed for emulated SYNCHRONIZE_CACHE
		 */
		if (cdb[0] == SYNCHRONIZE_CACHE) {
			sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
3165
			cmd->t_task_lba = transport_lba_32(cdb);
3166 3167
		} else {
			sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
3168
			cmd->t_task_lba = transport_lba_64(cdb);
3169 3170 3171 3172 3173 3174 3175 3176 3177 3178
		}
		if (sector_ret)
			goto out_unsupported_cdb;

		size = transport_get_size(sectors, cdb, cmd);
		cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;

		/*
		 * For TCM/pSCSI passthrough, skip cmd->transport_emulate_cdb()
		 */
3179
		if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV)
3180 3181 3182 3183 3184 3185 3186 3187
			break;
		/*
		 * Set SCF_EMULATE_CDB_ASYNC to ensure asynchronous operation
		 * for SYNCHRONIZE_CACHE* Immed=1 case in __transport_execute_tasks()
		 */
		cmd->se_cmd_flags |= SCF_EMULATE_CDB_ASYNC;
		/*
		 * Check to ensure that LBA + Range does not exceed past end of
3188
		 * device for IBLOCK and FILEIO ->do_sync_cache() backend calls
3189
		 */
3190 3191 3192 3193
		if ((cmd->t_task_lba != 0) || (sectors != 0)) {
			if (transport_cmd_get_valid_sectors(cmd) < 0)
				goto out_invalid_cdb_field;
		}
3194 3195 3196
		break;
	case UNMAP:
		size = get_unaligned_be16(&cdb[7]);
3197
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3198 3199 3200 3201 3202
		break;
	case WRITE_SAME_16:
		sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
		if (sector_ret)
			goto out_unsupported_cdb;
3203

3204
		if (sectors)
3205
			size = transport_get_size(1, cdb, cmd);
3206 3207 3208 3209
		else {
			pr_err("WSNZ=1, WRITE_SAME w/sectors=0 not supported\n");
			goto out_invalid_cdb_field;
		}
3210

3211
		cmd->t_task_lba = get_unaligned_be64(&cdb[2]);
3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;

		if (target_check_write_same_discard(&cdb[1], dev) < 0)
			goto out_invalid_cdb_field;
		break;
	case WRITE_SAME:
		sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
		if (sector_ret)
			goto out_unsupported_cdb;

		if (sectors)
3223
			size = transport_get_size(1, cdb, cmd);
3224 3225 3226
		else {
			pr_err("WSNZ=1, WRITE_SAME w/sectors=0 not supported\n");
			goto out_invalid_cdb_field;
3227
		}
3228 3229

		cmd->t_task_lba = get_unaligned_be32(&cdb[2]);
3230
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3231 3232 3233 3234 3235 3236
		/*
		 * Follow sbcr26 with WRITE_SAME (10) and check for the existence
		 * of byte 1 bit 3 UNMAP instead of original reserved field
		 */
		if (target_check_write_same_discard(&cdb[1], dev) < 0)
			goto out_invalid_cdb_field;
3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255
		break;
	case ALLOW_MEDIUM_REMOVAL:
	case GPCMD_CLOSE_TRACK:
	case ERASE:
	case INITIALIZE_ELEMENT_STATUS:
	case GPCMD_LOAD_UNLOAD:
	case REZERO_UNIT:
	case SEEK_10:
	case GPCMD_SET_SPEED:
	case SPACE:
	case START_STOP:
	case TEST_UNIT_READY:
	case VERIFY:
	case WRITE_FILEMARKS:
	case MOVE_MEDIUM:
		cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
		break;
	case REPORT_LUNS:
		cmd->transport_emulate_cdb =
3256
				transport_core_report_lun_response;
3257 3258 3259 3260 3261
		size = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9];
		/*
		 * Do implict HEAD_OF_QUEUE processing for REPORT_LUNS
		 * See spc4r17 section 5.3
		 */
3262
		if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
3263
			cmd->sam_task_attr = MSG_HEAD_TAG;
3264
		cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3265 3266
		break;
	default:
3267
		pr_warn("TARGET_CORE[%s]: Unsupported SCSI Opcode"
3268
			" 0x%02x, sending CHECK_CONDITION.\n",
3269
			cmd->se_tfo->get_fabric_name(), cdb[0]);
3270 3271 3272 3273
		goto out_unsupported_cdb;
	}

	if (size != cmd->data_length) {
3274
		pr_warn("TARGET_CORE[%s]: Expected Transfer Length:"
3275
			" %u does not match SCSI CDB Length: %u for SAM Opcode:"
3276
			" 0x%02x\n", cmd->se_tfo->get_fabric_name(),
3277 3278 3279 3280 3281
				cmd->data_length, size, cdb[0]);

		cmd->cmd_spdtl = size;

		if (cmd->data_direction == DMA_TO_DEVICE) {
3282
			pr_err("Rejecting underflow/overflow"
3283 3284 3285 3286 3287 3288 3289
					" WRITE data\n");
			goto out_invalid_cdb_field;
		}
		/*
		 * Reject READ_* or WRITE_* with overflow/underflow for
		 * type SCF_SCSI_DATA_SG_IO_CDB.
		 */
3290 3291
		if (!ret && (dev->se_sub_dev->se_dev_attrib.block_size != 512))  {
			pr_err("Failing OVERFLOW/UNDERFLOW for LBA op"
3292
				" CDB on non 512-byte sector setup subsystem"
3293
				" plugin: %s\n", dev->transport->name);
3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307
			/* Returns CHECK_CONDITION + INVALID_CDB_FIELD */
			goto out_invalid_cdb_field;
		}

		if (size > cmd->data_length) {
			cmd->se_cmd_flags |= SCF_OVERFLOW_BIT;
			cmd->residual_count = (size - cmd->data_length);
		} else {
			cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
			cmd->residual_count = (cmd->data_length - size);
		}
		cmd->data_length = size;
	}

3308 3309 3310 3311 3312
	/* Let's limit control cdbs to a page, for simplicity's sake. */
	if ((cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB) &&
	    size > PAGE_SIZE)
		goto out_invalid_cdb_field;

3313 3314 3315 3316 3317 3318
	transport_set_supported_SAM_opcode(cmd);
	return ret;

out_unsupported_cdb:
	cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
	cmd->scsi_sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
3319
	return -EINVAL;
3320 3321 3322
out_invalid_cdb_field:
	cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
	cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
3323
	return -EINVAL;
3324 3325 3326
}

/*
3327
 * Called from I/O completion to determine which dormant/delayed
3328 3329 3330 3331
 * and ordered cmds need to have their tasks added to the execution queue.
 */
static void transport_complete_task_attr(struct se_cmd *cmd)
{
3332
	struct se_device *dev = cmd->se_dev;
3333 3334 3335
	struct se_cmd *cmd_p, *cmd_tmp;
	int new_active_tasks = 0;

3336
	if (cmd->sam_task_attr == MSG_SIMPLE_TAG) {
3337 3338 3339
		atomic_dec(&dev->simple_cmds);
		smp_mb__after_atomic_dec();
		dev->dev_cur_ordered_id++;
3340
		pr_debug("Incremented dev->dev_cur_ordered_id: %u for"
3341 3342
			" SIMPLE: %u\n", dev->dev_cur_ordered_id,
			cmd->se_ordered_id);
3343
	} else if (cmd->sam_task_attr == MSG_HEAD_TAG) {
3344 3345 3346
		atomic_dec(&dev->dev_hoq_count);
		smp_mb__after_atomic_dec();
		dev->dev_cur_ordered_id++;
3347
		pr_debug("Incremented dev_cur_ordered_id: %u for"
3348 3349
			" HEAD_OF_QUEUE: %u\n", dev->dev_cur_ordered_id,
			cmd->se_ordered_id);
3350
	} else if (cmd->sam_task_attr == MSG_ORDERED_TAG) {
3351
		spin_lock(&dev->ordered_cmd_lock);
3352
		list_del(&cmd->se_ordered_node);
3353 3354 3355 3356 3357
		atomic_dec(&dev->dev_ordered_sync);
		smp_mb__after_atomic_dec();
		spin_unlock(&dev->ordered_cmd_lock);

		dev->dev_cur_ordered_id++;
3358
		pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED:"
3359 3360 3361 3362 3363 3364 3365 3366 3367
			" %u\n", dev->dev_cur_ordered_id, cmd->se_ordered_id);
	}
	/*
	 * Process all commands up to the last received
	 * ORDERED task attribute which requires another blocking
	 * boundary
	 */
	spin_lock(&dev->delayed_cmd_lock);
	list_for_each_entry_safe(cmd_p, cmd_tmp,
3368
			&dev->delayed_cmd_list, se_delayed_node) {
3369

3370
		list_del(&cmd_p->se_delayed_node);
3371 3372
		spin_unlock(&dev->delayed_cmd_lock);

3373
		pr_debug("Calling add_tasks() for"
3374 3375
			" cmd_p: 0x%02x Task Attr: 0x%02x"
			" Dormant -> Active, se_ordered_id: %u\n",
3376
			cmd_p->t_task_cdb[0],
3377 3378 3379 3380 3381 3382
			cmd_p->sam_task_attr, cmd_p->se_ordered_id);

		transport_add_tasks_from_cmd(cmd_p);
		new_active_tasks++;

		spin_lock(&dev->delayed_cmd_lock);
3383
		if (cmd_p->sam_task_attr == MSG_ORDERED_TAG)
3384 3385 3386 3387 3388 3389 3390 3391
			break;
	}
	spin_unlock(&dev->delayed_cmd_lock);
	/*
	 * If new tasks have become active, wake up the transport thread
	 * to do the processing of the Active tasks.
	 */
	if (new_active_tasks != 0)
3392
		wake_up_interruptible(&dev->dev_queue_obj.thread_wq);
3393 3394
}

3395
static void transport_complete_qf(struct se_cmd *cmd)
3396 3397 3398
{
	int ret = 0;

3399 3400 3401 3402 3403 3404 3405 3406
	if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
		transport_complete_task_attr(cmd);

	if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
		ret = cmd->se_tfo->queue_status(cmd);
		if (ret)
			goto out;
	}
3407 3408 3409 3410 3411 3412

	switch (cmd->data_direction) {
	case DMA_FROM_DEVICE:
		ret = cmd->se_tfo->queue_data_in(cmd);
		break;
	case DMA_TO_DEVICE:
3413
		if (cmd->t_bidi_data_sg) {
3414 3415
			ret = cmd->se_tfo->queue_data_in(cmd);
			if (ret < 0)
3416
				break;
3417 3418 3419 3420 3421 3422 3423 3424 3425
		}
		/* Fall through for DMA_TO_DEVICE */
	case DMA_NONE:
		ret = cmd->se_tfo->queue_status(cmd);
		break;
	default:
		break;
	}

3426 3427 3428 3429 3430 3431 3432
out:
	if (ret < 0) {
		transport_handle_queue_full(cmd, cmd->se_dev);
		return;
	}
	transport_lun_remove_cmd(cmd);
	transport_cmd_check_stop_to_fabric(cmd);
3433 3434 3435 3436
}

static void transport_handle_queue_full(
	struct se_cmd *cmd,
3437
	struct se_device *dev)
3438 3439 3440 3441 3442 3443 3444 3445 3446 3447
{
	spin_lock_irq(&dev->qf_cmd_lock);
	list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list);
	atomic_inc(&dev->dev_qf_count);
	smp_mb__after_atomic_inc();
	spin_unlock_irq(&cmd->se_dev->qf_cmd_lock);

	schedule_work(&cmd->se_dev->qf_work_queue);
}

3448
static void target_complete_ok_work(struct work_struct *work)
3449
{
3450
	struct se_cmd *cmd = container_of(work, struct se_cmd, work);
3451
	int reason = 0, ret;
3452

3453 3454 3455 3456 3457
	/*
	 * Check if we need to move delayed/dormant tasks from cmds on the
	 * delayed execution list after a HEAD_OF_QUEUE or ORDERED Task
	 * Attribute.
	 */
3458
	if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
3459
		transport_complete_task_attr(cmd);
3460 3461 3462 3463 3464 3465 3466
	/*
	 * Check to schedule QUEUE_FULL work, or execute an existing
	 * cmd->transport_qf_callback()
	 */
	if (atomic_read(&cmd->se_dev->dev_qf_count) != 0)
		schedule_work(&cmd->se_dev->qf_work_queue);

3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479
	/*
	 * Check if we need to retrieve a sense buffer from
	 * the struct se_cmd in question.
	 */
	if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
		if (transport_get_sense_data(cmd) < 0)
			reason = TCM_NON_EXISTENT_LUN;

		/*
		 * Only set when an struct se_task->task_scsi_status returned
		 * a non GOOD status.
		 */
		if (cmd->scsi_status) {
3480
			ret = transport_send_check_condition_and_sense(
3481
					cmd, reason, 1);
3482 3483 3484
			if (ret == -EAGAIN)
				goto queue_full;

3485 3486 3487 3488 3489 3490
			transport_lun_remove_cmd(cmd);
			transport_cmd_check_stop_to_fabric(cmd);
			return;
		}
	}
	/*
L
Lucas De Marchi 已提交
3491
	 * Check for a callback, used by amongst other things
3492 3493 3494 3495 3496 3497 3498 3499
	 * XDWRITE_READ_10 emulation.
	 */
	if (cmd->transport_complete_callback)
		cmd->transport_complete_callback(cmd);

	switch (cmd->data_direction) {
	case DMA_FROM_DEVICE:
		spin_lock(&cmd->se_lun->lun_sep_lock);
3500 3501
		if (cmd->se_lun->lun_sep) {
			cmd->se_lun->lun_sep->sep_stats.tx_data_octets +=
3502 3503 3504 3505
					cmd->data_length;
		}
		spin_unlock(&cmd->se_lun->lun_sep_lock);

3506 3507 3508
		ret = cmd->se_tfo->queue_data_in(cmd);
		if (ret == -EAGAIN)
			goto queue_full;
3509 3510 3511
		break;
	case DMA_TO_DEVICE:
		spin_lock(&cmd->se_lun->lun_sep_lock);
3512 3513
		if (cmd->se_lun->lun_sep) {
			cmd->se_lun->lun_sep->sep_stats.rx_data_octets +=
3514 3515 3516 3517 3518 3519
				cmd->data_length;
		}
		spin_unlock(&cmd->se_lun->lun_sep_lock);
		/*
		 * Check if we need to send READ payload for BIDI-COMMAND
		 */
3520
		if (cmd->t_bidi_data_sg) {
3521
			spin_lock(&cmd->se_lun->lun_sep_lock);
3522 3523
			if (cmd->se_lun->lun_sep) {
				cmd->se_lun->lun_sep->sep_stats.tx_data_octets +=
3524 3525 3526
					cmd->data_length;
			}
			spin_unlock(&cmd->se_lun->lun_sep_lock);
3527 3528 3529
			ret = cmd->se_tfo->queue_data_in(cmd);
			if (ret == -EAGAIN)
				goto queue_full;
3530 3531 3532 3533
			break;
		}
		/* Fall through for DMA_TO_DEVICE */
	case DMA_NONE:
3534 3535 3536
		ret = cmd->se_tfo->queue_status(cmd);
		if (ret == -EAGAIN)
			goto queue_full;
3537 3538 3539 3540 3541 3542 3543
		break;
	default:
		break;
	}

	transport_lun_remove_cmd(cmd);
	transport_cmd_check_stop_to_fabric(cmd);
3544 3545 3546
	return;

queue_full:
3547
	pr_debug("Handling complete_ok QUEUE_FULL: se_cmd: %p,"
3548
		" data_direction: %d\n", cmd, cmd->data_direction);
3549 3550
	cmd->t_state = TRANSPORT_COMPLETE_QF_OK;
	transport_handle_queue_full(cmd, cmd->se_dev);
3551 3552 3553 3554 3555 3556
}

static void transport_free_dev_tasks(struct se_cmd *cmd)
{
	struct se_task *task, *task_tmp;
	unsigned long flags;
3557
	LIST_HEAD(dispose_list);
3558

3559
	spin_lock_irqsave(&cmd->t_state_lock, flags);
3560
	list_for_each_entry_safe(task, task_tmp,
3561
				&cmd->t_task_list, t_list) {
3562 3563 3564 3565 3566 3567 3568
		if (!(task->task_flags & TF_ACTIVE))
			list_move_tail(&task->t_list, &dispose_list);
	}
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);

	while (!list_empty(&dispose_list)) {
		task = list_first_entry(&dispose_list, struct se_task, t_list);
3569

3570 3571 3572 3573 3574 3575 3576 3577
		/*
		 * We already cancelled all pending timers in
		 * transport_complete_task, but that was just a pure del_timer,
		 * so do a full del_timer_sync here to make sure any handler
		 * that was running at that point has finished execution.
		 */
		del_timer_sync(&task->task_timer);

3578 3579 3580 3581 3582
		kfree(task->task_sg_bidi);
		kfree(task->task_sg);

		list_del(&task->t_list);

3583
		cmd->se_dev->transport->free_task(task);
3584 3585 3586
	}
}

3587
static inline void transport_free_sgl(struct scatterlist *sgl, int nents)
3588
{
3589 3590
	struct scatterlist *sg;
	int count;
3591

3592 3593
	for_each_sg(sgl, sg, nents, count)
		__free_page(sg_page(sg));
3594

3595 3596
	kfree(sgl);
}
3597

3598 3599 3600 3601 3602 3603
static inline void transport_free_pages(struct se_cmd *cmd)
{
	if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC)
		return;

	transport_free_sgl(cmd->t_data_sg, cmd->t_data_nents);
3604 3605
	cmd->t_data_sg = NULL;
	cmd->t_data_nents = 0;
3606

3607
	transport_free_sgl(cmd->t_bidi_data_sg, cmd->t_bidi_data_nents);
3608 3609
	cmd->t_bidi_data_sg = NULL;
	cmd->t_bidi_data_nents = 0;
3610 3611
}

3612 3613 3614 3615 3616 3617
/**
 * transport_put_cmd - release a reference to a command
 * @cmd:       command to release
 *
 * This routine releases our reference to the command and frees it if possible.
 */
3618
static void transport_put_cmd(struct se_cmd *cmd)
3619 3620
{
	unsigned long flags;
3621
	int free_tasks = 0;
3622

3623
	spin_lock_irqsave(&cmd->t_state_lock, flags);
3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637
	if (atomic_read(&cmd->t_fe_count)) {
		if (!atomic_dec_and_test(&cmd->t_fe_count))
			goto out_busy;
	}

	if (atomic_read(&cmd->t_se_count)) {
		if (!atomic_dec_and_test(&cmd->t_se_count))
			goto out_busy;
	}

	if (atomic_read(&cmd->transport_dev_active)) {
		atomic_set(&cmd->transport_dev_active, 0);
		transport_all_task_dev_remove_state(cmd);
		free_tasks = 1;
3638
	}
3639
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3640

3641 3642
	if (free_tasks != 0)
		transport_free_dev_tasks(cmd);
3643

3644
	transport_free_pages(cmd);
3645
	transport_release_cmd(cmd);
3646
	return;
3647 3648
out_busy:
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3649 3650 3651
}

/*
3652 3653
 * transport_generic_map_mem_to_cmd - Use fabric-alloced pages instead of
 * allocating in the core.
3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664
 * @cmd:  Associated se_cmd descriptor
 * @mem:  SGL style memory for TCM WRITE / READ
 * @sg_mem_num: Number of SGL elements
 * @mem_bidi_in: SGL style memory for TCM BIDI READ
 * @sg_mem_bidi_num: Number of BIDI READ SGL elements
 *
 * Return: nonzero return cmd was rejected for -ENOMEM or inproper usage
 * of parameters.
 */
int transport_generic_map_mem_to_cmd(
	struct se_cmd *cmd,
3665 3666 3667 3668
	struct scatterlist *sgl,
	u32 sgl_count,
	struct scatterlist *sgl_bidi,
	u32 sgl_bidi_count)
3669
{
3670
	if (!sgl || !sgl_count)
3671 3672 3673 3674 3675
		return 0;

	if ((cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB) ||
	    (cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB)) {

3676 3677
		cmd->t_data_sg = sgl;
		cmd->t_data_nents = sgl_count;
3678

3679 3680 3681
		if (sgl_bidi && sgl_bidi_count) {
			cmd->t_bidi_data_sg = sgl_bidi;
			cmd->t_bidi_data_nents = sgl_bidi_count;
3682 3683 3684 3685 3686 3687 3688 3689 3690 3691
		}
		cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
	}

	return 0;
}
EXPORT_SYMBOL(transport_generic_map_mem_to_cmd);

static int transport_new_cmd_obj(struct se_cmd *cmd)
{
3692
	struct se_device *dev = cmd->se_dev;
3693
	int set_counts = 1, rc, task_cdbs;
3694

3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706
	/*
	 * Setup any BIDI READ tasks and memory from
	 * cmd->t_mem_bidi_list so the READ struct se_tasks
	 * are queued first for the non pSCSI passthrough case.
	 */
	if (cmd->t_bidi_data_sg &&
	    (dev->transport->transport_type != TRANSPORT_PLUGIN_PHBA_PDEV)) {
		rc = transport_allocate_tasks(cmd,
					      cmd->t_task_lba,
					      DMA_FROM_DEVICE,
					      cmd->t_bidi_data_sg,
					      cmd->t_bidi_data_nents);
3707
		if (rc <= 0) {
3708 3709
			cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
			cmd->scsi_sense_reason =
3710
				TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
3711
			return -EINVAL;
3712
		}
3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725
		atomic_inc(&cmd->t_fe_count);
		atomic_inc(&cmd->t_se_count);
		set_counts = 0;
	}
	/*
	 * Setup the tasks and memory from cmd->t_mem_list
	 * Note for BIDI transfers this will contain the WRITE payload
	 */
	task_cdbs = transport_allocate_tasks(cmd,
					     cmd->t_task_lba,
					     cmd->data_direction,
					     cmd->t_data_sg,
					     cmd->t_data_nents);
3726
	if (task_cdbs <= 0) {
3727 3728 3729
		cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
		cmd->scsi_sense_reason =
			TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
3730
		return -EINVAL;
3731
	}
3732

3733 3734 3735
	if (set_counts) {
		atomic_inc(&cmd->t_fe_count);
		atomic_inc(&cmd->t_se_count);
3736 3737
	}

3738 3739
	cmd->t_task_list_num = task_cdbs;

3740 3741 3742
	atomic_set(&cmd->t_task_cdbs_left, task_cdbs);
	atomic_set(&cmd->t_task_cdbs_ex_left, task_cdbs);
	atomic_set(&cmd->t_task_cdbs_timeout_left, task_cdbs);
3743 3744 3745
	return 0;
}

3746 3747
void *transport_kmap_first_data_page(struct se_cmd *cmd)
{
3748
	struct scatterlist *sg = cmd->t_data_sg;
3749

3750
	BUG_ON(!sg);
3751
	/*
3752 3753 3754
	 * We need to take into account a possible offset here for fabrics like
	 * tcm_loop who may be using a contig buffer from the SCSI midlayer for
	 * control CDBs passed as SGLs via transport_generic_map_mem_to_cmd()
3755
	 */
3756
	return kmap(sg_page(sg)) + sg->offset;
3757 3758 3759 3760 3761
}
EXPORT_SYMBOL(transport_kmap_first_data_page);

void transport_kunmap_first_data_page(struct se_cmd *cmd)
{
3762
	kunmap(sg_page(cmd->t_data_sg));
3763 3764 3765
}
EXPORT_SYMBOL(transport_kunmap_first_data_page);

3766
static int
3767
transport_generic_get_mem(struct se_cmd *cmd)
3768
{
3769 3770 3771 3772
	u32 length = cmd->data_length;
	unsigned int nents;
	struct page *page;
	int i = 0;
3773

3774 3775 3776 3777
	nents = DIV_ROUND_UP(length, PAGE_SIZE);
	cmd->t_data_sg = kmalloc(sizeof(struct scatterlist) * nents, GFP_KERNEL);
	if (!cmd->t_data_sg)
		return -ENOMEM;
3778

3779 3780
	cmd->t_data_nents = nents;
	sg_init_table(cmd->t_data_sg, nents);
3781

3782 3783 3784 3785 3786
	while (length) {
		u32 page_len = min_t(u32, length, PAGE_SIZE);
		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
		if (!page)
			goto out;
3787

3788 3789 3790
		sg_set_page(&cmd->t_data_sg[i], page, page_len, 0);
		length -= page_len;
		i++;
3791 3792 3793
	}
	return 0;

3794 3795 3796 3797
out:
	while (i >= 0) {
		__free_page(sg_page(&cmd->t_data_sg[i]));
		i--;
3798
	}
3799 3800 3801
	kfree(cmd->t_data_sg);
	cmd->t_data_sg = NULL;
	return -ENOMEM;
3802 3803
}

3804 3805
/* Reduce sectors if they are too long for the device */
static inline sector_t transport_limit_task_sectors(
3806 3807
	struct se_device *dev,
	unsigned long long lba,
3808
	sector_t sectors)
3809
{
3810
	sectors = min_t(sector_t, sectors, dev->se_sub_dev->se_dev_attrib.max_sectors);
3811

3812 3813 3814
	if (dev->transport->get_device_type(dev) == TYPE_DISK)
		if ((lba + sectors) > transport_dev_end_lba(dev))
			sectors = ((transport_dev_end_lba(dev) - lba) + 1);
3815

3816
	return sectors;
3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827
}


/*
 * This function can be used by HW target mode drivers to create a linked
 * scatterlist from all contiguously allocated struct se_task->task_sg[].
 * This is intended to be called during the completion path by TCM Core
 * when struct target_core_fabric_ops->check_task_sg_chaining is enabled.
 */
void transport_do_task_sg_chain(struct se_cmd *cmd)
{
3828 3829 3830 3831
	struct scatterlist *sg_first = NULL;
	struct scatterlist *sg_prev = NULL;
	int sg_prev_nents = 0;
	struct scatterlist *sg;
3832
	struct se_task *task;
3833
	u32 chained_nents = 0;
3834 3835
	int i;

3836 3837
	BUG_ON(!cmd->se_tfo->task_sg_chaining);

3838 3839
	/*
	 * Walk the struct se_task list and setup scatterlist chains
3840
	 * for each contiguously allocated struct se_task->task_sg[].
3841
	 */
3842
	list_for_each_entry(task, &cmd->t_task_list, t_list) {
3843
		if (!task->task_sg)
3844 3845
			continue;

3846 3847
		if (!sg_first) {
			sg_first = task->task_sg;
3848
			chained_nents = task->task_sg_nents;
3849
		} else {
3850
			sg_chain(sg_prev, sg_prev_nents, task->task_sg);
3851
			chained_nents += task->task_sg_nents;
3852
		}
3853 3854 3855
		/*
		 * For the padded tasks, use the extra SGL vector allocated
		 * in transport_allocate_data_tasks() for the sg_prev_nents
3856 3857 3858 3859 3860
		 * offset into sg_chain() above.
		 *
		 * We do not need the padding for the last task (or a single
		 * task), but in that case we will never use the sg_prev_nents
		 * value below which would be incorrect.
3861
		 */
3862
		sg_prev_nents = (task->task_sg_nents + 1);
3863
		sg_prev = task->task_sg;
3864 3865 3866 3867 3868
	}
	/*
	 * Setup the starting pointer and total t_tasks_sg_linked_no including
	 * padding SGs for linking and to mark the end.
	 */
3869
	cmd->t_tasks_sg_chained = sg_first;
3870
	cmd->t_tasks_sg_chained_no = chained_nents;
3871

3872
	pr_debug("Setup cmd: %p cmd->t_tasks_sg_chained: %p and"
3873 3874
		" t_tasks_sg_chained_no: %u\n", cmd, cmd->t_tasks_sg_chained,
		cmd->t_tasks_sg_chained_no);
3875

3876 3877
	for_each_sg(cmd->t_tasks_sg_chained, sg,
			cmd->t_tasks_sg_chained_no, i) {
3878

3879
		pr_debug("SG[%d]: %p page: %p length: %d offset: %d\n",
3880
			i, sg, sg_page(sg), sg->length, sg->offset);
3881
		if (sg_is_chain(sg))
3882
			pr_debug("SG: %p sg_is_chain=1\n", sg);
3883
		if (sg_is_last(sg))
3884
			pr_debug("SG: %p sg_is_last=1\n", sg);
3885 3886 3887 3888
	}
}
EXPORT_SYMBOL(transport_do_task_sg_chain);

3889 3890 3891
/*
 * Break up cmd into chunks transport can handle
 */
3892
static int transport_allocate_data_tasks(
3893 3894 3895
	struct se_cmd *cmd,
	unsigned long long lba,
	enum dma_data_direction data_direction,
3896 3897
	struct scatterlist *sgl,
	unsigned int sgl_nents)
3898 3899
{
	struct se_task *task;
3900
	struct se_device *dev = cmd->se_dev;
3901
	unsigned long flags;
3902
	int task_count, i;
3903
	sector_t sectors, dev_max_sectors = dev->se_sub_dev->se_dev_attrib.max_sectors;
3904 3905 3906
	u32 sector_size = dev->se_sub_dev->se_dev_attrib.block_size;
	struct scatterlist *sg;
	struct scatterlist *cmd_sg;
3907

3908 3909
	WARN_ON(cmd->data_length % sector_size);
	sectors = DIV_ROUND_UP(cmd->data_length, sector_size);
3910 3911
	task_count = DIV_ROUND_UP_SECTOR_T(sectors, dev_max_sectors);
	
3912 3913
	cmd_sg = sgl;
	for (i = 0; i < task_count; i++) {
3914
		unsigned int task_size, task_sg_nents_padded;
3915
		int count;
3916

3917
		task = transport_generic_get_task(cmd, data_direction);
3918
		if (!task)
3919
			return -ENOMEM;
3920 3921

		task->task_lba = lba;
3922 3923
		task->task_sectors = min(sectors, dev_max_sectors);
		task->task_size = task->task_sectors * sector_size;
3924

3925 3926 3927 3928 3929
		/*
		 * This now assumes that passed sg_ents are in PAGE_SIZE chunks
		 * in order to calculate the number per task SGL entries
		 */
		task->task_sg_nents = DIV_ROUND_UP(task->task_size, PAGE_SIZE);
3930
		/*
3931 3932 3933
		 * Check if the fabric module driver is requesting that all
		 * struct se_task->task_sg[] be chained together..  If so,
		 * then allocate an extra padding SG entry for linking and
3934 3935 3936
		 * marking the end of the chained SGL for every task except
		 * the last one for (task_count > 1) operation, or skipping
		 * the extra padding for the (task_count == 1) case.
3937
		 */
3938 3939 3940 3941
		if (cmd->se_tfo->task_sg_chaining && (i < (task_count - 1))) {
			task_sg_nents_padded = (task->task_sg_nents + 1);
		} else
			task_sg_nents_padded = task->task_sg_nents;
3942

3943
		task->task_sg = kmalloc(sizeof(struct scatterlist) *
3944
					task_sg_nents_padded, GFP_KERNEL);
3945 3946 3947 3948 3949
		if (!task->task_sg) {
			cmd->se_dev->transport->free_task(task);
			return -ENOMEM;
		}

3950
		sg_init_table(task->task_sg, task_sg_nents_padded);
3951

3952 3953 3954
		task_size = task->task_size;

		/* Build new sgl, only up to task_size */
3955
		for_each_sg(task->task_sg, sg, task->task_sg_nents, count) {
3956 3957 3958 3959 3960 3961
			if (cmd_sg->length > task_size)
				break;

			*sg = *cmd_sg;
			task_size -= cmd_sg->length;
			cmd_sg = sg_next(cmd_sg);
3962 3963
		}

3964 3965
		lba += task->task_sectors;
		sectors -= task->task_sectors;
3966

3967 3968 3969
		spin_lock_irqsave(&cmd->t_state_lock, flags);
		list_add_tail(&task->t_list, &cmd->t_task_list);
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3970 3971
	}

3972
	return task_count;
3973 3974 3975
}

static int
3976
transport_allocate_control_task(struct se_cmd *cmd)
3977 3978
{
	struct se_task *task;
3979
	unsigned long flags;
3980 3981 3982

	task = transport_generic_get_task(cmd, cmd->data_direction);
	if (!task)
3983
		return -ENOMEM;
3984

3985 3986 3987 3988 3989 3990 3991 3992 3993
	task->task_sg = kmalloc(sizeof(struct scatterlist) * cmd->t_data_nents,
				GFP_KERNEL);
	if (!task->task_sg) {
		cmd->se_dev->transport->free_task(task);
		return -ENOMEM;
	}

	memcpy(task->task_sg, cmd->t_data_sg,
	       sizeof(struct scatterlist) * cmd->t_data_nents);
3994
	task->task_size = cmd->data_length;
3995
	task->task_sg_nents = cmd->t_data_nents;
3996

3997 3998 3999
	spin_lock_irqsave(&cmd->t_state_lock, flags);
	list_add_tail(&task->t_list, &cmd->t_task_list);
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
4000

4001
	/* Success! Return number of tasks allocated */
4002
	return 1;
4003 4004 4005 4006 4007 4008 4009 4010 4011
}

static u32 transport_allocate_tasks(
	struct se_cmd *cmd,
	unsigned long long lba,
	enum dma_data_direction data_direction,
	struct scatterlist *sgl,
	unsigned int sgl_nents)
{
4012 4013 4014 4015
	if (cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB) {
		if (transport_cmd_get_valid_sectors(cmd) < 0)
			return -EINVAL;

4016 4017
		return transport_allocate_data_tasks(cmd, lba, data_direction,
						     sgl, sgl_nents);
4018
	} else
4019 4020
		return transport_allocate_control_task(cmd);

4021 4022
}

4023

4024 4025 4026 4027 4028 4029 4030 4031 4032
/*	 transport_generic_new_cmd(): Called from transport_processing_thread()
 *
 *	 Allocate storage transport resources from a set of values predefined
 *	 by transport_generic_cmd_sequencer() from the iSCSI Target RX process.
 *	 Any non zero return here is treated as an "out of resource' op here.
 */
	/*
	 * Generate struct se_task(s) and/or their payloads for this CDB.
	 */
4033
int transport_generic_new_cmd(struct se_cmd *cmd)
4034 4035 4036 4037 4038 4039
{
	int ret = 0;

	/*
	 * Determine is the TCM fabric module has already allocated physical
	 * memory, and is directly calling transport_generic_map_mem_to_cmd()
4040
	 * beforehand.
4041
	 */
4042 4043
	if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) &&
	    cmd->data_length) {
4044
		ret = transport_generic_get_mem(cmd);
4045 4046 4047
		if (ret < 0)
			return ret;
	}
4048 4049 4050 4051 4052 4053 4054
	/*
	 * Call transport_new_cmd_obj() to invoke transport_allocate_tasks() for
	 * control or data CDB types, and perform the map to backend subsystem
	 * code from SGL memory allocated here by transport_generic_get_mem(), or
	 * via pre-existing SGL memory setup explictly by fabric module code with
	 * transport_generic_map_mem_to_cmd().
	 */
4055 4056 4057 4058
	ret = transport_new_cmd_obj(cmd);
	if (ret < 0)
		return ret;
	/*
4059
	 * For WRITEs, let the fabric know its buffer is ready..
4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075
	 * This WRITE struct se_cmd (and all of its associated struct se_task's)
	 * will be added to the struct se_device execution queue after its WRITE
	 * data has arrived. (ie: It gets handled by the transport processing
	 * thread a second time)
	 */
	if (cmd->data_direction == DMA_TO_DEVICE) {
		transport_add_tasks_to_state_queue(cmd);
		return transport_generic_write_pending(cmd);
	}
	/*
	 * Everything else but a WRITE, add the struct se_cmd's struct se_task's
	 * to the execution queue.
	 */
	transport_execute_tasks(cmd);
	return 0;
}
4076
EXPORT_SYMBOL(transport_generic_new_cmd);
4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087

/*	transport_generic_process_write():
 *
 *
 */
void transport_generic_process_write(struct se_cmd *cmd)
{
	transport_execute_tasks(cmd);
}
EXPORT_SYMBOL(transport_generic_process_write);

4088
static void transport_write_pending_qf(struct se_cmd *cmd)
4089
{
4090 4091 4092 4093 4094
	if (cmd->se_tfo->write_pending(cmd) == -EAGAIN) {
		pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n",
			 cmd);
		transport_handle_queue_full(cmd, cmd->se_dev);
	}
4095 4096
}

4097 4098 4099 4100 4101
static int transport_generic_write_pending(struct se_cmd *cmd)
{
	unsigned long flags;
	int ret;

4102
	spin_lock_irqsave(&cmd->t_state_lock, flags);
4103
	cmd->t_state = TRANSPORT_WRITE_PENDING;
4104
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
4105

4106 4107
	/*
	 * Clear the se_cmd for WRITE_PENDING status in order to set
4108
	 * cmd->t_transport_active=0 so that transport_generic_handle_data
4109
	 * can be called from HW target mode interrupt code.  This is safe
4110
	 * to be called with transport_off=1 before the cmd->se_tfo->write_pending
4111 4112 4113 4114 4115 4116 4117 4118
	 * because the se_cmd->se_lun pointer is not being cleared.
	 */
	transport_cmd_check_stop(cmd, 1, 0);

	/*
	 * Call the fabric write_pending function here to let the
	 * frontend know that WRITE buffers are ready.
	 */
4119
	ret = cmd->se_tfo->write_pending(cmd);
4120 4121 4122
	if (ret == -EAGAIN)
		goto queue_full;
	else if (ret < 0)
4123 4124 4125
		return ret;

	return PYX_TRANSPORT_WRITE_PENDING;
4126 4127

queue_full:
4128
	pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd);
4129
	cmd->t_state = TRANSPORT_COMPLETE_QF_WP;
4130
	transport_handle_queue_full(cmd, cmd->se_dev);
4131
	return ret;
4132 4133
}

4134 4135 4136 4137 4138 4139 4140
/**
 * transport_release_cmd - free a command
 * @cmd:       command to free
 *
 * This routine unconditionally frees a command, and reference counting
 * or list removal must be done in the caller.
 */
4141
void transport_release_cmd(struct se_cmd *cmd)
4142
{
4143
	BUG_ON(!cmd->se_tfo);
4144

4145 4146 4147 4148
	if (cmd->se_tmr_req)
		core_tmr_release_req(cmd->se_tmr_req);
	if (cmd->t_task_cdb != cmd->__t_task_cdb)
		kfree(cmd->t_task_cdb);
4149
	cmd->se_tfo->release_cmd(cmd);
4150
}
4151
EXPORT_SYMBOL(transport_release_cmd);
4152

4153
void transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks)
4154
{
4155 4156 4157 4158
	if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD)) {
		if (wait_for_tasks && cmd->se_tmr_req)
			 transport_wait_for_tasks(cmd);

4159
		transport_release_cmd(cmd);
4160 4161 4162 4163
	} else {
		if (wait_for_tasks)
			transport_wait_for_tasks(cmd);

4164 4165
		core_dec_lacl_count(cmd->se_sess->se_node_acl, cmd);

4166
		if (cmd->se_lun)
4167 4168
			transport_lun_remove_cmd(cmd);

4169 4170
		transport_free_dev_tasks(cmd);

4171
		transport_put_cmd(cmd);
4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188
	}
}
EXPORT_SYMBOL(transport_generic_free_cmd);

/*	transport_lun_wait_for_tasks():
 *
 *	Called from ConfigFS context to stop the passed struct se_cmd to allow
 *	an struct se_lun to be successfully shutdown.
 */
static int transport_lun_wait_for_tasks(struct se_cmd *cmd, struct se_lun *lun)
{
	unsigned long flags;
	int ret;
	/*
	 * If the frontend has already requested this struct se_cmd to
	 * be stopped, we can safely ignore this struct se_cmd.
	 */
4189 4190 4191
	spin_lock_irqsave(&cmd->t_state_lock, flags);
	if (atomic_read(&cmd->t_transport_stop)) {
		atomic_set(&cmd->transport_lun_stop, 0);
4192
		pr_debug("ConfigFS ITT[0x%08x] - t_transport_stop =="
4193
			" TRUE, skipping\n", cmd->se_tfo->get_task_tag(cmd));
4194
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
4195
		transport_cmd_check_stop(cmd, 1, 0);
4196
		return -EPERM;
4197
	}
4198 4199
	atomic_set(&cmd->transport_lun_fe_stop, 1);
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
4200

4201
	wake_up_interruptible(&cmd->se_dev->dev_queue_obj.thread_wq);
4202 4203 4204

	ret = transport_stop_tasks_for_cmd(cmd);

4205 4206
	pr_debug("ConfigFS: cmd: %p t_tasks: %d stop tasks ret:"
			" %d\n", cmd, cmd->t_task_list_num, ret);
4207
	if (!ret) {
4208
		pr_debug("ConfigFS: ITT[0x%08x] - stopping cmd....\n",
4209
				cmd->se_tfo->get_task_tag(cmd));
4210
		wait_for_completion(&cmd->transport_lun_stop_comp);
4211
		pr_debug("ConfigFS: ITT[0x%08x] - stopped cmd....\n",
4212
				cmd->se_tfo->get_task_tag(cmd));
4213
	}
4214
	transport_remove_cmd_from_queue(cmd);
4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227

	return 0;
}

static void __transport_clear_lun_from_sessions(struct se_lun *lun)
{
	struct se_cmd *cmd = NULL;
	unsigned long lun_flags, cmd_flags;
	/*
	 * Do exception processing and return CHECK_CONDITION status to the
	 * Initiator Port.
	 */
	spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
4228 4229 4230 4231 4232
	while (!list_empty(&lun->lun_cmd_list)) {
		cmd = list_first_entry(&lun->lun_cmd_list,
		       struct se_cmd, se_lun_node);
		list_del(&cmd->se_lun_node);

4233
		atomic_set(&cmd->transport_lun_active, 0);
4234 4235 4236 4237 4238
		/*
		 * This will notify iscsi_target_transport.c:
		 * transport_cmd_check_stop() that a LUN shutdown is in
		 * progress for the iscsi_cmd_t.
		 */
4239
		spin_lock(&cmd->t_state_lock);
4240
		pr_debug("SE_LUN[%d] - Setting cmd->transport"
4241
			"_lun_stop for  ITT: 0x%08x\n",
4242 4243
			cmd->se_lun->unpacked_lun,
			cmd->se_tfo->get_task_tag(cmd));
4244 4245
		atomic_set(&cmd->transport_lun_stop, 1);
		spin_unlock(&cmd->t_state_lock);
4246 4247 4248

		spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags);

4249 4250
		if (!cmd->se_lun) {
			pr_err("ITT: 0x%08x, [i,t]_state: %u/%u\n",
4251 4252
				cmd->se_tfo->get_task_tag(cmd),
				cmd->se_tfo->get_cmd_state(cmd), cmd->t_state);
4253 4254 4255 4256 4257 4258
			BUG();
		}
		/*
		 * If the Storage engine still owns the iscsi_cmd_t, determine
		 * and/or stop its context.
		 */
4259
		pr_debug("SE_LUN[%d] - ITT: 0x%08x before transport"
4260 4261
			"_lun_wait_for_tasks()\n", cmd->se_lun->unpacked_lun,
			cmd->se_tfo->get_task_tag(cmd));
4262

4263
		if (transport_lun_wait_for_tasks(cmd, cmd->se_lun) < 0) {
4264 4265 4266 4267
			spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
			continue;
		}

4268
		pr_debug("SE_LUN[%d] - ITT: 0x%08x after transport_lun"
4269
			"_wait_for_tasks(): SUCCESS\n",
4270 4271
			cmd->se_lun->unpacked_lun,
			cmd->se_tfo->get_task_tag(cmd));
4272

4273
		spin_lock_irqsave(&cmd->t_state_lock, cmd_flags);
4274
		if (!atomic_read(&cmd->transport_dev_active)) {
4275
			spin_unlock_irqrestore(&cmd->t_state_lock, cmd_flags);
4276 4277
			goto check_cond;
		}
4278
		atomic_set(&cmd->transport_dev_active, 0);
4279
		transport_all_task_dev_remove_state(cmd);
4280
		spin_unlock_irqrestore(&cmd->t_state_lock, cmd_flags);
4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296

		transport_free_dev_tasks(cmd);
		/*
		 * The Storage engine stopped this struct se_cmd before it was
		 * send to the fabric frontend for delivery back to the
		 * Initiator Node.  Return this SCSI CDB back with an
		 * CHECK_CONDITION status.
		 */
check_cond:
		transport_send_check_condition_and_sense(cmd,
				TCM_NON_EXISTENT_LUN, 0);
		/*
		 *  If the fabric frontend is waiting for this iscsi_cmd_t to
		 * be released, notify the waiting thread now that LU has
		 * finished accessing it.
		 */
4297 4298
		spin_lock_irqsave(&cmd->t_state_lock, cmd_flags);
		if (atomic_read(&cmd->transport_lun_fe_stop)) {
4299
			pr_debug("SE_LUN[%d] - Detected FE stop for"
4300 4301
				" struct se_cmd: %p ITT: 0x%08x\n",
				lun->unpacked_lun,
4302
				cmd, cmd->se_tfo->get_task_tag(cmd));
4303

4304
			spin_unlock_irqrestore(&cmd->t_state_lock,
4305 4306
					cmd_flags);
			transport_cmd_check_stop(cmd, 1, 0);
4307
			complete(&cmd->transport_lun_fe_stop_comp);
4308 4309 4310
			spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
			continue;
		}
4311
		pr_debug("SE_LUN[%d] - ITT: 0x%08x finished processing\n",
4312
			lun->unpacked_lun, cmd->se_tfo->get_task_tag(cmd));
4313

4314
		spin_unlock_irqrestore(&cmd->t_state_lock, cmd_flags);
4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333
		spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
	}
	spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags);
}

static int transport_clear_lun_thread(void *p)
{
	struct se_lun *lun = (struct se_lun *)p;

	__transport_clear_lun_from_sessions(lun);
	complete(&lun->lun_shutdown_comp);

	return 0;
}

int transport_clear_lun_from_sessions(struct se_lun *lun)
{
	struct task_struct *kt;

4334
	kt = kthread_run(transport_clear_lun_thread, lun,
4335 4336
			"tcm_cl_%u", lun->unpacked_lun);
	if (IS_ERR(kt)) {
4337
		pr_err("Unable to start clear_lun thread\n");
4338
		return PTR_ERR(kt);
4339 4340 4341 4342 4343 4344
	}
	wait_for_completion(&lun->lun_shutdown_comp);

	return 0;
}

4345 4346 4347
/**
 * transport_wait_for_tasks - wait for completion to occur
 * @cmd:	command to wait
4348
 *
4349 4350
 * Called from frontend fabric context to wait for storage engine
 * to pause and/or release frontend generated struct se_cmd.
4351
 */
4352
void transport_wait_for_tasks(struct se_cmd *cmd)
4353 4354 4355
{
	unsigned long flags;

4356
	spin_lock_irqsave(&cmd->t_state_lock, flags);
4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368
	if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) && !(cmd->se_tmr_req)) {
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
		return;
	}
	/*
	 * Only perform a possible wait_for_tasks if SCF_SUPPORTED_SAM_OPCODE
	 * has been set in transport_set_supported_SAM_opcode().
	 */
	if (!(cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) && !cmd->se_tmr_req) {
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
		return;
	}
4369 4370 4371
	/*
	 * If we are already stopped due to an external event (ie: LUN shutdown)
	 * sleep until the connection can have the passed struct se_cmd back.
4372
	 * The cmd->transport_lun_stopped_sem will be upped by
4373 4374 4375
	 * transport_clear_lun_from_sessions() once the ConfigFS context caller
	 * has completed its operation on the struct se_cmd.
	 */
4376
	if (atomic_read(&cmd->transport_lun_stop)) {
4377

4378
		pr_debug("wait_for_tasks: Stopping"
4379
			" wait_for_completion(&cmd->t_tasktransport_lun_fe"
4380
			"_stop_comp); for ITT: 0x%08x\n",
4381
			cmd->se_tfo->get_task_tag(cmd));
4382 4383 4384 4385 4386 4387 4388
		/*
		 * There is a special case for WRITES where a FE exception +
		 * LUN shutdown means ConfigFS context is still sleeping on
		 * transport_lun_stop_comp in transport_lun_wait_for_tasks().
		 * We go ahead and up transport_lun_stop_comp just to be sure
		 * here.
		 */
4389 4390 4391 4392
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
		complete(&cmd->transport_lun_stop_comp);
		wait_for_completion(&cmd->transport_lun_fe_stop_comp);
		spin_lock_irqsave(&cmd->t_state_lock, flags);
4393 4394 4395 4396 4397 4398 4399

		transport_all_task_dev_remove_state(cmd);
		/*
		 * At this point, the frontend who was the originator of this
		 * struct se_cmd, now owns the structure and can be released through
		 * normal means below.
		 */
4400
		pr_debug("wait_for_tasks: Stopped"
4401
			" wait_for_completion(&cmd->t_tasktransport_lun_fe_"
4402
			"stop_comp); for ITT: 0x%08x\n",
4403
			cmd->se_tfo->get_task_tag(cmd));
4404

4405
		atomic_set(&cmd->transport_lun_stop, 0);
4406
	}
4407
	if (!atomic_read(&cmd->t_transport_active) ||
4408 4409 4410 4411
	     atomic_read(&cmd->t_transport_aborted)) {
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
		return;
	}
4412

4413
	atomic_set(&cmd->t_transport_stop, 1);
4414

4415
	pr_debug("wait_for_tasks: Stopping %p ITT: 0x%08x"
4416 4417 4418
		" i_state: %d, t_state: %d, t_transport_stop = TRUE\n",
		cmd, cmd->se_tfo->get_task_tag(cmd),
		cmd->se_tfo->get_cmd_state(cmd), cmd->t_state);
4419

4420
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
4421

4422
	wake_up_interruptible(&cmd->se_dev->dev_queue_obj.thread_wq);
4423

4424
	wait_for_completion(&cmd->t_transport_stop_comp);
4425

4426 4427 4428
	spin_lock_irqsave(&cmd->t_state_lock, flags);
	atomic_set(&cmd->t_transport_active, 0);
	atomic_set(&cmd->t_transport_stop, 0);
4429

4430
	pr_debug("wait_for_tasks: Stopped wait_for_compltion("
4431
		"&cmd->t_transport_stop_comp) for ITT: 0x%08x\n",
4432
		cmd->se_tfo->get_task_tag(cmd));
4433

4434
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
4435
}
4436
EXPORT_SYMBOL(transport_wait_for_tasks);
4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469

static int transport_get_sense_codes(
	struct se_cmd *cmd,
	u8 *asc,
	u8 *ascq)
{
	*asc = cmd->scsi_asc;
	*ascq = cmd->scsi_ascq;

	return 0;
}

static int transport_set_sense_codes(
	struct se_cmd *cmd,
	u8 asc,
	u8 ascq)
{
	cmd->scsi_asc = asc;
	cmd->scsi_ascq = ascq;

	return 0;
}

int transport_send_check_condition_and_sense(
	struct se_cmd *cmd,
	u8 reason,
	int from_transport)
{
	unsigned char *buffer = cmd->sense_buffer;
	unsigned long flags;
	int offset;
	u8 asc = 0, ascq = 0;

4470
	spin_lock_irqsave(&cmd->t_state_lock, flags);
4471
	if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
4472
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
4473 4474 4475
		return 0;
	}
	cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION;
4476
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488

	if (!reason && from_transport)
		goto after_reason;

	if (!from_transport)
		cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
	/*
	 * Data Segment and SenseLength of the fabric response PDU.
	 *
	 * TRANSPORT_SENSE_BUFFER is now set to SCSI_SENSE_BUFFERSIZE
	 * from include/scsi/scsi_cmnd.h
	 */
4489
	offset = cmd->se_tfo->set_fabric_sense_len(cmd,
4490 4491 4492 4493 4494 4495 4496
				TRANSPORT_SENSE_BUFFER);
	/*
	 * Actual SENSE DATA, see SPC-3 7.23.2  SPC_SENSE_KEY_OFFSET uses
	 * SENSE KEY values from include/scsi/scsi.h
	 */
	switch (reason) {
	case TCM_NON_EXISTENT_LUN:
4497 4498 4499 4500 4501 4502 4503
		/* CURRENT ERROR */
		buffer[offset] = 0x70;
		/* ILLEGAL REQUEST */
		buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
		/* LOGICAL UNIT NOT SUPPORTED */
		buffer[offset+SPC_ASC_KEY_OFFSET] = 0x25;
		break;
4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632
	case TCM_UNSUPPORTED_SCSI_OPCODE:
	case TCM_SECTOR_COUNT_TOO_MANY:
		/* CURRENT ERROR */
		buffer[offset] = 0x70;
		/* ILLEGAL REQUEST */
		buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
		/* INVALID COMMAND OPERATION CODE */
		buffer[offset+SPC_ASC_KEY_OFFSET] = 0x20;
		break;
	case TCM_UNKNOWN_MODE_PAGE:
		/* CURRENT ERROR */
		buffer[offset] = 0x70;
		/* ILLEGAL REQUEST */
		buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
		/* INVALID FIELD IN CDB */
		buffer[offset+SPC_ASC_KEY_OFFSET] = 0x24;
		break;
	case TCM_CHECK_CONDITION_ABORT_CMD:
		/* CURRENT ERROR */
		buffer[offset] = 0x70;
		/* ABORTED COMMAND */
		buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
		/* BUS DEVICE RESET FUNCTION OCCURRED */
		buffer[offset+SPC_ASC_KEY_OFFSET] = 0x29;
		buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x03;
		break;
	case TCM_INCORRECT_AMOUNT_OF_DATA:
		/* CURRENT ERROR */
		buffer[offset] = 0x70;
		/* ABORTED COMMAND */
		buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
		/* WRITE ERROR */
		buffer[offset+SPC_ASC_KEY_OFFSET] = 0x0c;
		/* NOT ENOUGH UNSOLICITED DATA */
		buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x0d;
		break;
	case TCM_INVALID_CDB_FIELD:
		/* CURRENT ERROR */
		buffer[offset] = 0x70;
		/* ABORTED COMMAND */
		buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
		/* INVALID FIELD IN CDB */
		buffer[offset+SPC_ASC_KEY_OFFSET] = 0x24;
		break;
	case TCM_INVALID_PARAMETER_LIST:
		/* CURRENT ERROR */
		buffer[offset] = 0x70;
		/* ABORTED COMMAND */
		buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
		/* INVALID FIELD IN PARAMETER LIST */
		buffer[offset+SPC_ASC_KEY_OFFSET] = 0x26;
		break;
	case TCM_UNEXPECTED_UNSOLICITED_DATA:
		/* CURRENT ERROR */
		buffer[offset] = 0x70;
		/* ABORTED COMMAND */
		buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
		/* WRITE ERROR */
		buffer[offset+SPC_ASC_KEY_OFFSET] = 0x0c;
		/* UNEXPECTED_UNSOLICITED_DATA */
		buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x0c;
		break;
	case TCM_SERVICE_CRC_ERROR:
		/* CURRENT ERROR */
		buffer[offset] = 0x70;
		/* ABORTED COMMAND */
		buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
		/* PROTOCOL SERVICE CRC ERROR */
		buffer[offset+SPC_ASC_KEY_OFFSET] = 0x47;
		/* N/A */
		buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x05;
		break;
	case TCM_SNACK_REJECTED:
		/* CURRENT ERROR */
		buffer[offset] = 0x70;
		/* ABORTED COMMAND */
		buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
		/* READ ERROR */
		buffer[offset+SPC_ASC_KEY_OFFSET] = 0x11;
		/* FAILED RETRANSMISSION REQUEST */
		buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x13;
		break;
	case TCM_WRITE_PROTECTED:
		/* CURRENT ERROR */
		buffer[offset] = 0x70;
		/* DATA PROTECT */
		buffer[offset+SPC_SENSE_KEY_OFFSET] = DATA_PROTECT;
		/* WRITE PROTECTED */
		buffer[offset+SPC_ASC_KEY_OFFSET] = 0x27;
		break;
	case TCM_CHECK_CONDITION_UNIT_ATTENTION:
		/* CURRENT ERROR */
		buffer[offset] = 0x70;
		/* UNIT ATTENTION */
		buffer[offset+SPC_SENSE_KEY_OFFSET] = UNIT_ATTENTION;
		core_scsi3_ua_for_check_condition(cmd, &asc, &ascq);
		buffer[offset+SPC_ASC_KEY_OFFSET] = asc;
		buffer[offset+SPC_ASCQ_KEY_OFFSET] = ascq;
		break;
	case TCM_CHECK_CONDITION_NOT_READY:
		/* CURRENT ERROR */
		buffer[offset] = 0x70;
		/* Not Ready */
		buffer[offset+SPC_SENSE_KEY_OFFSET] = NOT_READY;
		transport_get_sense_codes(cmd, &asc, &ascq);
		buffer[offset+SPC_ASC_KEY_OFFSET] = asc;
		buffer[offset+SPC_ASCQ_KEY_OFFSET] = ascq;
		break;
	case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
	default:
		/* CURRENT ERROR */
		buffer[offset] = 0x70;
		/* ILLEGAL REQUEST */
		buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
		/* LOGICAL UNIT COMMUNICATION FAILURE */
		buffer[offset+SPC_ASC_KEY_OFFSET] = 0x80;
		break;
	}
	/*
	 * This code uses linux/include/scsi/scsi.h SAM status codes!
	 */
	cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
	/*
	 * Automatically padded, this value is encoded in the fabric's
	 * data_length response PDU containing the SCSI defined sense data.
	 */
	cmd->scsi_sense_length  = TRANSPORT_SENSE_BUFFER + offset;

after_reason:
4633
	return cmd->se_tfo->queue_status(cmd);
4634 4635 4636 4637 4638 4639 4640
}
EXPORT_SYMBOL(transport_send_check_condition_and_sense);

int transport_check_aborted_status(struct se_cmd *cmd, int send_status)
{
	int ret = 0;

4641
	if (atomic_read(&cmd->t_transport_aborted) != 0) {
4642
		if (!send_status ||
4643 4644 4645
		     (cmd->se_cmd_flags & SCF_SENT_DELAYED_TAS))
			return 1;
#if 0
4646
		pr_debug("Sending delayed SAM_STAT_TASK_ABORTED"
4647
			" status for CDB: 0x%02x ITT: 0x%08x\n",
4648
			cmd->t_task_cdb[0],
4649
			cmd->se_tfo->get_task_tag(cmd));
4650 4651
#endif
		cmd->se_cmd_flags |= SCF_SENT_DELAYED_TAS;
4652
		cmd->se_tfo->queue_status(cmd);
4653 4654 4655 4656 4657 4658 4659 4660
		ret = 1;
	}
	return ret;
}
EXPORT_SYMBOL(transport_check_aborted_status);

void transport_send_task_abort(struct se_cmd *cmd)
{
4661 4662 4663 4664 4665 4666 4667 4668 4669
	unsigned long flags;

	spin_lock_irqsave(&cmd->t_state_lock, flags);
	if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
		return;
	}
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);

4670 4671 4672 4673 4674 4675 4676
	/*
	 * If there are still expected incoming fabric WRITEs, we wait
	 * until until they have completed before sending a TASK_ABORTED
	 * response.  This response with TASK_ABORTED status will be
	 * queued back to fabric module by transport_check_aborted_status().
	 */
	if (cmd->data_direction == DMA_TO_DEVICE) {
4677
		if (cmd->se_tfo->write_pending_status(cmd) != 0) {
4678
			atomic_inc(&cmd->t_transport_aborted);
4679 4680 4681 4682 4683 4684 4685 4686
			smp_mb__after_atomic_inc();
			cmd->scsi_status = SAM_STAT_TASK_ABORTED;
			transport_new_cmd_failure(cmd);
			return;
		}
	}
	cmd->scsi_status = SAM_STAT_TASK_ABORTED;
#if 0
4687
	pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x,"
4688
		" ITT: 0x%08x\n", cmd->t_task_cdb[0],
4689
		cmd->se_tfo->get_task_tag(cmd));
4690
#endif
4691
	cmd->se_tfo->queue_status(cmd);
4692 4693 4694 4695 4696 4697 4698 4699
}

/*	transport_generic_do_tmr():
 *
 *
 */
int transport_generic_do_tmr(struct se_cmd *cmd)
{
4700
	struct se_device *dev = cmd->se_dev;
4701 4702 4703 4704
	struct se_tmr_req *tmr = cmd->se_tmr_req;
	int ret;

	switch (tmr->function) {
4705
	case TMR_ABORT_TASK:
4706 4707
		tmr->response = TMR_FUNCTION_REJECTED;
		break;
4708 4709 4710
	case TMR_ABORT_TASK_SET:
	case TMR_CLEAR_ACA:
	case TMR_CLEAR_TASK_SET:
4711 4712
		tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED;
		break;
4713
	case TMR_LUN_RESET:
4714 4715 4716 4717
		ret = core_tmr_lun_reset(dev, tmr, NULL, NULL);
		tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE :
					 TMR_FUNCTION_REJECTED;
		break;
4718
	case TMR_TARGET_WARM_RESET:
4719 4720
		tmr->response = TMR_FUNCTION_REJECTED;
		break;
4721
	case TMR_TARGET_COLD_RESET:
4722 4723 4724
		tmr->response = TMR_FUNCTION_REJECTED;
		break;
	default:
4725
		pr_err("Uknown TMR function: 0x%02x.\n",
4726 4727 4728 4729 4730 4731
				tmr->function);
		tmr->response = TMR_FUNCTION_REJECTED;
		break;
	}

	cmd->t_state = TRANSPORT_ISTATE_PROCESSING;
4732
	cmd->se_tfo->queue_tm_rsp(cmd);
4733

4734
	transport_cmd_check_stop_to_fabric(cmd);
4735 4736 4737 4738 4739 4740 4741 4742 4743
	return 0;
}

/*	transport_processing_thread():
 *
 *
 */
static int transport_processing_thread(void *param)
{
4744
	int ret;
4745 4746 4747 4748 4749 4750
	struct se_cmd *cmd;
	struct se_device *dev = (struct se_device *) param;

	set_user_nice(current, -20);

	while (!kthread_should_stop()) {
4751 4752
		ret = wait_event_interruptible(dev->dev_queue_obj.thread_wq,
				atomic_read(&dev->dev_queue_obj.queue_cnt) ||
4753 4754 4755 4756 4757 4758 4759
				kthread_should_stop());
		if (ret < 0)
			goto out;

get_cmd:
		__transport_execute_tasks(dev);

4760 4761
		cmd = transport_get_cmd_from_queue(&dev->dev_queue_obj);
		if (!cmd)
4762 4763
			continue;

4764
		switch (cmd->t_state) {
4765 4766 4767
		case TRANSPORT_NEW_CMD:
			BUG();
			break;
4768
		case TRANSPORT_NEW_CMD_MAP:
4769 4770
			if (!cmd->se_tfo->new_cmd_map) {
				pr_err("cmd->se_tfo->new_cmd_map is"
4771 4772 4773
					" NULL for TRANSPORT_NEW_CMD_MAP\n");
				BUG();
			}
4774
			ret = cmd->se_tfo->new_cmd_map(cmd);
4775 4776
			if (ret < 0) {
				cmd->transport_error_status = ret;
4777
				transport_generic_request_failure(cmd,
4778 4779 4780 4781 4782
						0, (cmd->data_direction !=
						    DMA_TO_DEVICE));
				break;
			}
			ret = transport_generic_new_cmd(cmd);
4783 4784 4785
			if (ret == -EAGAIN)
				break;
			else if (ret < 0) {
4786
				cmd->transport_error_status = ret;
4787
				transport_generic_request_failure(cmd,
4788 4789 4790 4791 4792 4793 4794
					0, (cmd->data_direction !=
					 DMA_TO_DEVICE));
			}
			break;
		case TRANSPORT_PROCESS_WRITE:
			transport_generic_process_write(cmd);
			break;
4795
		case TRANSPORT_FREE_CMD_INTR:
4796
			transport_generic_free_cmd(cmd, 0);
4797
			break;
4798 4799 4800
		case TRANSPORT_PROCESS_TMR:
			transport_generic_do_tmr(cmd);
			break;
4801
		case TRANSPORT_COMPLETE_QF_WP:
4802 4803 4804 4805
			transport_write_pending_qf(cmd);
			break;
		case TRANSPORT_COMPLETE_QF_OK:
			transport_complete_qf(cmd);
4806
			break;
4807
		default:
4808 4809 4810
			pr_err("Unknown t_state: %d  for ITT: 0x%08x "
				"i_state: %d on SE LUN: %u\n",
				cmd->t_state,
4811 4812 4813
				cmd->se_tfo->get_task_tag(cmd),
				cmd->se_tfo->get_cmd_state(cmd),
				cmd->se_lun->unpacked_lun);
4814 4815 4816 4817 4818 4819 4820
			BUG();
		}

		goto get_cmd;
	}

out:
4821 4822
	WARN_ON(!list_empty(&dev->state_task_list));
	WARN_ON(!list_empty(&dev->dev_queue_obj.qobj_list));
4823 4824 4825
	dev->process_thread = NULL;
	return 0;
}