target_core_transport.c 77.2 KB
Newer Older
1 2 3 4 5
/*******************************************************************************
 * Filename:  target_core_transport.c
 *
 * This file contains the Generic Target Engine Core.
 *
6
 * (c) Copyright 2002-2012 RisingTide Systems LLC.
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
 *
 * Nicholas A. Bellinger <nab@kernel.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 ******************************************************************************/

#include <linux/net.h>
#include <linux/delay.h>
#include <linux/string.h>
#include <linux/timer.h>
#include <linux/slab.h>
#include <linux/blkdev.h>
#include <linux/spinlock.h>
#include <linux/kthread.h>
#include <linux/in.h>
#include <linux/cdrom.h>
36
#include <linux/module.h>
37
#include <linux/ratelimit.h>
38 39 40 41 42
#include <asm/unaligned.h>
#include <net/sock.h>
#include <net/tcp.h>
#include <scsi/scsi.h>
#include <scsi/scsi_cmnd.h>
43
#include <scsi/scsi_tcq.h>
44 45

#include <target/target_core_base.h>
46 47
#include <target/target_core_backend.h>
#include <target/target_core_fabric.h>
48 49
#include <target/target_core_configfs.h>

C
Christoph Hellwig 已提交
50
#include "target_core_internal.h"
51 52 53 54
#include "target_core_alua.h"
#include "target_core_pr.h"
#include "target_core_ua.h"

55
static struct workqueue_struct *target_completion_wq;
56 57 58 59 60 61 62 63 64
static struct kmem_cache *se_sess_cache;
struct kmem_cache *se_ua_cache;
struct kmem_cache *t10_pr_reg_cache;
struct kmem_cache *t10_alua_lu_gp_cache;
struct kmem_cache *t10_alua_lu_gp_mem_cache;
struct kmem_cache *t10_alua_tg_pt_gp_cache;
struct kmem_cache *t10_alua_tg_pt_gp_mem_cache;

static void transport_complete_task_attr(struct se_cmd *cmd);
65
static void transport_handle_queue_full(struct se_cmd *cmd,
66
		struct se_device *dev);
67
static int transport_generic_get_mem(struct se_cmd *cmd);
68
static int target_get_sess_cmd(struct se_session *, struct se_cmd *, bool);
69
static void transport_put_cmd(struct se_cmd *cmd);
70
static void target_complete_ok_work(struct work_struct *work);
71

72
int init_se_kmem_caches(void)
73 74 75 76
{
	se_sess_cache = kmem_cache_create("se_sess_cache",
			sizeof(struct se_session), __alignof__(struct se_session),
			0, NULL);
77 78
	if (!se_sess_cache) {
		pr_err("kmem_cache_create() for struct se_session"
79
				" failed\n");
80
		goto out;
81 82 83 84
	}
	se_ua_cache = kmem_cache_create("se_ua_cache",
			sizeof(struct se_ua), __alignof__(struct se_ua),
			0, NULL);
85 86
	if (!se_ua_cache) {
		pr_err("kmem_cache_create() for struct se_ua failed\n");
87
		goto out_free_sess_cache;
88 89 90 91
	}
	t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache",
			sizeof(struct t10_pr_registration),
			__alignof__(struct t10_pr_registration), 0, NULL);
92 93
	if (!t10_pr_reg_cache) {
		pr_err("kmem_cache_create() for struct t10_pr_registration"
94
				" failed\n");
95
		goto out_free_ua_cache;
96 97 98 99
	}
	t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache",
			sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp),
			0, NULL);
100 101
	if (!t10_alua_lu_gp_cache) {
		pr_err("kmem_cache_create() for t10_alua_lu_gp_cache"
102
				" failed\n");
103
		goto out_free_pr_reg_cache;
104 105 106 107
	}
	t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache",
			sizeof(struct t10_alua_lu_gp_member),
			__alignof__(struct t10_alua_lu_gp_member), 0, NULL);
108 109
	if (!t10_alua_lu_gp_mem_cache) {
		pr_err("kmem_cache_create() for t10_alua_lu_gp_mem_"
110
				"cache failed\n");
111
		goto out_free_lu_gp_cache;
112 113 114 115
	}
	t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache",
			sizeof(struct t10_alua_tg_pt_gp),
			__alignof__(struct t10_alua_tg_pt_gp), 0, NULL);
116 117
	if (!t10_alua_tg_pt_gp_cache) {
		pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_"
118
				"cache failed\n");
119
		goto out_free_lu_gp_mem_cache;
120 121 122 123 124 125
	}
	t10_alua_tg_pt_gp_mem_cache = kmem_cache_create(
			"t10_alua_tg_pt_gp_mem_cache",
			sizeof(struct t10_alua_tg_pt_gp_member),
			__alignof__(struct t10_alua_tg_pt_gp_member),
			0, NULL);
126 127
	if (!t10_alua_tg_pt_gp_mem_cache) {
		pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_"
128
				"mem_t failed\n");
129
		goto out_free_tg_pt_gp_cache;
130 131
	}

132 133 134 135 136
	target_completion_wq = alloc_workqueue("target_completion",
					       WQ_MEM_RECLAIM, 0);
	if (!target_completion_wq)
		goto out_free_tg_pt_gp_mem_cache;

137
	return 0;
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152

out_free_tg_pt_gp_mem_cache:
	kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache);
out_free_tg_pt_gp_cache:
	kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
out_free_lu_gp_mem_cache:
	kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
out_free_lu_gp_cache:
	kmem_cache_destroy(t10_alua_lu_gp_cache);
out_free_pr_reg_cache:
	kmem_cache_destroy(t10_pr_reg_cache);
out_free_ua_cache:
	kmem_cache_destroy(se_ua_cache);
out_free_sess_cache:
	kmem_cache_destroy(se_sess_cache);
153
out:
154
	return -ENOMEM;
155 156
}

157
void release_se_kmem_caches(void)
158
{
159
	destroy_workqueue(target_completion_wq);
160 161 162 163 164 165 166 167 168
	kmem_cache_destroy(se_sess_cache);
	kmem_cache_destroy(se_ua_cache);
	kmem_cache_destroy(t10_pr_reg_cache);
	kmem_cache_destroy(t10_alua_lu_gp_cache);
	kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
	kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
	kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache);
}

169 170 171
/* This code ensures unique mib indexes are handed out. */
static DEFINE_SPINLOCK(scsi_mib_index_lock);
static u32 scsi_mib_index[SCSI_INDEX_TYPE_MAX];
172 173 174 175 176 177 178 179

/*
 * Allocate a new row index for the entry type specified
 */
u32 scsi_get_new_index(scsi_index_t type)
{
	u32 new_index;

180
	BUG_ON((type < 0) || (type >= SCSI_INDEX_TYPE_MAX));
181

182 183 184
	spin_lock(&scsi_mib_index_lock);
	new_index = ++scsi_mib_index[type];
	spin_unlock(&scsi_mib_index_lock);
185 186 187 188

	return new_index;
}

189
void transport_subsystem_check_init(void)
190 191
{
	int ret;
192
	static int sub_api_initialized;
193

194 195 196
	if (sub_api_initialized)
		return;

197 198
	ret = request_module("target_core_iblock");
	if (ret != 0)
199
		pr_err("Unable to load target_core_iblock\n");
200 201 202

	ret = request_module("target_core_file");
	if (ret != 0)
203
		pr_err("Unable to load target_core_file\n");
204 205 206

	ret = request_module("target_core_pscsi");
	if (ret != 0)
207
		pr_err("Unable to load target_core_pscsi\n");
208

209
	sub_api_initialized = 1;
210 211 212 213 214 215 216
}

struct se_session *transport_init_session(void)
{
	struct se_session *se_sess;

	se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL);
217 218
	if (!se_sess) {
		pr_err("Unable to allocate struct se_session from"
219 220 221 222 223
				" se_sess_cache\n");
		return ERR_PTR(-ENOMEM);
	}
	INIT_LIST_HEAD(&se_sess->sess_list);
	INIT_LIST_HEAD(&se_sess->sess_acl_list);
224 225
	INIT_LIST_HEAD(&se_sess->sess_cmd_list);
	spin_lock_init(&se_sess->sess_cmd_lock);
226
	kref_init(&se_sess->sess_kref);
227 228 229 230 231 232

	return se_sess;
}
EXPORT_SYMBOL(transport_init_session);

/*
233
 * Called with spin_lock_irqsave(&struct se_portal_group->session_lock called.
234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
 */
void __transport_register_session(
	struct se_portal_group *se_tpg,
	struct se_node_acl *se_nacl,
	struct se_session *se_sess,
	void *fabric_sess_ptr)
{
	unsigned char buf[PR_REG_ISID_LEN];

	se_sess->se_tpg = se_tpg;
	se_sess->fabric_sess_ptr = fabric_sess_ptr;
	/*
	 * Used by struct se_node_acl's under ConfigFS to locate active se_session-t
	 *
	 * Only set for struct se_session's that will actually be moving I/O.
	 * eg: *NOT* discovery sessions.
	 */
	if (se_nacl) {
		/*
		 * If the fabric module supports an ISID based TransportID,
		 * save this value in binary from the fabric I_T Nexus now.
		 */
256
		if (se_tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) {
257
			memset(&buf[0], 0, PR_REG_ISID_LEN);
258
			se_tpg->se_tpg_tfo->sess_get_initiator_sid(se_sess,
259 260 261
					&buf[0], PR_REG_ISID_LEN);
			se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]);
		}
262 263
		kref_get(&se_nacl->acl_kref);

264 265 266 267 268 269 270 271 272 273 274 275 276
		spin_lock_irq(&se_nacl->nacl_sess_lock);
		/*
		 * The se_nacl->nacl_sess pointer will be set to the
		 * last active I_T Nexus for each struct se_node_acl.
		 */
		se_nacl->nacl_sess = se_sess;

		list_add_tail(&se_sess->sess_acl_list,
			      &se_nacl->acl_sess_list);
		spin_unlock_irq(&se_nacl->nacl_sess_lock);
	}
	list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list);

277
	pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n",
278
		se_tpg->se_tpg_tfo->get_fabric_name(), se_sess->fabric_sess_ptr);
279 280 281 282 283 284 285 286 287
}
EXPORT_SYMBOL(__transport_register_session);

void transport_register_session(
	struct se_portal_group *se_tpg,
	struct se_node_acl *se_nacl,
	struct se_session *se_sess,
	void *fabric_sess_ptr)
{
288 289 290
	unsigned long flags;

	spin_lock_irqsave(&se_tpg->session_lock, flags);
291
	__transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr);
292
	spin_unlock_irqrestore(&se_tpg->session_lock, flags);
293 294 295
}
EXPORT_SYMBOL(transport_register_session);

296
static void target_release_session(struct kref *kref)
297 298 299 300 301 302 303 304 305 306 307 308 309 310
{
	struct se_session *se_sess = container_of(kref,
			struct se_session, sess_kref);
	struct se_portal_group *se_tpg = se_sess->se_tpg;

	se_tpg->se_tpg_tfo->close_session(se_sess);
}

void target_get_session(struct se_session *se_sess)
{
	kref_get(&se_sess->sess_kref);
}
EXPORT_SYMBOL(target_get_session);

311
void target_put_session(struct se_session *se_sess)
312
{
313 314 315 316 317 318
	struct se_portal_group *tpg = se_sess->se_tpg;

	if (tpg->se_tpg_tfo->put_session != NULL) {
		tpg->se_tpg_tfo->put_session(se_sess);
		return;
	}
319
	kref_put(&se_sess->sess_kref, target_release_session);
320 321 322
}
EXPORT_SYMBOL(target_put_session);

323 324 325 326 327 328 329 330 331 332 333 334 335
static void target_complete_nacl(struct kref *kref)
{
	struct se_node_acl *nacl = container_of(kref,
				struct se_node_acl, acl_kref);

	complete(&nacl->acl_free_comp);
}

void target_put_nacl(struct se_node_acl *nacl)
{
	kref_put(&nacl->acl_kref, target_complete_nacl);
}

336 337 338
void transport_deregister_session_configfs(struct se_session *se_sess)
{
	struct se_node_acl *se_nacl;
339
	unsigned long flags;
340 341 342 343
	/*
	 * Used by struct se_node_acl's under ConfigFS to locate active struct se_session
	 */
	se_nacl = se_sess->se_node_acl;
344
	if (se_nacl) {
345
		spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
346 347
		if (se_nacl->acl_stop == 0)
			list_del(&se_sess->sess_acl_list);
348 349 350 351 352 353 354 355 356 357 358 359
		/*
		 * If the session list is empty, then clear the pointer.
		 * Otherwise, set the struct se_session pointer from the tail
		 * element of the per struct se_node_acl active session list.
		 */
		if (list_empty(&se_nacl->acl_sess_list))
			se_nacl->nacl_sess = NULL;
		else {
			se_nacl->nacl_sess = container_of(
					se_nacl->acl_sess_list.prev,
					struct se_session, sess_acl_list);
		}
360
		spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
361 362 363 364 365 366 367 368 369 370 371 372 373
	}
}
EXPORT_SYMBOL(transport_deregister_session_configfs);

void transport_free_session(struct se_session *se_sess)
{
	kmem_cache_free(se_sess_cache, se_sess);
}
EXPORT_SYMBOL(transport_free_session);

void transport_deregister_session(struct se_session *se_sess)
{
	struct se_portal_group *se_tpg = se_sess->se_tpg;
374
	struct target_core_fabric_ops *se_tfo;
375
	struct se_node_acl *se_nacl;
376
	unsigned long flags;
377
	bool comp_nacl = true;
378

379
	if (!se_tpg) {
380 381 382
		transport_free_session(se_sess);
		return;
	}
383
	se_tfo = se_tpg->se_tpg_tfo;
384

385
	spin_lock_irqsave(&se_tpg->session_lock, flags);
386 387 388
	list_del(&se_sess->sess_list);
	se_sess->se_tpg = NULL;
	se_sess->fabric_sess_ptr = NULL;
389
	spin_unlock_irqrestore(&se_tpg->session_lock, flags);
390 391 392 393 394 395

	/*
	 * Determine if we need to do extra work for this initiator node's
	 * struct se_node_acl if it had been previously dynamically generated.
	 */
	se_nacl = se_sess->se_node_acl;
396 397 398 399 400 401 402 403 404 405 406 407 408

	spin_lock_irqsave(&se_tpg->acl_node_lock, flags);
	if (se_nacl && se_nacl->dynamic_node_acl) {
		if (!se_tfo->tpg_check_demo_mode_cache(se_tpg)) {
			list_del(&se_nacl->acl_list);
			se_tpg->num_node_acls--;
			spin_unlock_irqrestore(&se_tpg->acl_node_lock, flags);
			core_tpg_wait_for_nacl_pr_ref(se_nacl);
			core_free_device_list_for_node(se_nacl, se_tpg);
			se_tfo->tpg_release_fabric_acl(se_tpg, se_nacl);

			comp_nacl = false;
			spin_lock_irqsave(&se_tpg->acl_node_lock, flags);
409 410
		}
	}
411
	spin_unlock_irqrestore(&se_tpg->acl_node_lock, flags);
412

413
	pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n",
414
		se_tpg->se_tpg_tfo->get_fabric_name());
415
	/*
416 417 418
	 * If last kref is dropping now for an explict NodeACL, awake sleeping
	 * ->acl_free_comp caller to wakeup configfs se_node_acl->acl_group
	 * removal context.
419 420
	 */
	if (se_nacl && comp_nacl == true)
421
		target_put_nacl(se_nacl);
422

423
	transport_free_session(se_sess);
424 425 426 427
}
EXPORT_SYMBOL(transport_deregister_session);

/*
428
 * Called with cmd->t_state_lock held.
429
 */
430
static void target_remove_from_state_list(struct se_cmd *cmd)
431
{
432
	struct se_device *dev = cmd->se_dev;
433 434
	unsigned long flags;

435 436
	if (!dev)
		return;
437

438 439
	if (cmd->transport_state & CMD_T_BUSY)
		return;
440

441 442 443 444
	spin_lock_irqsave(&dev->execute_task_lock, flags);
	if (cmd->state_active) {
		list_del(&cmd->state_list);
		cmd->state_active = false;
445
	}
446
	spin_unlock_irqrestore(&dev->execute_task_lock, flags);
447 448
}

449
static int transport_cmd_check_stop(struct se_cmd *cmd, bool remove_from_lists)
450 451 452
{
	unsigned long flags;

453
	spin_lock_irqsave(&cmd->t_state_lock, flags);
454 455 456 457
	/*
	 * Determine if IOCTL context caller in requesting the stopping of this
	 * command for LUN shutdown purposes.
	 */
458 459 460
	if (cmd->transport_state & CMD_T_LUN_STOP) {
		pr_debug("%s:%d CMD_T_LUN_STOP for ITT: 0x%08x\n",
			__func__, __LINE__, cmd->se_tfo->get_task_tag(cmd));
461

462
		cmd->transport_state &= ~CMD_T_ACTIVE;
463
		if (remove_from_lists)
464
			target_remove_from_state_list(cmd);
465
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
466

467
		complete(&cmd->transport_lun_stop_comp);
468 469
		return 1;
	}
470 471 472 473 474 475 476 477 478 479

	if (remove_from_lists) {
		target_remove_from_state_list(cmd);

		/*
		 * Clear struct se_cmd->se_lun before the handoff to FE.
		 */
		cmd->se_lun = NULL;
	}

480 481
	/*
	 * Determine if frontend context caller is requesting the stopping of
482
	 * this command for frontend exceptions.
483
	 */
484 485 486
	if (cmd->transport_state & CMD_T_STOP) {
		pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08x\n",
			__func__, __LINE__,
487
			cmd->se_tfo->get_task_tag(cmd));
488

489
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
490

491
		complete(&cmd->t_transport_stop_comp);
492 493
		return 1;
	}
494 495 496 497 498 499 500 501 502 503 504 505 506 507 508

	cmd->transport_state &= ~CMD_T_ACTIVE;
	if (remove_from_lists) {
		/*
		 * Some fabric modules like tcm_loop can release
		 * their internally allocated I/O reference now and
		 * struct se_cmd now.
		 *
		 * Fabric modules are expected to return '1' here if the
		 * se_cmd being passed is released at this point,
		 * or zero if not being released.
		 */
		if (cmd->se_tfo->check_stop_free != NULL) {
			spin_unlock_irqrestore(&cmd->t_state_lock, flags);
			return cmd->se_tfo->check_stop_free(cmd);
509
		}
510
	}
511

512
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
513 514 515 516 517
	return 0;
}

static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd)
{
518
	return transport_cmd_check_stop(cmd, true);
519 520 521 522
}

static void transport_lun_remove_cmd(struct se_cmd *cmd)
{
523
	struct se_lun *lun = cmd->se_lun;
524 525 526 527 528
	unsigned long flags;

	if (!lun)
		return;

529
	spin_lock_irqsave(&cmd->t_state_lock, flags);
530 531
	if (cmd->transport_state & CMD_T_DEV_ACTIVE) {
		cmd->transport_state &= ~CMD_T_DEV_ACTIVE;
532
		target_remove_from_state_list(cmd);
533
	}
534
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
535 536

	spin_lock_irqsave(&lun->lun_cmd_lock, flags);
537 538
	if (!list_empty(&cmd->se_lun_node))
		list_del_init(&cmd->se_lun_node);
539 540 541 542 543
	spin_unlock_irqrestore(&lun->lun_cmd_lock, flags);
}

void transport_cmd_finish_abort(struct se_cmd *cmd, int remove)
{
544
	if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
545
		transport_lun_remove_cmd(cmd);
546 547 548

	if (transport_cmd_check_stop_to_fabric(cmd))
		return;
549
	if (remove)
550
		transport_put_cmd(cmd);
551 552
}

553 554 555 556
static void target_complete_failure_work(struct work_struct *work)
{
	struct se_cmd *cmd = container_of(work, struct se_cmd, work);

557 558
	transport_generic_request_failure(cmd,
			TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE);
559 560
}

561
/*
562 563
 * Used when asking transport to copy Sense Data from the underlying
 * Linux/SCSI struct scsi_cmnd
564
 */
565
static unsigned char *transport_get_sense_buffer(struct se_cmd *cmd)
566 567 568 569 570 571
{
	struct se_device *dev = cmd->se_dev;

	WARN_ON(!cmd->se_lun);

	if (!dev)
572
		return NULL;
573

574 575
	if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION)
		return NULL;
576

577
	cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER;
578

579
	pr_debug("HBA_[%u]_PLUG[%s]: Requesting sense for SAM STATUS: 0x%02x\n",
580
		dev->se_hba->hba_id, dev->transport->name, cmd->scsi_status);
581
	return cmd->sense_buffer;
582 583
}

584
void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status)
585
{
586
	struct se_device *dev = cmd->se_dev;
587
	int success = scsi_status == GOOD;
588 589
	unsigned long flags;

590 591 592
	cmd->scsi_status = scsi_status;


593
	spin_lock_irqsave(&cmd->t_state_lock, flags);
594
	cmd->transport_state &= ~CMD_T_BUSY;
595 596

	if (dev && dev->transport->transport_complete) {
597 598 599 600
		dev->transport->transport_complete(cmd,
				cmd->t_data_sg,
				transport_get_sense_buffer(cmd));
		if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)
601 602 603 604
			success = 1;
	}

	/*
605
	 * See if we are waiting to complete for an exception condition.
606
	 */
607
	if (cmd->transport_state & CMD_T_REQUEST_STOP) {
608
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
609
		complete(&cmd->task_stop_comp);
610 611
		return;
	}
612 613

	if (!success)
614
		cmd->transport_state |= CMD_T_FAILED;
615

616 617 618 619 620 621 622 623 624 625
	/*
	 * Check for case where an explict ABORT_TASK has been received
	 * and transport_wait_for_tasks() will be waiting for completion..
	 */
	if (cmd->transport_state & CMD_T_ABORTED &&
	    cmd->transport_state & CMD_T_STOP) {
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
		complete(&cmd->t_transport_stop_comp);
		return;
	} else if (cmd->transport_state & CMD_T_FAILED) {
626
		INIT_WORK(&cmd->work, target_complete_failure_work);
627
	} else {
628
		INIT_WORK(&cmd->work, target_complete_ok_work);
629
	}
630 631

	cmd->t_state = TRANSPORT_COMPLETE;
632
	cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE);
633
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
634

635
	queue_work(target_completion_wq, &cmd->work);
636
}
637 638
EXPORT_SYMBOL(target_complete_cmd);

639
static void target_add_to_state_list(struct se_cmd *cmd)
640
{
641 642
	struct se_device *dev = cmd->se_dev;
	unsigned long flags;
643

644 645 646 647
	spin_lock_irqsave(&dev->execute_task_lock, flags);
	if (!cmd->state_active) {
		list_add_tail(&cmd->state_list, &dev->state_list);
		cmd->state_active = true;
648
	}
649
	spin_unlock_irqrestore(&dev->execute_task_lock, flags);
650 651
}

652
/*
653
 * Handle QUEUE_FULL / -EAGAIN and -ENOMEM status
654
 */
655 656
static void transport_write_pending_qf(struct se_cmd *cmd);
static void transport_complete_qf(struct se_cmd *cmd);
657

658
void target_qf_do_work(struct work_struct *work)
659 660 661
{
	struct se_device *dev = container_of(work, struct se_device,
					qf_work_queue);
662
	LIST_HEAD(qf_cmd_list);
663 664 665
	struct se_cmd *cmd, *cmd_tmp;

	spin_lock_irq(&dev->qf_cmd_lock);
666 667
	list_splice_init(&dev->qf_cmd_list, &qf_cmd_list);
	spin_unlock_irq(&dev->qf_cmd_lock);
668

669
	list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) {
670 671 672 673
		list_del(&cmd->se_qf_node);
		atomic_dec(&dev->dev_qf_count);
		smp_mb__after_atomic_dec();

674
		pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue"
675
			" context: %s\n", cmd->se_tfo->get_fabric_name(), cmd,
676
			(cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" :
677 678
			(cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING"
			: "UNKNOWN");
679

680 681 682 683
		if (cmd->t_state == TRANSPORT_COMPLETE_QF_WP)
			transport_write_pending_qf(cmd);
		else if (cmd->t_state == TRANSPORT_COMPLETE_QF_OK)
			transport_complete_qf(cmd);
684 685 686
	}
}

687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710
unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd)
{
	switch (cmd->data_direction) {
	case DMA_NONE:
		return "NONE";
	case DMA_FROM_DEVICE:
		return "READ";
	case DMA_TO_DEVICE:
		return "WRITE";
	case DMA_BIDIRECTIONAL:
		return "BIDI";
	default:
		break;
	}

	return "UNKNOWN";
}

void transport_dump_dev_state(
	struct se_device *dev,
	char *b,
	int *bl)
{
	*bl += sprintf(b + *bl, "Status: ");
711
	if (dev->export_count)
712
		*bl += sprintf(b + *bl, "ACTIVATED");
713
	else
714 715
		*bl += sprintf(b + *bl, "DEACTIVATED");

716
	*bl += sprintf(b + *bl, "  Max Queue Depth: %d", dev->queue_depth);
717
	*bl += sprintf(b + *bl, "  SectorSize: %u  HwMaxSectors: %u\n",
718 719
		dev->dev_attrib.block_size,
		dev->dev_attrib.hw_max_sectors);
720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772
	*bl += sprintf(b + *bl, "        ");
}

void transport_dump_vpd_proto_id(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
	int len;

	memset(buf, 0, VPD_TMP_BUF_SIZE);
	len = sprintf(buf, "T10 VPD Protocol Identifier: ");

	switch (vpd->protocol_identifier) {
	case 0x00:
		sprintf(buf+len, "Fibre Channel\n");
		break;
	case 0x10:
		sprintf(buf+len, "Parallel SCSI\n");
		break;
	case 0x20:
		sprintf(buf+len, "SSA\n");
		break;
	case 0x30:
		sprintf(buf+len, "IEEE 1394\n");
		break;
	case 0x40:
		sprintf(buf+len, "SCSI Remote Direct Memory Access"
				" Protocol\n");
		break;
	case 0x50:
		sprintf(buf+len, "Internet SCSI (iSCSI)\n");
		break;
	case 0x60:
		sprintf(buf+len, "SAS Serial SCSI Protocol\n");
		break;
	case 0x70:
		sprintf(buf+len, "Automation/Drive Interface Transport"
				" Protocol\n");
		break;
	case 0x80:
		sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n");
		break;
	default:
		sprintf(buf+len, "Unknown 0x%02x\n",
				vpd->protocol_identifier);
		break;
	}

	if (p_buf)
		strncpy(p_buf, buf, p_buf_len);
	else
773
		pr_debug("%s", buf);
774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797
}

void
transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83)
{
	/*
	 * Check if the Protocol Identifier Valid (PIV) bit is set..
	 *
	 * from spc3r23.pdf section 7.5.1
	 */
	 if (page_83[1] & 0x80) {
		vpd->protocol_identifier = (page_83[0] & 0xf0);
		vpd->protocol_identifier_set = 1;
		transport_dump_vpd_proto_id(vpd, NULL, 0);
	}
}
EXPORT_SYMBOL(transport_set_vpd_proto_id);

int transport_dump_vpd_assoc(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
798 799
	int ret = 0;
	int len;
800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815

	memset(buf, 0, VPD_TMP_BUF_SIZE);
	len = sprintf(buf, "T10 VPD Identifier Association: ");

	switch (vpd->association) {
	case 0x00:
		sprintf(buf+len, "addressed logical unit\n");
		break;
	case 0x10:
		sprintf(buf+len, "target port\n");
		break;
	case 0x20:
		sprintf(buf+len, "SCSI target device\n");
		break;
	default:
		sprintf(buf+len, "Unknown 0x%02x\n", vpd->association);
816
		ret = -EINVAL;
817 818 819 820 821 822
		break;
	}

	if (p_buf)
		strncpy(p_buf, buf, p_buf_len);
	else
823
		pr_debug("%s", buf);
824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845

	return ret;
}

int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83)
{
	/*
	 * The VPD identification association..
	 *
	 * from spc3r23.pdf Section 7.6.3.1 Table 297
	 */
	vpd->association = (page_83[1] & 0x30);
	return transport_dump_vpd_assoc(vpd, NULL, 0);
}
EXPORT_SYMBOL(transport_set_vpd_assoc);

int transport_dump_vpd_ident_type(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
846 847
	int ret = 0;
	int len;
848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873

	memset(buf, 0, VPD_TMP_BUF_SIZE);
	len = sprintf(buf, "T10 VPD Identifier Type: ");

	switch (vpd->device_identifier_type) {
	case 0x00:
		sprintf(buf+len, "Vendor specific\n");
		break;
	case 0x01:
		sprintf(buf+len, "T10 Vendor ID based\n");
		break;
	case 0x02:
		sprintf(buf+len, "EUI-64 based\n");
		break;
	case 0x03:
		sprintf(buf+len, "NAA\n");
		break;
	case 0x04:
		sprintf(buf+len, "Relative target port identifier\n");
		break;
	case 0x08:
		sprintf(buf+len, "SCSI name string\n");
		break;
	default:
		sprintf(buf+len, "Unsupported: 0x%02x\n",
				vpd->device_identifier_type);
874
		ret = -EINVAL;
875 876 877
		break;
	}

878 879 880
	if (p_buf) {
		if (p_buf_len < strlen(buf)+1)
			return -EINVAL;
881
		strncpy(p_buf, buf, p_buf_len);
882
	} else {
883
		pr_debug("%s", buf);
884
	}
885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926

	return ret;
}

int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83)
{
	/*
	 * The VPD identifier type..
	 *
	 * from spc3r23.pdf Section 7.6.3.1 Table 298
	 */
	vpd->device_identifier_type = (page_83[1] & 0x0f);
	return transport_dump_vpd_ident_type(vpd, NULL, 0);
}
EXPORT_SYMBOL(transport_set_vpd_ident_type);

int transport_dump_vpd_ident(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
	int ret = 0;

	memset(buf, 0, VPD_TMP_BUF_SIZE);

	switch (vpd->device_identifier_code_set) {
	case 0x01: /* Binary */
		sprintf(buf, "T10 VPD Binary Device Identifier: %s\n",
			&vpd->device_identifier[0]);
		break;
	case 0x02: /* ASCII */
		sprintf(buf, "T10 VPD ASCII Device Identifier: %s\n",
			&vpd->device_identifier[0]);
		break;
	case 0x03: /* UTF-8 */
		sprintf(buf, "T10 VPD UTF-8 Device Identifier: %s\n",
			&vpd->device_identifier[0]);
		break;
	default:
		sprintf(buf, "T10 VPD Device Identifier encoding unsupported:"
			" 0x%02x", vpd->device_identifier_code_set);
927
		ret = -EINVAL;
928 929 930 931 932 933
		break;
	}

	if (p_buf)
		strncpy(p_buf, buf, p_buf_len);
	else
934
		pr_debug("%s", buf);
935 936 937 938 939 940 941 942

	return ret;
}

int
transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83)
{
	static const char hex_str[] = "0123456789abcdef";
943
	int j = 0, i = 4; /* offset to start of the identifier */
944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975

	/*
	 * The VPD Code Set (encoding)
	 *
	 * from spc3r23.pdf Section 7.6.3.1 Table 296
	 */
	vpd->device_identifier_code_set = (page_83[0] & 0x0f);
	switch (vpd->device_identifier_code_set) {
	case 0x01: /* Binary */
		vpd->device_identifier[j++] =
				hex_str[vpd->device_identifier_type];
		while (i < (4 + page_83[3])) {
			vpd->device_identifier[j++] =
				hex_str[(page_83[i] & 0xf0) >> 4];
			vpd->device_identifier[j++] =
				hex_str[page_83[i] & 0x0f];
			i++;
		}
		break;
	case 0x02: /* ASCII */
	case 0x03: /* UTF-8 */
		while (i < (4 + page_83[3]))
			vpd->device_identifier[j++] = page_83[i++];
		break;
	default:
		break;
	}

	return transport_dump_vpd_ident(vpd, NULL, 0);
}
EXPORT_SYMBOL(transport_set_vpd_ident);

976 977
sense_reason_t
target_cmd_size_check(struct se_cmd *cmd, unsigned int size)
978 979 980 981 982 983 984 985 986 987 988 989 990 991
{
	struct se_device *dev = cmd->se_dev;

	if (cmd->unknown_data_length) {
		cmd->data_length = size;
	} else if (size != cmd->data_length) {
		pr_warn("TARGET_CORE[%s]: Expected Transfer Length:"
			" %u does not match SCSI CDB Length: %u for SAM Opcode:"
			" 0x%02x\n", cmd->se_tfo->get_fabric_name(),
				cmd->data_length, size, cmd->t_task_cdb[0]);

		if (cmd->data_direction == DMA_TO_DEVICE) {
			pr_err("Rejecting underflow/overflow"
					" WRITE data\n");
992
			return TCM_INVALID_CDB_FIELD;
993 994 995 996 997
		}
		/*
		 * Reject READ_* or WRITE_* with overflow/underflow for
		 * type SCF_SCSI_DATA_CDB.
		 */
998
		if (dev->dev_attrib.block_size != 512)  {
999 1000 1001 1002
			pr_err("Failing OVERFLOW/UNDERFLOW for LBA op"
				" CDB on non 512-byte sector setup subsystem"
				" plugin: %s\n", dev->transport->name);
			/* Returns CHECK_CONDITION + INVALID_CDB_FIELD */
1003
			return TCM_INVALID_CDB_FIELD;
1004
		}
1005 1006 1007 1008 1009 1010
		/*
		 * For the overflow case keep the existing fabric provided
		 * ->data_length.  Otherwise for the underflow case, reset
		 * ->data_length to the smaller SCSI expected data transfer
		 * length.
		 */
1011 1012 1013 1014 1015 1016
		if (size > cmd->data_length) {
			cmd->se_cmd_flags |= SCF_OVERFLOW_BIT;
			cmd->residual_count = (size - cmd->data_length);
		} else {
			cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
			cmd->residual_count = (cmd->data_length - size);
1017
			cmd->data_length = size;
1018 1019 1020 1021 1022 1023 1024
		}
	}

	return 0;

}

1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
/*
 * Used by fabric modules containing a local struct se_cmd within their
 * fabric dependent per I/O descriptor.
 */
void transport_init_se_cmd(
	struct se_cmd *cmd,
	struct target_core_fabric_ops *tfo,
	struct se_session *se_sess,
	u32 data_length,
	int data_direction,
	int task_attr,
	unsigned char *sense_buffer)
{
1038 1039
	INIT_LIST_HEAD(&cmd->se_lun_node);
	INIT_LIST_HEAD(&cmd->se_delayed_node);
1040
	INIT_LIST_HEAD(&cmd->se_qf_node);
1041
	INIT_LIST_HEAD(&cmd->se_cmd_list);
1042
	INIT_LIST_HEAD(&cmd->state_list);
1043 1044 1045
	init_completion(&cmd->transport_lun_fe_stop_comp);
	init_completion(&cmd->transport_lun_stop_comp);
	init_completion(&cmd->t_transport_stop_comp);
1046
	init_completion(&cmd->cmd_wait_comp);
1047
	init_completion(&cmd->task_stop_comp);
1048
	spin_lock_init(&cmd->t_state_lock);
1049
	cmd->transport_state = CMD_T_DEV_ACTIVE;
1050 1051 1052 1053 1054 1055 1056

	cmd->se_tfo = tfo;
	cmd->se_sess = se_sess;
	cmd->data_length = data_length;
	cmd->data_direction = data_direction;
	cmd->sam_task_attr = task_attr;
	cmd->sense_buffer = sense_buffer;
1057 1058

	cmd->state_active = false;
1059 1060 1061
}
EXPORT_SYMBOL(transport_init_se_cmd);

1062 1063
static sense_reason_t
transport_check_alloc_task_attr(struct se_cmd *cmd)
1064
{
1065 1066
	struct se_device *dev = cmd->se_dev;

1067 1068 1069 1070
	/*
	 * Check if SAM Task Attribute emulation is enabled for this
	 * struct se_device storage object
	 */
1071
	if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV)
1072 1073
		return 0;

1074
	if (cmd->sam_task_attr == MSG_ACA_TAG) {
1075
		pr_debug("SAM Task Attribute ACA"
1076
			" emulation is not supported\n");
1077
		return TCM_INVALID_CDB_FIELD;
1078 1079 1080 1081 1082
	}
	/*
	 * Used to determine when ORDERED commands should go from
	 * Dormant to Active status.
	 */
1083
	cmd->se_ordered_id = atomic_inc_return(&dev->dev_ordered_id);
1084
	smp_mb__after_atomic_inc();
1085
	pr_debug("Allocated se_ordered_id: %u for Task Attr: 0x%02x on %s\n",
1086
			cmd->se_ordered_id, cmd->sam_task_attr,
1087
			dev->transport->name);
1088 1089 1090
	return 0;
}

1091 1092
sense_reason_t
target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb)
1093
{
1094
	struct se_device *dev = cmd->se_dev;
1095
	unsigned long flags;
1096
	sense_reason_t ret;
1097 1098 1099 1100 1101 1102

	/*
	 * Ensure that the received CDB is less than the max (252 + 8) bytes
	 * for VARIABLE_LENGTH_CMD
	 */
	if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) {
1103
		pr_err("Received SCSI CDB with command_size: %d that"
1104 1105
			" exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
			scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE);
1106
		return TCM_INVALID_CDB_FIELD;
1107 1108 1109 1110 1111 1112
	}
	/*
	 * If the received CDB is larger than TCM_MAX_COMMAND_SIZE,
	 * allocate the additional extended CDB buffer now..  Otherwise
	 * setup the pointer from __t_task_cdb to t_task_cdb.
	 */
1113 1114
	if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) {
		cmd->t_task_cdb = kzalloc(scsi_command_size(cdb),
1115
						GFP_KERNEL);
1116 1117
		if (!cmd->t_task_cdb) {
			pr_err("Unable to allocate cmd->t_task_cdb"
1118
				" %u > sizeof(cmd->__t_task_cdb): %lu ops\n",
1119
				scsi_command_size(cdb),
1120
				(unsigned long)sizeof(cmd->__t_task_cdb));
1121
			return TCM_OUT_OF_RESOURCES;
1122 1123
		}
	} else
1124
		cmd->t_task_cdb = &cmd->__t_task_cdb[0];
1125
	/*
1126
	 * Copy the original CDB into cmd->
1127
	 */
1128
	memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb));
1129 1130 1131 1132

	/*
	 * Check for an existing UNIT ATTENTION condition
	 */
1133 1134 1135
	ret = target_scsi3_ua_check(cmd);
	if (ret)
		return ret;
1136

C
Christoph Hellwig 已提交
1137
	ret = target_alua_state_check(cmd);
1138 1139
	if (ret)
		return ret;
1140

1141
	ret = target_check_reservation(cmd);
1142
	if (ret)
1143
		return ret;
1144

1145
	ret = dev->transport->parse_cdb(cmd);
1146 1147 1148 1149 1150
	if (ret)
		return ret;

	ret = transport_check_alloc_task_attr(cmd);
	if (ret)
1151
		return ret;
1152 1153 1154 1155 1156

	spin_lock_irqsave(&cmd->t_state_lock, flags);
	cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE;
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);

1157 1158 1159 1160 1161 1162
	spin_lock(&cmd->se_lun->lun_sep_lock);
	if (cmd->se_lun->lun_sep)
		cmd->se_lun->lun_sep->sep_stats.cmd_pdus++;
	spin_unlock(&cmd->se_lun->lun_sep_lock);
	return 0;
}
1163
EXPORT_SYMBOL(target_setup_cmd_from_cdb);
1164

1165 1166 1167 1168 1169 1170 1171
/*
 * Used by fabric module frontends to queue tasks directly.
 * Many only be used from process context only
 */
int transport_handle_cdb_direct(
	struct se_cmd *cmd)
{
1172
	sense_reason_t ret;
1173

1174 1175
	if (!cmd->se_lun) {
		dump_stack();
1176
		pr_err("cmd->se_lun is NULL\n");
1177 1178 1179 1180
		return -EINVAL;
	}
	if (in_interrupt()) {
		dump_stack();
1181
		pr_err("transport_generic_handle_cdb cannot be called"
1182 1183 1184
				" from interrupt context\n");
		return -EINVAL;
	}
1185
	/*
1186 1187 1188
	 * Set TRANSPORT_NEW_CMD state and CMD_T_ACTIVE to ensure that
	 * outstanding descriptors are handled correctly during shutdown via
	 * transport_wait_for_tasks()
1189 1190 1191 1192 1193
	 *
	 * Also, we don't take cmd->t_state_lock here as we only expect
	 * this to be called for initial descriptor submission.
	 */
	cmd->t_state = TRANSPORT_NEW_CMD;
1194 1195
	cmd->transport_state |= CMD_T_ACTIVE;

1196 1197 1198 1199 1200 1201
	/*
	 * transport_generic_new_cmd() is already handling QUEUE_FULL,
	 * so follow TRANSPORT_NEW_CMD processing thread context usage
	 * and call transport_generic_request_failure() if necessary..
	 */
	ret = transport_generic_new_cmd(cmd);
1202 1203
	if (ret)
		transport_generic_request_failure(cmd, ret);
1204
	return 0;
1205 1206 1207
}
EXPORT_SYMBOL(transport_handle_cdb_direct);

1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236
static sense_reason_t
transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl,
		u32 sgl_count, struct scatterlist *sgl_bidi, u32 sgl_bidi_count)
{
	if (!sgl || !sgl_count)
		return 0;

	/*
	 * Reject SCSI data overflow with map_mem_to_cmd() as incoming
	 * scatterlists already have been set to follow what the fabric
	 * passes for the original expected data transfer length.
	 */
	if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
		pr_warn("Rejecting SCSI DATA overflow for fabric using"
			" SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC\n");
		return TCM_INVALID_CDB_FIELD;
	}

	cmd->t_data_sg = sgl;
	cmd->t_data_nents = sgl_count;

	if (sgl_bidi && sgl_bidi_count) {
		cmd->t_bidi_data_sg = sgl_bidi;
		cmd->t_bidi_data_nents = sgl_bidi_count;
	}
	cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
	return 0;
}

1237 1238 1239
/*
 * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized
 * 			 se_cmd + use pre-allocated SGL memory.
1240 1241 1242 1243 1244 1245 1246 1247 1248 1249
 *
 * @se_cmd: command descriptor to submit
 * @se_sess: associated se_sess for endpoint
 * @cdb: pointer to SCSI CDB
 * @sense: pointer to SCSI sense buffer
 * @unpacked_lun: unpacked LUN to reference for struct se_lun
 * @data_length: fabric expected data transfer length
 * @task_addr: SAM task attribute
 * @data_dir: DMA data direction
 * @flags: flags for command submission from target_sc_flags_tables
1250 1251 1252 1253
 * @sgl: struct scatterlist memory for unidirectional mapping
 * @sgl_count: scatterlist count for unidirectional mapping
 * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping
 * @sgl_bidi_count: scatterlist count for bidirectional READ mapping
1254
 *
1255 1256 1257 1258
 * Returns non zero to signal active I/O shutdown failure.  All other
 * setup exceptions will be returned as a SCSI CHECK_CONDITION response,
 * but still return zero here.
 *
1259 1260
 * This may only be called from process context, and also currently
 * assumes internal allocation of fabric payload buffer by target-core.
1261 1262
 */
int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess,
1263
		unsigned char *cdb, unsigned char *sense, u32 unpacked_lun,
1264 1265 1266
		u32 data_length, int task_attr, int data_dir, int flags,
		struct scatterlist *sgl, u32 sgl_count,
		struct scatterlist *sgl_bidi, u32 sgl_bidi_count)
1267 1268
{
	struct se_portal_group *se_tpg;
1269 1270
	sense_reason_t rc;
	int ret;
1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282

	se_tpg = se_sess->se_tpg;
	BUG_ON(!se_tpg);
	BUG_ON(se_cmd->se_tfo || se_cmd->se_sess);
	BUG_ON(in_interrupt());
	/*
	 * Initialize se_cmd for target operation.  From this point
	 * exceptions are handled by sending exception status via
	 * target_core_fabric_ops->queue_status() callback
	 */
	transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
				data_length, data_dir, task_attr, sense);
1283 1284
	if (flags & TARGET_SCF_UNKNOWN_SIZE)
		se_cmd->unknown_data_length = 1;
1285 1286 1287 1288 1289 1290
	/*
	 * Obtain struct se_cmd->cmd_kref reference and add new cmd to
	 * se_sess->sess_cmd_list.  A second kref_get here is necessary
	 * for fabrics using TARGET_SCF_ACK_KREF that expect a second
	 * kref_put() to happen during fabric packet acknowledgement.
	 */
1291 1292 1293
	ret = target_get_sess_cmd(se_sess, se_cmd, (flags & TARGET_SCF_ACK_KREF));
	if (ret)
		return ret;
1294 1295 1296 1297 1298 1299 1300 1301
	/*
	 * Signal bidirectional data payloads to target-core
	 */
	if (flags & TARGET_SCF_BIDI_OP)
		se_cmd->se_cmd_flags |= SCF_BIDI;
	/*
	 * Locate se_lun pointer and attach it to struct se_cmd
	 */
1302 1303 1304
	rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun);
	if (rc) {
		transport_send_check_condition_and_sense(se_cmd, rc, 0);
1305
		target_put_sess_cmd(se_sess, se_cmd);
1306
		return 0;
1307
	}
1308

1309
	rc = target_setup_cmd_from_cdb(se_cmd, cdb);
1310
	if (rc != 0) {
1311
		transport_generic_request_failure(se_cmd, rc);
1312
		return 0;
1313
	}
1314 1315 1316 1317 1318 1319 1320 1321
	/*
	 * When a non zero sgl_count has been passed perform SGL passthrough
	 * mapping for pre-allocated fabric memory instead of having target
	 * core perform an internal SGL allocation..
	 */
	if (sgl_count != 0) {
		BUG_ON(!sgl);

1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342
		/*
		 * A work-around for tcm_loop as some userspace code via
		 * scsi-generic do not memset their associated read buffers,
		 * so go ahead and do that here for type non-data CDBs.  Also
		 * note that this is currently guaranteed to be a single SGL
		 * for this case by target core in target_setup_cmd_from_cdb()
		 * -> transport_generic_cmd_sequencer().
		 */
		if (!(se_cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) &&
		     se_cmd->data_direction == DMA_FROM_DEVICE) {
			unsigned char *buf = NULL;

			if (sgl)
				buf = kmap(sg_page(sgl)) + sgl->offset;

			if (buf) {
				memset(buf, 0, sgl->length);
				kunmap(sg_page(sgl));
			}
		}

1343 1344 1345
		rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count,
				sgl_bidi, sgl_bidi_count);
		if (rc != 0) {
1346
			transport_generic_request_failure(se_cmd, rc);
1347 1348 1349
			return 0;
		}
	}
1350 1351 1352 1353 1354 1355
	/*
	 * Check if we need to delay processing because of ALUA
	 * Active/NonOptimized primary access state..
	 */
	core_alua_check_nonop_delay(se_cmd);

1356
	transport_handle_cdb_direct(se_cmd);
1357
	return 0;
1358
}
1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390
EXPORT_SYMBOL(target_submit_cmd_map_sgls);

/*
 * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd
 *
 * @se_cmd: command descriptor to submit
 * @se_sess: associated se_sess for endpoint
 * @cdb: pointer to SCSI CDB
 * @sense: pointer to SCSI sense buffer
 * @unpacked_lun: unpacked LUN to reference for struct se_lun
 * @data_length: fabric expected data transfer length
 * @task_addr: SAM task attribute
 * @data_dir: DMA data direction
 * @flags: flags for command submission from target_sc_flags_tables
 *
 * Returns non zero to signal active I/O shutdown failure.  All other
 * setup exceptions will be returned as a SCSI CHECK_CONDITION response,
 * but still return zero here.
 *
 * This may only be called from process context, and also currently
 * assumes internal allocation of fabric payload buffer by target-core.
 *
 * It also assumes interal target core SGL memory allocation.
 */
int target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess,
		unsigned char *cdb, unsigned char *sense, u32 unpacked_lun,
		u32 data_length, int task_attr, int data_dir, int flags)
{
	return target_submit_cmd_map_sgls(se_cmd, se_sess, cdb, sense,
			unpacked_lun, data_length, task_attr, data_dir,
			flags, NULL, 0, NULL, 0);
}
1391 1392
EXPORT_SYMBOL(target_submit_cmd);

1393 1394 1395 1396 1397 1398 1399 1400
static void target_complete_tmr_failure(struct work_struct *work)
{
	struct se_cmd *se_cmd = container_of(work, struct se_cmd, work);

	se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST;
	se_cmd->se_tfo->queue_tm_rsp(se_cmd);
}

1401 1402 1403 1404 1405 1406 1407 1408 1409 1410
/**
 * target_submit_tmr - lookup unpacked lun and submit uninitialized se_cmd
 *                     for TMR CDBs
 *
 * @se_cmd: command descriptor to submit
 * @se_sess: associated se_sess for endpoint
 * @sense: pointer to SCSI sense buffer
 * @unpacked_lun: unpacked LUN to reference for struct se_lun
 * @fabric_context: fabric context for TMR req
 * @tm_type: Type of TM request
1411 1412
 * @gfp: gfp type for caller
 * @tag: referenced task tag for TMR_ABORT_TASK
1413
 * @flags: submit cmd flags
1414 1415 1416 1417
 *
 * Callable from all contexts.
 **/

1418
int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess,
1419
		unsigned char *sense, u32 unpacked_lun,
1420 1421
		void *fabric_tmr_ptr, unsigned char tm_type,
		gfp_t gfp, unsigned int tag, int flags)
1422 1423 1424 1425 1426 1427 1428 1429 1430
{
	struct se_portal_group *se_tpg;
	int ret;

	se_tpg = se_sess->se_tpg;
	BUG_ON(!se_tpg);

	transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
			      0, DMA_NONE, MSG_SIMPLE_TAG, sense);
1431 1432 1433 1434
	/*
	 * FIXME: Currently expect caller to handle se_cmd->se_tmr_req
	 * allocation failure.
	 */
1435
	ret = core_tmr_alloc_req(se_cmd, fabric_tmr_ptr, tm_type, gfp);
1436 1437
	if (ret < 0)
		return -ENOMEM;
1438

1439 1440 1441
	if (tm_type == TMR_ABORT_TASK)
		se_cmd->se_tmr_req->ref_task_tag = tag;

1442
	/* See target_submit_cmd for commentary */
1443 1444 1445 1446 1447
	ret = target_get_sess_cmd(se_sess, se_cmd, (flags & TARGET_SCF_ACK_KREF));
	if (ret) {
		core_tmr_release_req(se_cmd->se_tmr_req);
		return ret;
	}
1448 1449 1450

	ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun);
	if (ret) {
1451 1452 1453 1454 1455 1456
		/*
		 * For callback during failure handling, push this work off
		 * to process context with TMR_LUN_DOES_NOT_EXIST status.
		 */
		INIT_WORK(&se_cmd->work, target_complete_tmr_failure);
		schedule_work(&se_cmd->work);
1457
		return 0;
1458 1459
	}
	transport_generic_handle_tmr(se_cmd);
1460
	return 0;
1461 1462 1463
}
EXPORT_SYMBOL(target_submit_tmr);

1464
/*
1465
 * If the cmd is active, request it to be stopped and sleep until it
1466 1467
 * has completed.
 */
1468
bool target_stop_cmd(struct se_cmd *cmd, unsigned long *flags)
1469 1470 1471
{
	bool was_active = false;

1472 1473
	if (cmd->transport_state & CMD_T_BUSY) {
		cmd->transport_state |= CMD_T_REQUEST_STOP;
1474 1475
		spin_unlock_irqrestore(&cmd->t_state_lock, *flags);

1476 1477 1478
		pr_debug("cmd %p waiting to complete\n", cmd);
		wait_for_completion(&cmd->task_stop_comp);
		pr_debug("cmd %p stopped successfully\n", cmd);
1479 1480

		spin_lock_irqsave(&cmd->t_state_lock, *flags);
1481 1482
		cmd->transport_state &= ~CMD_T_REQUEST_STOP;
		cmd->transport_state &= ~CMD_T_BUSY;
1483 1484 1485 1486 1487 1488
		was_active = true;
	}

	return was_active;
}

1489 1490 1491
/*
 * Handle SAM-esque emulation for generic transport request failures.
 */
1492 1493
void transport_generic_request_failure(struct se_cmd *cmd,
		sense_reason_t sense_reason)
1494
{
1495 1496
	int ret = 0;

1497
	pr_debug("-----[ Storage Engine Exception for cmd: %p ITT: 0x%08x"
1498
		" CDB: 0x%02x\n", cmd, cmd->se_tfo->get_task_tag(cmd),
1499
		cmd->t_task_cdb[0]);
1500
	pr_debug("-----[ i_state: %d t_state: %d sense_reason: %d\n",
1501
		cmd->se_tfo->get_cmd_state(cmd),
1502
		cmd->t_state, sense_reason);
1503
	pr_debug("-----[ CMD_T_ACTIVE: %d CMD_T_STOP: %d CMD_T_SENT: %d\n",
1504 1505 1506
		(cmd->transport_state & CMD_T_ACTIVE) != 0,
		(cmd->transport_state & CMD_T_STOP) != 0,
		(cmd->transport_state & CMD_T_SENT) != 0);
1507 1508 1509 1510

	/*
	 * For SAM Task Attribute emulation for failed struct se_cmd
	 */
1511
	transport_complete_task_attr(cmd);
1512

1513
	switch (sense_reason) {
1514 1515 1516 1517 1518 1519 1520
	case TCM_NON_EXISTENT_LUN:
	case TCM_UNSUPPORTED_SCSI_OPCODE:
	case TCM_INVALID_CDB_FIELD:
	case TCM_INVALID_PARAMETER_LIST:
	case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
	case TCM_UNKNOWN_MODE_PAGE:
	case TCM_WRITE_PROTECTED:
1521
	case TCM_ADDRESS_OUT_OF_RANGE:
1522 1523 1524
	case TCM_CHECK_CONDITION_ABORT_CMD:
	case TCM_CHECK_CONDITION_UNIT_ATTENTION:
	case TCM_CHECK_CONDITION_NOT_READY:
1525
		break;
1526 1527 1528
	case TCM_OUT_OF_RESOURCES:
		sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
		break;
1529
	case TCM_RESERVATION_CONFLICT:
1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543
		/*
		 * No SENSE Data payload for this case, set SCSI Status
		 * and queue the response to $FABRIC_MOD.
		 *
		 * Uses linux/include/scsi/scsi.h SAM status codes defs
		 */
		cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
		/*
		 * For UA Interlock Code 11b, a RESERVATION CONFLICT will
		 * establish a UNIT ATTENTION with PREVIOUS RESERVATION
		 * CONFLICT STATUS.
		 *
		 * See spc4r17, section 7.4.6 Control Mode Page, Table 349
		 */
1544
		if (cmd->se_sess &&
1545
		    cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl == 2)
1546
			core_scsi3_ua_allocate(cmd->se_sess->se_node_acl,
1547 1548 1549
				cmd->orig_fe_lun, 0x2C,
				ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);

1550
		ret = cmd->se_tfo->queue_status(cmd);
1551
		if (ret == -EAGAIN || ret == -ENOMEM)
1552
			goto queue_full;
1553 1554
		goto check_stop;
	default:
1555
		pr_err("Unknown transport error for CDB 0x%02x: %d\n",
1556 1557
			cmd->t_task_cdb[0], sense_reason);
		sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
1558 1559
		break;
	}
1560

1561
	ret = transport_send_check_condition_and_sense(cmd, sense_reason, 0);
1562 1563
	if (ret == -EAGAIN || ret == -ENOMEM)
		goto queue_full;
1564

1565 1566
check_stop:
	transport_lun_remove_cmd(cmd);
1567
	if (!transport_cmd_check_stop_to_fabric(cmd))
1568
		;
1569 1570 1571
	return;

queue_full:
1572 1573
	cmd->t_state = TRANSPORT_COMPLETE_QF_OK;
	transport_handle_queue_full(cmd, cmd->se_dev);
1574
}
1575
EXPORT_SYMBOL(transport_generic_request_failure);
1576

1577
static void __target_execute_cmd(struct se_cmd *cmd)
1578
{
1579
	sense_reason_t ret;
1580 1581 1582 1583 1584

	spin_lock_irq(&cmd->t_state_lock);
	cmd->transport_state |= (CMD_T_BUSY|CMD_T_SENT);
	spin_unlock_irq(&cmd->t_state_lock);

1585 1586 1587 1588 1589 1590
	if (cmd->execute_cmd) {
		ret = cmd->execute_cmd(cmd);
		if (ret) {
			spin_lock_irq(&cmd->t_state_lock);
			cmd->transport_state &= ~(CMD_T_BUSY|CMD_T_SENT);
			spin_unlock_irq(&cmd->t_state_lock);
1591

1592 1593
			transport_generic_request_failure(cmd, ret);
		}
1594 1595 1596
	}
}

1597
static bool target_handle_task_attr(struct se_cmd *cmd)
1598 1599 1600
{
	struct se_device *dev = cmd->se_dev;

1601 1602
	if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV)
		return false;
1603

1604
	/*
L
Lucas De Marchi 已提交
1605
	 * Check for the existence of HEAD_OF_QUEUE, and if true return 1
1606 1607
	 * to allow the passed struct se_cmd list of tasks to the front of the list.
	 */
1608 1609 1610 1611 1612
	switch (cmd->sam_task_attr) {
	case MSG_HEAD_TAG:
		pr_debug("Added HEAD_OF_QUEUE for CDB: 0x%02x, "
			 "se_ordered_id: %u\n",
			 cmd->t_task_cdb[0], cmd->se_ordered_id);
1613
		return false;
1614 1615
	case MSG_ORDERED_TAG:
		atomic_inc(&dev->dev_ordered_sync);
1616 1617
		smp_mb__after_atomic_inc();

1618 1619 1620 1621
		pr_debug("Added ORDERED for CDB: 0x%02x to ordered list, "
			 " se_ordered_id: %u\n",
			 cmd->t_task_cdb[0], cmd->se_ordered_id);

1622
		/*
1623 1624
		 * Execute an ORDERED command if no other older commands
		 * exist that need to be completed first.
1625
		 */
1626
		if (!atomic_read(&dev->simple_cmds))
1627
			return false;
1628 1629
		break;
	default:
1630 1631 1632
		/*
		 * For SIMPLE and UNTAGGED Task Attribute commands
		 */
1633
		atomic_inc(&dev->simple_cmds);
1634
		smp_mb__after_atomic_inc();
1635
		break;
1636
	}
1637

1638 1639
	if (atomic_read(&dev->dev_ordered_sync) == 0)
		return false;
1640

1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657
	spin_lock(&dev->delayed_cmd_lock);
	list_add_tail(&cmd->se_delayed_node, &dev->delayed_cmd_list);
	spin_unlock(&dev->delayed_cmd_lock);

	pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to"
		" delayed CMD list, se_ordered_id: %u\n",
		cmd->t_task_cdb[0], cmd->sam_task_attr,
		cmd->se_ordered_id);
	return true;
}

void target_execute_cmd(struct se_cmd *cmd)
{
	/*
	 * If the received CDB has aleady been aborted stop processing it here.
	 */
	if (transport_check_aborted_status(cmd, 1))
1658
		return;
1659 1660

	/*
1661 1662
	 * Determine if IOCTL context caller in requesting the stopping of this
	 * command for LUN shutdown purposes.
1663
	 */
1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692
	spin_lock_irq(&cmd->t_state_lock);
	if (cmd->transport_state & CMD_T_LUN_STOP) {
		pr_debug("%s:%d CMD_T_LUN_STOP for ITT: 0x%08x\n",
			__func__, __LINE__, cmd->se_tfo->get_task_tag(cmd));

		cmd->transport_state &= ~CMD_T_ACTIVE;
		spin_unlock_irq(&cmd->t_state_lock);
		complete(&cmd->transport_lun_stop_comp);
		return;
	}
	/*
	 * Determine if frontend context caller is requesting the stopping of
	 * this command for frontend exceptions.
	 */
	if (cmd->transport_state & CMD_T_STOP) {
		pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08x\n",
			__func__, __LINE__,
			cmd->se_tfo->get_task_tag(cmd));

		spin_unlock_irq(&cmd->t_state_lock);
		complete(&cmd->t_transport_stop_comp);
		return;
	}

	cmd->t_state = TRANSPORT_PROCESSING;
	spin_unlock_irq(&cmd->t_state_lock);

	if (!target_handle_task_attr(cmd))
		__target_execute_cmd(cmd);
1693
}
1694
EXPORT_SYMBOL(target_execute_cmd);
1695

1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722
/*
 * Process all commands up to the last received ORDERED task attribute which
 * requires another blocking boundary
 */
static void target_restart_delayed_cmds(struct se_device *dev)
{
	for (;;) {
		struct se_cmd *cmd;

		spin_lock(&dev->delayed_cmd_lock);
		if (list_empty(&dev->delayed_cmd_list)) {
			spin_unlock(&dev->delayed_cmd_lock);
			break;
		}

		cmd = list_entry(dev->delayed_cmd_list.next,
				 struct se_cmd, se_delayed_node);
		list_del(&cmd->se_delayed_node);
		spin_unlock(&dev->delayed_cmd_lock);

		__target_execute_cmd(cmd);

		if (cmd->sam_task_attr == MSG_ORDERED_TAG)
			break;
	}
}

1723
/*
1724
 * Called from I/O completion to determine which dormant/delayed
1725 1726 1727 1728
 * and ordered cmds need to have their tasks added to the execution queue.
 */
static void transport_complete_task_attr(struct se_cmd *cmd)
{
1729
	struct se_device *dev = cmd->se_dev;
1730

1731 1732 1733
	if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV)
		return;

1734
	if (cmd->sam_task_attr == MSG_SIMPLE_TAG) {
1735 1736 1737
		atomic_dec(&dev->simple_cmds);
		smp_mb__after_atomic_dec();
		dev->dev_cur_ordered_id++;
1738
		pr_debug("Incremented dev->dev_cur_ordered_id: %u for"
1739 1740
			" SIMPLE: %u\n", dev->dev_cur_ordered_id,
			cmd->se_ordered_id);
1741
	} else if (cmd->sam_task_attr == MSG_HEAD_TAG) {
1742
		dev->dev_cur_ordered_id++;
1743
		pr_debug("Incremented dev_cur_ordered_id: %u for"
1744 1745
			" HEAD_OF_QUEUE: %u\n", dev->dev_cur_ordered_id,
			cmd->se_ordered_id);
1746
	} else if (cmd->sam_task_attr == MSG_ORDERED_TAG) {
1747 1748 1749 1750
		atomic_dec(&dev->dev_ordered_sync);
		smp_mb__after_atomic_dec();

		dev->dev_cur_ordered_id++;
1751
		pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED:"
1752 1753 1754
			" %u\n", dev->dev_cur_ordered_id, cmd->se_ordered_id);
	}

1755
	target_restart_delayed_cmds(dev);
1756 1757
}

1758
static void transport_complete_qf(struct se_cmd *cmd)
1759 1760 1761
{
	int ret = 0;

1762
	transport_complete_task_attr(cmd);
1763 1764 1765 1766 1767 1768

	if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
		ret = cmd->se_tfo->queue_status(cmd);
		if (ret)
			goto out;
	}
1769 1770 1771 1772 1773 1774

	switch (cmd->data_direction) {
	case DMA_FROM_DEVICE:
		ret = cmd->se_tfo->queue_data_in(cmd);
		break;
	case DMA_TO_DEVICE:
1775
		if (cmd->t_bidi_data_sg) {
1776 1777
			ret = cmd->se_tfo->queue_data_in(cmd);
			if (ret < 0)
1778
				break;
1779 1780 1781 1782 1783 1784 1785 1786 1787
		}
		/* Fall through for DMA_TO_DEVICE */
	case DMA_NONE:
		ret = cmd->se_tfo->queue_status(cmd);
		break;
	default:
		break;
	}

1788 1789 1790 1791 1792 1793 1794
out:
	if (ret < 0) {
		transport_handle_queue_full(cmd, cmd->se_dev);
		return;
	}
	transport_lun_remove_cmd(cmd);
	transport_cmd_check_stop_to_fabric(cmd);
1795 1796 1797 1798
}

static void transport_handle_queue_full(
	struct se_cmd *cmd,
1799
	struct se_device *dev)
1800 1801 1802 1803 1804 1805 1806 1807 1808 1809
{
	spin_lock_irq(&dev->qf_cmd_lock);
	list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list);
	atomic_inc(&dev->dev_qf_count);
	smp_mb__after_atomic_inc();
	spin_unlock_irq(&cmd->se_dev->qf_cmd_lock);

	schedule_work(&cmd->se_dev->qf_work_queue);
}

1810
static void target_complete_ok_work(struct work_struct *work)
1811
{
1812
	struct se_cmd *cmd = container_of(work, struct se_cmd, work);
1813
	int ret;
1814

1815 1816 1817 1818 1819
	/*
	 * Check if we need to move delayed/dormant tasks from cmds on the
	 * delayed execution list after a HEAD_OF_QUEUE or ORDERED Task
	 * Attribute.
	 */
1820 1821
	transport_complete_task_attr(cmd);

1822 1823 1824 1825 1826 1827 1828
	/*
	 * Check to schedule QUEUE_FULL work, or execute an existing
	 * cmd->transport_qf_callback()
	 */
	if (atomic_read(&cmd->se_dev->dev_qf_count) != 0)
		schedule_work(&cmd->se_dev->qf_work_queue);

1829
	/*
1830
	 * Check if we need to send a sense buffer from
1831 1832 1833
	 * the struct se_cmd in question.
	 */
	if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
1834 1835 1836 1837 1838 1839 1840 1841 1842
		WARN_ON(!cmd->scsi_status);
		ret = transport_send_check_condition_and_sense(
					cmd, 0, 1);
		if (ret == -EAGAIN || ret == -ENOMEM)
			goto queue_full;

		transport_lun_remove_cmd(cmd);
		transport_cmd_check_stop_to_fabric(cmd);
		return;
1843 1844
	}
	/*
L
Lucas De Marchi 已提交
1845
	 * Check for a callback, used by amongst other things
1846 1847 1848 1849 1850 1851 1852 1853
	 * XDWRITE_READ_10 emulation.
	 */
	if (cmd->transport_complete_callback)
		cmd->transport_complete_callback(cmd);

	switch (cmd->data_direction) {
	case DMA_FROM_DEVICE:
		spin_lock(&cmd->se_lun->lun_sep_lock);
1854 1855
		if (cmd->se_lun->lun_sep) {
			cmd->se_lun->lun_sep->sep_stats.tx_data_octets +=
1856 1857 1858 1859
					cmd->data_length;
		}
		spin_unlock(&cmd->se_lun->lun_sep_lock);

1860
		ret = cmd->se_tfo->queue_data_in(cmd);
1861
		if (ret == -EAGAIN || ret == -ENOMEM)
1862
			goto queue_full;
1863 1864 1865
		break;
	case DMA_TO_DEVICE:
		spin_lock(&cmd->se_lun->lun_sep_lock);
1866 1867
		if (cmd->se_lun->lun_sep) {
			cmd->se_lun->lun_sep->sep_stats.rx_data_octets +=
1868 1869 1870 1871 1872 1873
				cmd->data_length;
		}
		spin_unlock(&cmd->se_lun->lun_sep_lock);
		/*
		 * Check if we need to send READ payload for BIDI-COMMAND
		 */
1874
		if (cmd->t_bidi_data_sg) {
1875
			spin_lock(&cmd->se_lun->lun_sep_lock);
1876 1877
			if (cmd->se_lun->lun_sep) {
				cmd->se_lun->lun_sep->sep_stats.tx_data_octets +=
1878 1879 1880
					cmd->data_length;
			}
			spin_unlock(&cmd->se_lun->lun_sep_lock);
1881
			ret = cmd->se_tfo->queue_data_in(cmd);
1882
			if (ret == -EAGAIN || ret == -ENOMEM)
1883
				goto queue_full;
1884 1885 1886 1887
			break;
		}
		/* Fall through for DMA_TO_DEVICE */
	case DMA_NONE:
1888
		ret = cmd->se_tfo->queue_status(cmd);
1889
		if (ret == -EAGAIN || ret == -ENOMEM)
1890
			goto queue_full;
1891 1892 1893 1894 1895 1896 1897
		break;
	default:
		break;
	}

	transport_lun_remove_cmd(cmd);
	transport_cmd_check_stop_to_fabric(cmd);
1898 1899 1900
	return;

queue_full:
1901
	pr_debug("Handling complete_ok QUEUE_FULL: se_cmd: %p,"
1902
		" data_direction: %d\n", cmd, cmd->data_direction);
1903 1904
	cmd->t_state = TRANSPORT_COMPLETE_QF_OK;
	transport_handle_queue_full(cmd, cmd->se_dev);
1905 1906
}

1907
static inline void transport_free_sgl(struct scatterlist *sgl, int nents)
1908
{
1909 1910
	struct scatterlist *sg;
	int count;
1911

1912 1913
	for_each_sg(sgl, sg, nents, count)
		__free_page(sg_page(sg));
1914

1915 1916
	kfree(sgl);
}
1917

1918 1919 1920 1921 1922 1923
static inline void transport_free_pages(struct se_cmd *cmd)
{
	if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC)
		return;

	transport_free_sgl(cmd->t_data_sg, cmd->t_data_nents);
1924 1925
	cmd->t_data_sg = NULL;
	cmd->t_data_nents = 0;
1926

1927
	transport_free_sgl(cmd->t_bidi_data_sg, cmd->t_bidi_data_nents);
1928 1929
	cmd->t_bidi_data_sg = NULL;
	cmd->t_bidi_data_nents = 0;
1930 1931
}

C
Christoph Hellwig 已提交
1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942
/**
 * transport_release_cmd - free a command
 * @cmd:       command to free
 *
 * This routine unconditionally frees a command, and reference counting
 * or list removal must be done in the caller.
 */
static void transport_release_cmd(struct se_cmd *cmd)
{
	BUG_ON(!cmd->se_tfo);

1943
	if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
C
Christoph Hellwig 已提交
1944 1945 1946 1947
		core_tmr_release_req(cmd->se_tmr_req);
	if (cmd->t_task_cdb != cmd->__t_task_cdb)
		kfree(cmd->t_task_cdb);
	/*
1948 1949
	 * If this cmd has been setup with target_get_sess_cmd(), drop
	 * the kref and call ->release_cmd() in kref callback.
C
Christoph Hellwig 已提交
1950
	 */
1951 1952 1953 1954
	 if (cmd->check_release != 0) {
		target_put_sess_cmd(cmd->se_sess, cmd);
		return;
	}
C
Christoph Hellwig 已提交
1955 1956 1957
	cmd->se_tfo->release_cmd(cmd);
}

1958 1959 1960 1961 1962 1963
/**
 * transport_put_cmd - release a reference to a command
 * @cmd:       command to release
 *
 * This routine releases our reference to the command and frees it if possible.
 */
1964
static void transport_put_cmd(struct se_cmd *cmd)
1965 1966 1967
{
	unsigned long flags;

1968
	spin_lock_irqsave(&cmd->t_state_lock, flags);
1969 1970 1971 1972
	if (atomic_read(&cmd->t_fe_count) &&
	    !atomic_dec_and_test(&cmd->t_fe_count)) {
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
		return;
1973 1974
	}

1975 1976
	if (cmd->transport_state & CMD_T_DEV_ACTIVE) {
		cmd->transport_state &= ~CMD_T_DEV_ACTIVE;
1977
		target_remove_from_state_list(cmd);
1978
	}
1979
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
1980 1981

	transport_free_pages(cmd);
1982
	transport_release_cmd(cmd);
1983
	return;
1984 1985
}

1986
void *transport_kmap_data_sg(struct se_cmd *cmd)
1987
{
1988
	struct scatterlist *sg = cmd->t_data_sg;
1989 1990
	struct page **pages;
	int i;
1991 1992

	/*
1993 1994 1995
	 * We need to take into account a possible offset here for fabrics like
	 * tcm_loop who may be using a contig buffer from the SCSI midlayer for
	 * control CDBs passed as SGLs via transport_generic_map_mem_to_cmd()
1996
	 */
1997 1998
	if (!cmd->t_data_nents)
		return NULL;
1999 2000 2001

	BUG_ON(!sg);
	if (cmd->t_data_nents == 1)
2002 2003 2004 2005
		return kmap(sg_page(sg)) + sg->offset;

	/* >1 page. use vmap */
	pages = kmalloc(sizeof(*pages) * cmd->t_data_nents, GFP_KERNEL);
2006
	if (!pages)
2007 2008 2009 2010 2011 2012 2013 2014 2015
		return NULL;

	/* convert sg[] to pages[] */
	for_each_sg(cmd->t_data_sg, sg, cmd->t_data_nents, i) {
		pages[i] = sg_page(sg);
	}

	cmd->t_data_vmap = vmap(pages, cmd->t_data_nents,  VM_MAP, PAGE_KERNEL);
	kfree(pages);
2016
	if (!cmd->t_data_vmap)
2017 2018 2019
		return NULL;

	return cmd->t_data_vmap + cmd->t_data_sg[0].offset;
2020
}
2021
EXPORT_SYMBOL(transport_kmap_data_sg);
2022

2023
void transport_kunmap_data_sg(struct se_cmd *cmd)
2024
{
2025
	if (!cmd->t_data_nents) {
2026
		return;
2027
	} else if (cmd->t_data_nents == 1) {
2028
		kunmap(sg_page(cmd->t_data_sg));
2029 2030
		return;
	}
2031 2032 2033

	vunmap(cmd->t_data_vmap);
	cmd->t_data_vmap = NULL;
2034
}
2035
EXPORT_SYMBOL(transport_kunmap_data_sg);
2036

2037
static int
2038
transport_generic_get_mem(struct se_cmd *cmd)
2039
{
2040 2041 2042
	u32 length = cmd->data_length;
	unsigned int nents;
	struct page *page;
2043
	gfp_t zero_flag;
2044
	int i = 0;
2045

2046 2047 2048 2049
	nents = DIV_ROUND_UP(length, PAGE_SIZE);
	cmd->t_data_sg = kmalloc(sizeof(struct scatterlist) * nents, GFP_KERNEL);
	if (!cmd->t_data_sg)
		return -ENOMEM;
2050

2051 2052
	cmd->t_data_nents = nents;
	sg_init_table(cmd->t_data_sg, nents);
2053

2054
	zero_flag = cmd->se_cmd_flags & SCF_SCSI_DATA_CDB ? 0 : __GFP_ZERO;
2055

2056 2057
	while (length) {
		u32 page_len = min_t(u32, length, PAGE_SIZE);
2058
		page = alloc_page(GFP_KERNEL | zero_flag);
2059 2060
		if (!page)
			goto out;
2061

2062 2063 2064
		sg_set_page(&cmd->t_data_sg[i], page, page_len, 0);
		length -= page_len;
		i++;
2065 2066 2067
	}
	return 0;

2068
out:
2069
	while (i > 0) {
2070
		i--;
2071
		__free_page(sg_page(&cmd->t_data_sg[i]));
2072
	}
2073 2074 2075
	kfree(cmd->t_data_sg);
	cmd->t_data_sg = NULL;
	return -ENOMEM;
2076 2077
}

2078
/*
2079 2080 2081
 * Allocate any required resources to execute the command.  For writes we
 * might not have the payload yet, so notify the fabric via a call to
 * ->write_pending instead. Otherwise place it on the execution queue.
2082
 */
2083 2084
sense_reason_t
transport_generic_new_cmd(struct se_cmd *cmd)
2085 2086 2087 2088 2089 2090
{
	int ret = 0;

	/*
	 * Determine is the TCM fabric module has already allocated physical
	 * memory, and is directly calling transport_generic_map_mem_to_cmd()
2091
	 * beforehand.
2092
	 */
2093 2094
	if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) &&
	    cmd->data_length) {
2095
		ret = transport_generic_get_mem(cmd);
2096
		if (ret < 0)
2097
			return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2098
	}
2099

2100 2101
	atomic_inc(&cmd->t_fe_count);

2102
	/*
2103 2104 2105
	 * If this command is not a write we can execute it right here,
	 * for write buffers we need to notify the fabric driver first
	 * and let it call back once the write buffers are ready.
2106
	 */
2107
	target_add_to_state_list(cmd);
2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122
	if (cmd->data_direction != DMA_TO_DEVICE) {
		target_execute_cmd(cmd);
		return 0;
	}

	spin_lock_irq(&cmd->t_state_lock);
	cmd->t_state = TRANSPORT_WRITE_PENDING;
	spin_unlock_irq(&cmd->t_state_lock);

	transport_cmd_check_stop(cmd, false);

	ret = cmd->se_tfo->write_pending(cmd);
	if (ret == -EAGAIN || ret == -ENOMEM)
		goto queue_full;

2123 2124 2125
	/* fabric drivers should only return -EAGAIN or -ENOMEM as error */
	WARN_ON(ret);

2126
	return (!ret) ? 0 : TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2127

2128 2129 2130 2131 2132
queue_full:
	pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd);
	cmd->t_state = TRANSPORT_COMPLETE_QF_WP;
	transport_handle_queue_full(cmd, cmd->se_dev);
	return 0;
2133
}
2134
EXPORT_SYMBOL(transport_generic_new_cmd);
2135

2136
static void transport_write_pending_qf(struct se_cmd *cmd)
2137
{
2138 2139 2140 2141
	int ret;

	ret = cmd->se_tfo->write_pending(cmd);
	if (ret == -EAGAIN || ret == -ENOMEM) {
2142 2143 2144 2145
		pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n",
			 cmd);
		transport_handle_queue_full(cmd, cmd->se_dev);
	}
2146 2147
}

2148
void transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks)
2149
{
2150
	if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD)) {
2151
		if (wait_for_tasks && (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
2152 2153
			 transport_wait_for_tasks(cmd);

2154
		transport_release_cmd(cmd);
2155 2156 2157 2158
	} else {
		if (wait_for_tasks)
			transport_wait_for_tasks(cmd);

2159 2160
		core_dec_lacl_count(cmd->se_sess->se_node_acl, cmd);

2161
		if (cmd->se_lun)
2162 2163
			transport_lun_remove_cmd(cmd);

2164
		transport_put_cmd(cmd);
2165 2166 2167 2168
	}
}
EXPORT_SYMBOL(transport_generic_free_cmd);

2169 2170 2171
/* target_get_sess_cmd - Add command to active ->sess_cmd_list
 * @se_sess:	session to reference
 * @se_cmd:	command descriptor to add
2172
 * @ack_kref:	Signal that fabric will perform an ack target_put_sess_cmd()
2173
 */
2174 2175
static int target_get_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd,
			       bool ack_kref)
2176 2177
{
	unsigned long flags;
2178
	int ret = 0;
2179

2180
	kref_init(&se_cmd->cmd_kref);
2181 2182 2183 2184 2185
	/*
	 * Add a second kref if the fabric caller is expecting to handle
	 * fabric acknowledgement that requires two target_put_sess_cmd()
	 * invocations before se_cmd descriptor release.
	 */
2186
	if (ack_kref == true) {
2187
		kref_get(&se_cmd->cmd_kref);
2188 2189
		se_cmd->se_cmd_flags |= SCF_ACK_KREF;
	}
2190

2191
	spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2192 2193 2194 2195
	if (se_sess->sess_tearing_down) {
		ret = -ESHUTDOWN;
		goto out;
	}
2196 2197
	list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list);
	se_cmd->check_release = 1;
2198 2199

out:
2200
	spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2201
	return ret;
2202 2203
}

2204
static void target_release_cmd_kref(struct kref *kref)
2205
{
2206 2207
	struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref);
	struct se_session *se_sess = se_cmd->se_sess;
2208 2209 2210 2211 2212
	unsigned long flags;

	spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
	if (list_empty(&se_cmd->se_cmd_list)) {
		spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2213
		se_cmd->se_tfo->release_cmd(se_cmd);
2214
		return;
2215 2216 2217 2218
	}
	if (se_sess->sess_tearing_down && se_cmd->cmd_wait_set) {
		spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
		complete(&se_cmd->cmd_wait_comp);
2219
		return;
2220 2221 2222 2223
	}
	list_del(&se_cmd->se_cmd_list);
	spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);

2224 2225 2226 2227 2228 2229 2230 2231 2232 2233
	se_cmd->se_tfo->release_cmd(se_cmd);
}

/* target_put_sess_cmd - Check for active I/O shutdown via kref_put
 * @se_sess:	session to reference
 * @se_cmd:	command descriptor to drop
 */
int target_put_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd)
{
	return kref_put(&se_cmd->cmd_kref, target_release_cmd_kref);
2234 2235 2236
}
EXPORT_SYMBOL(target_put_sess_cmd);

2237 2238 2239 2240
/* target_sess_cmd_list_set_waiting - Flag all commands in
 *         sess_cmd_list to complete cmd_wait_comp.  Set
 *         sess_tearing_down so no more commands are queued.
 * @se_sess:	session to flag
2241
 */
2242
void target_sess_cmd_list_set_waiting(struct se_session *se_sess)
2243 2244 2245 2246 2247 2248
{
	struct se_cmd *se_cmd;
	unsigned long flags;

	spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);

2249 2250
	WARN_ON(se_sess->sess_tearing_down);
	se_sess->sess_tearing_down = 1;
2251

2252
	list_for_each_entry(se_cmd, &se_sess->sess_cmd_list, se_cmd_list)
2253 2254 2255 2256
		se_cmd->cmd_wait_set = 1;

	spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
}
2257
EXPORT_SYMBOL(target_sess_cmd_list_set_waiting);
2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270

/* target_wait_for_sess_cmds - Wait for outstanding descriptors
 * @se_sess:    session to wait for active I/O
 * @wait_for_tasks:	Make extra transport_wait_for_tasks call
 */
void target_wait_for_sess_cmds(
	struct se_session *se_sess,
	int wait_for_tasks)
{
	struct se_cmd *se_cmd, *tmp_cmd;
	bool rc = false;

	list_for_each_entry_safe(se_cmd, tmp_cmd,
2271
				&se_sess->sess_cmd_list, se_cmd_list) {
2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301
		list_del(&se_cmd->se_cmd_list);

		pr_debug("Waiting for se_cmd: %p t_state: %d, fabric state:"
			" %d\n", se_cmd, se_cmd->t_state,
			se_cmd->se_tfo->get_cmd_state(se_cmd));

		if (wait_for_tasks) {
			pr_debug("Calling transport_wait_for_tasks se_cmd: %p t_state: %d,"
				" fabric state: %d\n", se_cmd, se_cmd->t_state,
				se_cmd->se_tfo->get_cmd_state(se_cmd));

			rc = transport_wait_for_tasks(se_cmd);

			pr_debug("After transport_wait_for_tasks se_cmd: %p t_state: %d,"
				" fabric state: %d\n", se_cmd, se_cmd->t_state,
				se_cmd->se_tfo->get_cmd_state(se_cmd));
		}

		if (!rc) {
			wait_for_completion(&se_cmd->cmd_wait_comp);
			pr_debug("After cmd_wait_comp: se_cmd: %p t_state: %d"
				" fabric state: %d\n", se_cmd, se_cmd->t_state,
				se_cmd->se_tfo->get_cmd_state(se_cmd));
		}

		se_cmd->se_tfo->release_cmd(se_cmd);
	}
}
EXPORT_SYMBOL(target_wait_for_sess_cmds);

2302 2303 2304 2305 2306 2307 2308 2309
/*	transport_lun_wait_for_tasks():
 *
 *	Called from ConfigFS context to stop the passed struct se_cmd to allow
 *	an struct se_lun to be successfully shutdown.
 */
static int transport_lun_wait_for_tasks(struct se_cmd *cmd, struct se_lun *lun)
{
	unsigned long flags;
2310 2311
	int ret = 0;

2312 2313 2314 2315
	/*
	 * If the frontend has already requested this struct se_cmd to
	 * be stopped, we can safely ignore this struct se_cmd.
	 */
2316
	spin_lock_irqsave(&cmd->t_state_lock, flags);
2317 2318 2319 2320 2321
	if (cmd->transport_state & CMD_T_STOP) {
		cmd->transport_state &= ~CMD_T_LUN_STOP;

		pr_debug("ConfigFS ITT[0x%08x] - CMD_T_STOP, skipping\n",
			 cmd->se_tfo->get_task_tag(cmd));
2322
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2323
		transport_cmd_check_stop(cmd, false);
2324
		return -EPERM;
2325
	}
2326
	cmd->transport_state |= CMD_T_LUN_FE_STOP;
2327
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2328

2329 2330 2331 2332 2333 2334 2335
	// XXX: audit task_flags checks.
	spin_lock_irqsave(&cmd->t_state_lock, flags);
	if ((cmd->transport_state & CMD_T_BUSY) &&
	    (cmd->transport_state & CMD_T_SENT)) {
		if (!target_stop_cmd(cmd, &flags))
			ret++;
	}
2336
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2337

2338 2339
	pr_debug("ConfigFS: cmd: %p stop tasks ret:"
			" %d\n", cmd, ret);
2340
	if (!ret) {
2341
		pr_debug("ConfigFS: ITT[0x%08x] - stopping cmd....\n",
2342
				cmd->se_tfo->get_task_tag(cmd));
2343
		wait_for_completion(&cmd->transport_lun_stop_comp);
2344
		pr_debug("ConfigFS: ITT[0x%08x] - stopped cmd....\n",
2345
				cmd->se_tfo->get_task_tag(cmd));
2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359
	}

	return 0;
}

static void __transport_clear_lun_from_sessions(struct se_lun *lun)
{
	struct se_cmd *cmd = NULL;
	unsigned long lun_flags, cmd_flags;
	/*
	 * Do exception processing and return CHECK_CONDITION status to the
	 * Initiator Port.
	 */
	spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
2360 2361 2362
	while (!list_empty(&lun->lun_cmd_list)) {
		cmd = list_first_entry(&lun->lun_cmd_list,
		       struct se_cmd, se_lun_node);
2363
		list_del_init(&cmd->se_lun_node);
2364

2365
		spin_lock(&cmd->t_state_lock);
2366
		pr_debug("SE_LUN[%d] - Setting cmd->transport"
2367
			"_lun_stop for  ITT: 0x%08x\n",
2368 2369
			cmd->se_lun->unpacked_lun,
			cmd->se_tfo->get_task_tag(cmd));
2370
		cmd->transport_state |= CMD_T_LUN_STOP;
2371
		spin_unlock(&cmd->t_state_lock);
2372 2373 2374

		spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags);

2375 2376
		if (!cmd->se_lun) {
			pr_err("ITT: 0x%08x, [i,t]_state: %u/%u\n",
2377 2378
				cmd->se_tfo->get_task_tag(cmd),
				cmd->se_tfo->get_cmd_state(cmd), cmd->t_state);
2379 2380 2381 2382 2383 2384
			BUG();
		}
		/*
		 * If the Storage engine still owns the iscsi_cmd_t, determine
		 * and/or stop its context.
		 */
2385
		pr_debug("SE_LUN[%d] - ITT: 0x%08x before transport"
2386 2387
			"_lun_wait_for_tasks()\n", cmd->se_lun->unpacked_lun,
			cmd->se_tfo->get_task_tag(cmd));
2388

2389
		if (transport_lun_wait_for_tasks(cmd, cmd->se_lun) < 0) {
2390 2391 2392 2393
			spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
			continue;
		}

2394
		pr_debug("SE_LUN[%d] - ITT: 0x%08x after transport_lun"
2395
			"_wait_for_tasks(): SUCCESS\n",
2396 2397
			cmd->se_lun->unpacked_lun,
			cmd->se_tfo->get_task_tag(cmd));
2398

2399
		spin_lock_irqsave(&cmd->t_state_lock, cmd_flags);
2400
		if (!(cmd->transport_state & CMD_T_DEV_ACTIVE)) {
2401
			spin_unlock_irqrestore(&cmd->t_state_lock, cmd_flags);
2402 2403
			goto check_cond;
		}
2404
		cmd->transport_state &= ~CMD_T_DEV_ACTIVE;
2405
		target_remove_from_state_list(cmd);
2406
		spin_unlock_irqrestore(&cmd->t_state_lock, cmd_flags);
2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421

		/*
		 * The Storage engine stopped this struct se_cmd before it was
		 * send to the fabric frontend for delivery back to the
		 * Initiator Node.  Return this SCSI CDB back with an
		 * CHECK_CONDITION status.
		 */
check_cond:
		transport_send_check_condition_and_sense(cmd,
				TCM_NON_EXISTENT_LUN, 0);
		/*
		 *  If the fabric frontend is waiting for this iscsi_cmd_t to
		 * be released, notify the waiting thread now that LU has
		 * finished accessing it.
		 */
2422
		spin_lock_irqsave(&cmd->t_state_lock, cmd_flags);
2423
		if (cmd->transport_state & CMD_T_LUN_FE_STOP) {
2424
			pr_debug("SE_LUN[%d] - Detected FE stop for"
2425 2426
				" struct se_cmd: %p ITT: 0x%08x\n",
				lun->unpacked_lun,
2427
				cmd, cmd->se_tfo->get_task_tag(cmd));
2428

2429
			spin_unlock_irqrestore(&cmd->t_state_lock,
2430
					cmd_flags);
2431
			transport_cmd_check_stop(cmd, false);
2432
			complete(&cmd->transport_lun_fe_stop_comp);
2433 2434 2435
			spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
			continue;
		}
2436
		pr_debug("SE_LUN[%d] - ITT: 0x%08x finished processing\n",
2437
			lun->unpacked_lun, cmd->se_tfo->get_task_tag(cmd));
2438

2439
		spin_unlock_irqrestore(&cmd->t_state_lock, cmd_flags);
2440 2441 2442 2443 2444 2445 2446
		spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
	}
	spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags);
}

static int transport_clear_lun_thread(void *p)
{
J
Jörn Engel 已提交
2447
	struct se_lun *lun = p;
2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458

	__transport_clear_lun_from_sessions(lun);
	complete(&lun->lun_shutdown_comp);

	return 0;
}

int transport_clear_lun_from_sessions(struct se_lun *lun)
{
	struct task_struct *kt;

2459
	kt = kthread_run(transport_clear_lun_thread, lun,
2460 2461
			"tcm_cl_%u", lun->unpacked_lun);
	if (IS_ERR(kt)) {
2462
		pr_err("Unable to start clear_lun thread\n");
2463
		return PTR_ERR(kt);
2464 2465 2466 2467 2468 2469
	}
	wait_for_completion(&lun->lun_shutdown_comp);

	return 0;
}

2470 2471 2472
/**
 * transport_wait_for_tasks - wait for completion to occur
 * @cmd:	command to wait
2473
 *
2474 2475
 * Called from frontend fabric context to wait for storage engine
 * to pause and/or release frontend generated struct se_cmd.
2476
 */
2477
bool transport_wait_for_tasks(struct se_cmd *cmd)
2478 2479 2480
{
	unsigned long flags;

2481
	spin_lock_irqsave(&cmd->t_state_lock, flags);
2482 2483
	if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) &&
	    !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
2484
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2485
		return false;
2486
	}
2487

2488 2489
	if (!(cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) &&
	    !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
2490
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2491
		return false;
2492
	}
2493 2494 2495
	/*
	 * If we are already stopped due to an external event (ie: LUN shutdown)
	 * sleep until the connection can have the passed struct se_cmd back.
2496
	 * The cmd->transport_lun_stopped_sem will be upped by
2497 2498 2499
	 * transport_clear_lun_from_sessions() once the ConfigFS context caller
	 * has completed its operation on the struct se_cmd.
	 */
2500
	if (cmd->transport_state & CMD_T_LUN_STOP) {
2501
		pr_debug("wait_for_tasks: Stopping"
2502
			" wait_for_completion(&cmd->t_tasktransport_lun_fe"
2503
			"_stop_comp); for ITT: 0x%08x\n",
2504
			cmd->se_tfo->get_task_tag(cmd));
2505 2506 2507 2508 2509 2510 2511
		/*
		 * There is a special case for WRITES where a FE exception +
		 * LUN shutdown means ConfigFS context is still sleeping on
		 * transport_lun_stop_comp in transport_lun_wait_for_tasks().
		 * We go ahead and up transport_lun_stop_comp just to be sure
		 * here.
		 */
2512 2513 2514 2515
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
		complete(&cmd->transport_lun_stop_comp);
		wait_for_completion(&cmd->transport_lun_fe_stop_comp);
		spin_lock_irqsave(&cmd->t_state_lock, flags);
2516

2517
		target_remove_from_state_list(cmd);
2518 2519 2520 2521 2522
		/*
		 * At this point, the frontend who was the originator of this
		 * struct se_cmd, now owns the structure and can be released through
		 * normal means below.
		 */
2523
		pr_debug("wait_for_tasks: Stopped"
2524
			" wait_for_completion(&cmd->t_tasktransport_lun_fe_"
2525
			"stop_comp); for ITT: 0x%08x\n",
2526
			cmd->se_tfo->get_task_tag(cmd));
2527

2528
		cmd->transport_state &= ~CMD_T_LUN_STOP;
2529
	}
2530

2531
	if (!(cmd->transport_state & CMD_T_ACTIVE)) {
2532
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2533
		return false;
2534
	}
2535

2536
	cmd->transport_state |= CMD_T_STOP;
2537

2538
	pr_debug("wait_for_tasks: Stopping %p ITT: 0x%08x"
2539
		" i_state: %d, t_state: %d, CMD_T_STOP\n",
2540 2541
		cmd, cmd->se_tfo->get_task_tag(cmd),
		cmd->se_tfo->get_cmd_state(cmd), cmd->t_state);
2542

2543
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2544

2545
	wait_for_completion(&cmd->t_transport_stop_comp);
2546

2547
	spin_lock_irqsave(&cmd->t_state_lock, flags);
2548
	cmd->transport_state &= ~(CMD_T_ACTIVE | CMD_T_STOP);
2549

2550
	pr_debug("wait_for_tasks: Stopped wait_for_completion("
2551
		"&cmd->t_transport_stop_comp) for ITT: 0x%08x\n",
2552
		cmd->se_tfo->get_task_tag(cmd));
2553

2554
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2555 2556

	return true;
2557
}
2558
EXPORT_SYMBOL(transport_wait_for_tasks);
2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570

static int transport_get_sense_codes(
	struct se_cmd *cmd,
	u8 *asc,
	u8 *ascq)
{
	*asc = cmd->scsi_asc;
	*ascq = cmd->scsi_ascq;

	return 0;
}

2571 2572 2573
int
transport_send_check_condition_and_sense(struct se_cmd *cmd,
		sense_reason_t reason, int from_transport)
2574 2575 2576 2577 2578
{
	unsigned char *buffer = cmd->sense_buffer;
	unsigned long flags;
	u8 asc = 0, ascq = 0;

2579
	spin_lock_irqsave(&cmd->t_state_lock, flags);
2580
	if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
2581
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2582 2583 2584
		return 0;
	}
	cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION;
2585
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2586 2587 2588 2589 2590 2591

	if (!reason && from_transport)
		goto after_reason;

	if (!from_transport)
		cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
2592

2593 2594 2595 2596 2597 2598
	/*
	 * Actual SENSE DATA, see SPC-3 7.23.2  SPC_SENSE_KEY_OFFSET uses
	 * SENSE KEY values from include/scsi/scsi.h
	 */
	switch (reason) {
	case TCM_NON_EXISTENT_LUN:
2599
		/* CURRENT ERROR */
2600 2601
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2602
		/* ILLEGAL REQUEST */
2603
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
2604
		/* LOGICAL UNIT NOT SUPPORTED */
2605
		buffer[SPC_ASC_KEY_OFFSET] = 0x25;
2606
		break;
2607 2608 2609
	case TCM_UNSUPPORTED_SCSI_OPCODE:
	case TCM_SECTOR_COUNT_TOO_MANY:
		/* CURRENT ERROR */
2610 2611
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2612
		/* ILLEGAL REQUEST */
2613
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
2614
		/* INVALID COMMAND OPERATION CODE */
2615
		buffer[SPC_ASC_KEY_OFFSET] = 0x20;
2616 2617 2618
		break;
	case TCM_UNKNOWN_MODE_PAGE:
		/* CURRENT ERROR */
2619 2620
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2621
		/* ILLEGAL REQUEST */
2622
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
2623
		/* INVALID FIELD IN CDB */
2624
		buffer[SPC_ASC_KEY_OFFSET] = 0x24;
2625 2626 2627
		break;
	case TCM_CHECK_CONDITION_ABORT_CMD:
		/* CURRENT ERROR */
2628 2629
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2630
		/* ABORTED COMMAND */
2631
		buffer[SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
2632
		/* BUS DEVICE RESET FUNCTION OCCURRED */
2633 2634
		buffer[SPC_ASC_KEY_OFFSET] = 0x29;
		buffer[SPC_ASCQ_KEY_OFFSET] = 0x03;
2635 2636 2637
		break;
	case TCM_INCORRECT_AMOUNT_OF_DATA:
		/* CURRENT ERROR */
2638 2639
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2640
		/* ABORTED COMMAND */
2641
		buffer[SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
2642
		/* WRITE ERROR */
2643
		buffer[SPC_ASC_KEY_OFFSET] = 0x0c;
2644
		/* NOT ENOUGH UNSOLICITED DATA */
2645
		buffer[SPC_ASCQ_KEY_OFFSET] = 0x0d;
2646 2647 2648
		break;
	case TCM_INVALID_CDB_FIELD:
		/* CURRENT ERROR */
2649 2650
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2651
		/* ILLEGAL REQUEST */
2652
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
2653
		/* INVALID FIELD IN CDB */
2654
		buffer[SPC_ASC_KEY_OFFSET] = 0x24;
2655 2656 2657
		break;
	case TCM_INVALID_PARAMETER_LIST:
		/* CURRENT ERROR */
2658 2659
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2660
		/* ILLEGAL REQUEST */
2661
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
2662
		/* INVALID FIELD IN PARAMETER LIST */
2663
		buffer[SPC_ASC_KEY_OFFSET] = 0x26;
2664 2665 2666
		break;
	case TCM_UNEXPECTED_UNSOLICITED_DATA:
		/* CURRENT ERROR */
2667 2668
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2669
		/* ABORTED COMMAND */
2670
		buffer[SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
2671
		/* WRITE ERROR */
2672
		buffer[SPC_ASC_KEY_OFFSET] = 0x0c;
2673
		/* UNEXPECTED_UNSOLICITED_DATA */
2674
		buffer[SPC_ASCQ_KEY_OFFSET] = 0x0c;
2675 2676 2677
		break;
	case TCM_SERVICE_CRC_ERROR:
		/* CURRENT ERROR */
2678 2679
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2680
		/* ABORTED COMMAND */
2681
		buffer[SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
2682
		/* PROTOCOL SERVICE CRC ERROR */
2683
		buffer[SPC_ASC_KEY_OFFSET] = 0x47;
2684
		/* N/A */
2685
		buffer[SPC_ASCQ_KEY_OFFSET] = 0x05;
2686 2687 2688
		break;
	case TCM_SNACK_REJECTED:
		/* CURRENT ERROR */
2689 2690
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2691
		/* ABORTED COMMAND */
2692
		buffer[SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
2693
		/* READ ERROR */
2694
		buffer[SPC_ASC_KEY_OFFSET] = 0x11;
2695
		/* FAILED RETRANSMISSION REQUEST */
2696
		buffer[SPC_ASCQ_KEY_OFFSET] = 0x13;
2697 2698 2699
		break;
	case TCM_WRITE_PROTECTED:
		/* CURRENT ERROR */
2700 2701
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2702
		/* DATA PROTECT */
2703
		buffer[SPC_SENSE_KEY_OFFSET] = DATA_PROTECT;
2704
		/* WRITE PROTECTED */
2705
		buffer[SPC_ASC_KEY_OFFSET] = 0x27;
2706
		break;
2707 2708
	case TCM_ADDRESS_OUT_OF_RANGE:
		/* CURRENT ERROR */
2709 2710
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2711
		/* ILLEGAL REQUEST */
2712
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
2713
		/* LOGICAL BLOCK ADDRESS OUT OF RANGE */
2714
		buffer[SPC_ASC_KEY_OFFSET] = 0x21;
2715
		break;
2716 2717
	case TCM_CHECK_CONDITION_UNIT_ATTENTION:
		/* CURRENT ERROR */
2718 2719
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2720
		/* UNIT ATTENTION */
2721
		buffer[SPC_SENSE_KEY_OFFSET] = UNIT_ATTENTION;
2722
		core_scsi3_ua_for_check_condition(cmd, &asc, &ascq);
2723 2724
		buffer[SPC_ASC_KEY_OFFSET] = asc;
		buffer[SPC_ASCQ_KEY_OFFSET] = ascq;
2725 2726 2727
		break;
	case TCM_CHECK_CONDITION_NOT_READY:
		/* CURRENT ERROR */
2728 2729
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2730
		/* Not Ready */
2731
		buffer[SPC_SENSE_KEY_OFFSET] = NOT_READY;
2732
		transport_get_sense_codes(cmd, &asc, &ascq);
2733 2734
		buffer[SPC_ASC_KEY_OFFSET] = asc;
		buffer[SPC_ASCQ_KEY_OFFSET] = ascq;
2735 2736 2737 2738
		break;
	case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
	default:
		/* CURRENT ERROR */
2739 2740
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2741
		/* ILLEGAL REQUEST */
2742
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
2743
		/* LOGICAL UNIT COMMUNICATION FAILURE */
2744
		buffer[SPC_ASC_KEY_OFFSET] = 0x80;
2745 2746 2747 2748 2749 2750 2751 2752 2753 2754
		break;
	}
	/*
	 * This code uses linux/include/scsi/scsi.h SAM status codes!
	 */
	cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
	/*
	 * Automatically padded, this value is encoded in the fabric's
	 * data_length response PDU containing the SCSI defined sense data.
	 */
2755
	cmd->scsi_sense_length  = TRANSPORT_SENSE_BUFFER;
2756 2757

after_reason:
2758
	return cmd->se_tfo->queue_status(cmd);
2759 2760 2761 2762 2763 2764 2765
}
EXPORT_SYMBOL(transport_send_check_condition_and_sense);

int transport_check_aborted_status(struct se_cmd *cmd, int send_status)
{
	int ret = 0;

2766
	if (cmd->transport_state & CMD_T_ABORTED) {
2767
		if (!send_status ||
2768 2769
		     (cmd->se_cmd_flags & SCF_SENT_DELAYED_TAS))
			return 1;
2770

2771
		pr_debug("Sending delayed SAM_STAT_TASK_ABORTED"
2772
			" status for CDB: 0x%02x ITT: 0x%08x\n",
2773
			cmd->t_task_cdb[0],
2774
			cmd->se_tfo->get_task_tag(cmd));
2775

2776
		cmd->se_cmd_flags |= SCF_SENT_DELAYED_TAS;
2777
		cmd->se_tfo->queue_status(cmd);
2778 2779 2780 2781 2782 2783 2784 2785
		ret = 1;
	}
	return ret;
}
EXPORT_SYMBOL(transport_check_aborted_status);

void transport_send_task_abort(struct se_cmd *cmd)
{
2786 2787 2788 2789 2790 2791 2792 2793 2794
	unsigned long flags;

	spin_lock_irqsave(&cmd->t_state_lock, flags);
	if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
		return;
	}
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);

2795 2796 2797 2798 2799 2800 2801
	/*
	 * If there are still expected incoming fabric WRITEs, we wait
	 * until until they have completed before sending a TASK_ABORTED
	 * response.  This response with TASK_ABORTED status will be
	 * queued back to fabric module by transport_check_aborted_status().
	 */
	if (cmd->data_direction == DMA_TO_DEVICE) {
2802
		if (cmd->se_tfo->write_pending_status(cmd) != 0) {
2803
			cmd->transport_state |= CMD_T_ABORTED;
2804 2805 2806 2807
			smp_mb__after_atomic_inc();
		}
	}
	cmd->scsi_status = SAM_STAT_TASK_ABORTED;
2808

2809
	pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x,"
2810
		" ITT: 0x%08x\n", cmd->t_task_cdb[0],
2811
		cmd->se_tfo->get_task_tag(cmd));
2812

2813
	cmd->se_tfo->queue_status(cmd);
2814 2815
}

2816
static void target_tmr_work(struct work_struct *work)
2817
{
2818
	struct se_cmd *cmd = container_of(work, struct se_cmd, work);
2819
	struct se_device *dev = cmd->se_dev;
2820 2821 2822 2823
	struct se_tmr_req *tmr = cmd->se_tmr_req;
	int ret;

	switch (tmr->function) {
2824
	case TMR_ABORT_TASK:
2825
		core_tmr_abort_task(dev, tmr, cmd->se_sess);
2826
		break;
2827 2828 2829
	case TMR_ABORT_TASK_SET:
	case TMR_CLEAR_ACA:
	case TMR_CLEAR_TASK_SET:
2830 2831
		tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED;
		break;
2832
	case TMR_LUN_RESET:
2833 2834 2835 2836
		ret = core_tmr_lun_reset(dev, tmr, NULL, NULL);
		tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE :
					 TMR_FUNCTION_REJECTED;
		break;
2837
	case TMR_TARGET_WARM_RESET:
2838 2839
		tmr->response = TMR_FUNCTION_REJECTED;
		break;
2840
	case TMR_TARGET_COLD_RESET:
2841 2842 2843
		tmr->response = TMR_FUNCTION_REJECTED;
		break;
	default:
2844
		pr_err("Uknown TMR function: 0x%02x.\n",
2845 2846 2847 2848 2849 2850
				tmr->function);
		tmr->response = TMR_FUNCTION_REJECTED;
		break;
	}

	cmd->t_state = TRANSPORT_ISTATE_PROCESSING;
2851
	cmd->se_tfo->queue_tm_rsp(cmd);
2852

2853
	transport_cmd_check_stop_to_fabric(cmd);
2854 2855
}

2856 2857
int transport_generic_handle_tmr(
	struct se_cmd *cmd)
2858
{
2859 2860
	INIT_WORK(&cmd->work, target_tmr_work);
	queue_work(cmd->se_dev->tmr_wq, &cmd->work);
2861 2862
	return 0;
}
2863
EXPORT_SYMBOL(transport_generic_handle_tmr);