target_core_transport.c 76.3 KB
Newer Older
1 2 3 4 5
/*******************************************************************************
 * Filename:  target_core_transport.c
 *
 * This file contains the Generic Target Engine Core.
 *
6
 * (c) Copyright 2002-2013 Datera, Inc.
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
 *
 * Nicholas A. Bellinger <nab@kernel.org>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 ******************************************************************************/

#include <linux/net.h>
#include <linux/delay.h>
#include <linux/string.h>
#include <linux/timer.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/kthread.h>
#include <linux/in.h>
#include <linux/cdrom.h>
35
#include <linux/module.h>
36
#include <linux/ratelimit.h>
37 38 39 40 41
#include <asm/unaligned.h>
#include <net/sock.h>
#include <net/tcp.h>
#include <scsi/scsi.h>
#include <scsi/scsi_cmnd.h>
42
#include <scsi/scsi_tcq.h>
43 44

#include <target/target_core_base.h>
45 46
#include <target/target_core_backend.h>
#include <target/target_core_fabric.h>
47 48
#include <target/target_core_configfs.h>

C
Christoph Hellwig 已提交
49
#include "target_core_internal.h"
50 51 52 53
#include "target_core_alua.h"
#include "target_core_pr.h"
#include "target_core_ua.h"

54 55 56
#define CREATE_TRACE_POINTS
#include <trace/events/target.h>

57
static struct workqueue_struct *target_completion_wq;
58 59 60 61 62 63 64
static struct kmem_cache *se_sess_cache;
struct kmem_cache *se_ua_cache;
struct kmem_cache *t10_pr_reg_cache;
struct kmem_cache *t10_alua_lu_gp_cache;
struct kmem_cache *t10_alua_lu_gp_mem_cache;
struct kmem_cache *t10_alua_tg_pt_gp_cache;
struct kmem_cache *t10_alua_tg_pt_gp_mem_cache;
65 66
struct kmem_cache *t10_alua_lba_map_cache;
struct kmem_cache *t10_alua_lba_map_mem_cache;
67 68

static void transport_complete_task_attr(struct se_cmd *cmd);
69
static void transport_handle_queue_full(struct se_cmd *cmd,
70
		struct se_device *dev);
71
static int transport_put_cmd(struct se_cmd *cmd);
72
static void target_complete_ok_work(struct work_struct *work);
73

74
int init_se_kmem_caches(void)
75 76 77 78
{
	se_sess_cache = kmem_cache_create("se_sess_cache",
			sizeof(struct se_session), __alignof__(struct se_session),
			0, NULL);
79 80
	if (!se_sess_cache) {
		pr_err("kmem_cache_create() for struct se_session"
81
				" failed\n");
82
		goto out;
83 84 85 86
	}
	se_ua_cache = kmem_cache_create("se_ua_cache",
			sizeof(struct se_ua), __alignof__(struct se_ua),
			0, NULL);
87 88
	if (!se_ua_cache) {
		pr_err("kmem_cache_create() for struct se_ua failed\n");
89
		goto out_free_sess_cache;
90 91 92 93
	}
	t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache",
			sizeof(struct t10_pr_registration),
			__alignof__(struct t10_pr_registration), 0, NULL);
94 95
	if (!t10_pr_reg_cache) {
		pr_err("kmem_cache_create() for struct t10_pr_registration"
96
				" failed\n");
97
		goto out_free_ua_cache;
98 99 100 101
	}
	t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache",
			sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp),
			0, NULL);
102 103
	if (!t10_alua_lu_gp_cache) {
		pr_err("kmem_cache_create() for t10_alua_lu_gp_cache"
104
				" failed\n");
105
		goto out_free_pr_reg_cache;
106 107 108 109
	}
	t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache",
			sizeof(struct t10_alua_lu_gp_member),
			__alignof__(struct t10_alua_lu_gp_member), 0, NULL);
110 111
	if (!t10_alua_lu_gp_mem_cache) {
		pr_err("kmem_cache_create() for t10_alua_lu_gp_mem_"
112
				"cache failed\n");
113
		goto out_free_lu_gp_cache;
114 115 116 117
	}
	t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache",
			sizeof(struct t10_alua_tg_pt_gp),
			__alignof__(struct t10_alua_tg_pt_gp), 0, NULL);
118 119
	if (!t10_alua_tg_pt_gp_cache) {
		pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_"
120
				"cache failed\n");
121
		goto out_free_lu_gp_mem_cache;
122 123 124 125 126 127
	}
	t10_alua_tg_pt_gp_mem_cache = kmem_cache_create(
			"t10_alua_tg_pt_gp_mem_cache",
			sizeof(struct t10_alua_tg_pt_gp_member),
			__alignof__(struct t10_alua_tg_pt_gp_member),
			0, NULL);
128 129
	if (!t10_alua_tg_pt_gp_mem_cache) {
		pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_"
130
				"mem_t failed\n");
131
		goto out_free_tg_pt_gp_cache;
132
	}
133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150
	t10_alua_lba_map_cache = kmem_cache_create(
			"t10_alua_lba_map_cache",
			sizeof(struct t10_alua_lba_map),
			__alignof__(struct t10_alua_lba_map), 0, NULL);
	if (!t10_alua_lba_map_cache) {
		pr_err("kmem_cache_create() for t10_alua_lba_map_"
				"cache failed\n");
		goto out_free_tg_pt_gp_mem_cache;
	}
	t10_alua_lba_map_mem_cache = kmem_cache_create(
			"t10_alua_lba_map_mem_cache",
			sizeof(struct t10_alua_lba_map_member),
			__alignof__(struct t10_alua_lba_map_member), 0, NULL);
	if (!t10_alua_lba_map_mem_cache) {
		pr_err("kmem_cache_create() for t10_alua_lba_map_mem_"
				"cache failed\n");
		goto out_free_lba_map_cache;
	}
151

152 153 154
	target_completion_wq = alloc_workqueue("target_completion",
					       WQ_MEM_RECLAIM, 0);
	if (!target_completion_wq)
155
		goto out_free_lba_map_mem_cache;
156

157
	return 0;
158

159 160 161 162
out_free_lba_map_mem_cache:
	kmem_cache_destroy(t10_alua_lba_map_mem_cache);
out_free_lba_map_cache:
	kmem_cache_destroy(t10_alua_lba_map_cache);
163 164 165 166 167 168 169 170 171 172 173 174 175 176
out_free_tg_pt_gp_mem_cache:
	kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache);
out_free_tg_pt_gp_cache:
	kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
out_free_lu_gp_mem_cache:
	kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
out_free_lu_gp_cache:
	kmem_cache_destroy(t10_alua_lu_gp_cache);
out_free_pr_reg_cache:
	kmem_cache_destroy(t10_pr_reg_cache);
out_free_ua_cache:
	kmem_cache_destroy(se_ua_cache);
out_free_sess_cache:
	kmem_cache_destroy(se_sess_cache);
177
out:
178
	return -ENOMEM;
179 180
}

181
void release_se_kmem_caches(void)
182
{
183
	destroy_workqueue(target_completion_wq);
184 185 186 187 188 189 190
	kmem_cache_destroy(se_sess_cache);
	kmem_cache_destroy(se_ua_cache);
	kmem_cache_destroy(t10_pr_reg_cache);
	kmem_cache_destroy(t10_alua_lu_gp_cache);
	kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
	kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
	kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache);
191 192
	kmem_cache_destroy(t10_alua_lba_map_cache);
	kmem_cache_destroy(t10_alua_lba_map_mem_cache);
193 194
}

195 196 197
/* This code ensures unique mib indexes are handed out. */
static DEFINE_SPINLOCK(scsi_mib_index_lock);
static u32 scsi_mib_index[SCSI_INDEX_TYPE_MAX];
198 199 200 201 202 203 204 205

/*
 * Allocate a new row index for the entry type specified
 */
u32 scsi_get_new_index(scsi_index_t type)
{
	u32 new_index;

206
	BUG_ON((type < 0) || (type >= SCSI_INDEX_TYPE_MAX));
207

208 209 210
	spin_lock(&scsi_mib_index_lock);
	new_index = ++scsi_mib_index[type];
	spin_unlock(&scsi_mib_index_lock);
211 212 213 214

	return new_index;
}

215
void transport_subsystem_check_init(void)
216 217
{
	int ret;
218
	static int sub_api_initialized;
219

220 221 222
	if (sub_api_initialized)
		return;

223 224
	ret = request_module("target_core_iblock");
	if (ret != 0)
225
		pr_err("Unable to load target_core_iblock\n");
226 227 228

	ret = request_module("target_core_file");
	if (ret != 0)
229
		pr_err("Unable to load target_core_file\n");
230 231 232

	ret = request_module("target_core_pscsi");
	if (ret != 0)
233
		pr_err("Unable to load target_core_pscsi\n");
234

235
	sub_api_initialized = 1;
236 237 238 239 240 241 242
}

struct se_session *transport_init_session(void)
{
	struct se_session *se_sess;

	se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL);
243 244
	if (!se_sess) {
		pr_err("Unable to allocate struct se_session from"
245 246 247 248 249
				" se_sess_cache\n");
		return ERR_PTR(-ENOMEM);
	}
	INIT_LIST_HEAD(&se_sess->sess_list);
	INIT_LIST_HEAD(&se_sess->sess_acl_list);
250
	INIT_LIST_HEAD(&se_sess->sess_cmd_list);
251
	INIT_LIST_HEAD(&se_sess->sess_wait_list);
252
	spin_lock_init(&se_sess->sess_cmd_lock);
253
	kref_init(&se_sess->sess_kref);
254 255 256 257 258

	return se_sess;
}
EXPORT_SYMBOL(transport_init_session);

259 260 261 262 263
int transport_alloc_session_tags(struct se_session *se_sess,
			         unsigned int tag_num, unsigned int tag_size)
{
	int rc;

264 265
	se_sess->sess_cmd_map = kzalloc(tag_num * tag_size,
					GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT);
266
	if (!se_sess->sess_cmd_map) {
267 268 269 270 271
		se_sess->sess_cmd_map = vzalloc(tag_num * tag_size);
		if (!se_sess->sess_cmd_map) {
			pr_err("Unable to allocate se_sess->sess_cmd_map\n");
			return -ENOMEM;
		}
272 273 274 275 276 277
	}

	rc = percpu_ida_init(&se_sess->sess_tag_pool, tag_num);
	if (rc < 0) {
		pr_err("Unable to init se_sess->sess_tag_pool,"
			" tag_num: %u\n", tag_num);
278 279 280 281
		if (is_vmalloc_addr(se_sess->sess_cmd_map))
			vfree(se_sess->sess_cmd_map);
		else
			kfree(se_sess->sess_cmd_map);
282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309
		se_sess->sess_cmd_map = NULL;
		return -ENOMEM;
	}

	return 0;
}
EXPORT_SYMBOL(transport_alloc_session_tags);

struct se_session *transport_init_session_tags(unsigned int tag_num,
					       unsigned int tag_size)
{
	struct se_session *se_sess;
	int rc;

	se_sess = transport_init_session();
	if (IS_ERR(se_sess))
		return se_sess;

	rc = transport_alloc_session_tags(se_sess, tag_num, tag_size);
	if (rc < 0) {
		transport_free_session(se_sess);
		return ERR_PTR(-ENOMEM);
	}

	return se_sess;
}
EXPORT_SYMBOL(transport_init_session_tags);

310
/*
311
 * Called with spin_lock_irqsave(&struct se_portal_group->session_lock called.
312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333
 */
void __transport_register_session(
	struct se_portal_group *se_tpg,
	struct se_node_acl *se_nacl,
	struct se_session *se_sess,
	void *fabric_sess_ptr)
{
	unsigned char buf[PR_REG_ISID_LEN];

	se_sess->se_tpg = se_tpg;
	se_sess->fabric_sess_ptr = fabric_sess_ptr;
	/*
	 * Used by struct se_node_acl's under ConfigFS to locate active se_session-t
	 *
	 * Only set for struct se_session's that will actually be moving I/O.
	 * eg: *NOT* discovery sessions.
	 */
	if (se_nacl) {
		/*
		 * If the fabric module supports an ISID based TransportID,
		 * save this value in binary from the fabric I_T Nexus now.
		 */
334
		if (se_tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) {
335
			memset(&buf[0], 0, PR_REG_ISID_LEN);
336
			se_tpg->se_tpg_tfo->sess_get_initiator_sid(se_sess,
337 338 339
					&buf[0], PR_REG_ISID_LEN);
			se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]);
		}
340 341
		kref_get(&se_nacl->acl_kref);

342 343 344 345 346 347 348 349 350 351 352 353 354
		spin_lock_irq(&se_nacl->nacl_sess_lock);
		/*
		 * The se_nacl->nacl_sess pointer will be set to the
		 * last active I_T Nexus for each struct se_node_acl.
		 */
		se_nacl->nacl_sess = se_sess;

		list_add_tail(&se_sess->sess_acl_list,
			      &se_nacl->acl_sess_list);
		spin_unlock_irq(&se_nacl->nacl_sess_lock);
	}
	list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list);

355
	pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n",
356
		se_tpg->se_tpg_tfo->get_fabric_name(), se_sess->fabric_sess_ptr);
357 358 359 360 361 362 363 364 365
}
EXPORT_SYMBOL(__transport_register_session);

void transport_register_session(
	struct se_portal_group *se_tpg,
	struct se_node_acl *se_nacl,
	struct se_session *se_sess,
	void *fabric_sess_ptr)
{
366 367 368
	unsigned long flags;

	spin_lock_irqsave(&se_tpg->session_lock, flags);
369
	__transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr);
370
	spin_unlock_irqrestore(&se_tpg->session_lock, flags);
371 372 373
}
EXPORT_SYMBOL(transport_register_session);

374
static void target_release_session(struct kref *kref)
375 376 377 378 379 380 381 382 383 384 385 386 387 388
{
	struct se_session *se_sess = container_of(kref,
			struct se_session, sess_kref);
	struct se_portal_group *se_tpg = se_sess->se_tpg;

	se_tpg->se_tpg_tfo->close_session(se_sess);
}

void target_get_session(struct se_session *se_sess)
{
	kref_get(&se_sess->sess_kref);
}
EXPORT_SYMBOL(target_get_session);

389
void target_put_session(struct se_session *se_sess)
390
{
391 392 393 394 395 396
	struct se_portal_group *tpg = se_sess->se_tpg;

	if (tpg->se_tpg_tfo->put_session != NULL) {
		tpg->se_tpg_tfo->put_session(se_sess);
		return;
	}
397
	kref_put(&se_sess->sess_kref, target_release_session);
398 399 400
}
EXPORT_SYMBOL(target_put_session);

401 402 403 404 405 406 407 408 409 410 411 412 413
static void target_complete_nacl(struct kref *kref)
{
	struct se_node_acl *nacl = container_of(kref,
				struct se_node_acl, acl_kref);

	complete(&nacl->acl_free_comp);
}

void target_put_nacl(struct se_node_acl *nacl)
{
	kref_put(&nacl->acl_kref, target_complete_nacl);
}

414 415 416
void transport_deregister_session_configfs(struct se_session *se_sess)
{
	struct se_node_acl *se_nacl;
417
	unsigned long flags;
418 419 420 421
	/*
	 * Used by struct se_node_acl's under ConfigFS to locate active struct se_session
	 */
	se_nacl = se_sess->se_node_acl;
422
	if (se_nacl) {
423
		spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
424 425
		if (se_nacl->acl_stop == 0)
			list_del(&se_sess->sess_acl_list);
426 427 428 429 430 431 432 433 434 435 436 437
		/*
		 * If the session list is empty, then clear the pointer.
		 * Otherwise, set the struct se_session pointer from the tail
		 * element of the per struct se_node_acl active session list.
		 */
		if (list_empty(&se_nacl->acl_sess_list))
			se_nacl->nacl_sess = NULL;
		else {
			se_nacl->nacl_sess = container_of(
					se_nacl->acl_sess_list.prev,
					struct se_session, sess_acl_list);
		}
438
		spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
439 440 441 442 443 444
	}
}
EXPORT_SYMBOL(transport_deregister_session_configfs);

void transport_free_session(struct se_session *se_sess)
{
445 446
	if (se_sess->sess_cmd_map) {
		percpu_ida_destroy(&se_sess->sess_tag_pool);
447 448 449 450
		if (is_vmalloc_addr(se_sess->sess_cmd_map))
			vfree(se_sess->sess_cmd_map);
		else
			kfree(se_sess->sess_cmd_map);
451
	}
452 453 454 455 456 457 458
	kmem_cache_free(se_sess_cache, se_sess);
}
EXPORT_SYMBOL(transport_free_session);

void transport_deregister_session(struct se_session *se_sess)
{
	struct se_portal_group *se_tpg = se_sess->se_tpg;
459
	struct target_core_fabric_ops *se_tfo;
460
	struct se_node_acl *se_nacl;
461
	unsigned long flags;
462
	bool comp_nacl = true;
463

464
	if (!se_tpg) {
465 466 467
		transport_free_session(se_sess);
		return;
	}
468
	se_tfo = se_tpg->se_tpg_tfo;
469

470
	spin_lock_irqsave(&se_tpg->session_lock, flags);
471 472 473
	list_del(&se_sess->sess_list);
	se_sess->se_tpg = NULL;
	se_sess->fabric_sess_ptr = NULL;
474
	spin_unlock_irqrestore(&se_tpg->session_lock, flags);
475 476 477 478 479 480

	/*
	 * Determine if we need to do extra work for this initiator node's
	 * struct se_node_acl if it had been previously dynamically generated.
	 */
	se_nacl = se_sess->se_node_acl;
481 482 483 484 485 486 487 488 489 490 491 492 493

	spin_lock_irqsave(&se_tpg->acl_node_lock, flags);
	if (se_nacl && se_nacl->dynamic_node_acl) {
		if (!se_tfo->tpg_check_demo_mode_cache(se_tpg)) {
			list_del(&se_nacl->acl_list);
			se_tpg->num_node_acls--;
			spin_unlock_irqrestore(&se_tpg->acl_node_lock, flags);
			core_tpg_wait_for_nacl_pr_ref(se_nacl);
			core_free_device_list_for_node(se_nacl, se_tpg);
			se_tfo->tpg_release_fabric_acl(se_tpg, se_nacl);

			comp_nacl = false;
			spin_lock_irqsave(&se_tpg->acl_node_lock, flags);
494 495
		}
	}
496
	spin_unlock_irqrestore(&se_tpg->acl_node_lock, flags);
497

498
	pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n",
499
		se_tpg->se_tpg_tfo->get_fabric_name());
500
	/*
501
	 * If last kref is dropping now for an explicit NodeACL, awake sleeping
502 503
	 * ->acl_free_comp caller to wakeup configfs se_node_acl->acl_group
	 * removal context.
504 505
	 */
	if (se_nacl && comp_nacl == true)
506
		target_put_nacl(se_nacl);
507

508
	transport_free_session(se_sess);
509 510 511 512
}
EXPORT_SYMBOL(transport_deregister_session);

/*
513
 * Called with cmd->t_state_lock held.
514
 */
515
static void target_remove_from_state_list(struct se_cmd *cmd)
516
{
517
	struct se_device *dev = cmd->se_dev;
518 519
	unsigned long flags;

520 521
	if (!dev)
		return;
522

523 524
	if (cmd->transport_state & CMD_T_BUSY)
		return;
525

526 527 528 529
	spin_lock_irqsave(&dev->execute_task_lock, flags);
	if (cmd->state_active) {
		list_del(&cmd->state_list);
		cmd->state_active = false;
530
	}
531
	spin_unlock_irqrestore(&dev->execute_task_lock, flags);
532 533
}

534 535
static int transport_cmd_check_stop(struct se_cmd *cmd, bool remove_from_lists,
				    bool write_pending)
536 537 538
{
	unsigned long flags;

539
	spin_lock_irqsave(&cmd->t_state_lock, flags);
540 541 542
	if (write_pending)
		cmd->t_state = TRANSPORT_WRITE_PENDING;

543 544 545 546 547 548 549 550 551
	if (remove_from_lists) {
		target_remove_from_state_list(cmd);

		/*
		 * Clear struct se_cmd->se_lun before the handoff to FE.
		 */
		cmd->se_lun = NULL;
	}

552 553
	/*
	 * Determine if frontend context caller is requesting the stopping of
554
	 * this command for frontend exceptions.
555
	 */
556 557 558
	if (cmd->transport_state & CMD_T_STOP) {
		pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08x\n",
			__func__, __LINE__,
559
			cmd->se_tfo->get_task_tag(cmd));
560

561
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
562

563
		complete(&cmd->t_transport_stop_comp);
564 565
		return 1;
	}
566 567 568 569 570 571 572 573 574 575 576 577 578 579 580

	cmd->transport_state &= ~CMD_T_ACTIVE;
	if (remove_from_lists) {
		/*
		 * Some fabric modules like tcm_loop can release
		 * their internally allocated I/O reference now and
		 * struct se_cmd now.
		 *
		 * Fabric modules are expected to return '1' here if the
		 * se_cmd being passed is released at this point,
		 * or zero if not being released.
		 */
		if (cmd->se_tfo->check_stop_free != NULL) {
			spin_unlock_irqrestore(&cmd->t_state_lock, flags);
			return cmd->se_tfo->check_stop_free(cmd);
581
		}
582
	}
583

584
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
585 586 587 588 589
	return 0;
}

static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd)
{
590
	return transport_cmd_check_stop(cmd, true, false);
591 592 593 594
}

static void transport_lun_remove_cmd(struct se_cmd *cmd)
{
595
	struct se_lun *lun = cmd->se_lun;
596

597
	if (!lun || !cmd->lun_ref_active)
598 599
		return;

600
	percpu_ref_put(&lun->lun_ref);
601 602 603 604 605 606
}

void transport_cmd_finish_abort(struct se_cmd *cmd, int remove)
{
	if (transport_cmd_check_stop_to_fabric(cmd))
		return;
607
	if (remove)
608
		transport_put_cmd(cmd);
609 610
}

611 612 613 614
static void target_complete_failure_work(struct work_struct *work)
{
	struct se_cmd *cmd = container_of(work, struct se_cmd, work);

615 616
	transport_generic_request_failure(cmd,
			TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE);
617 618
}

619
/*
620 621
 * Used when asking transport to copy Sense Data from the underlying
 * Linux/SCSI struct scsi_cmnd
622
 */
623
static unsigned char *transport_get_sense_buffer(struct se_cmd *cmd)
624 625 626 627 628 629
{
	struct se_device *dev = cmd->se_dev;

	WARN_ON(!cmd->se_lun);

	if (!dev)
630
		return NULL;
631

632 633
	if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION)
		return NULL;
634

635
	cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER;
636

637
	pr_debug("HBA_[%u]_PLUG[%s]: Requesting sense for SAM STATUS: 0x%02x\n",
638
		dev->se_hba->hba_id, dev->transport->name, cmd->scsi_status);
639
	return cmd->sense_buffer;
640 641
}

642
void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status)
643
{
644
	struct se_device *dev = cmd->se_dev;
645
	int success = scsi_status == GOOD;
646 647
	unsigned long flags;

648 649 650
	cmd->scsi_status = scsi_status;


651
	spin_lock_irqsave(&cmd->t_state_lock, flags);
652
	cmd->transport_state &= ~CMD_T_BUSY;
653 654

	if (dev && dev->transport->transport_complete) {
655 656 657 658
		dev->transport->transport_complete(cmd,
				cmd->t_data_sg,
				transport_get_sense_buffer(cmd));
		if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)
659 660 661 662
			success = 1;
	}

	/*
663
	 * See if we are waiting to complete for an exception condition.
664
	 */
665
	if (cmd->transport_state & CMD_T_REQUEST_STOP) {
666
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
667
		complete(&cmd->task_stop_comp);
668 669
		return;
	}
670 671

	if (!success)
672
		cmd->transport_state |= CMD_T_FAILED;
673

674
	/*
675
	 * Check for case where an explicit ABORT_TASK has been received
676 677 678 679 680 681 682 683
	 * and transport_wait_for_tasks() will be waiting for completion..
	 */
	if (cmd->transport_state & CMD_T_ABORTED &&
	    cmd->transport_state & CMD_T_STOP) {
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
		complete(&cmd->t_transport_stop_comp);
		return;
	} else if (cmd->transport_state & CMD_T_FAILED) {
684
		INIT_WORK(&cmd->work, target_complete_failure_work);
685
	} else {
686
		INIT_WORK(&cmd->work, target_complete_ok_work);
687
	}
688 689

	cmd->t_state = TRANSPORT_COMPLETE;
690
	cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE);
691
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
692

693
	queue_work(target_completion_wq, &cmd->work);
694
}
695 696
EXPORT_SYMBOL(target_complete_cmd);

697
static void target_add_to_state_list(struct se_cmd *cmd)
698
{
699 700
	struct se_device *dev = cmd->se_dev;
	unsigned long flags;
701

702 703 704 705
	spin_lock_irqsave(&dev->execute_task_lock, flags);
	if (!cmd->state_active) {
		list_add_tail(&cmd->state_list, &dev->state_list);
		cmd->state_active = true;
706
	}
707
	spin_unlock_irqrestore(&dev->execute_task_lock, flags);
708 709
}

710
/*
711
 * Handle QUEUE_FULL / -EAGAIN and -ENOMEM status
712
 */
713 714
static void transport_write_pending_qf(struct se_cmd *cmd);
static void transport_complete_qf(struct se_cmd *cmd);
715

716
void target_qf_do_work(struct work_struct *work)
717 718 719
{
	struct se_device *dev = container_of(work, struct se_device,
					qf_work_queue);
720
	LIST_HEAD(qf_cmd_list);
721 722 723
	struct se_cmd *cmd, *cmd_tmp;

	spin_lock_irq(&dev->qf_cmd_lock);
724 725
	list_splice_init(&dev->qf_cmd_list, &qf_cmd_list);
	spin_unlock_irq(&dev->qf_cmd_lock);
726

727
	list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) {
728 729 730 731
		list_del(&cmd->se_qf_node);
		atomic_dec(&dev->dev_qf_count);
		smp_mb__after_atomic_dec();

732
		pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue"
733
			" context: %s\n", cmd->se_tfo->get_fabric_name(), cmd,
734
			(cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" :
735 736
			(cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING"
			: "UNKNOWN");
737

738 739 740 741
		if (cmd->t_state == TRANSPORT_COMPLETE_QF_WP)
			transport_write_pending_qf(cmd);
		else if (cmd->t_state == TRANSPORT_COMPLETE_QF_OK)
			transport_complete_qf(cmd);
742 743 744
	}
}

745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768
unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd)
{
	switch (cmd->data_direction) {
	case DMA_NONE:
		return "NONE";
	case DMA_FROM_DEVICE:
		return "READ";
	case DMA_TO_DEVICE:
		return "WRITE";
	case DMA_BIDIRECTIONAL:
		return "BIDI";
	default:
		break;
	}

	return "UNKNOWN";
}

void transport_dump_dev_state(
	struct se_device *dev,
	char *b,
	int *bl)
{
	*bl += sprintf(b + *bl, "Status: ");
769
	if (dev->export_count)
770
		*bl += sprintf(b + *bl, "ACTIVATED");
771
	else
772 773
		*bl += sprintf(b + *bl, "DEACTIVATED");

774
	*bl += sprintf(b + *bl, "  Max Queue Depth: %d", dev->queue_depth);
775
	*bl += sprintf(b + *bl, "  SectorSize: %u  HwMaxSectors: %u\n",
776 777
		dev->dev_attrib.block_size,
		dev->dev_attrib.hw_max_sectors);
778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830
	*bl += sprintf(b + *bl, "        ");
}

void transport_dump_vpd_proto_id(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
	int len;

	memset(buf, 0, VPD_TMP_BUF_SIZE);
	len = sprintf(buf, "T10 VPD Protocol Identifier: ");

	switch (vpd->protocol_identifier) {
	case 0x00:
		sprintf(buf+len, "Fibre Channel\n");
		break;
	case 0x10:
		sprintf(buf+len, "Parallel SCSI\n");
		break;
	case 0x20:
		sprintf(buf+len, "SSA\n");
		break;
	case 0x30:
		sprintf(buf+len, "IEEE 1394\n");
		break;
	case 0x40:
		sprintf(buf+len, "SCSI Remote Direct Memory Access"
				" Protocol\n");
		break;
	case 0x50:
		sprintf(buf+len, "Internet SCSI (iSCSI)\n");
		break;
	case 0x60:
		sprintf(buf+len, "SAS Serial SCSI Protocol\n");
		break;
	case 0x70:
		sprintf(buf+len, "Automation/Drive Interface Transport"
				" Protocol\n");
		break;
	case 0x80:
		sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n");
		break;
	default:
		sprintf(buf+len, "Unknown 0x%02x\n",
				vpd->protocol_identifier);
		break;
	}

	if (p_buf)
		strncpy(p_buf, buf, p_buf_len);
	else
831
		pr_debug("%s", buf);
832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855
}

void
transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83)
{
	/*
	 * Check if the Protocol Identifier Valid (PIV) bit is set..
	 *
	 * from spc3r23.pdf section 7.5.1
	 */
	 if (page_83[1] & 0x80) {
		vpd->protocol_identifier = (page_83[0] & 0xf0);
		vpd->protocol_identifier_set = 1;
		transport_dump_vpd_proto_id(vpd, NULL, 0);
	}
}
EXPORT_SYMBOL(transport_set_vpd_proto_id);

int transport_dump_vpd_assoc(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
856 857
	int ret = 0;
	int len;
858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873

	memset(buf, 0, VPD_TMP_BUF_SIZE);
	len = sprintf(buf, "T10 VPD Identifier Association: ");

	switch (vpd->association) {
	case 0x00:
		sprintf(buf+len, "addressed logical unit\n");
		break;
	case 0x10:
		sprintf(buf+len, "target port\n");
		break;
	case 0x20:
		sprintf(buf+len, "SCSI target device\n");
		break;
	default:
		sprintf(buf+len, "Unknown 0x%02x\n", vpd->association);
874
		ret = -EINVAL;
875 876 877 878 879 880
		break;
	}

	if (p_buf)
		strncpy(p_buf, buf, p_buf_len);
	else
881
		pr_debug("%s", buf);
882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903

	return ret;
}

int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83)
{
	/*
	 * The VPD identification association..
	 *
	 * from spc3r23.pdf Section 7.6.3.1 Table 297
	 */
	vpd->association = (page_83[1] & 0x30);
	return transport_dump_vpd_assoc(vpd, NULL, 0);
}
EXPORT_SYMBOL(transport_set_vpd_assoc);

int transport_dump_vpd_ident_type(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
904 905
	int ret = 0;
	int len;
906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931

	memset(buf, 0, VPD_TMP_BUF_SIZE);
	len = sprintf(buf, "T10 VPD Identifier Type: ");

	switch (vpd->device_identifier_type) {
	case 0x00:
		sprintf(buf+len, "Vendor specific\n");
		break;
	case 0x01:
		sprintf(buf+len, "T10 Vendor ID based\n");
		break;
	case 0x02:
		sprintf(buf+len, "EUI-64 based\n");
		break;
	case 0x03:
		sprintf(buf+len, "NAA\n");
		break;
	case 0x04:
		sprintf(buf+len, "Relative target port identifier\n");
		break;
	case 0x08:
		sprintf(buf+len, "SCSI name string\n");
		break;
	default:
		sprintf(buf+len, "Unsupported: 0x%02x\n",
				vpd->device_identifier_type);
932
		ret = -EINVAL;
933 934 935
		break;
	}

936 937 938
	if (p_buf) {
		if (p_buf_len < strlen(buf)+1)
			return -EINVAL;
939
		strncpy(p_buf, buf, p_buf_len);
940
	} else {
941
		pr_debug("%s", buf);
942
	}
943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970

	return ret;
}

int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83)
{
	/*
	 * The VPD identifier type..
	 *
	 * from spc3r23.pdf Section 7.6.3.1 Table 298
	 */
	vpd->device_identifier_type = (page_83[1] & 0x0f);
	return transport_dump_vpd_ident_type(vpd, NULL, 0);
}
EXPORT_SYMBOL(transport_set_vpd_ident_type);

int transport_dump_vpd_ident(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
	int ret = 0;

	memset(buf, 0, VPD_TMP_BUF_SIZE);

	switch (vpd->device_identifier_code_set) {
	case 0x01: /* Binary */
971 972
		snprintf(buf, sizeof(buf),
			"T10 VPD Binary Device Identifier: %s\n",
973 974 975
			&vpd->device_identifier[0]);
		break;
	case 0x02: /* ASCII */
976 977
		snprintf(buf, sizeof(buf),
			"T10 VPD ASCII Device Identifier: %s\n",
978 979 980
			&vpd->device_identifier[0]);
		break;
	case 0x03: /* UTF-8 */
981 982
		snprintf(buf, sizeof(buf),
			"T10 VPD UTF-8 Device Identifier: %s\n",
983 984 985 986 987
			&vpd->device_identifier[0]);
		break;
	default:
		sprintf(buf, "T10 VPD Device Identifier encoding unsupported:"
			" 0x%02x", vpd->device_identifier_code_set);
988
		ret = -EINVAL;
989 990 991 992 993 994
		break;
	}

	if (p_buf)
		strncpy(p_buf, buf, p_buf_len);
	else
995
		pr_debug("%s", buf);
996 997 998 999 1000 1001 1002 1003

	return ret;
}

int
transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83)
{
	static const char hex_str[] = "0123456789abcdef";
1004
	int j = 0, i = 4; /* offset to start of the identifier */
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036

	/*
	 * The VPD Code Set (encoding)
	 *
	 * from spc3r23.pdf Section 7.6.3.1 Table 296
	 */
	vpd->device_identifier_code_set = (page_83[0] & 0x0f);
	switch (vpd->device_identifier_code_set) {
	case 0x01: /* Binary */
		vpd->device_identifier[j++] =
				hex_str[vpd->device_identifier_type];
		while (i < (4 + page_83[3])) {
			vpd->device_identifier[j++] =
				hex_str[(page_83[i] & 0xf0) >> 4];
			vpd->device_identifier[j++] =
				hex_str[page_83[i] & 0x0f];
			i++;
		}
		break;
	case 0x02: /* ASCII */
	case 0x03: /* UTF-8 */
		while (i < (4 + page_83[3]))
			vpd->device_identifier[j++] = page_83[i++];
		break;
	default:
		break;
	}

	return transport_dump_vpd_ident(vpd, NULL, 0);
}
EXPORT_SYMBOL(transport_set_vpd_ident);

1037 1038
sense_reason_t
target_cmd_size_check(struct se_cmd *cmd, unsigned int size)
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
{
	struct se_device *dev = cmd->se_dev;

	if (cmd->unknown_data_length) {
		cmd->data_length = size;
	} else if (size != cmd->data_length) {
		pr_warn("TARGET_CORE[%s]: Expected Transfer Length:"
			" %u does not match SCSI CDB Length: %u for SAM Opcode:"
			" 0x%02x\n", cmd->se_tfo->get_fabric_name(),
				cmd->data_length, size, cmd->t_task_cdb[0]);

		if (cmd->data_direction == DMA_TO_DEVICE) {
			pr_err("Rejecting underflow/overflow"
					" WRITE data\n");
1053
			return TCM_INVALID_CDB_FIELD;
1054 1055 1056 1057 1058
		}
		/*
		 * Reject READ_* or WRITE_* with overflow/underflow for
		 * type SCF_SCSI_DATA_CDB.
		 */
1059
		if (dev->dev_attrib.block_size != 512)  {
1060 1061 1062 1063
			pr_err("Failing OVERFLOW/UNDERFLOW for LBA op"
				" CDB on non 512-byte sector setup subsystem"
				" plugin: %s\n", dev->transport->name);
			/* Returns CHECK_CONDITION + INVALID_CDB_FIELD */
1064
			return TCM_INVALID_CDB_FIELD;
1065
		}
1066 1067 1068 1069 1070 1071
		/*
		 * For the overflow case keep the existing fabric provided
		 * ->data_length.  Otherwise for the underflow case, reset
		 * ->data_length to the smaller SCSI expected data transfer
		 * length.
		 */
1072 1073 1074 1075 1076 1077
		if (size > cmd->data_length) {
			cmd->se_cmd_flags |= SCF_OVERFLOW_BIT;
			cmd->residual_count = (size - cmd->data_length);
		} else {
			cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
			cmd->residual_count = (cmd->data_length - size);
1078
			cmd->data_length = size;
1079 1080 1081 1082 1083 1084 1085
		}
	}

	return 0;

}

1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098
/*
 * Used by fabric modules containing a local struct se_cmd within their
 * fabric dependent per I/O descriptor.
 */
void transport_init_se_cmd(
	struct se_cmd *cmd,
	struct target_core_fabric_ops *tfo,
	struct se_session *se_sess,
	u32 data_length,
	int data_direction,
	int task_attr,
	unsigned char *sense_buffer)
{
1099
	INIT_LIST_HEAD(&cmd->se_delayed_node);
1100
	INIT_LIST_HEAD(&cmd->se_qf_node);
1101
	INIT_LIST_HEAD(&cmd->se_cmd_list);
1102
	INIT_LIST_HEAD(&cmd->state_list);
1103
	init_completion(&cmd->t_transport_stop_comp);
1104
	init_completion(&cmd->cmd_wait_comp);
1105
	init_completion(&cmd->task_stop_comp);
1106
	spin_lock_init(&cmd->t_state_lock);
1107
	cmd->transport_state = CMD_T_DEV_ACTIVE;
1108 1109 1110 1111 1112 1113 1114

	cmd->se_tfo = tfo;
	cmd->se_sess = se_sess;
	cmd->data_length = data_length;
	cmd->data_direction = data_direction;
	cmd->sam_task_attr = task_attr;
	cmd->sense_buffer = sense_buffer;
1115 1116

	cmd->state_active = false;
1117 1118 1119
}
EXPORT_SYMBOL(transport_init_se_cmd);

1120 1121
static sense_reason_t
transport_check_alloc_task_attr(struct se_cmd *cmd)
1122
{
1123 1124
	struct se_device *dev = cmd->se_dev;

1125 1126 1127 1128
	/*
	 * Check if SAM Task Attribute emulation is enabled for this
	 * struct se_device storage object
	 */
1129
	if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV)
1130 1131
		return 0;

1132
	if (cmd->sam_task_attr == MSG_ACA_TAG) {
1133
		pr_debug("SAM Task Attribute ACA"
1134
			" emulation is not supported\n");
1135
		return TCM_INVALID_CDB_FIELD;
1136 1137 1138 1139 1140
	}
	/*
	 * Used to determine when ORDERED commands should go from
	 * Dormant to Active status.
	 */
1141
	cmd->se_ordered_id = atomic_inc_return(&dev->dev_ordered_id);
1142
	smp_mb__after_atomic_inc();
1143
	pr_debug("Allocated se_ordered_id: %u for Task Attr: 0x%02x on %s\n",
1144
			cmd->se_ordered_id, cmd->sam_task_attr,
1145
			dev->transport->name);
1146 1147 1148
	return 0;
}

1149 1150
sense_reason_t
target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb)
1151
{
1152
	struct se_device *dev = cmd->se_dev;
1153
	sense_reason_t ret;
1154 1155 1156 1157 1158 1159

	/*
	 * Ensure that the received CDB is less than the max (252 + 8) bytes
	 * for VARIABLE_LENGTH_CMD
	 */
	if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) {
1160
		pr_err("Received SCSI CDB with command_size: %d that"
1161 1162
			" exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
			scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE);
1163
		return TCM_INVALID_CDB_FIELD;
1164 1165 1166 1167 1168 1169
	}
	/*
	 * If the received CDB is larger than TCM_MAX_COMMAND_SIZE,
	 * allocate the additional extended CDB buffer now..  Otherwise
	 * setup the pointer from __t_task_cdb to t_task_cdb.
	 */
1170 1171
	if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) {
		cmd->t_task_cdb = kzalloc(scsi_command_size(cdb),
1172
						GFP_KERNEL);
1173 1174
		if (!cmd->t_task_cdb) {
			pr_err("Unable to allocate cmd->t_task_cdb"
1175
				" %u > sizeof(cmd->__t_task_cdb): %lu ops\n",
1176
				scsi_command_size(cdb),
1177
				(unsigned long)sizeof(cmd->__t_task_cdb));
1178
			return TCM_OUT_OF_RESOURCES;
1179 1180
		}
	} else
1181
		cmd->t_task_cdb = &cmd->__t_task_cdb[0];
1182
	/*
1183
	 * Copy the original CDB into cmd->
1184
	 */
1185
	memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb));
1186

1187 1188
	trace_target_sequencer_start(cmd);

1189 1190 1191
	/*
	 * Check for an existing UNIT ATTENTION condition
	 */
1192 1193 1194
	ret = target_scsi3_ua_check(cmd);
	if (ret)
		return ret;
1195

C
Christoph Hellwig 已提交
1196
	ret = target_alua_state_check(cmd);
1197 1198
	if (ret)
		return ret;
1199

1200
	ret = target_check_reservation(cmd);
1201 1202
	if (ret) {
		cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
1203
		return ret;
1204
	}
1205

1206
	ret = dev->transport->parse_cdb(cmd);
1207 1208 1209 1210 1211
	if (ret)
		return ret;

	ret = transport_check_alloc_task_attr(cmd);
	if (ret)
1212
		return ret;
1213 1214 1215

	cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE;

1216 1217 1218 1219 1220 1221
	spin_lock(&cmd->se_lun->lun_sep_lock);
	if (cmd->se_lun->lun_sep)
		cmd->se_lun->lun_sep->sep_stats.cmd_pdus++;
	spin_unlock(&cmd->se_lun->lun_sep_lock);
	return 0;
}
1222
EXPORT_SYMBOL(target_setup_cmd_from_cdb);
1223

1224 1225 1226 1227 1228 1229 1230
/*
 * Used by fabric module frontends to queue tasks directly.
 * Many only be used from process context only
 */
int transport_handle_cdb_direct(
	struct se_cmd *cmd)
{
1231
	sense_reason_t ret;
1232

1233 1234
	if (!cmd->se_lun) {
		dump_stack();
1235
		pr_err("cmd->se_lun is NULL\n");
1236 1237 1238 1239
		return -EINVAL;
	}
	if (in_interrupt()) {
		dump_stack();
1240
		pr_err("transport_generic_handle_cdb cannot be called"
1241 1242 1243
				" from interrupt context\n");
		return -EINVAL;
	}
1244
	/*
1245 1246 1247
	 * Set TRANSPORT_NEW_CMD state and CMD_T_ACTIVE to ensure that
	 * outstanding descriptors are handled correctly during shutdown via
	 * transport_wait_for_tasks()
1248 1249 1250 1251 1252
	 *
	 * Also, we don't take cmd->t_state_lock here as we only expect
	 * this to be called for initial descriptor submission.
	 */
	cmd->t_state = TRANSPORT_NEW_CMD;
1253 1254
	cmd->transport_state |= CMD_T_ACTIVE;

1255 1256 1257 1258 1259 1260
	/*
	 * transport_generic_new_cmd() is already handling QUEUE_FULL,
	 * so follow TRANSPORT_NEW_CMD processing thread context usage
	 * and call transport_generic_request_failure() if necessary..
	 */
	ret = transport_generic_new_cmd(cmd);
1261 1262
	if (ret)
		transport_generic_request_failure(cmd, ret);
1263
	return 0;
1264 1265 1266
}
EXPORT_SYMBOL(transport_handle_cdb_direct);

1267
sense_reason_t
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl,
		u32 sgl_count, struct scatterlist *sgl_bidi, u32 sgl_bidi_count)
{
	if (!sgl || !sgl_count)
		return 0;

	/*
	 * Reject SCSI data overflow with map_mem_to_cmd() as incoming
	 * scatterlists already have been set to follow what the fabric
	 * passes for the original expected data transfer length.
	 */
	if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
		pr_warn("Rejecting SCSI DATA overflow for fabric using"
			" SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC\n");
		return TCM_INVALID_CDB_FIELD;
	}

	cmd->t_data_sg = sgl;
	cmd->t_data_nents = sgl_count;

	if (sgl_bidi && sgl_bidi_count) {
		cmd->t_bidi_data_sg = sgl_bidi;
		cmd->t_bidi_data_nents = sgl_bidi_count;
	}
	cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
	return 0;
}

1296 1297 1298
/*
 * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized
 * 			 se_cmd + use pre-allocated SGL memory.
1299 1300 1301 1302 1303 1304 1305 1306 1307 1308
 *
 * @se_cmd: command descriptor to submit
 * @se_sess: associated se_sess for endpoint
 * @cdb: pointer to SCSI CDB
 * @sense: pointer to SCSI sense buffer
 * @unpacked_lun: unpacked LUN to reference for struct se_lun
 * @data_length: fabric expected data transfer length
 * @task_addr: SAM task attribute
 * @data_dir: DMA data direction
 * @flags: flags for command submission from target_sc_flags_tables
1309 1310 1311 1312
 * @sgl: struct scatterlist memory for unidirectional mapping
 * @sgl_count: scatterlist count for unidirectional mapping
 * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping
 * @sgl_bidi_count: scatterlist count for bidirectional READ mapping
1313 1314
 * @sgl_prot: struct scatterlist memory protection information
 * @sgl_prot_count: scatterlist count for protection information
1315
 *
1316 1317 1318 1319
 * Returns non zero to signal active I/O shutdown failure.  All other
 * setup exceptions will be returned as a SCSI CHECK_CONDITION response,
 * but still return zero here.
 *
1320 1321
 * This may only be called from process context, and also currently
 * assumes internal allocation of fabric payload buffer by target-core.
1322 1323
 */
int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess,
1324
		unsigned char *cdb, unsigned char *sense, u32 unpacked_lun,
1325 1326
		u32 data_length, int task_attr, int data_dir, int flags,
		struct scatterlist *sgl, u32 sgl_count,
1327 1328
		struct scatterlist *sgl_bidi, u32 sgl_bidi_count,
		struct scatterlist *sgl_prot, u32 sgl_prot_count)
1329 1330
{
	struct se_portal_group *se_tpg;
1331 1332
	sense_reason_t rc;
	int ret;
1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344

	se_tpg = se_sess->se_tpg;
	BUG_ON(!se_tpg);
	BUG_ON(se_cmd->se_tfo || se_cmd->se_sess);
	BUG_ON(in_interrupt());
	/*
	 * Initialize se_cmd for target operation.  From this point
	 * exceptions are handled by sending exception status via
	 * target_core_fabric_ops->queue_status() callback
	 */
	transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
				data_length, data_dir, task_attr, sense);
1345 1346
	if (flags & TARGET_SCF_UNKNOWN_SIZE)
		se_cmd->unknown_data_length = 1;
1347 1348 1349 1350 1351 1352
	/*
	 * Obtain struct se_cmd->cmd_kref reference and add new cmd to
	 * se_sess->sess_cmd_list.  A second kref_get here is necessary
	 * for fabrics using TARGET_SCF_ACK_KREF that expect a second
	 * kref_put() to happen during fabric packet acknowledgement.
	 */
1353 1354 1355
	ret = target_get_sess_cmd(se_sess, se_cmd, (flags & TARGET_SCF_ACK_KREF));
	if (ret)
		return ret;
1356 1357 1358 1359 1360 1361 1362 1363
	/*
	 * Signal bidirectional data payloads to target-core
	 */
	if (flags & TARGET_SCF_BIDI_OP)
		se_cmd->se_cmd_flags |= SCF_BIDI;
	/*
	 * Locate se_lun pointer and attach it to struct se_cmd
	 */
1364 1365 1366
	rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun);
	if (rc) {
		transport_send_check_condition_and_sense(se_cmd, rc, 0);
1367
		target_put_sess_cmd(se_sess, se_cmd);
1368
		return 0;
1369
	}
1370 1371 1372 1373 1374 1375 1376 1377
	/*
	 * Save pointers for SGLs containing protection information,
	 * if present.
	 */
	if (sgl_prot_count) {
		se_cmd->t_prot_sg = sgl_prot;
		se_cmd->t_prot_nents = sgl_prot_count;
	}
1378

1379
	rc = target_setup_cmd_from_cdb(se_cmd, cdb);
1380
	if (rc != 0) {
1381
		transport_generic_request_failure(se_cmd, rc);
1382
		return 0;
1383
	}
1384 1385 1386 1387 1388 1389 1390 1391
	/*
	 * When a non zero sgl_count has been passed perform SGL passthrough
	 * mapping for pre-allocated fabric memory instead of having target
	 * core perform an internal SGL allocation..
	 */
	if (sgl_count != 0) {
		BUG_ON(!sgl);

1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412
		/*
		 * A work-around for tcm_loop as some userspace code via
		 * scsi-generic do not memset their associated read buffers,
		 * so go ahead and do that here for type non-data CDBs.  Also
		 * note that this is currently guaranteed to be a single SGL
		 * for this case by target core in target_setup_cmd_from_cdb()
		 * -> transport_generic_cmd_sequencer().
		 */
		if (!(se_cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) &&
		     se_cmd->data_direction == DMA_FROM_DEVICE) {
			unsigned char *buf = NULL;

			if (sgl)
				buf = kmap(sg_page(sgl)) + sgl->offset;

			if (buf) {
				memset(buf, 0, sgl->length);
				kunmap(sg_page(sgl));
			}
		}

1413 1414 1415
		rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count,
				sgl_bidi, sgl_bidi_count);
		if (rc != 0) {
1416
			transport_generic_request_failure(se_cmd, rc);
1417 1418 1419
			return 0;
		}
	}
1420

1421 1422 1423 1424 1425 1426
	/*
	 * Check if we need to delay processing because of ALUA
	 * Active/NonOptimized primary access state..
	 */
	core_alua_check_nonop_delay(se_cmd);

1427
	transport_handle_cdb_direct(se_cmd);
1428
	return 0;
1429
}
1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459
EXPORT_SYMBOL(target_submit_cmd_map_sgls);

/*
 * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd
 *
 * @se_cmd: command descriptor to submit
 * @se_sess: associated se_sess for endpoint
 * @cdb: pointer to SCSI CDB
 * @sense: pointer to SCSI sense buffer
 * @unpacked_lun: unpacked LUN to reference for struct se_lun
 * @data_length: fabric expected data transfer length
 * @task_addr: SAM task attribute
 * @data_dir: DMA data direction
 * @flags: flags for command submission from target_sc_flags_tables
 *
 * Returns non zero to signal active I/O shutdown failure.  All other
 * setup exceptions will be returned as a SCSI CHECK_CONDITION response,
 * but still return zero here.
 *
 * This may only be called from process context, and also currently
 * assumes internal allocation of fabric payload buffer by target-core.
 *
 * It also assumes interal target core SGL memory allocation.
 */
int target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess,
		unsigned char *cdb, unsigned char *sense, u32 unpacked_lun,
		u32 data_length, int task_attr, int data_dir, int flags)
{
	return target_submit_cmd_map_sgls(se_cmd, se_sess, cdb, sense,
			unpacked_lun, data_length, task_attr, data_dir,
1460
			flags, NULL, 0, NULL, 0, NULL, 0);
1461
}
1462 1463
EXPORT_SYMBOL(target_submit_cmd);

1464 1465 1466 1467 1468 1469
static void target_complete_tmr_failure(struct work_struct *work)
{
	struct se_cmd *se_cmd = container_of(work, struct se_cmd, work);

	se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST;
	se_cmd->se_tfo->queue_tm_rsp(se_cmd);
1470 1471

	transport_cmd_check_stop_to_fabric(se_cmd);
1472 1473
}

1474 1475 1476 1477 1478 1479 1480 1481 1482 1483
/**
 * target_submit_tmr - lookup unpacked lun and submit uninitialized se_cmd
 *                     for TMR CDBs
 *
 * @se_cmd: command descriptor to submit
 * @se_sess: associated se_sess for endpoint
 * @sense: pointer to SCSI sense buffer
 * @unpacked_lun: unpacked LUN to reference for struct se_lun
 * @fabric_context: fabric context for TMR req
 * @tm_type: Type of TM request
1484 1485
 * @gfp: gfp type for caller
 * @tag: referenced task tag for TMR_ABORT_TASK
1486
 * @flags: submit cmd flags
1487 1488 1489 1490
 *
 * Callable from all contexts.
 **/

1491
int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess,
1492
		unsigned char *sense, u32 unpacked_lun,
1493 1494
		void *fabric_tmr_ptr, unsigned char tm_type,
		gfp_t gfp, unsigned int tag, int flags)
1495 1496 1497 1498 1499 1500 1501 1502 1503
{
	struct se_portal_group *se_tpg;
	int ret;

	se_tpg = se_sess->se_tpg;
	BUG_ON(!se_tpg);

	transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
			      0, DMA_NONE, MSG_SIMPLE_TAG, sense);
1504 1505 1506 1507
	/*
	 * FIXME: Currently expect caller to handle se_cmd->se_tmr_req
	 * allocation failure.
	 */
1508
	ret = core_tmr_alloc_req(se_cmd, fabric_tmr_ptr, tm_type, gfp);
1509 1510
	if (ret < 0)
		return -ENOMEM;
1511

1512 1513 1514
	if (tm_type == TMR_ABORT_TASK)
		se_cmd->se_tmr_req->ref_task_tag = tag;

1515
	/* See target_submit_cmd for commentary */
1516 1517 1518 1519 1520
	ret = target_get_sess_cmd(se_sess, se_cmd, (flags & TARGET_SCF_ACK_KREF));
	if (ret) {
		core_tmr_release_req(se_cmd->se_tmr_req);
		return ret;
	}
1521 1522 1523

	ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun);
	if (ret) {
1524 1525 1526 1527 1528 1529
		/*
		 * For callback during failure handling, push this work off
		 * to process context with TMR_LUN_DOES_NOT_EXIST status.
		 */
		INIT_WORK(&se_cmd->work, target_complete_tmr_failure);
		schedule_work(&se_cmd->work);
1530
		return 0;
1531 1532
	}
	transport_generic_handle_tmr(se_cmd);
1533
	return 0;
1534 1535 1536
}
EXPORT_SYMBOL(target_submit_tmr);

1537
/*
1538
 * If the cmd is active, request it to be stopped and sleep until it
1539 1540
 * has completed.
 */
1541
bool target_stop_cmd(struct se_cmd *cmd, unsigned long *flags)
1542 1543 1544
{
	bool was_active = false;

1545 1546
	if (cmd->transport_state & CMD_T_BUSY) {
		cmd->transport_state |= CMD_T_REQUEST_STOP;
1547 1548
		spin_unlock_irqrestore(&cmd->t_state_lock, *flags);

1549 1550 1551
		pr_debug("cmd %p waiting to complete\n", cmd);
		wait_for_completion(&cmd->task_stop_comp);
		pr_debug("cmd %p stopped successfully\n", cmd);
1552 1553

		spin_lock_irqsave(&cmd->t_state_lock, *flags);
1554 1555
		cmd->transport_state &= ~CMD_T_REQUEST_STOP;
		cmd->transport_state &= ~CMD_T_BUSY;
1556 1557 1558 1559 1560 1561
		was_active = true;
	}

	return was_active;
}

1562 1563 1564
/*
 * Handle SAM-esque emulation for generic transport request failures.
 */
1565 1566
void transport_generic_request_failure(struct se_cmd *cmd,
		sense_reason_t sense_reason)
1567
{
1568 1569
	int ret = 0;

1570
	pr_debug("-----[ Storage Engine Exception for cmd: %p ITT: 0x%08x"
1571
		" CDB: 0x%02x\n", cmd, cmd->se_tfo->get_task_tag(cmd),
1572
		cmd->t_task_cdb[0]);
1573
	pr_debug("-----[ i_state: %d t_state: %d sense_reason: %d\n",
1574
		cmd->se_tfo->get_cmd_state(cmd),
1575
		cmd->t_state, sense_reason);
1576
	pr_debug("-----[ CMD_T_ACTIVE: %d CMD_T_STOP: %d CMD_T_SENT: %d\n",
1577 1578 1579
		(cmd->transport_state & CMD_T_ACTIVE) != 0,
		(cmd->transport_state & CMD_T_STOP) != 0,
		(cmd->transport_state & CMD_T_SENT) != 0);
1580 1581 1582 1583

	/*
	 * For SAM Task Attribute emulation for failed struct se_cmd
	 */
1584
	transport_complete_task_attr(cmd);
1585 1586 1587 1588 1589 1590 1591
	/*
	 * Handle special case for COMPARE_AND_WRITE failure, where the
	 * callback is expected to drop the per device ->caw_mutex.
	 */
	if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) &&
	     cmd->transport_complete_callback)
		cmd->transport_complete_callback(cmd);
1592

1593
	switch (sense_reason) {
1594 1595 1596 1597
	case TCM_NON_EXISTENT_LUN:
	case TCM_UNSUPPORTED_SCSI_OPCODE:
	case TCM_INVALID_CDB_FIELD:
	case TCM_INVALID_PARAMETER_LIST:
1598
	case TCM_PARAMETER_LIST_LENGTH_ERROR:
1599 1600 1601
	case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
	case TCM_UNKNOWN_MODE_PAGE:
	case TCM_WRITE_PROTECTED:
1602
	case TCM_ADDRESS_OUT_OF_RANGE:
1603 1604 1605
	case TCM_CHECK_CONDITION_ABORT_CMD:
	case TCM_CHECK_CONDITION_UNIT_ATTENTION:
	case TCM_CHECK_CONDITION_NOT_READY:
1606
		break;
1607 1608 1609
	case TCM_OUT_OF_RESOURCES:
		sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
		break;
1610
	case TCM_RESERVATION_CONFLICT:
1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624
		/*
		 * No SENSE Data payload for this case, set SCSI Status
		 * and queue the response to $FABRIC_MOD.
		 *
		 * Uses linux/include/scsi/scsi.h SAM status codes defs
		 */
		cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
		/*
		 * For UA Interlock Code 11b, a RESERVATION CONFLICT will
		 * establish a UNIT ATTENTION with PREVIOUS RESERVATION
		 * CONFLICT STATUS.
		 *
		 * See spc4r17, section 7.4.6 Control Mode Page, Table 349
		 */
1625
		if (cmd->se_sess &&
1626
		    cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl == 2)
1627
			core_scsi3_ua_allocate(cmd->se_sess->se_node_acl,
1628 1629 1630
				cmd->orig_fe_lun, 0x2C,
				ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);

1631 1632
		trace_target_cmd_complete(cmd);
		ret = cmd->se_tfo-> queue_status(cmd);
1633
		if (ret == -EAGAIN || ret == -ENOMEM)
1634
			goto queue_full;
1635 1636
		goto check_stop;
	default:
1637
		pr_err("Unknown transport error for CDB 0x%02x: %d\n",
1638 1639
			cmd->t_task_cdb[0], sense_reason);
		sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
1640 1641
		break;
	}
1642

1643
	ret = transport_send_check_condition_and_sense(cmd, sense_reason, 0);
1644 1645
	if (ret == -EAGAIN || ret == -ENOMEM)
		goto queue_full;
1646

1647 1648
check_stop:
	transport_lun_remove_cmd(cmd);
1649
	if (!transport_cmd_check_stop_to_fabric(cmd))
1650
		;
1651 1652 1653
	return;

queue_full:
1654 1655
	cmd->t_state = TRANSPORT_COMPLETE_QF_OK;
	transport_handle_queue_full(cmd, cmd->se_dev);
1656
}
1657
EXPORT_SYMBOL(transport_generic_request_failure);
1658

1659
void __target_execute_cmd(struct se_cmd *cmd)
1660
{
1661
	sense_reason_t ret;
1662

1663 1664 1665 1666 1667 1668
	if (cmd->execute_cmd) {
		ret = cmd->execute_cmd(cmd);
		if (ret) {
			spin_lock_irq(&cmd->t_state_lock);
			cmd->transport_state &= ~(CMD_T_BUSY|CMD_T_SENT);
			spin_unlock_irq(&cmd->t_state_lock);
1669

1670 1671
			transport_generic_request_failure(cmd, ret);
		}
1672 1673 1674
	}
}

1675
static bool target_handle_task_attr(struct se_cmd *cmd)
1676 1677 1678
{
	struct se_device *dev = cmd->se_dev;

1679 1680
	if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV)
		return false;
1681

1682
	/*
L
Lucas De Marchi 已提交
1683
	 * Check for the existence of HEAD_OF_QUEUE, and if true return 1
1684 1685
	 * to allow the passed struct se_cmd list of tasks to the front of the list.
	 */
1686 1687 1688 1689 1690
	switch (cmd->sam_task_attr) {
	case MSG_HEAD_TAG:
		pr_debug("Added HEAD_OF_QUEUE for CDB: 0x%02x, "
			 "se_ordered_id: %u\n",
			 cmd->t_task_cdb[0], cmd->se_ordered_id);
1691
		return false;
1692 1693
	case MSG_ORDERED_TAG:
		atomic_inc(&dev->dev_ordered_sync);
1694 1695
		smp_mb__after_atomic_inc();

1696 1697 1698 1699
		pr_debug("Added ORDERED for CDB: 0x%02x to ordered list, "
			 " se_ordered_id: %u\n",
			 cmd->t_task_cdb[0], cmd->se_ordered_id);

1700
		/*
1701 1702
		 * Execute an ORDERED command if no other older commands
		 * exist that need to be completed first.
1703
		 */
1704
		if (!atomic_read(&dev->simple_cmds))
1705
			return false;
1706 1707
		break;
	default:
1708 1709 1710
		/*
		 * For SIMPLE and UNTAGGED Task Attribute commands
		 */
1711
		atomic_inc(&dev->simple_cmds);
1712
		smp_mb__after_atomic_inc();
1713
		break;
1714
	}
1715

1716 1717
	if (atomic_read(&dev->dev_ordered_sync) == 0)
		return false;
1718

1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734
	spin_lock(&dev->delayed_cmd_lock);
	list_add_tail(&cmd->se_delayed_node, &dev->delayed_cmd_list);
	spin_unlock(&dev->delayed_cmd_lock);

	pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to"
		" delayed CMD list, se_ordered_id: %u\n",
		cmd->t_task_cdb[0], cmd->sam_task_attr,
		cmd->se_ordered_id);
	return true;
}

void target_execute_cmd(struct se_cmd *cmd)
{
	/*
	 * If the received CDB has aleady been aborted stop processing it here.
	 */
1735
	if (transport_check_aborted_status(cmd, 1))
1736
		return;
1737

1738 1739 1740 1741
	/*
	 * Determine if frontend context caller is requesting the stopping of
	 * this command for frontend exceptions.
	 */
1742
	spin_lock_irq(&cmd->t_state_lock);
1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753
	if (cmd->transport_state & CMD_T_STOP) {
		pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08x\n",
			__func__, __LINE__,
			cmd->se_tfo->get_task_tag(cmd));

		spin_unlock_irq(&cmd->t_state_lock);
		complete(&cmd->t_transport_stop_comp);
		return;
	}

	cmd->t_state = TRANSPORT_PROCESSING;
1754
	cmd->transport_state |= CMD_T_ACTIVE|CMD_T_BUSY|CMD_T_SENT;
1755 1756
	spin_unlock_irq(&cmd->t_state_lock);

1757 1758 1759 1760 1761 1762 1763 1764
	if (target_handle_task_attr(cmd)) {
		spin_lock_irq(&cmd->t_state_lock);
		cmd->transport_state &= ~CMD_T_BUSY|CMD_T_SENT;
		spin_unlock_irq(&cmd->t_state_lock);
		return;
	}

	__target_execute_cmd(cmd);
1765
}
1766
EXPORT_SYMBOL(target_execute_cmd);
1767

1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794
/*
 * Process all commands up to the last received ORDERED task attribute which
 * requires another blocking boundary
 */
static void target_restart_delayed_cmds(struct se_device *dev)
{
	for (;;) {
		struct se_cmd *cmd;

		spin_lock(&dev->delayed_cmd_lock);
		if (list_empty(&dev->delayed_cmd_list)) {
			spin_unlock(&dev->delayed_cmd_lock);
			break;
		}

		cmd = list_entry(dev->delayed_cmd_list.next,
				 struct se_cmd, se_delayed_node);
		list_del(&cmd->se_delayed_node);
		spin_unlock(&dev->delayed_cmd_lock);

		__target_execute_cmd(cmd);

		if (cmd->sam_task_attr == MSG_ORDERED_TAG)
			break;
	}
}

1795
/*
1796
 * Called from I/O completion to determine which dormant/delayed
1797 1798 1799 1800
 * and ordered cmds need to have their tasks added to the execution queue.
 */
static void transport_complete_task_attr(struct se_cmd *cmd)
{
1801
	struct se_device *dev = cmd->se_dev;
1802

1803 1804 1805
	if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV)
		return;

1806
	if (cmd->sam_task_attr == MSG_SIMPLE_TAG) {
1807 1808 1809
		atomic_dec(&dev->simple_cmds);
		smp_mb__after_atomic_dec();
		dev->dev_cur_ordered_id++;
1810
		pr_debug("Incremented dev->dev_cur_ordered_id: %u for"
1811 1812
			" SIMPLE: %u\n", dev->dev_cur_ordered_id,
			cmd->se_ordered_id);
1813
	} else if (cmd->sam_task_attr == MSG_HEAD_TAG) {
1814
		dev->dev_cur_ordered_id++;
1815
		pr_debug("Incremented dev_cur_ordered_id: %u for"
1816 1817
			" HEAD_OF_QUEUE: %u\n", dev->dev_cur_ordered_id,
			cmd->se_ordered_id);
1818
	} else if (cmd->sam_task_attr == MSG_ORDERED_TAG) {
1819 1820 1821 1822
		atomic_dec(&dev->dev_ordered_sync);
		smp_mb__after_atomic_dec();

		dev->dev_cur_ordered_id++;
1823
		pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED:"
1824 1825 1826
			" %u\n", dev->dev_cur_ordered_id, cmd->se_ordered_id);
	}

1827
	target_restart_delayed_cmds(dev);
1828 1829
}

1830
static void transport_complete_qf(struct se_cmd *cmd)
1831 1832 1833
{
	int ret = 0;

1834
	transport_complete_task_attr(cmd);
1835 1836

	if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
1837
		trace_target_cmd_complete(cmd);
1838 1839 1840 1841
		ret = cmd->se_tfo->queue_status(cmd);
		if (ret)
			goto out;
	}
1842 1843 1844

	switch (cmd->data_direction) {
	case DMA_FROM_DEVICE:
1845
		trace_target_cmd_complete(cmd);
1846 1847 1848
		ret = cmd->se_tfo->queue_data_in(cmd);
		break;
	case DMA_TO_DEVICE:
1849
		if (cmd->se_cmd_flags & SCF_BIDI) {
1850 1851
			ret = cmd->se_tfo->queue_data_in(cmd);
			if (ret < 0)
1852
				break;
1853 1854 1855
		}
		/* Fall through for DMA_TO_DEVICE */
	case DMA_NONE:
1856
		trace_target_cmd_complete(cmd);
1857 1858 1859 1860 1861 1862
		ret = cmd->se_tfo->queue_status(cmd);
		break;
	default:
		break;
	}

1863 1864 1865 1866 1867 1868 1869
out:
	if (ret < 0) {
		transport_handle_queue_full(cmd, cmd->se_dev);
		return;
	}
	transport_lun_remove_cmd(cmd);
	transport_cmd_check_stop_to_fabric(cmd);
1870 1871 1872 1873
}

static void transport_handle_queue_full(
	struct se_cmd *cmd,
1874
	struct se_device *dev)
1875 1876 1877 1878 1879 1880 1881 1882 1883 1884
{
	spin_lock_irq(&dev->qf_cmd_lock);
	list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list);
	atomic_inc(&dev->dev_qf_count);
	smp_mb__after_atomic_inc();
	spin_unlock_irq(&cmd->se_dev->qf_cmd_lock);

	schedule_work(&cmd->se_dev->qf_work_queue);
}

1885
static void target_complete_ok_work(struct work_struct *work)
1886
{
1887
	struct se_cmd *cmd = container_of(work, struct se_cmd, work);
1888
	int ret;
1889

1890 1891 1892 1893 1894
	/*
	 * Check if we need to move delayed/dormant tasks from cmds on the
	 * delayed execution list after a HEAD_OF_QUEUE or ORDERED Task
	 * Attribute.
	 */
1895 1896
	transport_complete_task_attr(cmd);

1897 1898 1899 1900 1901 1902 1903
	/*
	 * Check to schedule QUEUE_FULL work, or execute an existing
	 * cmd->transport_qf_callback()
	 */
	if (atomic_read(&cmd->se_dev->dev_qf_count) != 0)
		schedule_work(&cmd->se_dev->qf_work_queue);

1904
	/*
1905
	 * Check if we need to send a sense buffer from
1906 1907 1908
	 * the struct se_cmd in question.
	 */
	if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
1909 1910 1911 1912 1913 1914 1915 1916 1917
		WARN_ON(!cmd->scsi_status);
		ret = transport_send_check_condition_and_sense(
					cmd, 0, 1);
		if (ret == -EAGAIN || ret == -ENOMEM)
			goto queue_full;

		transport_lun_remove_cmd(cmd);
		transport_cmd_check_stop_to_fabric(cmd);
		return;
1918 1919
	}
	/*
L
Lucas De Marchi 已提交
1920
	 * Check for a callback, used by amongst other things
1921
	 * XDWRITE_READ_10 and COMPARE_AND_WRITE emulation.
1922
	 */
1923 1924 1925 1926
	if (cmd->transport_complete_callback) {
		sense_reason_t rc;

		rc = cmd->transport_complete_callback(cmd);
1927
		if (!rc && !(cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE_POST)) {
1928
			return;
1929 1930 1931 1932 1933
		} else if (rc) {
			ret = transport_send_check_condition_and_sense(cmd,
						rc, 0);
			if (ret == -EAGAIN || ret == -ENOMEM)
				goto queue_full;
1934

1935 1936 1937 1938
			transport_lun_remove_cmd(cmd);
			transport_cmd_check_stop_to_fabric(cmd);
			return;
		}
1939
	}
1940 1941 1942 1943

	switch (cmd->data_direction) {
	case DMA_FROM_DEVICE:
		spin_lock(&cmd->se_lun->lun_sep_lock);
1944 1945
		if (cmd->se_lun->lun_sep) {
			cmd->se_lun->lun_sep->sep_stats.tx_data_octets +=
1946 1947 1948 1949
					cmd->data_length;
		}
		spin_unlock(&cmd->se_lun->lun_sep_lock);

1950
		trace_target_cmd_complete(cmd);
1951
		ret = cmd->se_tfo->queue_data_in(cmd);
1952
		if (ret == -EAGAIN || ret == -ENOMEM)
1953
			goto queue_full;
1954 1955 1956
		break;
	case DMA_TO_DEVICE:
		spin_lock(&cmd->se_lun->lun_sep_lock);
1957 1958
		if (cmd->se_lun->lun_sep) {
			cmd->se_lun->lun_sep->sep_stats.rx_data_octets +=
1959 1960 1961 1962 1963 1964
				cmd->data_length;
		}
		spin_unlock(&cmd->se_lun->lun_sep_lock);
		/*
		 * Check if we need to send READ payload for BIDI-COMMAND
		 */
1965
		if (cmd->se_cmd_flags & SCF_BIDI) {
1966
			spin_lock(&cmd->se_lun->lun_sep_lock);
1967 1968
			if (cmd->se_lun->lun_sep) {
				cmd->se_lun->lun_sep->sep_stats.tx_data_octets +=
1969 1970 1971
					cmd->data_length;
			}
			spin_unlock(&cmd->se_lun->lun_sep_lock);
1972
			ret = cmd->se_tfo->queue_data_in(cmd);
1973
			if (ret == -EAGAIN || ret == -ENOMEM)
1974
				goto queue_full;
1975 1976 1977 1978
			break;
		}
		/* Fall through for DMA_TO_DEVICE */
	case DMA_NONE:
1979
		trace_target_cmd_complete(cmd);
1980
		ret = cmd->se_tfo->queue_status(cmd);
1981
		if (ret == -EAGAIN || ret == -ENOMEM)
1982
			goto queue_full;
1983 1984 1985 1986 1987 1988 1989
		break;
	default:
		break;
	}

	transport_lun_remove_cmd(cmd);
	transport_cmd_check_stop_to_fabric(cmd);
1990 1991 1992
	return;

queue_full:
1993
	pr_debug("Handling complete_ok QUEUE_FULL: se_cmd: %p,"
1994
		" data_direction: %d\n", cmd, cmd->data_direction);
1995 1996
	cmd->t_state = TRANSPORT_COMPLETE_QF_OK;
	transport_handle_queue_full(cmd, cmd->se_dev);
1997 1998
}

1999
static inline void transport_free_sgl(struct scatterlist *sgl, int nents)
2000
{
2001 2002
	struct scatterlist *sg;
	int count;
2003

2004 2005
	for_each_sg(sgl, sg, nents, count)
		__free_page(sg_page(sg));
2006

2007 2008
	kfree(sgl);
}
2009

2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025
static inline void transport_reset_sgl_orig(struct se_cmd *cmd)
{
	/*
	 * Check for saved t_data_sg that may be used for COMPARE_AND_WRITE
	 * emulation, and free + reset pointers if necessary..
	 */
	if (!cmd->t_data_sg_orig)
		return;

	kfree(cmd->t_data_sg);
	cmd->t_data_sg = cmd->t_data_sg_orig;
	cmd->t_data_sg_orig = NULL;
	cmd->t_data_nents = cmd->t_data_nents_orig;
	cmd->t_data_nents_orig = 0;
}

2026 2027
static inline void transport_free_pages(struct se_cmd *cmd)
{
2028 2029
	if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
		transport_reset_sgl_orig(cmd);
2030
		return;
2031 2032
	}
	transport_reset_sgl_orig(cmd);
2033 2034

	transport_free_sgl(cmd->t_data_sg, cmd->t_data_nents);
2035 2036
	cmd->t_data_sg = NULL;
	cmd->t_data_nents = 0;
2037

2038
	transport_free_sgl(cmd->t_bidi_data_sg, cmd->t_bidi_data_nents);
2039 2040
	cmd->t_bidi_data_sg = NULL;
	cmd->t_bidi_data_nents = 0;
2041 2042
}

C
Christoph Hellwig 已提交
2043 2044 2045 2046 2047 2048 2049
/**
 * transport_release_cmd - free a command
 * @cmd:       command to free
 *
 * This routine unconditionally frees a command, and reference counting
 * or list removal must be done in the caller.
 */
2050
static int transport_release_cmd(struct se_cmd *cmd)
C
Christoph Hellwig 已提交
2051 2052 2053
{
	BUG_ON(!cmd->se_tfo);

2054
	if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
C
Christoph Hellwig 已提交
2055 2056 2057 2058
		core_tmr_release_req(cmd->se_tmr_req);
	if (cmd->t_task_cdb != cmd->__t_task_cdb)
		kfree(cmd->t_task_cdb);
	/*
2059 2060
	 * If this cmd has been setup with target_get_sess_cmd(), drop
	 * the kref and call ->release_cmd() in kref callback.
C
Christoph Hellwig 已提交
2061
	 */
2062
	return target_put_sess_cmd(cmd->se_sess, cmd);
C
Christoph Hellwig 已提交
2063 2064
}

2065 2066 2067 2068 2069 2070
/**
 * transport_put_cmd - release a reference to a command
 * @cmd:       command to release
 *
 * This routine releases our reference to the command and frees it if possible.
 */
2071
static int transport_put_cmd(struct se_cmd *cmd)
2072 2073
{
	transport_free_pages(cmd);
2074
	return transport_release_cmd(cmd);
2075 2076
}

2077
void *transport_kmap_data_sg(struct se_cmd *cmd)
2078
{
2079
	struct scatterlist *sg = cmd->t_data_sg;
2080 2081
	struct page **pages;
	int i;
2082 2083

	/*
2084 2085 2086
	 * We need to take into account a possible offset here for fabrics like
	 * tcm_loop who may be using a contig buffer from the SCSI midlayer for
	 * control CDBs passed as SGLs via transport_generic_map_mem_to_cmd()
2087
	 */
2088 2089
	if (!cmd->t_data_nents)
		return NULL;
2090 2091 2092

	BUG_ON(!sg);
	if (cmd->t_data_nents == 1)
2093 2094 2095 2096
		return kmap(sg_page(sg)) + sg->offset;

	/* >1 page. use vmap */
	pages = kmalloc(sizeof(*pages) * cmd->t_data_nents, GFP_KERNEL);
2097
	if (!pages)
2098 2099 2100 2101 2102 2103 2104 2105 2106
		return NULL;

	/* convert sg[] to pages[] */
	for_each_sg(cmd->t_data_sg, sg, cmd->t_data_nents, i) {
		pages[i] = sg_page(sg);
	}

	cmd->t_data_vmap = vmap(pages, cmd->t_data_nents,  VM_MAP, PAGE_KERNEL);
	kfree(pages);
2107
	if (!cmd->t_data_vmap)
2108 2109 2110
		return NULL;

	return cmd->t_data_vmap + cmd->t_data_sg[0].offset;
2111
}
2112
EXPORT_SYMBOL(transport_kmap_data_sg);
2113

2114
void transport_kunmap_data_sg(struct se_cmd *cmd)
2115
{
2116
	if (!cmd->t_data_nents) {
2117
		return;
2118
	} else if (cmd->t_data_nents == 1) {
2119
		kunmap(sg_page(cmd->t_data_sg));
2120 2121
		return;
	}
2122 2123 2124

	vunmap(cmd->t_data_vmap);
	cmd->t_data_vmap = NULL;
2125
}
2126
EXPORT_SYMBOL(transport_kunmap_data_sg);
2127

2128
int
2129 2130
target_alloc_sgl(struct scatterlist **sgl, unsigned int *nents, u32 length,
		 bool zero_page)
2131
{
2132
	struct scatterlist *sg;
2133
	struct page *page;
2134 2135
	gfp_t zero_flag = (zero_page) ? __GFP_ZERO : 0;
	unsigned int nent;
2136
	int i = 0;
2137

2138 2139 2140
	nent = DIV_ROUND_UP(length, PAGE_SIZE);
	sg = kmalloc(sizeof(struct scatterlist) * nent, GFP_KERNEL);
	if (!sg)
2141
		return -ENOMEM;
2142

2143
	sg_init_table(sg, nent);
2144

2145 2146
	while (length) {
		u32 page_len = min_t(u32, length, PAGE_SIZE);
2147
		page = alloc_page(GFP_KERNEL | zero_flag);
2148 2149
		if (!page)
			goto out;
2150

2151
		sg_set_page(&sg[i], page, page_len, 0);
2152 2153
		length -= page_len;
		i++;
2154
	}
2155 2156
	*sgl = sg;
	*nents = nent;
2157 2158
	return 0;

2159
out:
2160
	while (i > 0) {
2161
		i--;
2162
		__free_page(sg_page(&sg[i]));
2163
	}
2164
	kfree(sg);
2165
	return -ENOMEM;
2166 2167
}

2168
/*
2169 2170 2171
 * Allocate any required resources to execute the command.  For writes we
 * might not have the payload yet, so notify the fabric via a call to
 * ->write_pending instead. Otherwise place it on the execution queue.
2172
 */
2173 2174
sense_reason_t
transport_generic_new_cmd(struct se_cmd *cmd)
2175 2176 2177 2178 2179 2180
{
	int ret = 0;

	/*
	 * Determine is the TCM fabric module has already allocated physical
	 * memory, and is directly calling transport_generic_map_mem_to_cmd()
2181
	 * beforehand.
2182
	 */
2183 2184
	if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) &&
	    cmd->data_length) {
2185 2186
		bool zero_flag = !(cmd->se_cmd_flags & SCF_SCSI_DATA_CDB);

2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203
		if ((cmd->se_cmd_flags & SCF_BIDI) ||
		    (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)) {
			u32 bidi_length;

			if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)
				bidi_length = cmd->t_task_nolb *
					      cmd->se_dev->dev_attrib.block_size;
			else
				bidi_length = cmd->data_length;

			ret = target_alloc_sgl(&cmd->t_bidi_data_sg,
					       &cmd->t_bidi_data_nents,
					       bidi_length, zero_flag);
			if (ret < 0)
				return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
		}

2204 2205
		ret = target_alloc_sgl(&cmd->t_data_sg, &cmd->t_data_nents,
				       cmd->data_length, zero_flag);
2206
		if (ret < 0)
2207
			return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2208 2209
	}
	/*
2210 2211 2212
	 * If this command is not a write we can execute it right here,
	 * for write buffers we need to notify the fabric driver first
	 * and let it call back once the write buffers are ready.
2213
	 */
2214
	target_add_to_state_list(cmd);
2215 2216 2217 2218
	if (cmd->data_direction != DMA_TO_DEVICE) {
		target_execute_cmd(cmd);
		return 0;
	}
2219
	transport_cmd_check_stop(cmd, false, true);
2220 2221 2222 2223 2224

	ret = cmd->se_tfo->write_pending(cmd);
	if (ret == -EAGAIN || ret == -ENOMEM)
		goto queue_full;

2225 2226 2227
	/* fabric drivers should only return -EAGAIN or -ENOMEM as error */
	WARN_ON(ret);

2228
	return (!ret) ? 0 : TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2229

2230 2231 2232 2233 2234
queue_full:
	pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd);
	cmd->t_state = TRANSPORT_COMPLETE_QF_WP;
	transport_handle_queue_full(cmd, cmd->se_dev);
	return 0;
2235
}
2236
EXPORT_SYMBOL(transport_generic_new_cmd);
2237

2238
static void transport_write_pending_qf(struct se_cmd *cmd)
2239
{
2240 2241 2242 2243
	int ret;

	ret = cmd->se_tfo->write_pending(cmd);
	if (ret == -EAGAIN || ret == -ENOMEM) {
2244 2245 2246 2247
		pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n",
			 cmd);
		transport_handle_queue_full(cmd, cmd->se_dev);
	}
2248 2249
}

2250
int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks)
2251
{
2252
	unsigned long flags;
2253 2254
	int ret = 0;

2255
	if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD)) {
2256
		if (wait_for_tasks && (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
2257 2258
			 transport_wait_for_tasks(cmd);

2259
		ret = transport_release_cmd(cmd);
2260 2261 2262
	} else {
		if (wait_for_tasks)
			transport_wait_for_tasks(cmd);
2263 2264 2265 2266 2267 2268 2269 2270 2271 2272
		/*
		 * Handle WRITE failure case where transport_generic_new_cmd()
		 * has already added se_cmd to state_list, but fabric has
		 * failed command before I/O submission.
		 */
		if (cmd->state_active) {
			spin_lock_irqsave(&cmd->t_state_lock, flags);
			target_remove_from_state_list(cmd);
			spin_unlock_irqrestore(&cmd->t_state_lock, flags);
		}
2273

2274
		if (cmd->se_lun)
2275 2276
			transport_lun_remove_cmd(cmd);

2277
		ret = transport_put_cmd(cmd);
2278
	}
2279
	return ret;
2280 2281 2282
}
EXPORT_SYMBOL(transport_generic_free_cmd);

2283 2284 2285
/* target_get_sess_cmd - Add command to active ->sess_cmd_list
 * @se_sess:	session to reference
 * @se_cmd:	command descriptor to add
2286
 * @ack_kref:	Signal that fabric will perform an ack target_put_sess_cmd()
2287
 */
2288
int target_get_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd,
2289
			       bool ack_kref)
2290 2291
{
	unsigned long flags;
2292
	int ret = 0;
2293

2294
	kref_init(&se_cmd->cmd_kref);
2295 2296 2297 2298 2299
	/*
	 * Add a second kref if the fabric caller is expecting to handle
	 * fabric acknowledgement that requires two target_put_sess_cmd()
	 * invocations before se_cmd descriptor release.
	 */
2300
	if (ack_kref == true) {
2301
		kref_get(&se_cmd->cmd_kref);
2302 2303
		se_cmd->se_cmd_flags |= SCF_ACK_KREF;
	}
2304

2305
	spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2306 2307 2308 2309
	if (se_sess->sess_tearing_down) {
		ret = -ESHUTDOWN;
		goto out;
	}
2310
	list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list);
2311
out:
2312
	spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2313
	return ret;
2314
}
2315
EXPORT_SYMBOL(target_get_sess_cmd);
2316

2317
static void target_release_cmd_kref(struct kref *kref)
2318
{
2319 2320
	struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref);
	struct se_session *se_sess = se_cmd->se_sess;
2321 2322

	if (list_empty(&se_cmd->se_cmd_list)) {
2323
		spin_unlock(&se_sess->sess_cmd_lock);
2324
		se_cmd->se_tfo->release_cmd(se_cmd);
2325
		return;
2326 2327
	}
	if (se_sess->sess_tearing_down && se_cmd->cmd_wait_set) {
2328
		spin_unlock(&se_sess->sess_cmd_lock);
2329
		complete(&se_cmd->cmd_wait_comp);
2330
		return;
2331 2332
	}
	list_del(&se_cmd->se_cmd_list);
2333
	spin_unlock(&se_sess->sess_cmd_lock);
2334

2335 2336 2337 2338 2339 2340 2341 2342 2343
	se_cmd->se_tfo->release_cmd(se_cmd);
}

/* target_put_sess_cmd - Check for active I/O shutdown via kref_put
 * @se_sess:	session to reference
 * @se_cmd:	command descriptor to drop
 */
int target_put_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd)
{
2344 2345
	return kref_put_spinlock_irqsave(&se_cmd->cmd_kref, target_release_cmd_kref,
			&se_sess->sess_cmd_lock);
2346 2347 2348
}
EXPORT_SYMBOL(target_put_sess_cmd);

2349 2350 2351 2352
/* target_sess_cmd_list_set_waiting - Flag all commands in
 *         sess_cmd_list to complete cmd_wait_comp.  Set
 *         sess_tearing_down so no more commands are queued.
 * @se_sess:	session to flag
2353
 */
2354
void target_sess_cmd_list_set_waiting(struct se_session *se_sess)
2355 2356 2357 2358 2359
{
	struct se_cmd *se_cmd;
	unsigned long flags;

	spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2360 2361 2362 2363
	if (se_sess->sess_tearing_down) {
		spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
		return;
	}
2364
	se_sess->sess_tearing_down = 1;
2365
	list_splice_init(&se_sess->sess_cmd_list, &se_sess->sess_wait_list);
2366

2367
	list_for_each_entry(se_cmd, &se_sess->sess_wait_list, se_cmd_list)
2368 2369 2370 2371
		se_cmd->cmd_wait_set = 1;

	spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
}
2372
EXPORT_SYMBOL(target_sess_cmd_list_set_waiting);
2373 2374 2375 2376

/* target_wait_for_sess_cmds - Wait for outstanding descriptors
 * @se_sess:    session to wait for active I/O
 */
2377
void target_wait_for_sess_cmds(struct se_session *se_sess)
2378 2379
{
	struct se_cmd *se_cmd, *tmp_cmd;
2380
	unsigned long flags;
2381 2382

	list_for_each_entry_safe(se_cmd, tmp_cmd,
2383
				&se_sess->sess_wait_list, se_cmd_list) {
2384 2385 2386 2387 2388 2389
		list_del(&se_cmd->se_cmd_list);

		pr_debug("Waiting for se_cmd: %p t_state: %d, fabric state:"
			" %d\n", se_cmd, se_cmd->t_state,
			se_cmd->se_tfo->get_cmd_state(se_cmd));

2390 2391 2392 2393
		wait_for_completion(&se_cmd->cmd_wait_comp);
		pr_debug("After cmd_wait_comp: se_cmd: %p t_state: %d"
			" fabric state: %d\n", se_cmd, se_cmd->t_state,
			se_cmd->se_tfo->get_cmd_state(se_cmd));
2394 2395 2396

		se_cmd->se_tfo->release_cmd(se_cmd);
	}
2397 2398 2399 2400 2401

	spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
	WARN_ON(!list_empty(&se_sess->sess_cmd_list));
	spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);

2402 2403 2404
}
EXPORT_SYMBOL(target_wait_for_sess_cmds);

2405
static int transport_clear_lun_ref_thread(void *p)
2406
{
J
Jörn Engel 已提交
2407
	struct se_lun *lun = p;
2408

2409 2410 2411
	percpu_ref_kill(&lun->lun_ref);

	wait_for_completion(&lun->lun_ref_comp);
2412 2413 2414 2415 2416
	complete(&lun->lun_shutdown_comp);

	return 0;
}

2417
int transport_clear_lun_ref(struct se_lun *lun)
2418 2419 2420
{
	struct task_struct *kt;

2421
	kt = kthread_run(transport_clear_lun_ref_thread, lun,
2422 2423
			"tcm_cl_%u", lun->unpacked_lun);
	if (IS_ERR(kt)) {
2424
		pr_err("Unable to start clear_lun thread\n");
2425
		return PTR_ERR(kt);
2426 2427 2428 2429 2430 2431
	}
	wait_for_completion(&lun->lun_shutdown_comp);

	return 0;
}

2432 2433 2434
/**
 * transport_wait_for_tasks - wait for completion to occur
 * @cmd:	command to wait
2435
 *
2436 2437
 * Called from frontend fabric context to wait for storage engine
 * to pause and/or release frontend generated struct se_cmd.
2438
 */
2439
bool transport_wait_for_tasks(struct se_cmd *cmd)
2440 2441 2442
{
	unsigned long flags;

2443
	spin_lock_irqsave(&cmd->t_state_lock, flags);
2444 2445
	if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) &&
	    !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
2446
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2447
		return false;
2448
	}
2449

2450 2451
	if (!(cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) &&
	    !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
2452
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2453
		return false;
2454
	}
2455

2456
	if (!(cmd->transport_state & CMD_T_ACTIVE)) {
2457
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2458
		return false;
2459
	}
2460

2461
	cmd->transport_state |= CMD_T_STOP;
2462

2463
	pr_debug("wait_for_tasks: Stopping %p ITT: 0x%08x"
2464
		" i_state: %d, t_state: %d, CMD_T_STOP\n",
2465 2466
		cmd, cmd->se_tfo->get_task_tag(cmd),
		cmd->se_tfo->get_cmd_state(cmd), cmd->t_state);
2467

2468
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2469

2470
	wait_for_completion(&cmd->t_transport_stop_comp);
2471

2472
	spin_lock_irqsave(&cmd->t_state_lock, flags);
2473
	cmd->transport_state &= ~(CMD_T_ACTIVE | CMD_T_STOP);
2474

2475
	pr_debug("wait_for_tasks: Stopped wait_for_completion("
2476
		"&cmd->t_transport_stop_comp) for ITT: 0x%08x\n",
2477
		cmd->se_tfo->get_task_tag(cmd));
2478

2479
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2480 2481

	return true;
2482
}
2483
EXPORT_SYMBOL(transport_wait_for_tasks);
2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495

static int transport_get_sense_codes(
	struct se_cmd *cmd,
	u8 *asc,
	u8 *ascq)
{
	*asc = cmd->scsi_asc;
	*ascq = cmd->scsi_ascq;

	return 0;
}

2496 2497 2498
int
transport_send_check_condition_and_sense(struct se_cmd *cmd,
		sense_reason_t reason, int from_transport)
2499 2500 2501 2502 2503
{
	unsigned char *buffer = cmd->sense_buffer;
	unsigned long flags;
	u8 asc = 0, ascq = 0;

2504
	spin_lock_irqsave(&cmd->t_state_lock, flags);
2505
	if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
2506
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2507 2508 2509
		return 0;
	}
	cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION;
2510
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2511 2512 2513 2514 2515 2516

	if (!reason && from_transport)
		goto after_reason;

	if (!from_transport)
		cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
2517

2518 2519 2520 2521 2522
	/*
	 * Actual SENSE DATA, see SPC-3 7.23.2  SPC_SENSE_KEY_OFFSET uses
	 * SENSE KEY values from include/scsi/scsi.h
	 */
	switch (reason) {
H
Hannes Reinecke 已提交
2523 2524 2525 2526 2527 2528 2529 2530 2531 2532
	case TCM_NO_SENSE:
		/* CURRENT ERROR */
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
		/* Not Ready */
		buffer[SPC_SENSE_KEY_OFFSET] = NOT_READY;
		/* NO ADDITIONAL SENSE INFORMATION */
		buffer[SPC_ASC_KEY_OFFSET] = 0;
		buffer[SPC_ASCQ_KEY_OFFSET] = 0;
		break;
2533
	case TCM_NON_EXISTENT_LUN:
2534
		/* CURRENT ERROR */
2535 2536
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2537
		/* ILLEGAL REQUEST */
2538
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
2539
		/* LOGICAL UNIT NOT SUPPORTED */
2540
		buffer[SPC_ASC_KEY_OFFSET] = 0x25;
2541
		break;
2542 2543 2544
	case TCM_UNSUPPORTED_SCSI_OPCODE:
	case TCM_SECTOR_COUNT_TOO_MANY:
		/* CURRENT ERROR */
2545 2546
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2547
		/* ILLEGAL REQUEST */
2548
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
2549
		/* INVALID COMMAND OPERATION CODE */
2550
		buffer[SPC_ASC_KEY_OFFSET] = 0x20;
2551 2552 2553
		break;
	case TCM_UNKNOWN_MODE_PAGE:
		/* CURRENT ERROR */
2554 2555
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2556
		/* ILLEGAL REQUEST */
2557
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
2558
		/* INVALID FIELD IN CDB */
2559
		buffer[SPC_ASC_KEY_OFFSET] = 0x24;
2560 2561 2562
		break;
	case TCM_CHECK_CONDITION_ABORT_CMD:
		/* CURRENT ERROR */
2563 2564
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2565
		/* ABORTED COMMAND */
2566
		buffer[SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
2567
		/* BUS DEVICE RESET FUNCTION OCCURRED */
2568 2569
		buffer[SPC_ASC_KEY_OFFSET] = 0x29;
		buffer[SPC_ASCQ_KEY_OFFSET] = 0x03;
2570 2571 2572
		break;
	case TCM_INCORRECT_AMOUNT_OF_DATA:
		/* CURRENT ERROR */
2573 2574
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2575
		/* ABORTED COMMAND */
2576
		buffer[SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
2577
		/* WRITE ERROR */
2578
		buffer[SPC_ASC_KEY_OFFSET] = 0x0c;
2579
		/* NOT ENOUGH UNSOLICITED DATA */
2580
		buffer[SPC_ASCQ_KEY_OFFSET] = 0x0d;
2581 2582 2583
		break;
	case TCM_INVALID_CDB_FIELD:
		/* CURRENT ERROR */
2584 2585
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2586
		/* ILLEGAL REQUEST */
2587
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
2588
		/* INVALID FIELD IN CDB */
2589
		buffer[SPC_ASC_KEY_OFFSET] = 0x24;
2590 2591 2592
		break;
	case TCM_INVALID_PARAMETER_LIST:
		/* CURRENT ERROR */
2593 2594
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2595
		/* ILLEGAL REQUEST */
2596
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
2597
		/* INVALID FIELD IN PARAMETER LIST */
2598
		buffer[SPC_ASC_KEY_OFFSET] = 0x26;
2599
		break;
2600 2601 2602 2603 2604 2605 2606 2607 2608
	case TCM_PARAMETER_LIST_LENGTH_ERROR:
		/* CURRENT ERROR */
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
		/* ILLEGAL REQUEST */
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
		/* PARAMETER LIST LENGTH ERROR */
		buffer[SPC_ASC_KEY_OFFSET] = 0x1a;
		break;
2609 2610
	case TCM_UNEXPECTED_UNSOLICITED_DATA:
		/* CURRENT ERROR */
2611 2612
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2613
		/* ABORTED COMMAND */
2614
		buffer[SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
2615
		/* WRITE ERROR */
2616
		buffer[SPC_ASC_KEY_OFFSET] = 0x0c;
2617
		/* UNEXPECTED_UNSOLICITED_DATA */
2618
		buffer[SPC_ASCQ_KEY_OFFSET] = 0x0c;
2619 2620 2621
		break;
	case TCM_SERVICE_CRC_ERROR:
		/* CURRENT ERROR */
2622 2623
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2624
		/* ABORTED COMMAND */
2625
		buffer[SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
2626
		/* PROTOCOL SERVICE CRC ERROR */
2627
		buffer[SPC_ASC_KEY_OFFSET] = 0x47;
2628
		/* N/A */
2629
		buffer[SPC_ASCQ_KEY_OFFSET] = 0x05;
2630 2631 2632
		break;
	case TCM_SNACK_REJECTED:
		/* CURRENT ERROR */
2633 2634
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2635
		/* ABORTED COMMAND */
2636
		buffer[SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
2637
		/* READ ERROR */
2638
		buffer[SPC_ASC_KEY_OFFSET] = 0x11;
2639
		/* FAILED RETRANSMISSION REQUEST */
2640
		buffer[SPC_ASCQ_KEY_OFFSET] = 0x13;
2641 2642 2643
		break;
	case TCM_WRITE_PROTECTED:
		/* CURRENT ERROR */
2644 2645
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2646
		/* DATA PROTECT */
2647
		buffer[SPC_SENSE_KEY_OFFSET] = DATA_PROTECT;
2648
		/* WRITE PROTECTED */
2649
		buffer[SPC_ASC_KEY_OFFSET] = 0x27;
2650
		break;
2651 2652
	case TCM_ADDRESS_OUT_OF_RANGE:
		/* CURRENT ERROR */
2653 2654
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2655
		/* ILLEGAL REQUEST */
2656
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
2657
		/* LOGICAL BLOCK ADDRESS OUT OF RANGE */
2658
		buffer[SPC_ASC_KEY_OFFSET] = 0x21;
2659
		break;
2660 2661
	case TCM_CHECK_CONDITION_UNIT_ATTENTION:
		/* CURRENT ERROR */
2662 2663
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2664
		/* UNIT ATTENTION */
2665
		buffer[SPC_SENSE_KEY_OFFSET] = UNIT_ATTENTION;
2666
		core_scsi3_ua_for_check_condition(cmd, &asc, &ascq);
2667 2668
		buffer[SPC_ASC_KEY_OFFSET] = asc;
		buffer[SPC_ASCQ_KEY_OFFSET] = ascq;
2669 2670 2671
		break;
	case TCM_CHECK_CONDITION_NOT_READY:
		/* CURRENT ERROR */
2672 2673
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2674
		/* Not Ready */
2675
		buffer[SPC_SENSE_KEY_OFFSET] = NOT_READY;
2676
		transport_get_sense_codes(cmd, &asc, &ascq);
2677 2678
		buffer[SPC_ASC_KEY_OFFSET] = asc;
		buffer[SPC_ASCQ_KEY_OFFSET] = ascq;
2679
		break;
2680 2681 2682 2683 2684 2685 2686 2687 2688
	case TCM_MISCOMPARE_VERIFY:
		/* CURRENT ERROR */
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
		buffer[SPC_SENSE_KEY_OFFSET] = MISCOMPARE;
		/* MISCOMPARE DURING VERIFY OPERATION */
		buffer[SPC_ASC_KEY_OFFSET] = 0x1d;
		buffer[SPC_ASCQ_KEY_OFFSET] = 0x00;
		break;
2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718
	case TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED:
		/* CURRENT ERROR */
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
		/* ILLEGAL REQUEST */
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
		/* LOGICAL BLOCK GUARD CHECK FAILED */
		buffer[SPC_ASC_KEY_OFFSET] = 0x10;
		buffer[SPC_ASCQ_KEY_OFFSET] = 0x01;
		break;
	case TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED:
		/* CURRENT ERROR */
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
		/* ILLEGAL REQUEST */
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
		/* LOGICAL BLOCK APPLICATION TAG CHECK FAILED */
		buffer[SPC_ASC_KEY_OFFSET] = 0x10;
		buffer[SPC_ASCQ_KEY_OFFSET] = 0x02;
		break;
	case TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED:
		/* CURRENT ERROR */
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
		/* ILLEGAL REQUEST */
		buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
		/* LOGICAL BLOCK REFERENCE TAG CHECK FAILED */
		buffer[SPC_ASC_KEY_OFFSET] = 0x10;
		buffer[SPC_ASCQ_KEY_OFFSET] = 0x03;
		break;
2719 2720 2721
	case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
	default:
		/* CURRENT ERROR */
2722 2723
		buffer[0] = 0x70;
		buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10;
2724 2725 2726 2727 2728 2729 2730
		/*
		 * Returning ILLEGAL REQUEST would cause immediate IO errors on
		 * Solaris initiators.  Returning NOT READY instead means the
		 * operations will be retried a finite number of times and we
		 * can survive intermittent errors.
		 */
		buffer[SPC_SENSE_KEY_OFFSET] = NOT_READY;
2731
		/* LOGICAL UNIT COMMUNICATION FAILURE */
2732
		buffer[SPC_ASC_KEY_OFFSET] = 0x08;
2733 2734 2735 2736 2737 2738 2739 2740 2741 2742
		break;
	}
	/*
	 * This code uses linux/include/scsi/scsi.h SAM status codes!
	 */
	cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
	/*
	 * Automatically padded, this value is encoded in the fabric's
	 * data_length response PDU containing the SCSI defined sense data.
	 */
2743
	cmd->scsi_sense_length  = TRANSPORT_SENSE_BUFFER;
2744 2745

after_reason:
2746
	trace_target_cmd_complete(cmd);
2747
	return cmd->se_tfo->queue_status(cmd);
2748 2749 2750 2751 2752
}
EXPORT_SYMBOL(transport_send_check_condition_and_sense);

int transport_check_aborted_status(struct se_cmd *cmd, int send_status)
{
2753 2754
	if (!(cmd->transport_state & CMD_T_ABORTED))
		return 0;
2755

2756 2757
	if (!send_status || (cmd->se_cmd_flags & SCF_SENT_DELAYED_TAS))
		return 1;
2758

2759 2760
	pr_debug("Sending delayed SAM_STAT_TASK_ABORTED status for CDB: 0x%02x ITT: 0x%08x\n",
		 cmd->t_task_cdb[0], cmd->se_tfo->get_task_tag(cmd));
2761

2762
	cmd->se_cmd_flags |= SCF_SENT_DELAYED_TAS;
2763
	cmd->scsi_status = SAM_STAT_TASK_ABORTED;
2764
	trace_target_cmd_complete(cmd);
2765 2766 2767
	cmd->se_tfo->queue_status(cmd);

	return 1;
2768 2769 2770 2771 2772
}
EXPORT_SYMBOL(transport_check_aborted_status);

void transport_send_task_abort(struct se_cmd *cmd)
{
2773 2774 2775
	unsigned long flags;

	spin_lock_irqsave(&cmd->t_state_lock, flags);
2776
	if (cmd->se_cmd_flags & (SCF_SENT_CHECK_CONDITION | SCF_SENT_DELAYED_TAS)) {
2777 2778 2779 2780 2781
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
		return;
	}
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);

2782 2783 2784 2785 2786 2787 2788
	/*
	 * If there are still expected incoming fabric WRITEs, we wait
	 * until until they have completed before sending a TASK_ABORTED
	 * response.  This response with TASK_ABORTED status will be
	 * queued back to fabric module by transport_check_aborted_status().
	 */
	if (cmd->data_direction == DMA_TO_DEVICE) {
2789
		if (cmd->se_tfo->write_pending_status(cmd) != 0) {
2790
			cmd->transport_state |= CMD_T_ABORTED;
2791
			smp_mb__after_atomic_inc();
2792
			return;
2793 2794 2795
		}
	}
	cmd->scsi_status = SAM_STAT_TASK_ABORTED;
2796

2797 2798
	transport_lun_remove_cmd(cmd);

2799
	pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x,"
2800
		" ITT: 0x%08x\n", cmd->t_task_cdb[0],
2801
		cmd->se_tfo->get_task_tag(cmd));
2802

2803
	trace_target_cmd_complete(cmd);
2804
	cmd->se_tfo->queue_status(cmd);
2805 2806
}

2807
static void target_tmr_work(struct work_struct *work)
2808
{
2809
	struct se_cmd *cmd = container_of(work, struct se_cmd, work);
2810
	struct se_device *dev = cmd->se_dev;
2811 2812 2813 2814
	struct se_tmr_req *tmr = cmd->se_tmr_req;
	int ret;

	switch (tmr->function) {
2815
	case TMR_ABORT_TASK:
2816
		core_tmr_abort_task(dev, tmr, cmd->se_sess);
2817
		break;
2818 2819 2820
	case TMR_ABORT_TASK_SET:
	case TMR_CLEAR_ACA:
	case TMR_CLEAR_TASK_SET:
2821 2822
		tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED;
		break;
2823
	case TMR_LUN_RESET:
2824 2825 2826 2827
		ret = core_tmr_lun_reset(dev, tmr, NULL, NULL);
		tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE :
					 TMR_FUNCTION_REJECTED;
		break;
2828
	case TMR_TARGET_WARM_RESET:
2829 2830
		tmr->response = TMR_FUNCTION_REJECTED;
		break;
2831
	case TMR_TARGET_COLD_RESET:
2832 2833 2834
		tmr->response = TMR_FUNCTION_REJECTED;
		break;
	default:
2835
		pr_err("Uknown TMR function: 0x%02x.\n",
2836 2837 2838 2839 2840 2841
				tmr->function);
		tmr->response = TMR_FUNCTION_REJECTED;
		break;
	}

	cmd->t_state = TRANSPORT_ISTATE_PROCESSING;
2842
	cmd->se_tfo->queue_tm_rsp(cmd);
2843

2844
	transport_cmd_check_stop_to_fabric(cmd);
2845 2846
}

2847 2848
int transport_generic_handle_tmr(
	struct se_cmd *cmd)
2849
{
2850 2851
	INIT_WORK(&cmd->work, target_tmr_work);
	queue_work(cmd->se_dev->tmr_wq, &cmd->work);
2852 2853
	return 0;
}
2854
EXPORT_SYMBOL(transport_generic_handle_tmr);