target_core_transport.c 91.3 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-or-later
2 3 4 5 6
/*******************************************************************************
 * Filename:  target_core_transport.c
 *
 * This file contains the Generic Target Engine Core.
 *
7
 * (c) Copyright 2002-2013 Datera, Inc.
8 9 10 11 12 13 14 15 16 17 18 19 20 21
 *
 * Nicholas A. Bellinger <nab@kernel.org>
 *
 ******************************************************************************/

#include <linux/net.h>
#include <linux/delay.h>
#include <linux/string.h>
#include <linux/timer.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/kthread.h>
#include <linux/in.h>
#include <linux/cdrom.h>
22
#include <linux/module.h>
23
#include <linux/ratelimit.h>
24
#include <linux/vmalloc.h>
25 26 27
#include <asm/unaligned.h>
#include <net/sock.h>
#include <net/tcp.h>
28
#include <scsi/scsi_proto.h>
29
#include <scsi/scsi_common.h>
30 31

#include <target/target_core_base.h>
32 33
#include <target/target_core_backend.h>
#include <target/target_core_fabric.h>
34

C
Christoph Hellwig 已提交
35
#include "target_core_internal.h"
36 37 38 39
#include "target_core_alua.h"
#include "target_core_pr.h"
#include "target_core_ua.h"

40 41 42
#define CREATE_TRACE_POINTS
#include <trace/events/target.h>

43
static struct workqueue_struct *target_completion_wq;
44 45 46 47 48 49
static struct kmem_cache *se_sess_cache;
struct kmem_cache *se_ua_cache;
struct kmem_cache *t10_pr_reg_cache;
struct kmem_cache *t10_alua_lu_gp_cache;
struct kmem_cache *t10_alua_lu_gp_mem_cache;
struct kmem_cache *t10_alua_tg_pt_gp_cache;
50 51
struct kmem_cache *t10_alua_lba_map_cache;
struct kmem_cache *t10_alua_lba_map_mem_cache;
52 53

static void transport_complete_task_attr(struct se_cmd *cmd);
54
static void translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason);
55
static void transport_handle_queue_full(struct se_cmd *cmd,
56
		struct se_device *dev, int err, bool write_pending);
57
static void target_complete_ok_work(struct work_struct *work);
58

59
int init_se_kmem_caches(void)
60 61 62 63
{
	se_sess_cache = kmem_cache_create("se_sess_cache",
			sizeof(struct se_session), __alignof__(struct se_session),
			0, NULL);
64 65
	if (!se_sess_cache) {
		pr_err("kmem_cache_create() for struct se_session"
66
				" failed\n");
67
		goto out;
68 69 70 71
	}
	se_ua_cache = kmem_cache_create("se_ua_cache",
			sizeof(struct se_ua), __alignof__(struct se_ua),
			0, NULL);
72 73
	if (!se_ua_cache) {
		pr_err("kmem_cache_create() for struct se_ua failed\n");
74
		goto out_free_sess_cache;
75 76 77 78
	}
	t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache",
			sizeof(struct t10_pr_registration),
			__alignof__(struct t10_pr_registration), 0, NULL);
79 80
	if (!t10_pr_reg_cache) {
		pr_err("kmem_cache_create() for struct t10_pr_registration"
81
				" failed\n");
82
		goto out_free_ua_cache;
83 84 85 86
	}
	t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache",
			sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp),
			0, NULL);
87 88
	if (!t10_alua_lu_gp_cache) {
		pr_err("kmem_cache_create() for t10_alua_lu_gp_cache"
89
				" failed\n");
90
		goto out_free_pr_reg_cache;
91 92 93 94
	}
	t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache",
			sizeof(struct t10_alua_lu_gp_member),
			__alignof__(struct t10_alua_lu_gp_member), 0, NULL);
95 96
	if (!t10_alua_lu_gp_mem_cache) {
		pr_err("kmem_cache_create() for t10_alua_lu_gp_mem_"
97
				"cache failed\n");
98
		goto out_free_lu_gp_cache;
99 100 101 102
	}
	t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache",
			sizeof(struct t10_alua_tg_pt_gp),
			__alignof__(struct t10_alua_tg_pt_gp), 0, NULL);
103 104
	if (!t10_alua_tg_pt_gp_cache) {
		pr_err("kmem_cache_create() for t10_alua_tg_pt_gp_"
105
				"cache failed\n");
106
		goto out_free_lu_gp_mem_cache;
107
	}
108 109 110 111 112 113 114
	t10_alua_lba_map_cache = kmem_cache_create(
			"t10_alua_lba_map_cache",
			sizeof(struct t10_alua_lba_map),
			__alignof__(struct t10_alua_lba_map), 0, NULL);
	if (!t10_alua_lba_map_cache) {
		pr_err("kmem_cache_create() for t10_alua_lba_map_"
				"cache failed\n");
115
		goto out_free_tg_pt_gp_cache;
116 117 118 119 120 121 122 123 124 125
	}
	t10_alua_lba_map_mem_cache = kmem_cache_create(
			"t10_alua_lba_map_mem_cache",
			sizeof(struct t10_alua_lba_map_member),
			__alignof__(struct t10_alua_lba_map_member), 0, NULL);
	if (!t10_alua_lba_map_mem_cache) {
		pr_err("kmem_cache_create() for t10_alua_lba_map_mem_"
				"cache failed\n");
		goto out_free_lba_map_cache;
	}
126

127 128 129
	target_completion_wq = alloc_workqueue("target_completion",
					       WQ_MEM_RECLAIM, 0);
	if (!target_completion_wq)
130
		goto out_free_lba_map_mem_cache;
131

132
	return 0;
133

134 135 136 137
out_free_lba_map_mem_cache:
	kmem_cache_destroy(t10_alua_lba_map_mem_cache);
out_free_lba_map_cache:
	kmem_cache_destroy(t10_alua_lba_map_cache);
138 139 140 141 142 143 144 145 146 147 148 149
out_free_tg_pt_gp_cache:
	kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
out_free_lu_gp_mem_cache:
	kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
out_free_lu_gp_cache:
	kmem_cache_destroy(t10_alua_lu_gp_cache);
out_free_pr_reg_cache:
	kmem_cache_destroy(t10_pr_reg_cache);
out_free_ua_cache:
	kmem_cache_destroy(se_ua_cache);
out_free_sess_cache:
	kmem_cache_destroy(se_sess_cache);
150
out:
151
	return -ENOMEM;
152 153
}

154
void release_se_kmem_caches(void)
155
{
156
	destroy_workqueue(target_completion_wq);
157 158 159 160 161 162
	kmem_cache_destroy(se_sess_cache);
	kmem_cache_destroy(se_ua_cache);
	kmem_cache_destroy(t10_pr_reg_cache);
	kmem_cache_destroy(t10_alua_lu_gp_cache);
	kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
	kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
163 164
	kmem_cache_destroy(t10_alua_lba_map_cache);
	kmem_cache_destroy(t10_alua_lba_map_mem_cache);
165 166
}

167 168 169
/* This code ensures unique mib indexes are handed out. */
static DEFINE_SPINLOCK(scsi_mib_index_lock);
static u32 scsi_mib_index[SCSI_INDEX_TYPE_MAX];
170 171 172 173 174 175 176 177

/*
 * Allocate a new row index for the entry type specified
 */
u32 scsi_get_new_index(scsi_index_t type)
{
	u32 new_index;

178
	BUG_ON((type < 0) || (type >= SCSI_INDEX_TYPE_MAX));
179

180 181 182
	spin_lock(&scsi_mib_index_lock);
	new_index = ++scsi_mib_index[type];
	spin_unlock(&scsi_mib_index_lock);
183 184 185 186

	return new_index;
}

187
void transport_subsystem_check_init(void)
188 189
{
	int ret;
190
	static int sub_api_initialized;
191

192 193 194
	if (sub_api_initialized)
		return;

195
	ret = IS_ENABLED(CONFIG_TCM_IBLOCK) && request_module("target_core_iblock");
196
	if (ret != 0)
197
		pr_err("Unable to load target_core_iblock\n");
198

199
	ret = IS_ENABLED(CONFIG_TCM_FILEIO) && request_module("target_core_file");
200
	if (ret != 0)
201
		pr_err("Unable to load target_core_file\n");
202

203
	ret = IS_ENABLED(CONFIG_TCM_PSCSI) && request_module("target_core_pscsi");
204
	if (ret != 0)
205
		pr_err("Unable to load target_core_pscsi\n");
206

207
	ret = IS_ENABLED(CONFIG_TCM_USER2) && request_module("target_core_user");
208 209 210
	if (ret != 0)
		pr_err("Unable to load target_core_user\n");

211
	sub_api_initialized = 1;
212 213
}

214 215 216 217 218 219 220
static void target_release_sess_cmd_refcnt(struct percpu_ref *ref)
{
	struct se_session *sess = container_of(ref, typeof(*sess), cmd_count);

	wake_up(&sess->cmd_list_wq);
}

221 222 223 224 225 226
/**
 * transport_init_session - initialize a session object
 * @se_sess: Session object pointer.
 *
 * The caller must have zero-initialized @se_sess before calling this function.
 */
227
int transport_init_session(struct se_session *se_sess)
228 229 230 231 232
{
	INIT_LIST_HEAD(&se_sess->sess_list);
	INIT_LIST_HEAD(&se_sess->sess_acl_list);
	INIT_LIST_HEAD(&se_sess->sess_cmd_list);
	spin_lock_init(&se_sess->sess_cmd_lock);
233
	init_waitqueue_head(&se_sess->cmd_list_wq);
234 235
	return percpu_ref_init(&se_sess->cmd_count,
			       target_release_sess_cmd_refcnt, 0, GFP_KERNEL);
236 237 238
}
EXPORT_SYMBOL(transport_init_session);

239 240 241 242 243
/**
 * transport_alloc_session - allocate a session object and initialize it
 * @sup_prot_ops: bitmask that defines which T10-PI modes are supported.
 */
struct se_session *transport_alloc_session(enum target_prot_op sup_prot_ops)
244 245
{
	struct se_session *se_sess;
246
	int ret;
247 248

	se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL);
249 250
	if (!se_sess) {
		pr_err("Unable to allocate struct se_session from"
251 252 253
				" se_sess_cache\n");
		return ERR_PTR(-ENOMEM);
	}
254 255
	ret = transport_init_session(se_sess);
	if (ret < 0) {
256
		kmem_cache_free(se_sess_cache, se_sess);
257 258
		return ERR_PTR(ret);
	}
259
	se_sess->sup_prot_ops = sup_prot_ops;
260 261 262

	return se_sess;
}
263 264 265 266 267 268 269 270 271
EXPORT_SYMBOL(transport_alloc_session);

/**
 * transport_alloc_session_tags - allocate target driver private data
 * @se_sess:  Session pointer.
 * @tag_num:  Maximum number of in-flight commands between initiator and target.
 * @tag_size: Size in bytes of the private data a target driver associates with
 *	      each command.
 */
272 273 274 275 276
int transport_alloc_session_tags(struct se_session *se_sess,
			         unsigned int tag_num, unsigned int tag_size)
{
	int rc;

277 278
	se_sess->sess_cmd_map = kvcalloc(tag_size, tag_num,
					 GFP_KERNEL | __GFP_RETRY_MAYFAIL);
279
	if (!se_sess->sess_cmd_map) {
280 281
		pr_err("Unable to allocate se_sess->sess_cmd_map\n");
		return -ENOMEM;
282 283
	}

284 285
	rc = sbitmap_queue_init_node(&se_sess->sess_tag_pool, tag_num, -1,
			false, GFP_KERNEL, NUMA_NO_NODE);
286 287 288
	if (rc < 0) {
		pr_err("Unable to init se_sess->sess_tag_pool,"
			" tag_num: %u\n", tag_num);
289
		kvfree(se_sess->sess_cmd_map);
290 291 292 293 294 295 296 297
		se_sess->sess_cmd_map = NULL;
		return -ENOMEM;
	}

	return 0;
}
EXPORT_SYMBOL(transport_alloc_session_tags);

298 299 300 301 302 303 304
/**
 * transport_init_session_tags - allocate a session and target driver private data
 * @tag_num:  Maximum number of in-flight commands between initiator and target.
 * @tag_size: Size in bytes of the private data a target driver associates with
 *	      each command.
 * @sup_prot_ops: bitmask that defines which T10-PI modes are supported.
 */
305 306 307
static struct se_session *
transport_init_session_tags(unsigned int tag_num, unsigned int tag_size,
			    enum target_prot_op sup_prot_ops)
308 309 310 311
{
	struct se_session *se_sess;
	int rc;

312 313 314 315 316 317 318 319 320 321 322
	if (tag_num != 0 && !tag_size) {
		pr_err("init_session_tags called with percpu-ida tag_num:"
		       " %u, but zero tag_size\n", tag_num);
		return ERR_PTR(-EINVAL);
	}
	if (!tag_num && tag_size) {
		pr_err("init_session_tags called with percpu-ida tag_size:"
		       " %u, but zero tag_num\n", tag_size);
		return ERR_PTR(-EINVAL);
	}

323
	se_sess = transport_alloc_session(sup_prot_ops);
324 325 326 327 328 329 330 331 332 333 334 335
	if (IS_ERR(se_sess))
		return se_sess;

	rc = transport_alloc_session_tags(se_sess, tag_num, tag_size);
	if (rc < 0) {
		transport_free_session(se_sess);
		return ERR_PTR(-ENOMEM);
	}

	return se_sess;
}

336
/*
337
 * Called with spin_lock_irqsave(&struct se_portal_group->session_lock called.
338 339 340 341 342 343 344
 */
void __transport_register_session(
	struct se_portal_group *se_tpg,
	struct se_node_acl *se_nacl,
	struct se_session *se_sess,
	void *fabric_sess_ptr)
{
345
	const struct target_core_fabric_ops *tfo = se_tpg->se_tpg_tfo;
346
	unsigned char buf[PR_REG_ISID_LEN];
347
	unsigned long flags;
348 349 350 351 352 353 354 355 356 357

	se_sess->se_tpg = se_tpg;
	se_sess->fabric_sess_ptr = fabric_sess_ptr;
	/*
	 * Used by struct se_node_acl's under ConfigFS to locate active se_session-t
	 *
	 * Only set for struct se_session's that will actually be moving I/O.
	 * eg: *NOT* discovery sessions.
	 */
	if (se_nacl) {
358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
		/*
		 *
		 * Determine if fabric allows for T10-PI feature bits exposed to
		 * initiators for device backends with !dev->dev_attrib.pi_prot_type.
		 *
		 * If so, then always save prot_type on a per se_node_acl node
		 * basis and re-instate the previous sess_prot_type to avoid
		 * disabling PI from below any previously initiator side
		 * registered LUNs.
		 */
		if (se_nacl->saved_prot_type)
			se_sess->sess_prot_type = se_nacl->saved_prot_type;
		else if (tfo->tpg_check_prot_fabric_only)
			se_sess->sess_prot_type = se_nacl->saved_prot_type =
					tfo->tpg_check_prot_fabric_only(se_tpg);
373 374 375 376
		/*
		 * If the fabric module supports an ISID based TransportID,
		 * save this value in binary from the fabric I_T Nexus now.
		 */
377
		if (se_tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) {
378
			memset(&buf[0], 0, PR_REG_ISID_LEN);
379
			se_tpg->se_tpg_tfo->sess_get_initiator_sid(se_sess,
380 381 382
					&buf[0], PR_REG_ISID_LEN);
			se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]);
		}
383

384
		spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
385 386 387 388 389 390 391 392
		/*
		 * The se_nacl->nacl_sess pointer will be set to the
		 * last active I_T Nexus for each struct se_node_acl.
		 */
		se_nacl->nacl_sess = se_sess;

		list_add_tail(&se_sess->sess_acl_list,
			      &se_nacl->acl_sess_list);
393
		spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
394 395 396
	}
	list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list);

397
	pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n",
398
		se_tpg->se_tpg_tfo->fabric_name, se_sess->fabric_sess_ptr);
399 400 401 402 403 404 405 406 407
}
EXPORT_SYMBOL(__transport_register_session);

void transport_register_session(
	struct se_portal_group *se_tpg,
	struct se_node_acl *se_nacl,
	struct se_session *se_sess,
	void *fabric_sess_ptr)
{
408 409 410
	unsigned long flags;

	spin_lock_irqsave(&se_tpg->session_lock, flags);
411
	__transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr);
412
	spin_unlock_irqrestore(&se_tpg->session_lock, flags);
413 414 415
}
EXPORT_SYMBOL(transport_register_session);

416
struct se_session *
417
target_setup_session(struct se_portal_group *tpg,
418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
		     unsigned int tag_num, unsigned int tag_size,
		     enum target_prot_op prot_op,
		     const char *initiatorname, void *private,
		     int (*callback)(struct se_portal_group *,
				     struct se_session *, void *))
{
	struct se_session *sess;

	/*
	 * If the fabric driver is using percpu-ida based pre allocation
	 * of I/O descriptor tags, go ahead and perform that setup now..
	 */
	if (tag_num != 0)
		sess = transport_init_session_tags(tag_num, tag_size, prot_op);
	else
433
		sess = transport_alloc_session(prot_op);
434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458

	if (IS_ERR(sess))
		return sess;

	sess->se_node_acl = core_tpg_check_initiator_node_acl(tpg,
					(unsigned char *)initiatorname);
	if (!sess->se_node_acl) {
		transport_free_session(sess);
		return ERR_PTR(-EACCES);
	}
	/*
	 * Go ahead and perform any remaining fabric setup that is
	 * required before transport_register_session().
	 */
	if (callback != NULL) {
		int rc = callback(tpg, sess, private);
		if (rc) {
			transport_free_session(sess);
			return ERR_PTR(rc);
		}
	}

	transport_register_session(tpg, sess->se_node_acl, sess, private);
	return sess;
}
459
EXPORT_SYMBOL(target_setup_session);
460

461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484
ssize_t target_show_dynamic_sessions(struct se_portal_group *se_tpg, char *page)
{
	struct se_session *se_sess;
	ssize_t len = 0;

	spin_lock_bh(&se_tpg->session_lock);
	list_for_each_entry(se_sess, &se_tpg->tpg_sess_list, sess_list) {
		if (!se_sess->se_node_acl)
			continue;
		if (!se_sess->se_node_acl->dynamic_node_acl)
			continue;
		if (strlen(se_sess->se_node_acl->initiatorname) + 1 + len > PAGE_SIZE)
			break;

		len += snprintf(page + len, PAGE_SIZE - len, "%s\n",
				se_sess->se_node_acl->initiatorname);
		len += 1; /* Include NULL terminator */
	}
	spin_unlock_bh(&se_tpg->session_lock);

	return len;
}
EXPORT_SYMBOL(target_show_dynamic_sessions);

485 486 487 488
static void target_complete_nacl(struct kref *kref)
{
	struct se_node_acl *nacl = container_of(kref,
				struct se_node_acl, acl_kref);
489
	struct se_portal_group *se_tpg = nacl->se_tpg;
490

491 492 493 494 495 496
	if (!nacl->dynamic_stop) {
		complete(&nacl->acl_free_comp);
		return;
	}

	mutex_lock(&se_tpg->acl_node_mutex);
497
	list_del_init(&nacl->acl_list);
498 499 500 501 502
	mutex_unlock(&se_tpg->acl_node_mutex);

	core_tpg_wait_for_nacl_pr_ref(nacl);
	core_free_device_list_for_node(nacl, se_tpg);
	kfree(nacl);
503 504 505 506 507 508
}

void target_put_nacl(struct se_node_acl *nacl)
{
	kref_put(&nacl->acl_kref, target_complete_nacl);
}
509
EXPORT_SYMBOL(target_put_nacl);
510

511 512 513
void transport_deregister_session_configfs(struct se_session *se_sess)
{
	struct se_node_acl *se_nacl;
514
	unsigned long flags;
515 516 517 518
	/*
	 * Used by struct se_node_acl's under ConfigFS to locate active struct se_session
	 */
	se_nacl = se_sess->se_node_acl;
519
	if (se_nacl) {
520
		spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
C
Christoph Hellwig 已提交
521 522
		if (!list_empty(&se_sess->sess_acl_list))
			list_del_init(&se_sess->sess_acl_list);
523 524 525 526 527 528 529 530 531 532 533 534
		/*
		 * If the session list is empty, then clear the pointer.
		 * Otherwise, set the struct se_session pointer from the tail
		 * element of the per struct se_node_acl active session list.
		 */
		if (list_empty(&se_nacl->acl_sess_list))
			se_nacl->nacl_sess = NULL;
		else {
			se_nacl->nacl_sess = container_of(
					se_nacl->acl_sess_list.prev,
					struct se_session, sess_acl_list);
		}
535
		spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
536 537 538 539 540 541
	}
}
EXPORT_SYMBOL(transport_deregister_session_configfs);

void transport_free_session(struct se_session *se_sess)
{
542
	struct se_node_acl *se_nacl = se_sess->se_node_acl;
543

544 545 546 547 548
	/*
	 * Drop the se_node_acl->nacl_kref obtained from within
	 * core_tpg_get_initiator_node_acl().
	 */
	if (se_nacl) {
549 550 551 552
		struct se_portal_group *se_tpg = se_nacl->se_tpg;
		const struct target_core_fabric_ops *se_tfo = se_tpg->se_tpg_tfo;
		unsigned long flags;

553
		se_sess->se_node_acl = NULL;
554 555 556 557 558 559 560 561 562 563 564 565 566 567 568

		/*
		 * Also determine if we need to drop the extra ->cmd_kref if
		 * it had been previously dynamically generated, and
		 * the endpoint is not caching dynamic ACLs.
		 */
		mutex_lock(&se_tpg->acl_node_mutex);
		if (se_nacl->dynamic_node_acl &&
		    !se_tfo->tpg_check_demo_mode_cache(se_tpg)) {
			spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
			if (list_empty(&se_nacl->acl_sess_list))
				se_nacl->dynamic_stop = true;
			spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);

			if (se_nacl->dynamic_stop)
569
				list_del_init(&se_nacl->acl_list);
570 571 572 573 574 575
		}
		mutex_unlock(&se_tpg->acl_node_mutex);

		if (se_nacl->dynamic_stop)
			target_put_nacl(se_nacl);

576 577
		target_put_nacl(se_nacl);
	}
578
	if (se_sess->sess_cmd_map) {
579
		sbitmap_queue_free(&se_sess->sess_tag_pool);
580
		kvfree(se_sess->sess_cmd_map);
581
	}
582
	percpu_ref_exit(&se_sess->cmd_count);
583 584 585 586 587 588 589
	kmem_cache_free(se_sess_cache, se_sess);
}
EXPORT_SYMBOL(transport_free_session);

void transport_deregister_session(struct se_session *se_sess)
{
	struct se_portal_group *se_tpg = se_sess->se_tpg;
590
	unsigned long flags;
591

592
	if (!se_tpg) {
593 594 595 596
		transport_free_session(se_sess);
		return;
	}

597
	spin_lock_irqsave(&se_tpg->session_lock, flags);
598 599 600
	list_del(&se_sess->sess_list);
	se_sess->se_tpg = NULL;
	se_sess->fabric_sess_ptr = NULL;
601
	spin_unlock_irqrestore(&se_tpg->session_lock, flags);
602

603
	pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n",
604
		se_tpg->se_tpg_tfo->fabric_name);
605
	/*
606
	 * If last kref is dropping now for an explicit NodeACL, awake sleeping
607
	 * ->acl_free_comp caller to wakeup configfs se_node_acl->acl_group
608
	 * removal context from within transport_free_session() code.
609 610 611
	 *
	 * For dynamic ACL, target_put_nacl() uses target_complete_nacl()
	 * to release all remaining generate_node_acl=1 created ACL resources.
612 613
	 */

614
	transport_free_session(se_sess);
615 616 617
}
EXPORT_SYMBOL(transport_deregister_session);

618 619 620 621 622 623 624
void target_remove_session(struct se_session *se_sess)
{
	transport_deregister_session_configfs(se_sess);
	transport_deregister_session(se_sess);
}
EXPORT_SYMBOL(target_remove_session);

625
static void target_remove_from_state_list(struct se_cmd *cmd)
626
{
627
	struct se_device *dev = cmd->se_dev;
628 629
	unsigned long flags;

630 631
	if (!dev)
		return;
632

633 634 635 636
	spin_lock_irqsave(&dev->execute_task_lock, flags);
	if (cmd->state_active) {
		list_del(&cmd->state_list);
		cmd->state_active = false;
637
	}
638
	spin_unlock_irqrestore(&dev->execute_task_lock, flags);
639 640
}

641 642 643 644 645 646 647
/*
 * This function is called by the target core after the target core has
 * finished processing a SCSI command or SCSI TMF. Both the regular command
 * processing code and the code for aborting commands can call this
 * function. CMD_T_STOP is set if and only if another thread is waiting
 * inside transport_wait_for_tasks() for t_transport_stop_comp.
 */
648
static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd)
649 650 651
{
	unsigned long flags;

652
	target_remove_from_state_list(cmd);
653

654
	spin_lock_irqsave(&cmd->t_state_lock, flags);
655 656
	/*
	 * Determine if frontend context caller is requesting the stopping of
657
	 * this command for frontend exceptions.
658
	 */
659
	if (cmd->transport_state & CMD_T_STOP) {
660 661
		pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
			__func__, __LINE__, cmd->tag);
662

663
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
664

665
		complete_all(&cmd->t_transport_stop_comp);
666 667
		return 1;
	}
668
	cmd->transport_state &= ~CMD_T_ACTIVE;
669
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
670

671 672 673 674 675 676 677
	/*
	 * Some fabric modules like tcm_loop can release their internally
	 * allocated I/O reference and struct se_cmd now.
	 *
	 * Fabric modules are expected to return '1' here if the se_cmd being
	 * passed is released at this point, or zero if not being released.
	 */
678
	return cmd->se_tfo->check_stop_free(cmd);
679 680
}

681 682 683 684
static void target_complete_failure_work(struct work_struct *work)
{
	struct se_cmd *cmd = container_of(work, struct se_cmd, work);

685 686
	transport_generic_request_failure(cmd,
			TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE);
687 688
}

689
/*
690 691
 * Used when asking transport to copy Sense Data from the underlying
 * Linux/SCSI struct scsi_cmnd
692
 */
693
static unsigned char *transport_get_sense_buffer(struct se_cmd *cmd)
694 695 696 697 698 699
{
	struct se_device *dev = cmd->se_dev;

	WARN_ON(!cmd->se_lun);

	if (!dev)
700
		return NULL;
701

702 703
	if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION)
		return NULL;
704

705
	cmd->scsi_sense_length = TRANSPORT_SENSE_BUFFER;
706

707
	pr_debug("HBA_[%u]_PLUG[%s]: Requesting sense for SAM STATUS: 0x%02x\n",
708
		dev->se_hba->hba_id, dev->transport->name, cmd->scsi_status);
709
	return cmd->sense_buffer;
710 711
}

712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729
void transport_copy_sense_to_cmd(struct se_cmd *cmd, unsigned char *sense)
{
	unsigned char *cmd_sense_buf;
	unsigned long flags;

	spin_lock_irqsave(&cmd->t_state_lock, flags);
	cmd_sense_buf = transport_get_sense_buffer(cmd);
	if (!cmd_sense_buf) {
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
		return;
	}

	cmd->se_cmd_flags |= SCF_TRANSPORT_TASK_SENSE;
	memcpy(cmd_sense_buf, sense, cmd->scsi_sense_length);
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
}
EXPORT_SYMBOL(transport_copy_sense_to_cmd);

730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801
static void target_handle_abort(struct se_cmd *cmd)
{
	bool tas = cmd->transport_state & CMD_T_TAS;
	bool ack_kref = cmd->se_cmd_flags & SCF_ACK_KREF;
	int ret;

	pr_debug("tag %#llx: send_abort_response = %d\n", cmd->tag, tas);

	if (tas) {
		if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
			cmd->scsi_status = SAM_STAT_TASK_ABORTED;
			pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x, ITT: 0x%08llx\n",
				 cmd->t_task_cdb[0], cmd->tag);
			trace_target_cmd_complete(cmd);
			ret = cmd->se_tfo->queue_status(cmd);
			if (ret) {
				transport_handle_queue_full(cmd, cmd->se_dev,
							    ret, false);
				return;
			}
		} else {
			cmd->se_tmr_req->response = TMR_FUNCTION_REJECTED;
			cmd->se_tfo->queue_tm_rsp(cmd);
		}
	} else {
		/*
		 * Allow the fabric driver to unmap any resources before
		 * releasing the descriptor via TFO->release_cmd().
		 */
		cmd->se_tfo->aborted_task(cmd);
		if (ack_kref)
			WARN_ON_ONCE(target_put_sess_cmd(cmd) != 0);
		/*
		 * To do: establish a unit attention condition on the I_T
		 * nexus associated with cmd. See also the paragraph "Aborting
		 * commands" in SAM.
		 */
	}

	WARN_ON_ONCE(kref_read(&cmd->cmd_kref) == 0);

	transport_cmd_check_stop_to_fabric(cmd);
}

static void target_abort_work(struct work_struct *work)
{
	struct se_cmd *cmd = container_of(work, struct se_cmd, work);

	target_handle_abort(cmd);
}

static bool target_cmd_interrupted(struct se_cmd *cmd)
{
	int post_ret;

	if (cmd->transport_state & CMD_T_ABORTED) {
		if (cmd->transport_complete_callback)
			cmd->transport_complete_callback(cmd, false, &post_ret);
		INIT_WORK(&cmd->work, target_abort_work);
		queue_work(target_completion_wq, &cmd->work);
		return true;
	} else if (cmd->transport_state & CMD_T_STOP) {
		if (cmd->transport_complete_callback)
			cmd->transport_complete_callback(cmd, false, &post_ret);
		complete_all(&cmd->t_transport_stop_comp);
		return true;
	}

	return false;
}

/* May be called from interrupt context so must not sleep. */
802
void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status)
803
{
804
	int success;
805 806
	unsigned long flags;

807 808 809
	if (target_cmd_interrupted(cmd))
		return;

810 811
	cmd->scsi_status = scsi_status;

812
	spin_lock_irqsave(&cmd->t_state_lock, flags);
813 814
	switch (cmd->scsi_status) {
	case SAM_STAT_CHECK_CONDITION:
815
		if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)
816
			success = 1;
817 818 819 820
		else
			success = 0;
		break;
	default:
821
		success = 1;
822
		break;
823 824
	}

825
	cmd->t_state = TRANSPORT_COMPLETE;
826
	cmd->transport_state |= (CMD_T_COMPLETE | CMD_T_ACTIVE);
827
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
828

829 830
	INIT_WORK(&cmd->work, success ? target_complete_ok_work :
		  target_complete_failure_work);
831
	if (cmd->se_cmd_flags & SCF_USE_CPUID)
832
		queue_work_on(cmd->cpuid, target_completion_wq, &cmd->work);
833 834
	else
		queue_work(target_completion_wq, &cmd->work);
835
}
836 837
EXPORT_SYMBOL(target_complete_cmd);

838 839
void target_complete_cmd_with_length(struct se_cmd *cmd, u8 scsi_status, int length)
{
840 841 842
	if ((scsi_status == SAM_STAT_GOOD ||
	     cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) &&
	    length < cmd->data_length) {
843 844 845 846 847 848 849 850 851 852 853 854 855 856
		if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
			cmd->residual_count += cmd->data_length - length;
		} else {
			cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
			cmd->residual_count = cmd->data_length - length;
		}

		cmd->data_length = length;
	}

	target_complete_cmd(cmd, scsi_status);
}
EXPORT_SYMBOL(target_complete_cmd_with_length);

857
static void target_add_to_state_list(struct se_cmd *cmd)
858
{
859 860
	struct se_device *dev = cmd->se_dev;
	unsigned long flags;
861

862 863 864 865
	spin_lock_irqsave(&dev->execute_task_lock, flags);
	if (!cmd->state_active) {
		list_add_tail(&cmd->state_list, &dev->state_list);
		cmd->state_active = true;
866
	}
867
	spin_unlock_irqrestore(&dev->execute_task_lock, flags);
868 869
}

870
/*
871
 * Handle QUEUE_FULL / -EAGAIN and -ENOMEM status
872
 */
873 874
static void transport_write_pending_qf(struct se_cmd *cmd);
static void transport_complete_qf(struct se_cmd *cmd);
875

876
void target_qf_do_work(struct work_struct *work)
877 878 879
{
	struct se_device *dev = container_of(work, struct se_device,
					qf_work_queue);
880
	LIST_HEAD(qf_cmd_list);
881 882 883
	struct se_cmd *cmd, *cmd_tmp;

	spin_lock_irq(&dev->qf_cmd_lock);
884 885
	list_splice_init(&dev->qf_cmd_list, &qf_cmd_list);
	spin_unlock_irq(&dev->qf_cmd_lock);
886

887
	list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) {
888
		list_del(&cmd->se_qf_node);
889
		atomic_dec_mb(&dev->dev_qf_count);
890

891
		pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue"
892
			" context: %s\n", cmd->se_tfo->fabric_name, cmd,
893
			(cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" :
894 895
			(cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING"
			: "UNKNOWN");
896

897 898
		if (cmd->t_state == TRANSPORT_COMPLETE_QF_WP)
			transport_write_pending_qf(cmd);
899 900
		else if (cmd->t_state == TRANSPORT_COMPLETE_QF_OK ||
			 cmd->t_state == TRANSPORT_COMPLETE_QF_ERR)
901
			transport_complete_qf(cmd);
902 903 904
	}
}

905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928
unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd)
{
	switch (cmd->data_direction) {
	case DMA_NONE:
		return "NONE";
	case DMA_FROM_DEVICE:
		return "READ";
	case DMA_TO_DEVICE:
		return "WRITE";
	case DMA_BIDIRECTIONAL:
		return "BIDI";
	default:
		break;
	}

	return "UNKNOWN";
}

void transport_dump_dev_state(
	struct se_device *dev,
	char *b,
	int *bl)
{
	*bl += sprintf(b + *bl, "Status: ");
929
	if (dev->export_count)
930
		*bl += sprintf(b + *bl, "ACTIVATED");
931
	else
932 933
		*bl += sprintf(b + *bl, "DEACTIVATED");

934
	*bl += sprintf(b + *bl, "  Max Queue Depth: %d", dev->queue_depth);
935
	*bl += sprintf(b + *bl, "  SectorSize: %u  HwMaxSectors: %u\n",
936 937
		dev->dev_attrib.block_size,
		dev->dev_attrib.hw_max_sectors);
938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990
	*bl += sprintf(b + *bl, "        ");
}

void transport_dump_vpd_proto_id(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
	int len;

	memset(buf, 0, VPD_TMP_BUF_SIZE);
	len = sprintf(buf, "T10 VPD Protocol Identifier: ");

	switch (vpd->protocol_identifier) {
	case 0x00:
		sprintf(buf+len, "Fibre Channel\n");
		break;
	case 0x10:
		sprintf(buf+len, "Parallel SCSI\n");
		break;
	case 0x20:
		sprintf(buf+len, "SSA\n");
		break;
	case 0x30:
		sprintf(buf+len, "IEEE 1394\n");
		break;
	case 0x40:
		sprintf(buf+len, "SCSI Remote Direct Memory Access"
				" Protocol\n");
		break;
	case 0x50:
		sprintf(buf+len, "Internet SCSI (iSCSI)\n");
		break;
	case 0x60:
		sprintf(buf+len, "SAS Serial SCSI Protocol\n");
		break;
	case 0x70:
		sprintf(buf+len, "Automation/Drive Interface Transport"
				" Protocol\n");
		break;
	case 0x80:
		sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n");
		break;
	default:
		sprintf(buf+len, "Unknown 0x%02x\n",
				vpd->protocol_identifier);
		break;
	}

	if (p_buf)
		strncpy(p_buf, buf, p_buf_len);
	else
991
		pr_debug("%s", buf);
992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
}

void
transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83)
{
	/*
	 * Check if the Protocol Identifier Valid (PIV) bit is set..
	 *
	 * from spc3r23.pdf section 7.5.1
	 */
	 if (page_83[1] & 0x80) {
		vpd->protocol_identifier = (page_83[0] & 0xf0);
		vpd->protocol_identifier_set = 1;
		transport_dump_vpd_proto_id(vpd, NULL, 0);
	}
}
EXPORT_SYMBOL(transport_set_vpd_proto_id);

int transport_dump_vpd_assoc(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
1016 1017
	int ret = 0;
	int len;
1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033

	memset(buf, 0, VPD_TMP_BUF_SIZE);
	len = sprintf(buf, "T10 VPD Identifier Association: ");

	switch (vpd->association) {
	case 0x00:
		sprintf(buf+len, "addressed logical unit\n");
		break;
	case 0x10:
		sprintf(buf+len, "target port\n");
		break;
	case 0x20:
		sprintf(buf+len, "SCSI target device\n");
		break;
	default:
		sprintf(buf+len, "Unknown 0x%02x\n", vpd->association);
1034
		ret = -EINVAL;
1035 1036 1037 1038 1039 1040
		break;
	}

	if (p_buf)
		strncpy(p_buf, buf, p_buf_len);
	else
1041
		pr_debug("%s", buf);
1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063

	return ret;
}

int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83)
{
	/*
	 * The VPD identification association..
	 *
	 * from spc3r23.pdf Section 7.6.3.1 Table 297
	 */
	vpd->association = (page_83[1] & 0x30);
	return transport_dump_vpd_assoc(vpd, NULL, 0);
}
EXPORT_SYMBOL(transport_set_vpd_assoc);

int transport_dump_vpd_ident_type(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
1064 1065
	int ret = 0;
	int len;
1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091

	memset(buf, 0, VPD_TMP_BUF_SIZE);
	len = sprintf(buf, "T10 VPD Identifier Type: ");

	switch (vpd->device_identifier_type) {
	case 0x00:
		sprintf(buf+len, "Vendor specific\n");
		break;
	case 0x01:
		sprintf(buf+len, "T10 Vendor ID based\n");
		break;
	case 0x02:
		sprintf(buf+len, "EUI-64 based\n");
		break;
	case 0x03:
		sprintf(buf+len, "NAA\n");
		break;
	case 0x04:
		sprintf(buf+len, "Relative target port identifier\n");
		break;
	case 0x08:
		sprintf(buf+len, "SCSI name string\n");
		break;
	default:
		sprintf(buf+len, "Unsupported: 0x%02x\n",
				vpd->device_identifier_type);
1092
		ret = -EINVAL;
1093 1094 1095
		break;
	}

1096 1097 1098
	if (p_buf) {
		if (p_buf_len < strlen(buf)+1)
			return -EINVAL;
1099
		strncpy(p_buf, buf, p_buf_len);
1100
	} else {
1101
		pr_debug("%s", buf);
1102
	}
1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130

	return ret;
}

int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83)
{
	/*
	 * The VPD identifier type..
	 *
	 * from spc3r23.pdf Section 7.6.3.1 Table 298
	 */
	vpd->device_identifier_type = (page_83[1] & 0x0f);
	return transport_dump_vpd_ident_type(vpd, NULL, 0);
}
EXPORT_SYMBOL(transport_set_vpd_ident_type);

int transport_dump_vpd_ident(
	struct t10_vpd *vpd,
	unsigned char *p_buf,
	int p_buf_len)
{
	unsigned char buf[VPD_TMP_BUF_SIZE];
	int ret = 0;

	memset(buf, 0, VPD_TMP_BUF_SIZE);

	switch (vpd->device_identifier_code_set) {
	case 0x01: /* Binary */
1131 1132
		snprintf(buf, sizeof(buf),
			"T10 VPD Binary Device Identifier: %s\n",
1133 1134 1135
			&vpd->device_identifier[0]);
		break;
	case 0x02: /* ASCII */
1136 1137
		snprintf(buf, sizeof(buf),
			"T10 VPD ASCII Device Identifier: %s\n",
1138 1139 1140
			&vpd->device_identifier[0]);
		break;
	case 0x03: /* UTF-8 */
1141 1142
		snprintf(buf, sizeof(buf),
			"T10 VPD UTF-8 Device Identifier: %s\n",
1143 1144 1145 1146 1147
			&vpd->device_identifier[0]);
		break;
	default:
		sprintf(buf, "T10 VPD Device Identifier encoding unsupported:"
			" 0x%02x", vpd->device_identifier_code_set);
1148
		ret = -EINVAL;
1149 1150 1151 1152 1153 1154
		break;
	}

	if (p_buf)
		strncpy(p_buf, buf, p_buf_len);
	else
1155
		pr_debug("%s", buf);
1156 1157 1158 1159 1160 1161 1162 1163

	return ret;
}

int
transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83)
{
	static const char hex_str[] = "0123456789abcdef";
1164
	int j = 0, i = 4; /* offset to start of the identifier */
1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196

	/*
	 * The VPD Code Set (encoding)
	 *
	 * from spc3r23.pdf Section 7.6.3.1 Table 296
	 */
	vpd->device_identifier_code_set = (page_83[0] & 0x0f);
	switch (vpd->device_identifier_code_set) {
	case 0x01: /* Binary */
		vpd->device_identifier[j++] =
				hex_str[vpd->device_identifier_type];
		while (i < (4 + page_83[3])) {
			vpd->device_identifier[j++] =
				hex_str[(page_83[i] & 0xf0) >> 4];
			vpd->device_identifier[j++] =
				hex_str[page_83[i] & 0x0f];
			i++;
		}
		break;
	case 0x02: /* ASCII */
	case 0x03: /* UTF-8 */
		while (i < (4 + page_83[3]))
			vpd->device_identifier[j++] = page_83[i++];
		break;
	default:
		break;
	}

	return transport_dump_vpd_ident(vpd, NULL, 0);
}
EXPORT_SYMBOL(transport_set_vpd_ident);

1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245
static sense_reason_t
target_check_max_data_sg_nents(struct se_cmd *cmd, struct se_device *dev,
			       unsigned int size)
{
	u32 mtl;

	if (!cmd->se_tfo->max_data_sg_nents)
		return TCM_NO_SENSE;
	/*
	 * Check if fabric enforced maximum SGL entries per I/O descriptor
	 * exceeds se_cmd->data_length.  If true, set SCF_UNDERFLOW_BIT +
	 * residual_count and reduce original cmd->data_length to maximum
	 * length based on single PAGE_SIZE entry scatter-lists.
	 */
	mtl = (cmd->se_tfo->max_data_sg_nents * PAGE_SIZE);
	if (cmd->data_length > mtl) {
		/*
		 * If an existing CDB overflow is present, calculate new residual
		 * based on CDB size minus fabric maximum transfer length.
		 *
		 * If an existing CDB underflow is present, calculate new residual
		 * based on original cmd->data_length minus fabric maximum transfer
		 * length.
		 *
		 * Otherwise, set the underflow residual based on cmd->data_length
		 * minus fabric maximum transfer length.
		 */
		if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
			cmd->residual_count = (size - mtl);
		} else if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
			u32 orig_dl = size + cmd->residual_count;
			cmd->residual_count = (orig_dl - mtl);
		} else {
			cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
			cmd->residual_count = (cmd->data_length - mtl);
		}
		cmd->data_length = mtl;
		/*
		 * Reset sbc_check_prot() calculated protection payload
		 * length based upon the new smaller MTL.
		 */
		if (cmd->prot_length) {
			u32 sectors = (mtl / dev->dev_attrib.block_size);
			cmd->prot_length = dev->prot_length * sectors;
		}
	}
	return TCM_NO_SENSE;
}

1246 1247
sense_reason_t
target_cmd_size_check(struct se_cmd *cmd, unsigned int size)
1248 1249 1250 1251 1252 1253
{
	struct se_device *dev = cmd->se_dev;

	if (cmd->unknown_data_length) {
		cmd->data_length = size;
	} else if (size != cmd->data_length) {
1254
		pr_warn_ratelimited("TARGET_CORE[%s]: Expected Transfer Length:"
1255
			" %u does not match SCSI CDB Length: %u for SAM Opcode:"
1256
			" 0x%02x\n", cmd->se_tfo->fabric_name,
1257 1258
				cmd->data_length, size, cmd->t_task_cdb[0]);

1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275
		if (cmd->data_direction == DMA_TO_DEVICE) {
			if (cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) {
				pr_err_ratelimited("Rejecting underflow/overflow"
						   " for WRITE data CDB\n");
				return TCM_INVALID_CDB_FIELD;
			}
			/*
			 * Some fabric drivers like iscsi-target still expect to
			 * always reject overflow writes.  Reject this case until
			 * full fabric driver level support for overflow writes
			 * is introduced tree-wide.
			 */
			if (size > cmd->data_length) {
				pr_err_ratelimited("Rejecting overflow for"
						   " WRITE control CDB\n");
				return TCM_INVALID_CDB_FIELD;
			}
1276 1277 1278 1279 1280
		}
		/*
		 * Reject READ_* or WRITE_* with overflow/underflow for
		 * type SCF_SCSI_DATA_CDB.
		 */
1281
		if (dev->dev_attrib.block_size != 512)  {
1282 1283 1284 1285
			pr_err("Failing OVERFLOW/UNDERFLOW for LBA op"
				" CDB on non 512-byte sector setup subsystem"
				" plugin: %s\n", dev->transport->name);
			/* Returns CHECK_CONDITION + INVALID_CDB_FIELD */
1286
			return TCM_INVALID_CDB_FIELD;
1287
		}
1288 1289 1290 1291 1292 1293
		/*
		 * For the overflow case keep the existing fabric provided
		 * ->data_length.  Otherwise for the underflow case, reset
		 * ->data_length to the smaller SCSI expected data transfer
		 * length.
		 */
1294 1295 1296 1297 1298 1299
		if (size > cmd->data_length) {
			cmd->se_cmd_flags |= SCF_OVERFLOW_BIT;
			cmd->residual_count = (size - cmd->data_length);
		} else {
			cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
			cmd->residual_count = (cmd->data_length - size);
1300
			cmd->data_length = size;
1301 1302 1303
		}
	}

1304
	return target_check_max_data_sg_nents(cmd, dev, size);
1305 1306 1307

}

1308 1309 1310
/*
 * Used by fabric modules containing a local struct se_cmd within their
 * fabric dependent per I/O descriptor.
1311 1312
 *
 * Preserves the value of @cmd->tag.
1313 1314 1315
 */
void transport_init_se_cmd(
	struct se_cmd *cmd,
1316
	const struct target_core_fabric_ops *tfo,
1317 1318 1319 1320 1321 1322
	struct se_session *se_sess,
	u32 data_length,
	int data_direction,
	int task_attr,
	unsigned char *sense_buffer)
{
1323
	INIT_LIST_HEAD(&cmd->se_delayed_node);
1324
	INIT_LIST_HEAD(&cmd->se_qf_node);
1325
	INIT_LIST_HEAD(&cmd->se_cmd_list);
1326
	INIT_LIST_HEAD(&cmd->state_list);
1327
	init_completion(&cmd->t_transport_stop_comp);
1328 1329
	cmd->free_compl = NULL;
	cmd->abrt_compl = NULL;
1330
	spin_lock_init(&cmd->t_state_lock);
1331
	INIT_WORK(&cmd->work, NULL);
1332
	kref_init(&cmd->cmd_kref);
1333 1334 1335 1336 1337 1338 1339

	cmd->se_tfo = tfo;
	cmd->se_sess = se_sess;
	cmd->data_length = data_length;
	cmd->data_direction = data_direction;
	cmd->sam_task_attr = task_attr;
	cmd->sense_buffer = sense_buffer;
1340 1341

	cmd->state_active = false;
1342 1343 1344
}
EXPORT_SYMBOL(transport_init_se_cmd);

1345 1346
static sense_reason_t
transport_check_alloc_task_attr(struct se_cmd *cmd)
1347
{
1348 1349
	struct se_device *dev = cmd->se_dev;

1350 1351 1352 1353
	/*
	 * Check if SAM Task Attribute emulation is enabled for this
	 * struct se_device storage object
	 */
1354
	if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
1355 1356
		return 0;

C
Christoph Hellwig 已提交
1357
	if (cmd->sam_task_attr == TCM_ACA_TAG) {
1358
		pr_debug("SAM Task Attribute ACA"
1359
			" emulation is not supported\n");
1360
		return TCM_INVALID_CDB_FIELD;
1361
	}
1362

1363 1364 1365
	return 0;
}

1366 1367
sense_reason_t
target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb)
1368
{
1369
	struct se_device *dev = cmd->se_dev;
1370
	sense_reason_t ret;
1371 1372 1373 1374 1375 1376

	/*
	 * Ensure that the received CDB is less than the max (252 + 8) bytes
	 * for VARIABLE_LENGTH_CMD
	 */
	if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) {
1377
		pr_err("Received SCSI CDB with command_size: %d that"
1378 1379
			" exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
			scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE);
1380
		return TCM_INVALID_CDB_FIELD;
1381 1382 1383 1384 1385 1386
	}
	/*
	 * If the received CDB is larger than TCM_MAX_COMMAND_SIZE,
	 * allocate the additional extended CDB buffer now..  Otherwise
	 * setup the pointer from __t_task_cdb to t_task_cdb.
	 */
1387 1388
	if (scsi_command_size(cdb) > sizeof(cmd->__t_task_cdb)) {
		cmd->t_task_cdb = kzalloc(scsi_command_size(cdb),
1389
						GFP_KERNEL);
1390 1391
		if (!cmd->t_task_cdb) {
			pr_err("Unable to allocate cmd->t_task_cdb"
1392
				" %u > sizeof(cmd->__t_task_cdb): %lu ops\n",
1393
				scsi_command_size(cdb),
1394
				(unsigned long)sizeof(cmd->__t_task_cdb));
1395
			return TCM_OUT_OF_RESOURCES;
1396 1397
		}
	} else
1398
		cmd->t_task_cdb = &cmd->__t_task_cdb[0];
1399
	/*
1400
	 * Copy the original CDB into cmd->
1401
	 */
1402
	memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb));
1403

1404 1405
	trace_target_sequencer_start(cmd);

1406
	ret = dev->transport->parse_cdb(cmd);
1407 1408
	if (ret == TCM_UNSUPPORTED_SCSI_OPCODE)
		pr_warn_ratelimited("%s/%s: Unsupported SCSI Opcode 0x%02x, sending CHECK_CONDITION.\n",
1409
				    cmd->se_tfo->fabric_name,
1410 1411
				    cmd->se_sess->se_node_acl->initiatorname,
				    cmd->t_task_cdb[0]);
1412 1413 1414 1415 1416
	if (ret)
		return ret;

	ret = transport_check_alloc_task_attr(cmd);
	if (ret)
1417
		return ret;
1418 1419

	cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE;
1420
	atomic_long_inc(&cmd->se_lun->lun_stats.cmd_pdus);
1421 1422
	return 0;
}
1423
EXPORT_SYMBOL(target_setup_cmd_from_cdb);
1424

1425 1426
/*
 * Used by fabric module frontends to queue tasks directly.
1427
 * May only be used from process context.
1428 1429 1430 1431
 */
int transport_handle_cdb_direct(
	struct se_cmd *cmd)
{
1432
	sense_reason_t ret;
1433

1434 1435
	if (!cmd->se_lun) {
		dump_stack();
1436
		pr_err("cmd->se_lun is NULL\n");
1437 1438 1439 1440
		return -EINVAL;
	}
	if (in_interrupt()) {
		dump_stack();
1441
		pr_err("transport_generic_handle_cdb cannot be called"
1442 1443 1444
				" from interrupt context\n");
		return -EINVAL;
	}
1445
	/*
1446 1447 1448
	 * Set TRANSPORT_NEW_CMD state and CMD_T_ACTIVE to ensure that
	 * outstanding descriptors are handled correctly during shutdown via
	 * transport_wait_for_tasks()
1449 1450 1451 1452 1453
	 *
	 * Also, we don't take cmd->t_state_lock here as we only expect
	 * this to be called for initial descriptor submission.
	 */
	cmd->t_state = TRANSPORT_NEW_CMD;
1454 1455
	cmd->transport_state |= CMD_T_ACTIVE;

1456 1457 1458 1459 1460 1461
	/*
	 * transport_generic_new_cmd() is already handling QUEUE_FULL,
	 * so follow TRANSPORT_NEW_CMD processing thread context usage
	 * and call transport_generic_request_failure() if necessary..
	 */
	ret = transport_generic_new_cmd(cmd);
1462 1463
	if (ret)
		transport_generic_request_failure(cmd, ret);
1464
	return 0;
1465 1466 1467
}
EXPORT_SYMBOL(transport_handle_cdb_direct);

1468
sense_reason_t
1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487
transport_generic_map_mem_to_cmd(struct se_cmd *cmd, struct scatterlist *sgl,
		u32 sgl_count, struct scatterlist *sgl_bidi, u32 sgl_bidi_count)
{
	if (!sgl || !sgl_count)
		return 0;

	/*
	 * Reject SCSI data overflow with map_mem_to_cmd() as incoming
	 * scatterlists already have been set to follow what the fabric
	 * passes for the original expected data transfer length.
	 */
	if (cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
		pr_warn("Rejecting SCSI DATA overflow for fabric using"
			" SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC\n");
		return TCM_INVALID_CDB_FIELD;
	}

	cmd->t_data_sg = sgl;
	cmd->t_data_nents = sgl_count;
1488 1489
	cmd->t_bidi_data_sg = sgl_bidi;
	cmd->t_bidi_data_nents = sgl_bidi_count;
1490 1491 1492 1493 1494

	cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
	return 0;
}

1495
/**
1496 1497
 * target_submit_cmd_map_sgls - lookup unpacked lun and submit uninitialized
 * 			 se_cmd + use pre-allocated SGL memory.
1498 1499 1500 1501 1502 1503 1504
 *
 * @se_cmd: command descriptor to submit
 * @se_sess: associated se_sess for endpoint
 * @cdb: pointer to SCSI CDB
 * @sense: pointer to SCSI sense buffer
 * @unpacked_lun: unpacked LUN to reference for struct se_lun
 * @data_length: fabric expected data transfer length
1505
 * @task_attr: SAM task attribute
1506 1507
 * @data_dir: DMA data direction
 * @flags: flags for command submission from target_sc_flags_tables
1508 1509 1510 1511
 * @sgl: struct scatterlist memory for unidirectional mapping
 * @sgl_count: scatterlist count for unidirectional mapping
 * @sgl_bidi: struct scatterlist memory for bidirectional READ mapping
 * @sgl_bidi_count: scatterlist count for bidirectional READ mapping
1512 1513
 * @sgl_prot: struct scatterlist memory protection information
 * @sgl_prot_count: scatterlist count for protection information
1514
 *
1515 1516
 * Task tags are supported if the caller has set @se_cmd->tag.
 *
1517 1518 1519 1520
 * Returns non zero to signal active I/O shutdown failure.  All other
 * setup exceptions will be returned as a SCSI CHECK_CONDITION response,
 * but still return zero here.
 *
1521 1522
 * This may only be called from process context, and also currently
 * assumes internal allocation of fabric payload buffer by target-core.
1523 1524
 */
int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess,
H
Hannes Reinecke 已提交
1525
		unsigned char *cdb, unsigned char *sense, u64 unpacked_lun,
1526 1527
		u32 data_length, int task_attr, int data_dir, int flags,
		struct scatterlist *sgl, u32 sgl_count,
1528 1529
		struct scatterlist *sgl_bidi, u32 sgl_bidi_count,
		struct scatterlist *sgl_prot, u32 sgl_prot_count)
1530 1531
{
	struct se_portal_group *se_tpg;
1532 1533
	sense_reason_t rc;
	int ret;
1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545

	se_tpg = se_sess->se_tpg;
	BUG_ON(!se_tpg);
	BUG_ON(se_cmd->se_tfo || se_cmd->se_sess);
	BUG_ON(in_interrupt());
	/*
	 * Initialize se_cmd for target operation.  From this point
	 * exceptions are handled by sending exception status via
	 * target_core_fabric_ops->queue_status() callback
	 */
	transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
				data_length, data_dir, task_attr, sense);
1546 1547 1548 1549 1550 1551

	if (flags & TARGET_SCF_USE_CPUID)
		se_cmd->se_cmd_flags |= SCF_USE_CPUID;
	else
		se_cmd->cpuid = WORK_CPU_UNBOUND;

1552 1553
	if (flags & TARGET_SCF_UNKNOWN_SIZE)
		se_cmd->unknown_data_length = 1;
1554 1555 1556 1557 1558 1559
	/*
	 * Obtain struct se_cmd->cmd_kref reference and add new cmd to
	 * se_sess->sess_cmd_list.  A second kref_get here is necessary
	 * for fabrics using TARGET_SCF_ACK_KREF that expect a second
	 * kref_put() to happen during fabric packet acknowledgement.
	 */
1560
	ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF);
1561 1562
	if (ret)
		return ret;
1563 1564 1565 1566 1567 1568 1569 1570
	/*
	 * Signal bidirectional data payloads to target-core
	 */
	if (flags & TARGET_SCF_BIDI_OP)
		se_cmd->se_cmd_flags |= SCF_BIDI;
	/*
	 * Locate se_lun pointer and attach it to struct se_cmd
	 */
1571 1572 1573
	rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun);
	if (rc) {
		transport_send_check_condition_and_sense(se_cmd, rc, 0);
1574
		target_put_sess_cmd(se_cmd);
1575
		return 0;
1576
	}
1577 1578 1579 1580 1581 1582 1583

	rc = target_setup_cmd_from_cdb(se_cmd, cdb);
	if (rc != 0) {
		transport_generic_request_failure(se_cmd, rc);
		return 0;
	}

1584 1585 1586 1587 1588 1589 1590
	/*
	 * Save pointers for SGLs containing protection information,
	 * if present.
	 */
	if (sgl_prot_count) {
		se_cmd->t_prot_sg = sgl_prot;
		se_cmd->t_prot_nents = sgl_prot_count;
1591
		se_cmd->se_cmd_flags |= SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC;
1592
	}
1593

1594 1595 1596 1597 1598 1599 1600 1601
	/*
	 * When a non zero sgl_count has been passed perform SGL passthrough
	 * mapping for pre-allocated fabric memory instead of having target
	 * core perform an internal SGL allocation..
	 */
	if (sgl_count != 0) {
		BUG_ON(!sgl);

1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622
		/*
		 * A work-around for tcm_loop as some userspace code via
		 * scsi-generic do not memset their associated read buffers,
		 * so go ahead and do that here for type non-data CDBs.  Also
		 * note that this is currently guaranteed to be a single SGL
		 * for this case by target core in target_setup_cmd_from_cdb()
		 * -> transport_generic_cmd_sequencer().
		 */
		if (!(se_cmd->se_cmd_flags & SCF_SCSI_DATA_CDB) &&
		     se_cmd->data_direction == DMA_FROM_DEVICE) {
			unsigned char *buf = NULL;

			if (sgl)
				buf = kmap(sg_page(sgl)) + sgl->offset;

			if (buf) {
				memset(buf, 0, sgl->length);
				kunmap(sg_page(sgl));
			}
		}

1623 1624 1625
		rc = transport_generic_map_mem_to_cmd(se_cmd, sgl, sgl_count,
				sgl_bidi, sgl_bidi_count);
		if (rc != 0) {
1626
			transport_generic_request_failure(se_cmd, rc);
1627 1628 1629
			return 0;
		}
	}
1630

1631 1632 1633 1634 1635 1636
	/*
	 * Check if we need to delay processing because of ALUA
	 * Active/NonOptimized primary access state..
	 */
	core_alua_check_nonop_delay(se_cmd);

1637
	transport_handle_cdb_direct(se_cmd);
1638
	return 0;
1639
}
1640 1641
EXPORT_SYMBOL(target_submit_cmd_map_sgls);

1642
/**
1643 1644 1645 1646 1647 1648 1649 1650
 * target_submit_cmd - lookup unpacked lun and submit uninitialized se_cmd
 *
 * @se_cmd: command descriptor to submit
 * @se_sess: associated se_sess for endpoint
 * @cdb: pointer to SCSI CDB
 * @sense: pointer to SCSI sense buffer
 * @unpacked_lun: unpacked LUN to reference for struct se_lun
 * @data_length: fabric expected data transfer length
1651
 * @task_attr: SAM task attribute
1652 1653 1654
 * @data_dir: DMA data direction
 * @flags: flags for command submission from target_sc_flags_tables
 *
1655 1656
 * Task tags are supported if the caller has set @se_cmd->tag.
 *
1657 1658 1659 1660 1661 1662 1663 1664 1665 1666
 * Returns non zero to signal active I/O shutdown failure.  All other
 * setup exceptions will be returned as a SCSI CHECK_CONDITION response,
 * but still return zero here.
 *
 * This may only be called from process context, and also currently
 * assumes internal allocation of fabric payload buffer by target-core.
 *
 * It also assumes interal target core SGL memory allocation.
 */
int target_submit_cmd(struct se_cmd *se_cmd, struct se_session *se_sess,
H
Hannes Reinecke 已提交
1667
		unsigned char *cdb, unsigned char *sense, u64 unpacked_lun,
1668 1669 1670 1671
		u32 data_length, int task_attr, int data_dir, int flags)
{
	return target_submit_cmd_map_sgls(se_cmd, se_sess, cdb, sense,
			unpacked_lun, data_length, task_attr, data_dir,
1672
			flags, NULL, 0, NULL, 0, NULL, 0);
1673
}
1674 1675
EXPORT_SYMBOL(target_submit_cmd);

1676 1677 1678 1679 1680 1681
static void target_complete_tmr_failure(struct work_struct *work)
{
	struct se_cmd *se_cmd = container_of(work, struct se_cmd, work);

	se_cmd->se_tmr_req->response = TMR_LUN_DOES_NOT_EXIST;
	se_cmd->se_tfo->queue_tm_rsp(se_cmd);
1682 1683

	transport_cmd_check_stop_to_fabric(se_cmd);
1684 1685
}

1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708
static bool target_lookup_lun_from_tag(struct se_session *se_sess, u64 tag,
				       u64 *unpacked_lun)
{
	struct se_cmd *se_cmd;
	unsigned long flags;
	bool ret = false;

	spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
	list_for_each_entry(se_cmd, &se_sess->sess_cmd_list, se_cmd_list) {
		if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
			continue;

		if (se_cmd->tag == tag) {
			*unpacked_lun = se_cmd->orig_fe_lun;
			ret = true;
			break;
		}
	}
	spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);

	return ret;
}

1709 1710 1711 1712 1713 1714 1715 1716
/**
 * target_submit_tmr - lookup unpacked lun and submit uninitialized se_cmd
 *                     for TMR CDBs
 *
 * @se_cmd: command descriptor to submit
 * @se_sess: associated se_sess for endpoint
 * @sense: pointer to SCSI sense buffer
 * @unpacked_lun: unpacked LUN to reference for struct se_lun
1717
 * @fabric_tmr_ptr: fabric context for TMR req
1718
 * @tm_type: Type of TM request
1719 1720
 * @gfp: gfp type for caller
 * @tag: referenced task tag for TMR_ABORT_TASK
1721
 * @flags: submit cmd flags
1722 1723 1724 1725
 *
 * Callable from all contexts.
 **/

1726
int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess,
H
Hannes Reinecke 已提交
1727
		unsigned char *sense, u64 unpacked_lun,
1728
		void *fabric_tmr_ptr, unsigned char tm_type,
1729
		gfp_t gfp, u64 tag, int flags)
1730 1731 1732 1733 1734 1735 1736 1737
{
	struct se_portal_group *se_tpg;
	int ret;

	se_tpg = se_sess->se_tpg;
	BUG_ON(!se_tpg);

	transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
C
Christoph Hellwig 已提交
1738
			      0, DMA_NONE, TCM_SIMPLE_TAG, sense);
1739 1740 1741 1742
	/*
	 * FIXME: Currently expect caller to handle se_cmd->se_tmr_req
	 * allocation failure.
	 */
1743
	ret = core_tmr_alloc_req(se_cmd, fabric_tmr_ptr, tm_type, gfp);
1744 1745
	if (ret < 0)
		return -ENOMEM;
1746

1747 1748 1749
	if (tm_type == TMR_ABORT_TASK)
		se_cmd->se_tmr_req->ref_task_tag = tag;

1750
	/* See target_submit_cmd for commentary */
1751
	ret = target_get_sess_cmd(se_cmd, flags & TARGET_SCF_ACK_KREF);
1752 1753 1754 1755
	if (ret) {
		core_tmr_release_req(se_cmd->se_tmr_req);
		return ret;
	}
1756 1757 1758 1759 1760 1761 1762 1763 1764
	/*
	 * If this is ABORT_TASK with no explicit fabric provided LUN,
	 * go ahead and search active session tags for a match to figure
	 * out unpacked_lun for the original se_cmd.
	 */
	if (tm_type == TMR_ABORT_TASK && (flags & TARGET_SCF_LOOKUP_LUN_FROM_TAG)) {
		if (!target_lookup_lun_from_tag(se_sess, tag, &unpacked_lun))
			goto failure;
	}
1765 1766

	ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun);
1767 1768 1769
	if (ret)
		goto failure;

1770
	transport_generic_handle_tmr(se_cmd);
1771
	return 0;
1772 1773 1774 1775 1776 1777 1778 1779 1780

	/*
	 * For callback during failure handling, push this work off
	 * to process context with TMR_LUN_DOES_NOT_EXIST status.
	 */
failure:
	INIT_WORK(&se_cmd->work, target_complete_tmr_failure);
	schedule_work(&se_cmd->work);
	return 0;
1781 1782 1783
}
EXPORT_SYMBOL(target_submit_tmr);

1784 1785 1786
/*
 * Handle SAM-esque emulation for generic transport request failures.
 */
1787 1788
void transport_generic_request_failure(struct se_cmd *cmd,
		sense_reason_t sense_reason)
1789
{
1790
	int ret = 0, post_ret;
1791

1792 1793 1794
	pr_debug("-----[ Storage Engine Exception; sense_reason %d\n",
		 sense_reason);
	target_show_cmd("-----[ ", cmd);
1795 1796 1797 1798

	/*
	 * For SAM Task Attribute emulation for failed struct se_cmd
	 */
1799
	transport_complete_task_attr(cmd);
1800

1801
	if (cmd->transport_complete_callback)
1802
		cmd->transport_complete_callback(cmd, false, &post_ret);
1803

1804 1805 1806
	if (cmd->transport_state & CMD_T_ABORTED) {
		INIT_WORK(&cmd->work, target_abort_work);
		queue_work(target_completion_wq, &cmd->work);
1807
		return;
1808
	}
1809

1810
	switch (sense_reason) {
1811 1812 1813 1814
	case TCM_NON_EXISTENT_LUN:
	case TCM_UNSUPPORTED_SCSI_OPCODE:
	case TCM_INVALID_CDB_FIELD:
	case TCM_INVALID_PARAMETER_LIST:
1815
	case TCM_PARAMETER_LIST_LENGTH_ERROR:
1816 1817 1818
	case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
	case TCM_UNKNOWN_MODE_PAGE:
	case TCM_WRITE_PROTECTED:
1819
	case TCM_ADDRESS_OUT_OF_RANGE:
1820 1821 1822
	case TCM_CHECK_CONDITION_ABORT_CMD:
	case TCM_CHECK_CONDITION_UNIT_ATTENTION:
	case TCM_CHECK_CONDITION_NOT_READY:
1823 1824 1825
	case TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED:
	case TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED:
	case TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED:
1826
	case TCM_COPY_TARGET_DEVICE_NOT_REACHABLE:
1827 1828 1829 1830
	case TCM_TOO_MANY_TARGET_DESCS:
	case TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE:
	case TCM_TOO_MANY_SEGMENT_DESCS:
	case TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE:
1831
		break;
1832
	case TCM_OUT_OF_RESOURCES:
1833 1834
		cmd->scsi_status = SAM_STAT_TASK_SET_FULL;
		goto queue_status;
1835 1836 1837
	case TCM_LUN_BUSY:
		cmd->scsi_status = SAM_STAT_BUSY;
		goto queue_status;
1838
	case TCM_RESERVATION_CONFLICT:
1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852
		/*
		 * No SENSE Data payload for this case, set SCSI Status
		 * and queue the response to $FABRIC_MOD.
		 *
		 * Uses linux/include/scsi/scsi.h SAM status codes defs
		 */
		cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
		/*
		 * For UA Interlock Code 11b, a RESERVATION CONFLICT will
		 * establish a UNIT ATTENTION with PREVIOUS RESERVATION
		 * CONFLICT STATUS.
		 *
		 * See spc4r17, section 7.4.6 Control Mode Page, Table 349
		 */
1853
		if (cmd->se_sess &&
1854 1855 1856 1857 1858
		    cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl == 2) {
			target_ua_allocate_lun(cmd->se_sess->se_node_acl,
					       cmd->orig_fe_lun, 0x2C,
					ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);
		}
1859 1860

		goto queue_status;
1861
	default:
1862
		pr_err("Unknown transport error for CDB 0x%02x: %d\n",
1863 1864
			cmd->t_task_cdb[0], sense_reason);
		sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
1865 1866
		break;
	}
1867

1868
	ret = transport_send_check_condition_and_sense(cmd, sense_reason, 0);
1869
	if (ret)
1870
		goto queue_full;
1871

1872
check_stop:
A
Andy Grover 已提交
1873
	transport_cmd_check_stop_to_fabric(cmd);
1874 1875
	return;

1876 1877 1878 1879 1880
queue_status:
	trace_target_cmd_complete(cmd);
	ret = cmd->se_tfo->queue_status(cmd);
	if (!ret)
		goto check_stop;
1881
queue_full:
1882
	transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
1883
}
1884
EXPORT_SYMBOL(transport_generic_request_failure);
1885

1886
void __target_execute_cmd(struct se_cmd *cmd, bool do_checks)
1887
{
1888
	sense_reason_t ret;
1889

1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907
	if (!cmd->execute_cmd) {
		ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
		goto err;
	}
	if (do_checks) {
		/*
		 * Check for an existing UNIT ATTENTION condition after
		 * target_handle_task_attr() has done SAM task attr
		 * checking, and possibly have already defered execution
		 * out to target_restart_delayed_cmds() context.
		 */
		ret = target_scsi3_ua_check(cmd);
		if (ret)
			goto err;

		ret = target_alua_state_check(cmd);
		if (ret)
			goto err;
1908

1909 1910 1911 1912
		ret = target_check_reservation(cmd);
		if (ret) {
			cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
			goto err;
1913
		}
1914
	}
1915 1916 1917 1918 1919 1920

	ret = cmd->execute_cmd(cmd);
	if (!ret)
		return;
err:
	spin_lock_irq(&cmd->t_state_lock);
1921
	cmd->transport_state &= ~CMD_T_SENT;
1922 1923 1924
	spin_unlock_irq(&cmd->t_state_lock);

	transport_generic_request_failure(cmd, ret);
1925 1926
}

1927 1928
static int target_write_prot_action(struct se_cmd *cmd)
{
1929
	u32 sectors;
1930 1931 1932 1933 1934 1935 1936 1937 1938 1939
	/*
	 * Perform WRITE_INSERT of PI using software emulation when backend
	 * device has PI enabled, if the transport has not already generated
	 * PI using hardware WRITE_INSERT offload.
	 */
	switch (cmd->prot_op) {
	case TARGET_PROT_DOUT_INSERT:
		if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_INSERT))
			sbc_dif_generate(cmd);
		break;
1940 1941 1942 1943 1944
	case TARGET_PROT_DOUT_STRIP:
		if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DOUT_STRIP)
			break;

		sectors = cmd->data_length >> ilog2(cmd->se_dev->dev_attrib.block_size);
1945 1946
		cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba,
					     sectors, 0, cmd->t_prot_sg, 0);
1947 1948
		if (unlikely(cmd->pi_err)) {
			spin_lock_irq(&cmd->t_state_lock);
1949
			cmd->transport_state &= ~CMD_T_SENT;
1950 1951 1952 1953 1954
			spin_unlock_irq(&cmd->t_state_lock);
			transport_generic_request_failure(cmd, cmd->pi_err);
			return -1;
		}
		break;
1955 1956 1957 1958 1959 1960 1961
	default:
		break;
	}

	return 0;
}

1962
static bool target_handle_task_attr(struct se_cmd *cmd)
1963 1964 1965
{
	struct se_device *dev = cmd->se_dev;

1966
	if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
1967
		return false;
1968

1969 1970
	cmd->se_cmd_flags |= SCF_TASK_ATTR_SET;

1971
	/*
L
Lucas De Marchi 已提交
1972
	 * Check for the existence of HEAD_OF_QUEUE, and if true return 1
1973 1974
	 * to allow the passed struct se_cmd list of tasks to the front of the list.
	 */
1975
	switch (cmd->sam_task_attr) {
C
Christoph Hellwig 已提交
1976
	case TCM_HEAD_TAG:
1977 1978
		pr_debug("Added HEAD_OF_QUEUE for CDB: 0x%02x\n",
			 cmd->t_task_cdb[0]);
1979
		return false;
C
Christoph Hellwig 已提交
1980
	case TCM_ORDERED_TAG:
1981
		atomic_inc_mb(&dev->dev_ordered_sync);
1982

1983 1984
		pr_debug("Added ORDERED for CDB: 0x%02x to ordered list\n",
			 cmd->t_task_cdb[0]);
1985

1986
		/*
1987 1988
		 * Execute an ORDERED command if no other older commands
		 * exist that need to be completed first.
1989
		 */
1990
		if (!atomic_read(&dev->simple_cmds))
1991
			return false;
1992 1993
		break;
	default:
1994 1995 1996
		/*
		 * For SIMPLE and UNTAGGED Task Attribute commands
		 */
1997
		atomic_inc_mb(&dev->simple_cmds);
1998
		break;
1999
	}
2000

2001 2002
	if (atomic_read(&dev->dev_ordered_sync) == 0)
		return false;
2003

2004 2005 2006 2007
	spin_lock(&dev->delayed_cmd_lock);
	list_add_tail(&cmd->se_delayed_node, &dev->delayed_cmd_list);
	spin_unlock(&dev->delayed_cmd_lock);

2008 2009
	pr_debug("Added CDB: 0x%02x Task Attr: 0x%02x to delayed CMD listn",
		cmd->t_task_cdb[0], cmd->sam_task_attr);
2010 2011 2012 2013 2014 2015 2016 2017
	return true;
}

void target_execute_cmd(struct se_cmd *cmd)
{
	/*
	 * Determine if frontend context caller is requesting the stopping of
	 * this command for frontend exceptions.
2018
	 *
2019
	 * If the received CDB has already been aborted stop processing it here.
2020
	 */
2021
	if (target_cmd_interrupted(cmd))
2022 2023
		return;

2024
	spin_lock_irq(&cmd->t_state_lock);
2025
	cmd->t_state = TRANSPORT_PROCESSING;
2026
	cmd->transport_state |= CMD_T_ACTIVE | CMD_T_SENT;
2027
	spin_unlock_irq(&cmd->t_state_lock);
2028 2029 2030

	if (target_write_prot_action(cmd))
		return;
2031

2032 2033
	if (target_handle_task_attr(cmd)) {
		spin_lock_irq(&cmd->t_state_lock);
2034
		cmd->transport_state &= ~CMD_T_SENT;
2035 2036 2037 2038
		spin_unlock_irq(&cmd->t_state_lock);
		return;
	}

2039
	__target_execute_cmd(cmd, true);
2040
}
2041
EXPORT_SYMBOL(target_execute_cmd);
2042

2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062
/*
 * Process all commands up to the last received ORDERED task attribute which
 * requires another blocking boundary
 */
static void target_restart_delayed_cmds(struct se_device *dev)
{
	for (;;) {
		struct se_cmd *cmd;

		spin_lock(&dev->delayed_cmd_lock);
		if (list_empty(&dev->delayed_cmd_list)) {
			spin_unlock(&dev->delayed_cmd_lock);
			break;
		}

		cmd = list_entry(dev->delayed_cmd_list.next,
				 struct se_cmd, se_delayed_node);
		list_del(&cmd->se_delayed_node);
		spin_unlock(&dev->delayed_cmd_lock);

2063 2064
		cmd->transport_state |= CMD_T_SENT;

2065
		__target_execute_cmd(cmd, true);
2066

C
Christoph Hellwig 已提交
2067
		if (cmd->sam_task_attr == TCM_ORDERED_TAG)
2068 2069 2070 2071
			break;
	}
}

2072
/*
2073
 * Called from I/O completion to determine which dormant/delayed
2074 2075 2076 2077
 * and ordered cmds need to have their tasks added to the execution queue.
 */
static void transport_complete_task_attr(struct se_cmd *cmd)
{
2078
	struct se_device *dev = cmd->se_dev;
2079

2080
	if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
2081 2082
		return;

2083 2084 2085
	if (!(cmd->se_cmd_flags & SCF_TASK_ATTR_SET))
		goto restart;

C
Christoph Hellwig 已提交
2086
	if (cmd->sam_task_attr == TCM_SIMPLE_TAG) {
2087
		atomic_dec_mb(&dev->simple_cmds);
2088
		dev->dev_cur_ordered_id++;
C
Christoph Hellwig 已提交
2089
	} else if (cmd->sam_task_attr == TCM_HEAD_TAG) {
2090
		dev->dev_cur_ordered_id++;
2091 2092
		pr_debug("Incremented dev_cur_ordered_id: %u for HEAD_OF_QUEUE\n",
			 dev->dev_cur_ordered_id);
C
Christoph Hellwig 已提交
2093
	} else if (cmd->sam_task_attr == TCM_ORDERED_TAG) {
2094
		atomic_dec_mb(&dev->dev_ordered_sync);
2095 2096

		dev->dev_cur_ordered_id++;
2097 2098
		pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED\n",
			 dev->dev_cur_ordered_id);
2099
	}
2100 2101
	cmd->se_cmd_flags &= ~SCF_TASK_ATTR_SET;

2102
restart:
2103
	target_restart_delayed_cmds(dev);
2104 2105
}

2106
static void transport_complete_qf(struct se_cmd *cmd)
2107 2108 2109
{
	int ret = 0;

2110
	transport_complete_task_attr(cmd);
2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122
	/*
	 * If a fabric driver ->write_pending() or ->queue_data_in() callback
	 * has returned neither -ENOMEM or -EAGAIN, assume it's fatal and
	 * the same callbacks should not be retried.  Return CHECK_CONDITION
	 * if a scsi_status is not already set.
	 *
	 * If a fabric driver ->queue_status() has returned non zero, always
	 * keep retrying no matter what..
	 */
	if (cmd->t_state == TRANSPORT_COMPLETE_QF_ERR) {
		if (cmd->scsi_status)
			goto queue_status;
2123

2124 2125
		translate_sense_reason(cmd, TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE);
		goto queue_status;
2126
	}
2127

2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138
	/*
	 * Check if we need to send a sense buffer from
	 * the struct se_cmd in question. We do NOT want
	 * to take this path of the IO has been marked as
	 * needing to be treated like a "normal read". This
	 * is the case if it's a tape read, and either the
	 * FM, EOM, or ILI bits are set, but there is no
	 * sense data.
	 */
	if (!(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) &&
	    cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE)
2139 2140
		goto queue_status;

2141 2142
	switch (cmd->data_direction) {
	case DMA_FROM_DEVICE:
2143 2144 2145
		/* queue status if not treating this as a normal read */
		if (cmd->scsi_status &&
		    !(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL))
2146 2147
			goto queue_status;

2148
		trace_target_cmd_complete(cmd);
2149 2150 2151
		ret = cmd->se_tfo->queue_data_in(cmd);
		break;
	case DMA_TO_DEVICE:
2152
		if (cmd->se_cmd_flags & SCF_BIDI) {
2153
			ret = cmd->se_tfo->queue_data_in(cmd);
2154
			break;
2155
		}
2156
		/* fall through */
2157
	case DMA_NONE:
2158
queue_status:
2159
		trace_target_cmd_complete(cmd);
2160 2161 2162 2163 2164 2165
		ret = cmd->se_tfo->queue_status(cmd);
		break;
	default:
		break;
	}

2166
	if (ret < 0) {
2167
		transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
2168 2169 2170
		return;
	}
	transport_cmd_check_stop_to_fabric(cmd);
2171 2172
}

2173 2174
static void transport_handle_queue_full(struct se_cmd *cmd, struct se_device *dev,
					int err, bool write_pending)
2175
{
2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191
	/*
	 * -EAGAIN or -ENOMEM signals retry of ->write_pending() and/or
	 * ->queue_data_in() callbacks from new process context.
	 *
	 * Otherwise for other errors, transport_complete_qf() will send
	 * CHECK_CONDITION via ->queue_status() instead of attempting to
	 * retry associated fabric driver data-transfer callbacks.
	 */
	if (err == -EAGAIN || err == -ENOMEM) {
		cmd->t_state = (write_pending) ? TRANSPORT_COMPLETE_QF_WP :
						 TRANSPORT_COMPLETE_QF_OK;
	} else {
		pr_warn_ratelimited("Got unknown fabric queue status: %d\n", err);
		cmd->t_state = TRANSPORT_COMPLETE_QF_ERR;
	}

2192 2193
	spin_lock_irq(&dev->qf_cmd_lock);
	list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list);
2194
	atomic_inc_mb(&dev->dev_qf_count);
2195 2196 2197 2198 2199
	spin_unlock_irq(&cmd->se_dev->qf_cmd_lock);

	schedule_work(&cmd->se_dev->qf_work_queue);
}

2200
static bool target_read_prot_action(struct se_cmd *cmd)
2201
{
2202 2203 2204
	switch (cmd->prot_op) {
	case TARGET_PROT_DIN_STRIP:
		if (!(cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_STRIP)) {
2205 2206 2207 2208 2209 2210 2211
			u32 sectors = cmd->data_length >>
				  ilog2(cmd->se_dev->dev_attrib.block_size);

			cmd->pi_err = sbc_dif_verify(cmd, cmd->t_task_lba,
						     sectors, 0, cmd->t_prot_sg,
						     0);
			if (cmd->pi_err)
2212
				return true;
2213
		}
2214
		break;
2215 2216 2217 2218 2219 2220
	case TARGET_PROT_DIN_INSERT:
		if (cmd->se_sess->sup_prot_ops & TARGET_PROT_DIN_INSERT)
			break;

		sbc_dif_generate(cmd);
		break;
2221 2222
	default:
		break;
2223 2224 2225 2226 2227
	}

	return false;
}

2228
static void target_complete_ok_work(struct work_struct *work)
2229
{
2230
	struct se_cmd *cmd = container_of(work, struct se_cmd, work);
2231
	int ret;
2232

2233 2234 2235 2236 2237
	/*
	 * Check if we need to move delayed/dormant tasks from cmds on the
	 * delayed execution list after a HEAD_OF_QUEUE or ORDERED Task
	 * Attribute.
	 */
2238 2239
	transport_complete_task_attr(cmd);

2240 2241 2242 2243 2244 2245 2246
	/*
	 * Check to schedule QUEUE_FULL work, or execute an existing
	 * cmd->transport_qf_callback()
	 */
	if (atomic_read(&cmd->se_dev->dev_qf_count) != 0)
		schedule_work(&cmd->se_dev->qf_work_queue);

2247
	/*
2248
	 * Check if we need to send a sense buffer from
2249 2250 2251 2252 2253 2254
	 * the struct se_cmd in question. We do NOT want
	 * to take this path of the IO has been marked as
	 * needing to be treated like a "normal read". This
	 * is the case if it's a tape read, and either the
	 * FM, EOM, or ILI bits are set, but there is no
	 * sense data.
2255
	 */
2256 2257
	if (!(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL) &&
	    cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
2258 2259 2260
		WARN_ON(!cmd->scsi_status);
		ret = transport_send_check_condition_and_sense(
					cmd, 0, 1);
2261
		if (ret)
2262 2263 2264 2265
			goto queue_full;

		transport_cmd_check_stop_to_fabric(cmd);
		return;
2266 2267
	}
	/*
L
Lucas De Marchi 已提交
2268
	 * Check for a callback, used by amongst other things
2269
	 * XDWRITE_READ_10 and COMPARE_AND_WRITE emulation.
2270
	 */
2271 2272
	if (cmd->transport_complete_callback) {
		sense_reason_t rc;
2273 2274 2275
		bool caw = (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE);
		bool zero_dl = !(cmd->data_length);
		int post_ret = 0;
2276

2277 2278 2279
		rc = cmd->transport_complete_callback(cmd, true, &post_ret);
		if (!rc && !post_ret) {
			if (caw && zero_dl)
2280 2281
				goto queue_rsp;

2282
			return;
2283 2284 2285
		} else if (rc) {
			ret = transport_send_check_condition_and_sense(cmd,
						rc, 0);
2286
			if (ret)
2287
				goto queue_full;
2288

2289 2290 2291
			transport_cmd_check_stop_to_fabric(cmd);
			return;
		}
2292
	}
2293

2294
queue_rsp:
2295 2296
	switch (cmd->data_direction) {
	case DMA_FROM_DEVICE:
2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308
		/*
		 * if this is a READ-type IO, but SCSI status
		 * is set, then skip returning data and just
		 * return the status -- unless this IO is marked
		 * as needing to be treated as a normal read,
		 * in which case we want to go ahead and return
		 * the data. This happens, for example, for tape
		 * reads with the FM, EOM, or ILI bits set, with
		 * no sense data.
		 */
		if (cmd->scsi_status &&
		    !(cmd->se_cmd_flags & SCF_TREAT_READ_AS_NORMAL))
2309 2310
			goto queue_status;

2311 2312
		atomic_long_add(cmd->data_length,
				&cmd->se_lun->lun_stats.tx_data_octets);
2313 2314 2315 2316 2317
		/*
		 * Perform READ_STRIP of PI using software emulation when
		 * backend had PI enabled, if the transport will not be
		 * performing hardware READ_STRIP offload.
		 */
2318
		if (target_read_prot_action(cmd)) {
2319 2320
			ret = transport_send_check_condition_and_sense(cmd,
						cmd->pi_err, 0);
2321
			if (ret)
2322 2323 2324 2325 2326
				goto queue_full;

			transport_cmd_check_stop_to_fabric(cmd);
			return;
		}
2327

2328
		trace_target_cmd_complete(cmd);
2329
		ret = cmd->se_tfo->queue_data_in(cmd);
2330
		if (ret)
2331
			goto queue_full;
2332 2333
		break;
	case DMA_TO_DEVICE:
2334 2335
		atomic_long_add(cmd->data_length,
				&cmd->se_lun->lun_stats.rx_data_octets);
2336 2337 2338
		/*
		 * Check if we need to send READ payload for BIDI-COMMAND
		 */
2339
		if (cmd->se_cmd_flags & SCF_BIDI) {
2340 2341
			atomic_long_add(cmd->data_length,
					&cmd->se_lun->lun_stats.tx_data_octets);
2342
			ret = cmd->se_tfo->queue_data_in(cmd);
2343
			if (ret)
2344
				goto queue_full;
2345 2346
			break;
		}
2347
		/* fall through */
2348
	case DMA_NONE:
2349
queue_status:
2350
		trace_target_cmd_complete(cmd);
2351
		ret = cmd->se_tfo->queue_status(cmd);
2352
		if (ret)
2353
			goto queue_full;
2354 2355 2356 2357 2358 2359
		break;
	default:
		break;
	}

	transport_cmd_check_stop_to_fabric(cmd);
2360 2361 2362
	return;

queue_full:
2363
	pr_debug("Handling complete_ok QUEUE_FULL: se_cmd: %p,"
2364
		" data_direction: %d\n", cmd, cmd->data_direction);
2365 2366

	transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
2367 2368
}

2369
void target_free_sgl(struct scatterlist *sgl, int nents)
2370
{
2371
	sgl_free_n_order(sgl, nents, 0);
2372
}
2373
EXPORT_SYMBOL(target_free_sgl);
2374

2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390
static inline void transport_reset_sgl_orig(struct se_cmd *cmd)
{
	/*
	 * Check for saved t_data_sg that may be used for COMPARE_AND_WRITE
	 * emulation, and free + reset pointers if necessary..
	 */
	if (!cmd->t_data_sg_orig)
		return;

	kfree(cmd->t_data_sg);
	cmd->t_data_sg = cmd->t_data_sg_orig;
	cmd->t_data_sg_orig = NULL;
	cmd->t_data_nents = cmd->t_data_nents_orig;
	cmd->t_data_nents_orig = 0;
}

2391 2392
static inline void transport_free_pages(struct se_cmd *cmd)
{
2393
	if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) {
2394
		target_free_sgl(cmd->t_prot_sg, cmd->t_prot_nents);
2395 2396 2397 2398
		cmd->t_prot_sg = NULL;
		cmd->t_prot_nents = 0;
	}

2399
	if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
2400 2401 2402 2403 2404
		/*
		 * Release special case READ buffer payload required for
		 * SG_TO_MEM_NOALLOC to function with COMPARE_AND_WRITE
		 */
		if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) {
2405
			target_free_sgl(cmd->t_bidi_data_sg,
2406 2407 2408 2409
					   cmd->t_bidi_data_nents);
			cmd->t_bidi_data_sg = NULL;
			cmd->t_bidi_data_nents = 0;
		}
2410
		transport_reset_sgl_orig(cmd);
2411
		return;
2412 2413
	}
	transport_reset_sgl_orig(cmd);
2414

2415
	target_free_sgl(cmd->t_data_sg, cmd->t_data_nents);
2416 2417
	cmd->t_data_sg = NULL;
	cmd->t_data_nents = 0;
2418

2419
	target_free_sgl(cmd->t_bidi_data_sg, cmd->t_bidi_data_nents);
2420 2421
	cmd->t_bidi_data_sg = NULL;
	cmd->t_bidi_data_nents = 0;
2422 2423
}

2424
void *transport_kmap_data_sg(struct se_cmd *cmd)
2425
{
2426
	struct scatterlist *sg = cmd->t_data_sg;
2427 2428
	struct page **pages;
	int i;
2429 2430

	/*
2431 2432 2433
	 * We need to take into account a possible offset here for fabrics like
	 * tcm_loop who may be using a contig buffer from the SCSI midlayer for
	 * control CDBs passed as SGLs via transport_generic_map_mem_to_cmd()
2434
	 */
2435 2436
	if (!cmd->t_data_nents)
		return NULL;
2437 2438 2439

	BUG_ON(!sg);
	if (cmd->t_data_nents == 1)
2440 2441 2442
		return kmap(sg_page(sg)) + sg->offset;

	/* >1 page. use vmap */
2443
	pages = kmalloc_array(cmd->t_data_nents, sizeof(*pages), GFP_KERNEL);
2444
	if (!pages)
2445 2446 2447 2448 2449 2450 2451 2452 2453
		return NULL;

	/* convert sg[] to pages[] */
	for_each_sg(cmd->t_data_sg, sg, cmd->t_data_nents, i) {
		pages[i] = sg_page(sg);
	}

	cmd->t_data_vmap = vmap(pages, cmd->t_data_nents,  VM_MAP, PAGE_KERNEL);
	kfree(pages);
2454
	if (!cmd->t_data_vmap)
2455 2456 2457
		return NULL;

	return cmd->t_data_vmap + cmd->t_data_sg[0].offset;
2458
}
2459
EXPORT_SYMBOL(transport_kmap_data_sg);
2460

2461
void transport_kunmap_data_sg(struct se_cmd *cmd)
2462
{
2463
	if (!cmd->t_data_nents) {
2464
		return;
2465
	} else if (cmd->t_data_nents == 1) {
2466
		kunmap(sg_page(cmd->t_data_sg));
2467 2468
		return;
	}
2469 2470 2471

	vunmap(cmd->t_data_vmap);
	cmd->t_data_vmap = NULL;
2472
}
2473
EXPORT_SYMBOL(transport_kunmap_data_sg);
2474

2475
int
2476
target_alloc_sgl(struct scatterlist **sgl, unsigned int *nents, u32 length,
2477
		 bool zero_page, bool chainable)
2478
{
2479
	gfp_t gfp = GFP_KERNEL | (zero_page ? __GFP_ZERO : 0);
2480

2481 2482
	*sgl = sgl_alloc_order(length, 0, chainable, gfp, nents);
	return *sgl ? 0 : -ENOMEM;
2483
}
2484
EXPORT_SYMBOL(target_alloc_sgl);
2485

2486
/*
2487 2488 2489
 * Allocate any required resources to execute the command.  For writes we
 * might not have the payload yet, so notify the fabric via a call to
 * ->write_pending instead. Otherwise place it on the execution queue.
2490
 */
2491 2492
sense_reason_t
transport_generic_new_cmd(struct se_cmd *cmd)
2493
{
2494
	unsigned long flags;
2495
	int ret = 0;
2496
	bool zero_flag = !(cmd->se_cmd_flags & SCF_SCSI_DATA_CDB);
2497

2498 2499 2500
	if (cmd->prot_op != TARGET_PROT_NORMAL &&
	    !(cmd->se_cmd_flags & SCF_PASSTHROUGH_PROT_SG_TO_MEM_NOALLOC)) {
		ret = target_alloc_sgl(&cmd->t_prot_sg, &cmd->t_prot_nents,
2501
				       cmd->prot_length, true, false);
2502 2503 2504 2505
		if (ret < 0)
			return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
	}

2506
	/*
2507
	 * Determine if the TCM fabric module has already allocated physical
2508
	 * memory, and is directly calling transport_generic_map_mem_to_cmd()
2509
	 * beforehand.
2510
	 */
2511 2512
	if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) &&
	    cmd->data_length) {
2513

2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525
		if ((cmd->se_cmd_flags & SCF_BIDI) ||
		    (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)) {
			u32 bidi_length;

			if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE)
				bidi_length = cmd->t_task_nolb *
					      cmd->se_dev->dev_attrib.block_size;
			else
				bidi_length = cmd->data_length;

			ret = target_alloc_sgl(&cmd->t_bidi_data_sg,
					       &cmd->t_bidi_data_nents,
2526
					       bidi_length, zero_flag, false);
2527 2528 2529 2530
			if (ret < 0)
				return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
		}

2531
		ret = target_alloc_sgl(&cmd->t_data_sg, &cmd->t_data_nents,
2532
				       cmd->data_length, zero_flag, false);
2533
		if (ret < 0)
2534
			return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545
	} else if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) &&
		    cmd->data_length) {
		/*
		 * Special case for COMPARE_AND_WRITE with fabrics
		 * using SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC.
		 */
		u32 caw_length = cmd->t_task_nolb *
				 cmd->se_dev->dev_attrib.block_size;

		ret = target_alloc_sgl(&cmd->t_bidi_data_sg,
				       &cmd->t_bidi_data_nents,
2546
				       caw_length, zero_flag, false);
2547 2548
		if (ret < 0)
			return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2549 2550
	}
	/*
2551 2552 2553
	 * If this command is not a write we can execute it right here,
	 * for write buffers we need to notify the fabric driver first
	 * and let it call back once the write buffers are ready.
2554
	 */
2555
	target_add_to_state_list(cmd);
2556
	if (cmd->data_direction != DMA_TO_DEVICE || cmd->data_length == 0) {
2557 2558 2559
		target_execute_cmd(cmd);
		return 0;
	}
2560 2561 2562 2563 2564 2565 2566

	spin_lock_irqsave(&cmd->t_state_lock, flags);
	cmd->t_state = TRANSPORT_WRITE_PENDING;
	/*
	 * Determine if frontend context caller is requesting the stopping of
	 * this command for frontend exceptions.
	 */
2567 2568
	if (cmd->transport_state & CMD_T_STOP &&
	    !cmd->se_tfo->write_pending_must_be_called) {
2569 2570 2571 2572 2573 2574
		pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
			 __func__, __LINE__, cmd->tag);

		spin_unlock_irqrestore(&cmd->t_state_lock, flags);

		complete_all(&cmd->t_transport_stop_comp);
2575
		return 0;
2576 2577 2578
	}
	cmd->transport_state &= ~CMD_T_ACTIVE;
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
2579 2580

	ret = cmd->se_tfo->write_pending(cmd);
2581
	if (ret)
2582 2583
		goto queue_full;

2584
	return 0;
2585

2586 2587
queue_full:
	pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n", cmd);
2588
	transport_handle_queue_full(cmd, cmd->se_dev, ret, true);
2589
	return 0;
2590
}
2591
EXPORT_SYMBOL(transport_generic_new_cmd);
2592

2593
static void transport_write_pending_qf(struct se_cmd *cmd)
2594
{
2595
	unsigned long flags;
2596
	int ret;
2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608
	bool stop;

	spin_lock_irqsave(&cmd->t_state_lock, flags);
	stop = (cmd->transport_state & (CMD_T_STOP | CMD_T_ABORTED));
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);

	if (stop) {
		pr_debug("%s:%d CMD_T_STOP|CMD_T_ABORTED for ITT: 0x%08llx\n",
			__func__, __LINE__, cmd->tag);
		complete_all(&cmd->t_transport_stop_comp);
		return;
	}
2609 2610

	ret = cmd->se_tfo->write_pending(cmd);
2611
	if (ret) {
2612 2613
		pr_debug("Handling write_pending QUEUE__FULL: se_cmd: %p\n",
			 cmd);
2614
		transport_handle_queue_full(cmd, cmd->se_dev, ret, true);
2615
	}
2616 2617
}

2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630
static bool
__transport_wait_for_tasks(struct se_cmd *, bool, bool *, bool *,
			   unsigned long *flags);

static void target_wait_free_cmd(struct se_cmd *cmd, bool *aborted, bool *tas)
{
	unsigned long flags;

	spin_lock_irqsave(&cmd->t_state_lock, flags);
	__transport_wait_for_tasks(cmd, true, aborted, tas, &flags);
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
}

2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644
/*
 * Call target_put_sess_cmd() and wait until target_release_cmd_kref(@cmd) has
 * finished.
 */
void target_put_cmd_and_wait(struct se_cmd *cmd)
{
	DECLARE_COMPLETION_ONSTACK(compl);

	WARN_ON_ONCE(cmd->abrt_compl);
	cmd->abrt_compl = &compl;
	target_put_sess_cmd(cmd);
	wait_for_completion(&compl);
}

2645 2646 2647 2648
/*
 * This function is called by frontend drivers after processing of a command
 * has finished.
 *
2649 2650 2651
 * The protocol for ensuring that either the regular frontend command
 * processing flow or target_handle_abort() code drops one reference is as
 * follows:
2652
 * - Calling .queue_data_in(), .queue_status() or queue_tm_rsp() will cause
2653 2654
 *   the frontend driver to call this function synchronously or asynchronously.
 *   That will cause one reference to be dropped.
2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665
 * - During regular command processing the target core sets CMD_T_COMPLETE
 *   before invoking one of the .queue_*() functions.
 * - The code that aborts commands skips commands and TMFs for which
 *   CMD_T_COMPLETE has been set.
 * - CMD_T_ABORTED is set atomically after the CMD_T_COMPLETE check for
 *   commands that will be aborted.
 * - If the CMD_T_ABORTED flag is set but CMD_T_TAS has not been set
 *   transport_generic_free_cmd() skips its call to target_put_sess_cmd().
 * - For aborted commands for which CMD_T_TAS has been set .queue_status() will
 *   be called and will drop a reference.
 * - For aborted commands for which CMD_T_TAS has not been set .aborted_task()
2666
 *   will be called. target_handle_abort() will drop the final reference.
2667
 */
2668
int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks)
2669
{
2670
	DECLARE_COMPLETION_ONSTACK(compl);
2671
	int ret = 0;
2672
	bool aborted = false, tas = false;
2673

2674 2675 2676 2677
	if (wait_for_tasks)
		target_wait_free_cmd(cmd, &aborted, &tas);

	if (cmd->se_cmd_flags & SCF_SE_LUN_CMD) {
2678 2679 2680 2681 2682
		/*
		 * Handle WRITE failure case where transport_generic_new_cmd()
		 * has already added se_cmd to state_list, but fabric has
		 * failed command before I/O submission.
		 */
2683
		if (cmd->state_active)
2684
			target_remove_from_state_list(cmd);
2685
	}
2686
	if (aborted)
2687
		cmd->free_compl = &compl;
2688
	ret = target_put_sess_cmd(cmd);
2689 2690
	if (aborted) {
		pr_debug("Detected CMD_T_ABORTED for ITT: %llu\n", cmd->tag);
2691
		wait_for_completion(&compl);
2692
		ret = 1;
2693
	}
2694
	return ret;
2695 2696 2697
}
EXPORT_SYMBOL(transport_generic_free_cmd);

2698 2699
/**
 * target_get_sess_cmd - Add command to active ->sess_cmd_list
2700
 * @se_cmd:	command descriptor to add
2701
 * @ack_kref:	Signal that fabric will perform an ack target_put_sess_cmd()
2702
 */
2703
int target_get_sess_cmd(struct se_cmd *se_cmd, bool ack_kref)
2704
{
2705
	struct se_session *se_sess = se_cmd->se_sess;
2706
	unsigned long flags;
2707
	int ret = 0;
2708

2709 2710 2711 2712 2713
	/*
	 * Add a second kref if the fabric caller is expecting to handle
	 * fabric acknowledgement that requires two target_put_sess_cmd()
	 * invocations before se_cmd descriptor release.
	 */
2714
	if (ack_kref) {
2715 2716 2717
		if (!kref_get_unless_zero(&se_cmd->cmd_kref))
			return -EINVAL;

2718 2719
		se_cmd->se_cmd_flags |= SCF_ACK_KREF;
	}
2720

2721
	spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2722 2723 2724 2725
	if (se_sess->sess_tearing_down) {
		ret = -ESHUTDOWN;
		goto out;
	}
2726
	list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list);
2727
	percpu_ref_get(&se_sess->cmd_count);
2728
out:
2729
	spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2730 2731

	if (ret && ack_kref)
2732
		target_put_sess_cmd(se_cmd);
2733

2734
	return ret;
2735
}
2736
EXPORT_SYMBOL(target_get_sess_cmd);
2737

2738 2739 2740 2741 2742 2743 2744 2745 2746 2747
static void target_free_cmd_mem(struct se_cmd *cmd)
{
	transport_free_pages(cmd);

	if (cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)
		core_tmr_release_req(cmd->se_tmr_req);
	if (cmd->t_task_cdb != cmd->__t_task_cdb)
		kfree(cmd->t_task_cdb);
}

2748
static void target_release_cmd_kref(struct kref *kref)
2749
{
2750 2751
	struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref);
	struct se_session *se_sess = se_cmd->se_sess;
2752 2753
	struct completion *free_compl = se_cmd->free_compl;
	struct completion *abrt_compl = se_cmd->abrt_compl;
2754
	unsigned long flags;
2755

2756 2757 2758
	if (se_cmd->lun_ref_active)
		percpu_ref_put(&se_cmd->se_lun->lun_ref);

2759 2760
	if (se_sess) {
		spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2761
		list_del_init(&se_cmd->se_cmd_list);
2762
		spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2763 2764
	}

2765
	target_free_cmd_mem(se_cmd);
2766
	se_cmd->se_tfo->release_cmd(se_cmd);
2767 2768 2769 2770
	if (free_compl)
		complete(free_compl);
	if (abrt_compl)
		complete(abrt_compl);
2771 2772

	percpu_ref_put(&se_sess->cmd_count);
2773 2774
}

2775 2776 2777 2778 2779 2780
/**
 * target_put_sess_cmd - decrease the command reference count
 * @se_cmd:	command to drop a reference from
 *
 * Returns 1 if and only if this target_put_sess_cmd() call caused the
 * refcount to drop to zero. Returns zero otherwise.
2781
 */
2782
int target_put_sess_cmd(struct se_cmd *se_cmd)
2783
{
2784
	return kref_put(&se_cmd->cmd_kref, target_release_cmd_kref);
2785 2786 2787
}
EXPORT_SYMBOL(target_put_sess_cmd);

2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889
static const char *data_dir_name(enum dma_data_direction d)
{
	switch (d) {
	case DMA_BIDIRECTIONAL:	return "BIDI";
	case DMA_TO_DEVICE:	return "WRITE";
	case DMA_FROM_DEVICE:	return "READ";
	case DMA_NONE:		return "NONE";
	}

	return "(?)";
}

static const char *cmd_state_name(enum transport_state_table t)
{
	switch (t) {
	case TRANSPORT_NO_STATE:	return "NO_STATE";
	case TRANSPORT_NEW_CMD:		return "NEW_CMD";
	case TRANSPORT_WRITE_PENDING:	return "WRITE_PENDING";
	case TRANSPORT_PROCESSING:	return "PROCESSING";
	case TRANSPORT_COMPLETE:	return "COMPLETE";
	case TRANSPORT_ISTATE_PROCESSING:
					return "ISTATE_PROCESSING";
	case TRANSPORT_COMPLETE_QF_WP:	return "COMPLETE_QF_WP";
	case TRANSPORT_COMPLETE_QF_OK:	return "COMPLETE_QF_OK";
	case TRANSPORT_COMPLETE_QF_ERR:	return "COMPLETE_QF_ERR";
	}

	return "(?)";
}

static void target_append_str(char **str, const char *txt)
{
	char *prev = *str;

	*str = *str ? kasprintf(GFP_ATOMIC, "%s,%s", *str, txt) :
		kstrdup(txt, GFP_ATOMIC);
	kfree(prev);
}

/*
 * Convert a transport state bitmask into a string. The caller is
 * responsible for freeing the returned pointer.
 */
static char *target_ts_to_str(u32 ts)
{
	char *str = NULL;

	if (ts & CMD_T_ABORTED)
		target_append_str(&str, "aborted");
	if (ts & CMD_T_ACTIVE)
		target_append_str(&str, "active");
	if (ts & CMD_T_COMPLETE)
		target_append_str(&str, "complete");
	if (ts & CMD_T_SENT)
		target_append_str(&str, "sent");
	if (ts & CMD_T_STOP)
		target_append_str(&str, "stop");
	if (ts & CMD_T_FABRIC_STOP)
		target_append_str(&str, "fabric_stop");

	return str;
}

static const char *target_tmf_name(enum tcm_tmreq_table tmf)
{
	switch (tmf) {
	case TMR_ABORT_TASK:		return "ABORT_TASK";
	case TMR_ABORT_TASK_SET:	return "ABORT_TASK_SET";
	case TMR_CLEAR_ACA:		return "CLEAR_ACA";
	case TMR_CLEAR_TASK_SET:	return "CLEAR_TASK_SET";
	case TMR_LUN_RESET:		return "LUN_RESET";
	case TMR_TARGET_WARM_RESET:	return "TARGET_WARM_RESET";
	case TMR_TARGET_COLD_RESET:	return "TARGET_COLD_RESET";
	case TMR_UNKNOWN:		break;
	}
	return "(?)";
}

void target_show_cmd(const char *pfx, struct se_cmd *cmd)
{
	char *ts_str = target_ts_to_str(cmd->transport_state);
	const u8 *cdb = cmd->t_task_cdb;
	struct se_tmr_req *tmf = cmd->se_tmr_req;

	if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
		pr_debug("%scmd %#02x:%#02x with tag %#llx dir %s i_state %d t_state %s len %d refcnt %d transport_state %s\n",
			 pfx, cdb[0], cdb[1], cmd->tag,
			 data_dir_name(cmd->data_direction),
			 cmd->se_tfo->get_cmd_state(cmd),
			 cmd_state_name(cmd->t_state), cmd->data_length,
			 kref_read(&cmd->cmd_kref), ts_str);
	} else {
		pr_debug("%stmf %s with tag %#llx ref_task_tag %#llx i_state %d t_state %s refcnt %d transport_state %s\n",
			 pfx, target_tmf_name(tmf->function), cmd->tag,
			 tmf->ref_task_tag, cmd->se_tfo->get_cmd_state(cmd),
			 cmd_state_name(cmd->t_state),
			 kref_read(&cmd->cmd_kref), ts_str);
	}
	kfree(ts_str);
}
EXPORT_SYMBOL(target_show_cmd);

2890
/**
2891
 * target_sess_cmd_list_set_waiting - Set sess_tearing_down so no new commands are queued.
2892
 * @se_sess:	session to flag
2893
 */
2894
void target_sess_cmd_list_set_waiting(struct se_session *se_sess)
2895 2896 2897 2898
{
	unsigned long flags;

	spin_lock_irqsave(&se_sess->sess_cmd_lock, flags);
2899
	se_sess->sess_tearing_down = 1;
2900
	spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags);
2901 2902

	percpu_ref_kill(&se_sess->cmd_count);
2903
}
2904
EXPORT_SYMBOL(target_sess_cmd_list_set_waiting);
2905

2906
/**
2907
 * target_wait_for_sess_cmds - Wait for outstanding commands
2908 2909
 * @se_sess:    session to wait for active I/O
 */
2910
void target_wait_for_sess_cmds(struct se_session *se_sess)
2911
{
2912 2913
	struct se_cmd *cmd;
	int ret;
2914

2915 2916 2917
	WARN_ON_ONCE(!se_sess->sess_tearing_down);

	do {
2918 2919 2920
		ret = wait_event_timeout(se_sess->cmd_list_wq,
				percpu_ref_is_zero(&se_sess->cmd_count),
				180 * HZ);
2921 2922 2923 2924
		list_for_each_entry(cmd, &se_sess->sess_cmd_list, se_cmd_list)
			target_show_cmd("session shutdown: still waiting for ",
					cmd);
	} while (ret <= 0);
2925 2926 2927
}
EXPORT_SYMBOL(target_wait_for_sess_cmds);

2928 2929 2930 2931
/*
 * Prevent that new percpu_ref_tryget_live() calls succeed and wait until
 * all references to the LUN have been released. Called during LUN shutdown.
 */
2932
void transport_clear_lun_ref(struct se_lun *lun)
2933
{
2934
	percpu_ref_kill(&lun->lun_ref);
2935
	wait_for_completion(&lun->lun_shutdown_comp);
2936 2937
}

2938 2939 2940 2941 2942
static bool
__transport_wait_for_tasks(struct se_cmd *cmd, bool fabric_stop,
			   bool *aborted, bool *tas, unsigned long *flags)
	__releases(&cmd->t_state_lock)
	__acquires(&cmd->t_state_lock)
2943 2944
{

2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956
	assert_spin_locked(&cmd->t_state_lock);
	WARN_ON_ONCE(!irqs_disabled());

	if (fabric_stop)
		cmd->transport_state |= CMD_T_FABRIC_STOP;

	if (cmd->transport_state & CMD_T_ABORTED)
		*aborted = true;

	if (cmd->transport_state & CMD_T_TAS)
		*tas = true;

2957
	if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) &&
2958
	    !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
2959
		return false;
2960

2961
	if (!(cmd->se_cmd_flags & SCF_SUPPORTED_SAM_OPCODE) &&
2962
	    !(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB))
2963
		return false;
2964

2965 2966 2967 2968
	if (!(cmd->transport_state & CMD_T_ACTIVE))
		return false;

	if (fabric_stop && *aborted)
2969
		return false;
2970

2971
	cmd->transport_state |= CMD_T_STOP;
2972

2973
	target_show_cmd("wait_for_tasks: Stopping ", cmd);
2974

2975
	spin_unlock_irqrestore(&cmd->t_state_lock, *flags);
2976

2977 2978 2979
	while (!wait_for_completion_timeout(&cmd->t_transport_stop_comp,
					    180 * HZ))
		target_show_cmd("wait for tasks: ", cmd);
2980

2981
	spin_lock_irqsave(&cmd->t_state_lock, *flags);
2982
	cmd->transport_state &= ~(CMD_T_ACTIVE | CMD_T_STOP);
2983

2984 2985
	pr_debug("wait_for_tasks: Stopped wait_for_completion(&cmd->"
		 "t_transport_stop_comp) for ITT: 0x%08llx\n", cmd->tag);
2986

2987 2988 2989 2990
	return true;
}

/**
2991 2992
 * transport_wait_for_tasks - set CMD_T_STOP and wait for t_transport_stop_comp
 * @cmd: command to wait on
2993 2994 2995 2996 2997 2998 2999 3000
 */
bool transport_wait_for_tasks(struct se_cmd *cmd)
{
	unsigned long flags;
	bool ret, aborted = false, tas = false;

	spin_lock_irqsave(&cmd->t_state_lock, flags);
	ret = __transport_wait_for_tasks(cmd, false, &aborted, &tas, &flags);
3001
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3002

3003
	return ret;
3004
}
3005
EXPORT_SYMBOL(transport_wait_for_tasks);
3006

3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051
struct sense_info {
	u8 key;
	u8 asc;
	u8 ascq;
	bool add_sector_info;
};

static const struct sense_info sense_info_table[] = {
	[TCM_NO_SENSE] = {
		.key = NOT_READY
	},
	[TCM_NON_EXISTENT_LUN] = {
		.key = ILLEGAL_REQUEST,
		.asc = 0x25 /* LOGICAL UNIT NOT SUPPORTED */
	},
	[TCM_UNSUPPORTED_SCSI_OPCODE] = {
		.key = ILLEGAL_REQUEST,
		.asc = 0x20, /* INVALID COMMAND OPERATION CODE */
	},
	[TCM_SECTOR_COUNT_TOO_MANY] = {
		.key = ILLEGAL_REQUEST,
		.asc = 0x20, /* INVALID COMMAND OPERATION CODE */
	},
	[TCM_UNKNOWN_MODE_PAGE] = {
		.key = ILLEGAL_REQUEST,
		.asc = 0x24, /* INVALID FIELD IN CDB */
	},
	[TCM_CHECK_CONDITION_ABORT_CMD] = {
		.key = ABORTED_COMMAND,
		.asc = 0x29, /* BUS DEVICE RESET FUNCTION OCCURRED */
		.ascq = 0x03,
	},
	[TCM_INCORRECT_AMOUNT_OF_DATA] = {
		.key = ABORTED_COMMAND,
		.asc = 0x0c, /* WRITE ERROR */
		.ascq = 0x0d, /* NOT ENOUGH UNSOLICITED DATA */
	},
	[TCM_INVALID_CDB_FIELD] = {
		.key = ILLEGAL_REQUEST,
		.asc = 0x24, /* INVALID FIELD IN CDB */
	},
	[TCM_INVALID_PARAMETER_LIST] = {
		.key = ILLEGAL_REQUEST,
		.asc = 0x26, /* INVALID FIELD IN PARAMETER LIST */
	},
3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071
	[TCM_TOO_MANY_TARGET_DESCS] = {
		.key = ILLEGAL_REQUEST,
		.asc = 0x26,
		.ascq = 0x06, /* TOO MANY TARGET DESCRIPTORS */
	},
	[TCM_UNSUPPORTED_TARGET_DESC_TYPE_CODE] = {
		.key = ILLEGAL_REQUEST,
		.asc = 0x26,
		.ascq = 0x07, /* UNSUPPORTED TARGET DESCRIPTOR TYPE CODE */
	},
	[TCM_TOO_MANY_SEGMENT_DESCS] = {
		.key = ILLEGAL_REQUEST,
		.asc = 0x26,
		.ascq = 0x08, /* TOO MANY SEGMENT DESCRIPTORS */
	},
	[TCM_UNSUPPORTED_SEGMENT_DESC_TYPE_CODE] = {
		.key = ILLEGAL_REQUEST,
		.asc = 0x26,
		.ascq = 0x09, /* UNSUPPORTED SEGMENT DESCRIPTOR TYPE CODE */
	},
3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110
	[TCM_PARAMETER_LIST_LENGTH_ERROR] = {
		.key = ILLEGAL_REQUEST,
		.asc = 0x1a, /* PARAMETER LIST LENGTH ERROR */
	},
	[TCM_UNEXPECTED_UNSOLICITED_DATA] = {
		.key = ILLEGAL_REQUEST,
		.asc = 0x0c, /* WRITE ERROR */
		.ascq = 0x0c, /* UNEXPECTED_UNSOLICITED_DATA */
	},
	[TCM_SERVICE_CRC_ERROR] = {
		.key = ABORTED_COMMAND,
		.asc = 0x47, /* PROTOCOL SERVICE CRC ERROR */
		.ascq = 0x05, /* N/A */
	},
	[TCM_SNACK_REJECTED] = {
		.key = ABORTED_COMMAND,
		.asc = 0x11, /* READ ERROR */
		.ascq = 0x13, /* FAILED RETRANSMISSION REQUEST */
	},
	[TCM_WRITE_PROTECTED] = {
		.key = DATA_PROTECT,
		.asc = 0x27, /* WRITE PROTECTED */
	},
	[TCM_ADDRESS_OUT_OF_RANGE] = {
		.key = ILLEGAL_REQUEST,
		.asc = 0x21, /* LOGICAL BLOCK ADDRESS OUT OF RANGE */
	},
	[TCM_CHECK_CONDITION_UNIT_ATTENTION] = {
		.key = UNIT_ATTENTION,
	},
	[TCM_CHECK_CONDITION_NOT_READY] = {
		.key = NOT_READY,
	},
	[TCM_MISCOMPARE_VERIFY] = {
		.key = MISCOMPARE,
		.asc = 0x1d, /* MISCOMPARE DURING VERIFY OPERATION */
		.ascq = 0x00,
	},
	[TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED] = {
3111
		.key = ABORTED_COMMAND,
3112 3113 3114 3115 3116
		.asc = 0x10,
		.ascq = 0x01, /* LOGICAL BLOCK GUARD CHECK FAILED */
		.add_sector_info = true,
	},
	[TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED] = {
3117
		.key = ABORTED_COMMAND,
3118 3119 3120 3121 3122
		.asc = 0x10,
		.ascq = 0x02, /* LOGICAL BLOCK APPLICATION TAG CHECK FAILED */
		.add_sector_info = true,
	},
	[TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED] = {
3123
		.key = ABORTED_COMMAND,
3124 3125 3126 3127
		.asc = 0x10,
		.ascq = 0x03, /* LOGICAL BLOCK REFERENCE TAG CHECK FAILED */
		.add_sector_info = true,
	},
3128 3129 3130 3131 3132 3133
	[TCM_COPY_TARGET_DEVICE_NOT_REACHABLE] = {
		.key = COPY_ABORTED,
		.asc = 0x0d,
		.ascq = 0x02, /* COPY TARGET DEVICE NOT REACHABLE */

	},
3134 3135 3136 3137 3138 3139 3140 3141 3142 3143
	[TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE] = {
		/*
		 * Returning ILLEGAL REQUEST would cause immediate IO errors on
		 * Solaris initiators.  Returning NOT READY instead means the
		 * operations will be retried a finite number of times and we
		 * can survive intermittent errors.
		 */
		.key = NOT_READY,
		.asc = 0x08, /* LOGICAL UNIT COMMUNICATION FAILURE */
	},
3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158
	[TCM_INSUFFICIENT_REGISTRATION_RESOURCES] = {
		/*
		 * From spc4r22 section5.7.7,5.7.8
		 * If a PERSISTENT RESERVE OUT command with a REGISTER service action
		 * or a REGISTER AND IGNORE EXISTING KEY service action or
		 * REGISTER AND MOVE service actionis attempted,
		 * but there are insufficient device server resources to complete the
		 * operation, then the command shall be terminated with CHECK CONDITION
		 * status, with the sense key set to ILLEGAL REQUEST,and the additonal
		 * sense code set to INSUFFICIENT REGISTRATION RESOURCES.
		 */
		.key = ILLEGAL_REQUEST,
		.asc = 0x55,
		.ascq = 0x04, /* INSUFFICIENT REGISTRATION RESOURCES */
	},
3159 3160
};

3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171
/**
 * translate_sense_reason - translate a sense reason into T10 key, asc and ascq
 * @cmd: SCSI command in which the resulting sense buffer or SCSI status will
 *   be stored.
 * @reason: LIO sense reason code. If this argument has the value
 *   TCM_CHECK_CONDITION_UNIT_ATTENTION, try to dequeue a unit attention. If
 *   dequeuing a unit attention fails due to multiple commands being processed
 *   concurrently, set the command status to BUSY.
 *
 * Return: 0 upon success or -EINVAL if the sense buffer is too small.
 */
3172
static void translate_sense_reason(struct se_cmd *cmd, sense_reason_t reason)
3173 3174 3175 3176
{
	const struct sense_info *si;
	u8 *buffer = cmd->sense_buffer;
	int r = (__force int)reason;
3177
	u8 key, asc, ascq;
3178
	bool desc_format = target_sense_desc_format(cmd->se_dev);
3179 3180 3181 3182 3183 3184 3185

	if (r < ARRAY_SIZE(sense_info_table) && sense_info_table[r].key)
		si = &sense_info_table[r];
	else
		si = &sense_info_table[(__force int)
				       TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE];

3186
	key = si->key;
3187
	if (reason == TCM_CHECK_CONDITION_UNIT_ATTENTION) {
3188 3189 3190 3191 3192
		if (!core_scsi3_ua_for_check_condition(cmd, &key, &asc,
						       &ascq)) {
			cmd->scsi_status = SAM_STAT_BUSY;
			return;
		}
3193 3194 3195 3196 3197 3198 3199 3200
	} else if (si->asc == 0) {
		WARN_ON_ONCE(cmd->scsi_asc == 0);
		asc = cmd->scsi_asc;
		ascq = cmd->scsi_ascq;
	} else {
		asc = si->asc;
		ascq = si->ascq;
	}
3201

3202 3203 3204
	cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
	cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
	cmd->scsi_sense_length  = TRANSPORT_SENSE_BUFFER;
3205
	scsi_build_sense_buffer(desc_format, buffer, key, asc, ascq);
3206
	if (si->add_sector_info)
3207 3208 3209
		WARN_ON_ONCE(scsi_set_sense_information(buffer,
							cmd->scsi_sense_length,
							cmd->bad_sector) < 0);
3210 3211
}

3212 3213 3214
int
transport_send_check_condition_and_sense(struct se_cmd *cmd,
		sense_reason_t reason, int from_transport)
3215 3216 3217
{
	unsigned long flags;

3218 3219
	WARN_ON_ONCE(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB);

3220
	spin_lock_irqsave(&cmd->t_state_lock, flags);
3221
	if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
3222
		spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3223 3224 3225
		return 0;
	}
	cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION;
3226
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3227

3228 3229
	if (!from_transport)
		translate_sense_reason(cmd, reason);
3230

3231
	trace_target_cmd_complete(cmd);
3232
	return cmd->se_tfo->queue_status(cmd);
3233 3234 3235
}
EXPORT_SYMBOL(transport_send_check_condition_and_sense);

3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251
/**
 * target_send_busy - Send SCSI BUSY status back to the initiator
 * @cmd: SCSI command for which to send a BUSY reply.
 *
 * Note: Only call this function if target_submit_cmd*() failed.
 */
int target_send_busy(struct se_cmd *cmd)
{
	WARN_ON_ONCE(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB);

	cmd->scsi_status = SAM_STAT_BUSY;
	trace_target_cmd_complete(cmd);
	return cmd->se_tfo->queue_status(cmd);
}
EXPORT_SYMBOL(target_send_busy);

3252
static void target_tmr_work(struct work_struct *work)
3253
{
3254
	struct se_cmd *cmd = container_of(work, struct se_cmd, work);
3255
	struct se_device *dev = cmd->se_dev;
3256 3257 3258
	struct se_tmr_req *tmr = cmd->se_tmr_req;
	int ret;

3259 3260
	if (cmd->transport_state & CMD_T_ABORTED)
		goto aborted;
3261

3262
	switch (tmr->function) {
3263
	case TMR_ABORT_TASK:
3264
		core_tmr_abort_task(dev, tmr, cmd->se_sess);
3265
		break;
3266 3267 3268
	case TMR_ABORT_TASK_SET:
	case TMR_CLEAR_ACA:
	case TMR_CLEAR_TASK_SET:
3269 3270
		tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED;
		break;
3271
	case TMR_LUN_RESET:
3272 3273 3274
		ret = core_tmr_lun_reset(dev, tmr, NULL, NULL);
		tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE :
					 TMR_FUNCTION_REJECTED;
3275 3276 3277 3278 3279
		if (tmr->response == TMR_FUNCTION_COMPLETE) {
			target_ua_allocate_lun(cmd->se_sess->se_node_acl,
					       cmd->orig_fe_lun, 0x29,
					       ASCQ_29H_BUS_DEVICE_RESET_FUNCTION_OCCURRED);
		}
3280
		break;
3281
	case TMR_TARGET_WARM_RESET:
3282 3283
		tmr->response = TMR_FUNCTION_REJECTED;
		break;
3284
	case TMR_TARGET_COLD_RESET:
3285 3286 3287
		tmr->response = TMR_FUNCTION_REJECTED;
		break;
	default:
3288
		pr_err("Unknown TMR function: 0x%02x.\n",
3289 3290 3291 3292 3293
				tmr->function);
		tmr->response = TMR_FUNCTION_REJECTED;
		break;
	}

3294 3295
	if (cmd->transport_state & CMD_T_ABORTED)
		goto aborted;
3296

3297
	cmd->se_tfo->queue_tm_rsp(cmd);
3298

3299
	transport_cmd_check_stop_to_fabric(cmd);
3300 3301 3302 3303
	return;

aborted:
	target_handle_abort(cmd);
3304 3305
}

3306 3307
int transport_generic_handle_tmr(
	struct se_cmd *cmd)
3308
{
3309
	unsigned long flags;
3310
	bool aborted = false;
3311 3312

	spin_lock_irqsave(&cmd->t_state_lock, flags);
3313 3314 3315 3316 3317 3318
	if (cmd->transport_state & CMD_T_ABORTED) {
		aborted = true;
	} else {
		cmd->t_state = TRANSPORT_ISTATE_PROCESSING;
		cmd->transport_state |= CMD_T_ACTIVE;
	}
3319 3320
	spin_unlock_irqrestore(&cmd->t_state_lock, flags);

3321
	if (aborted) {
3322 3323 3324 3325
		pr_warn_ratelimited("handle_tmr caught CMD_T_ABORTED TMR %d ref_tag: %llu tag: %llu\n",
				    cmd->se_tmr_req->function,
				    cmd->se_tmr_req->ref_task_tag, cmd->tag);
		target_handle_abort(cmd);
3326 3327 3328
		return 0;
	}

3329
	INIT_WORK(&cmd->work, target_tmr_work);
3330
	schedule_work(&cmd->work);
3331 3332
	return 0;
}
3333
EXPORT_SYMBOL(transport_generic_handle_tmr);
3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352

bool
target_check_wce(struct se_device *dev)
{
	bool wce = false;

	if (dev->transport->get_write_cache)
		wce = dev->transport->get_write_cache(dev);
	else if (dev->dev_attrib.emulate_write_cache > 0)
		wce = true;

	return wce;
}

bool
target_check_fua(struct se_device *dev)
{
	return target_check_wce(dev) && dev->dev_attrib.emulate_fua_write > 0;
}