scsi_lib.c 86.0 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2 3
 * Copyright (C) 1999 Eric Youngdale
 * Copyright (C) 2014 Christoph Hellwig
L
Linus Torvalds 已提交
4 5 6 7 8 9 10 11
 *
 *  SCSI queueing library.
 *      Initial versions: Eric Youngdale (eric@andante.org).
 *                        Based upon conversations with large numbers
 *                        of people at Linux Expo.
 */

#include <linux/bio.h>
J
James Bottomley 已提交
12
#include <linux/bitops.h>
L
Linus Torvalds 已提交
13 14 15
#include <linux/blkdev.h>
#include <linux/completion.h>
#include <linux/kernel.h>
16
#include <linux/export.h>
L
Linus Torvalds 已提交
17 18 19
#include <linux/init.h>
#include <linux/pci.h>
#include <linux/delay.h>
20
#include <linux/hardirq.h>
J
Jens Axboe 已提交
21
#include <linux/scatterlist.h>
22
#include <linux/blk-mq.h>
23
#include <linux/ratelimit.h>
H
Hannes Reinecke 已提交
24
#include <asm/unaligned.h>
L
Linus Torvalds 已提交
25 26

#include <scsi/scsi.h>
27
#include <scsi/scsi_cmnd.h>
L
Linus Torvalds 已提交
28 29 30 31 32
#include <scsi/scsi_dbg.h>
#include <scsi/scsi_device.h>
#include <scsi/scsi_driver.h>
#include <scsi/scsi_eh.h>
#include <scsi/scsi_host.h>
33
#include <scsi/scsi_transport.h> /* __scsi_init_queue() */
34
#include <scsi/scsi_dh.h>
L
Linus Torvalds 已提交
35

36 37
#include <trace/events/scsi.h>

38
#include "scsi_debugfs.h"
L
Linus Torvalds 已提交
39 40 41
#include "scsi_priv.h"
#include "scsi_logging.h"

42
static struct kmem_cache *scsi_sdb_cache;
43 44 45
static struct kmem_cache *scsi_sense_cache;
static struct kmem_cache *scsi_sense_isadma_cache;
static DEFINE_MUTEX(scsi_sense_cache_mutex);
L
Linus Torvalds 已提交
46

47 48 49 50 51 52 53
static inline struct kmem_cache *
scsi_select_sense_cache(struct Scsi_Host *shost)
{
	return shost->unchecked_isa_dma ?
		scsi_sense_isadma_cache : scsi_sense_cache;
}

54
static void scsi_free_sense_buffer(struct Scsi_Host *shost,
55 56 57 58 59
		unsigned char *sense_buffer)
{
	kmem_cache_free(scsi_select_sense_cache(shost), sense_buffer);
}

60 61
static unsigned char *scsi_alloc_sense_buffer(struct Scsi_Host *shost,
	gfp_t gfp_mask, int numa_node)
62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
{
	return kmem_cache_alloc_node(scsi_select_sense_cache(shost), gfp_mask,
			numa_node);
}

int scsi_init_sense_cache(struct Scsi_Host *shost)
{
	struct kmem_cache *cache;
	int ret = 0;

	cache = scsi_select_sense_cache(shost);
	if (cache)
		return 0;

	mutex_lock(&scsi_sense_cache_mutex);
	if (shost->unchecked_isa_dma) {
		scsi_sense_isadma_cache =
			kmem_cache_create("scsi_sense_cache(DMA)",
			SCSI_SENSE_BUFFERSIZE, 0,
			SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA, NULL);
		if (!scsi_sense_isadma_cache)
			ret = -ENOMEM;
	} else {
		scsi_sense_cache =
			kmem_cache_create("scsi_sense_cache",
			SCSI_SENSE_BUFFERSIZE, 0, SLAB_HWCACHE_ALIGN, NULL);
		if (!scsi_sense_cache)
			ret = -ENOMEM;
	}

	mutex_unlock(&scsi_sense_cache_mutex);
	return ret;
}
95

J
Jens Axboe 已提交
96 97 98 99 100 101 102
/*
 * When to reinvoke queueing after a resource shortage. It's 3 msecs to
 * not change behaviour from the previous unplug mechanism, experimentation
 * may prove this needs changing.
 */
#define SCSI_QUEUE_DELAY	3

103 104
static void
scsi_set_blocked(struct scsi_cmnd *cmd, int reason)
L
Linus Torvalds 已提交
105 106 107
{
	struct Scsi_Host *host = cmd->device->host;
	struct scsi_device *device = cmd->device;
108
	struct scsi_target *starget = scsi_target(device);
L
Linus Torvalds 已提交
109 110

	/*
111
	 * Set the appropriate busy bit for the device/host.
L
Linus Torvalds 已提交
112 113 114 115 116 117 118 119 120 121 122
	 *
	 * If the host/device isn't busy, assume that something actually
	 * completed, and that we should be able to queue a command now.
	 *
	 * Note that the prior mid-layer assumption that any host could
	 * always queue at least one command is now broken.  The mid-layer
	 * will implement a user specifiable stall (see
	 * scsi_host.max_host_blocked and scsi_device.max_device_blocked)
	 * if a command is requeued with no other commands outstanding
	 * either for the device or for the host.
	 */
123 124
	switch (reason) {
	case SCSI_MLQUEUE_HOST_BUSY:
125
		atomic_set(&host->host_blocked, host->max_host_blocked);
126 127
		break;
	case SCSI_MLQUEUE_DEVICE_BUSY:
128
	case SCSI_MLQUEUE_EH_RETRY:
129 130
		atomic_set(&device->device_blocked,
			   device->max_device_blocked);
131 132
		break;
	case SCSI_MLQUEUE_TARGET_BUSY:
133 134
		atomic_set(&starget->target_blocked,
			   starget->max_target_blocked);
135 136
		break;
	}
137 138
}

139 140 141 142
static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd)
{
	struct scsi_device *sdev = cmd->device;

143
	blk_mq_requeue_request(cmd->request, true);
144 145 146
	put_device(&sdev->sdev_gendev);
}

147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
/**
 * __scsi_queue_insert - private queue insertion
 * @cmd: The SCSI command being requeued
 * @reason:  The reason for the requeue
 * @unbusy: Whether the queue should be unbusied
 *
 * This is a private queue insertion.  The public interface
 * scsi_queue_insert() always assumes the queue should be unbusied
 * because it's always called before the completion.  This function is
 * for a requeue after completion, which should only occur in this
 * file.
 */
static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, int unbusy)
{
	struct scsi_device *device = cmd->device;
	struct request_queue *q = device->request_queue;
	unsigned long flags;

	SCSI_LOG_MLQUEUE(1, scmd_printk(KERN_INFO, cmd,
		"Inserting command %p into mlqueue\n", cmd));

	scsi_set_blocked(cmd, reason);
L
Linus Torvalds 已提交
169 170 171 172 173

	/*
	 * Decrement the counters, since these commands are no longer
	 * active on the host/device.
	 */
174 175
	if (unbusy)
		scsi_device_unbusy(device);
L
Linus Torvalds 已提交
176 177

	/*
178
	 * Requeue this command.  It will go before all other commands
179 180 181
	 * that are already in the queue. Schedule requeue work under
	 * lock such that the kblockd_schedule_work() call happens
	 * before blk_cleanup_queue() finishes.
J
Jens Axboe 已提交
182
	 */
183
	cmd->result = 0;
184 185 186 187
	if (q->mq_ops) {
		scsi_mq_requeue_cmd(cmd);
		return;
	}
188
	spin_lock_irqsave(q->queue_lock, flags);
J
James Bottomley 已提交
189
	blk_requeue_request(q, cmd->request);
190
	kblockd_schedule_work(&device->requeue_work);
191
	spin_unlock_irqrestore(q->queue_lock, flags);
L
Linus Torvalds 已提交
192 193
}

194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
/*
 * Function:    scsi_queue_insert()
 *
 * Purpose:     Insert a command in the midlevel queue.
 *
 * Arguments:   cmd    - command that we are adding to queue.
 *              reason - why we are inserting command to queue.
 *
 * Lock status: Assumed that lock is not held upon entry.
 *
 * Returns:     Nothing.
 *
 * Notes:       We do this for one of two cases.  Either the host is busy
 *              and it cannot accept any more commands for the time being,
 *              or the device returned QUEUE_FULL and can accept no more
 *              commands.
 * Notes:       This could be called either from an interrupt context or a
 *              normal process context.
 */
213
void scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
214
{
215
	__scsi_queue_insert(cmd, reason, 1);
216
}
217

218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233

/**
 * scsi_execute - insert request and wait for the result
 * @sdev:	scsi device
 * @cmd:	scsi command
 * @data_direction: data direction
 * @buffer:	data buffer
 * @bufflen:	len of buffer
 * @sense:	optional sense buffer
 * @sshdr:	optional decoded sense header
 * @timeout:	request timeout in seconds
 * @retries:	number of times to retry request
 * @flags:	flags for ->cmd_flags
 * @rq_flags:	flags for ->rq_flags
 * @resid:	optional residual length
 *
234 235
 * Returns the scsi_cmnd result field if a command was executed, or a negative
 * Linux error code if we didn't get that far.
236 237
 */
int scsi_execute(struct scsi_device *sdev, const unsigned char *cmd,
238
		 int data_direction, void *buffer, unsigned bufflen,
239 240 241
		 unsigned char *sense, struct scsi_sense_hdr *sshdr,
		 int timeout, int retries, u64 flags, req_flags_t rq_flags,
		 int *resid)
242 243
{
	struct request *req;
244
	struct scsi_request *rq;
245 246
	int ret = DRIVER_ERROR << 24;

247 248 249
	req = blk_get_request(sdev->request_queue,
			data_direction == DMA_TO_DEVICE ?
			REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, __GFP_RECLAIM);
250
	if (IS_ERR(req))
251
		return ret;
252 253
	rq = scsi_req(req);
	scsi_req_init(req);
254 255

	if (bufflen &&	blk_rq_map_kern(sdev->request_queue, req,
256
					buffer, bufflen, __GFP_RECLAIM))
257 258
		goto out;

259 260
	rq->cmd_len = COMMAND_SIZE(cmd[0]);
	memcpy(rq->cmd, cmd, rq->cmd_len);
261
	rq->retries = retries;
262
	req->timeout = timeout;
263 264
	req->cmd_flags |= flags;
	req->rq_flags |= rq_flags | RQF_QUIET | RQF_PREEMPT;
265 266 267 268 269 270

	/*
	 * head injection *required* here otherwise quiesce won't work
	 */
	blk_execute_rq(req->q, NULL, req, 1);

271 272 273 274 275 276
	/*
	 * Some devices (USB mass-storage in particular) may transfer
	 * garbage data together with a residue indicating that the data
	 * is invalid.  Prevent the garbage from being misinterpreted
	 * and prevent security leaks by zeroing out the excess data.
	 */
277 278
	if (unlikely(rq->resid_len > 0 && rq->resid_len <= bufflen))
		memset(buffer + (bufflen - rq->resid_len), 0, rq->resid_len);
279

280
	if (resid)
281 282 283
		*resid = rq->resid_len;
	if (sense && rq->sense_len)
		memcpy(sense, rq->sense, SCSI_SENSE_BUFFERSIZE);
284 285
	if (sshdr)
		scsi_normalize_sense(rq->sense, rq->sense_len, sshdr);
286
	ret = rq->result;
287 288 289 290 291
 out:
	blk_put_request(req);

	return ret;
}
292
EXPORT_SYMBOL(scsi_execute);
293

L
Linus Torvalds 已提交
294 295 296 297 298 299 300 301 302 303 304
/*
 * Function:    scsi_init_cmd_errh()
 *
 * Purpose:     Initialize cmd fields related to error handling.
 *
 * Arguments:   cmd	- command that is ready to be queued.
 *
 * Notes:       This function has the job of initializing a number of
 *              fields related to error handling.   Typically this will
 *              be called once for each command, as required.
 */
305
static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
L
Linus Torvalds 已提交
306 307
{
	cmd->serial_number = 0;
B
Boaz Harrosh 已提交
308
	scsi_set_resid(cmd, 0);
309
	memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
L
Linus Torvalds 已提交
310
	if (cmd->cmd_len == 0)
311
		cmd->cmd_len = scsi_command_size(cmd->cmnd);
L
Linus Torvalds 已提交
312 313 314 315 316
}

void scsi_device_unbusy(struct scsi_device *sdev)
{
	struct Scsi_Host *shost = sdev->host;
317
	struct scsi_target *starget = scsi_target(sdev);
L
Linus Torvalds 已提交
318 319
	unsigned long flags;

320
	atomic_dec(&shost->host_busy);
321 322
	if (starget->can_queue > 0)
		atomic_dec(&starget->target_busy);
323

324
	if (unlikely(scsi_host_in_recovery(shost) &&
325 326
		     (shost->host_failed || shost->host_eh_scheduled))) {
		spin_lock_irqsave(shost->host_lock, flags);
L
Linus Torvalds 已提交
327
		scsi_eh_wakeup(shost);
328 329 330
		spin_unlock_irqrestore(shost->host_lock, flags);
	}

331
	atomic_dec(&sdev->device_busy);
L
Linus Torvalds 已提交
332 333
}

334 335 336 337 338 339 340 341
static void scsi_kick_queue(struct request_queue *q)
{
	if (q->mq_ops)
		blk_mq_start_hw_queues(q);
	else
		blk_run_queue(q);
}

L
Linus Torvalds 已提交
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365
/*
 * Called for single_lun devices on IO completion. Clear starget_sdev_user,
 * and call blk_run_queue for all the scsi_devices on the target -
 * including current_sdev first.
 *
 * Called with *no* scsi locks held.
 */
static void scsi_single_lun_run(struct scsi_device *current_sdev)
{
	struct Scsi_Host *shost = current_sdev->host;
	struct scsi_device *sdev, *tmp;
	struct scsi_target *starget = scsi_target(current_sdev);
	unsigned long flags;

	spin_lock_irqsave(shost->host_lock, flags);
	starget->starget_sdev_user = NULL;
	spin_unlock_irqrestore(shost->host_lock, flags);

	/*
	 * Call blk_run_queue for all LUNs on the target, starting with
	 * current_sdev. We race with others (to set starget_sdev_user),
	 * but in most cases, we will be first. Ideally, each LU on the
	 * target would get some limited time or requests on the target.
	 */
366
	scsi_kick_queue(current_sdev->request_queue);
L
Linus Torvalds 已提交
367 368 369 370 371 372 373 374 375 376 377 378

	spin_lock_irqsave(shost->host_lock, flags);
	if (starget->starget_sdev_user)
		goto out;
	list_for_each_entry_safe(sdev, tmp, &starget->devices,
			same_target_siblings) {
		if (sdev == current_sdev)
			continue;
		if (scsi_device_get(sdev))
			continue;

		spin_unlock_irqrestore(shost->host_lock, flags);
379
		scsi_kick_queue(sdev->request_queue);
L
Linus Torvalds 已提交
380 381 382 383 384 385 386 387
		spin_lock_irqsave(shost->host_lock, flags);
	
		scsi_device_put(sdev);
	}
 out:
	spin_unlock_irqrestore(shost->host_lock, flags);
}

388
static inline bool scsi_device_is_busy(struct scsi_device *sdev)
389
{
390 391 392 393 394
	if (atomic_read(&sdev->device_busy) >= sdev->queue_depth)
		return true;
	if (atomic_read(&sdev->device_blocked) > 0)
		return true;
	return false;
395 396
}

397
static inline bool scsi_target_is_busy(struct scsi_target *starget)
398
{
399 400 401 402 403 404
	if (starget->can_queue > 0) {
		if (atomic_read(&starget->target_busy) >= starget->can_queue)
			return true;
		if (atomic_read(&starget->target_blocked) > 0)
			return true;
	}
405
	return false;
406 407
}

408
static inline bool scsi_host_is_busy(struct Scsi_Host *shost)
409
{
410 411 412 413 414 415 416 417
	if (shost->can_queue > 0 &&
	    atomic_read(&shost->host_busy) >= shost->can_queue)
		return true;
	if (atomic_read(&shost->host_blocked) > 0)
		return true;
	if (shost->host_self_blocked)
		return true;
	return false;
418 419
}

420
static void scsi_starved_list_run(struct Scsi_Host *shost)
L
Linus Torvalds 已提交
421
{
422
	LIST_HEAD(starved_list);
423
	struct scsi_device *sdev;
L
Linus Torvalds 已提交
424 425 426
	unsigned long flags;

	spin_lock_irqsave(shost->host_lock, flags);
427 428 429
	list_splice_init(&shost->starved_list, &starved_list);

	while (!list_empty(&starved_list)) {
430 431
		struct request_queue *slq;

L
Linus Torvalds 已提交
432 433 434 435 436 437 438 439 440 441
		/*
		 * As long as shost is accepting commands and we have
		 * starved queues, call blk_run_queue. scsi_request_fn
		 * drops the queue_lock and can add us back to the
		 * starved_list.
		 *
		 * host_lock protects the starved_list and starved_entry.
		 * scsi_request_fn must get the host_lock before checking
		 * or modifying starved_list or starved_entry.
		 */
442
		if (scsi_host_is_busy(shost))
443 444
			break;

445 446 447
		sdev = list_entry(starved_list.next,
				  struct scsi_device, starved_entry);
		list_del_init(&sdev->starved_entry);
448 449 450 451 452 453
		if (scsi_target_is_busy(scsi_target(sdev))) {
			list_move_tail(&sdev->starved_entry,
				       &shost->starved_list);
			continue;
		}

454 455 456 457 458 459 460 461 462 463 464 465 466 467 468
		/*
		 * Once we drop the host lock, a racing scsi_remove_device()
		 * call may remove the sdev from the starved list and destroy
		 * it and the queue.  Mitigate by taking a reference to the
		 * queue and never touching the sdev again after we drop the
		 * host lock.  Note: if __scsi_remove_device() invokes
		 * blk_cleanup_queue() before the queue is run from this
		 * function then blk_run_queue() will return immediately since
		 * blk_cleanup_queue() marks the queue with QUEUE_FLAG_DYING.
		 */
		slq = sdev->request_queue;
		if (!blk_get_queue(slq))
			continue;
		spin_unlock_irqrestore(shost->host_lock, flags);

469
		scsi_kick_queue(slq);
470 471 472
		blk_put_queue(slq);

		spin_lock_irqsave(shost->host_lock, flags);
L
Linus Torvalds 已提交
473
	}
474 475
	/* put any unprocessed entries back */
	list_splice(&starved_list, &shost->starved_list);
L
Linus Torvalds 已提交
476
	spin_unlock_irqrestore(shost->host_lock, flags);
477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498
}

/*
 * Function:   scsi_run_queue()
 *
 * Purpose:    Select a proper request queue to serve next
 *
 * Arguments:  q       - last request's queue
 *
 * Returns:     Nothing
 *
 * Notes:      The previous command was completely finished, start
 *             a new one if possible.
 */
static void scsi_run_queue(struct request_queue *q)
{
	struct scsi_device *sdev = q->queuedata;

	if (scsi_target(sdev)->single_lun)
		scsi_single_lun_run(sdev);
	if (!list_empty(&sdev->host->starved_list))
		scsi_starved_list_run(sdev->host);
L
Linus Torvalds 已提交
499

500
	if (q->mq_ops)
501
		blk_mq_run_hw_queues(q, false);
502 503
	else
		blk_run_queue(q);
L
Linus Torvalds 已提交
504 505
}

506 507 508 509 510 511 512 513 514 515
void scsi_requeue_run_queue(struct work_struct *work)
{
	struct scsi_device *sdev;
	struct request_queue *q;

	sdev = container_of(work, struct scsi_device, requeue_work);
	q = sdev->request_queue;
	scsi_run_queue(q);
}

L
Linus Torvalds 已提交
516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531
/*
 * Function:	scsi_requeue_command()
 *
 * Purpose:	Handle post-processing of completed commands.
 *
 * Arguments:	q	- queue to operate on
 *		cmd	- command that may need to be requeued.
 *
 * Returns:	Nothing
 *
 * Notes:	After command completion, there may be blocks left
 *		over which weren't finished by the previous command
 *		this can be for a number of reasons - the main one is
 *		I/O errors in the middle of the request, in which case
 *		we need to request the blocks that come after the bad
 *		sector.
532
 * Notes:	Upon return, cmd is a stale pointer.
L
Linus Torvalds 已提交
533 534 535
 */
static void scsi_requeue_command(struct request_queue *q, struct scsi_cmnd *cmd)
{
536
	struct scsi_device *sdev = cmd->device;
537
	struct request *req = cmd->request;
538 539 540
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
541 542 543
	blk_unprep_request(req);
	req->special = NULL;
	scsi_put_command(cmd);
544
	blk_requeue_request(q, req);
545
	spin_unlock_irqrestore(q->queue_lock, flags);
L
Linus Torvalds 已提交
546 547

	scsi_run_queue(q);
548 549

	put_device(&sdev->sdev_gendev);
L
Linus Torvalds 已提交
550 551 552 553 554 555 556 557 558 559
}

void scsi_run_host_queues(struct Scsi_Host *shost)
{
	struct scsi_device *sdev;

	shost_for_each_device(sdev, shost)
		scsi_run_queue(sdev->request_queue);
}

560 561
static void scsi_uninit_cmd(struct scsi_cmnd *cmd)
{
562
	if (!blk_rq_is_passthrough(cmd->request)) {
563 564 565 566 567 568 569 570 571
		struct scsi_driver *drv = scsi_cmd_to_driver(cmd);

		if (drv->uninit_command)
			drv->uninit_command(cmd);
	}
}

static void scsi_mq_free_sgtables(struct scsi_cmnd *cmd)
{
572 573
	struct scsi_data_buffer *sdb;

574
	if (cmd->sdb.table.nents)
575
		sg_free_table_chained(&cmd->sdb.table, true);
576 577 578
	if (cmd->request->next_rq) {
		sdb = cmd->request->next_rq->special;
		if (sdb)
579
			sg_free_table_chained(&sdb->table, true);
580
	}
581
	if (scsi_prot_sg_count(cmd))
582
		sg_free_table_chained(&cmd->prot_sdb->table, true);
583 584 585 586 587
}

static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd)
{
	struct scsi_device *sdev = cmd->device;
588
	struct Scsi_Host *shost = sdev->host;
589 590 591 592 593
	unsigned long flags;

	scsi_mq_free_sgtables(cmd);
	scsi_uninit_cmd(cmd);

594 595 596 597 598 599
	if (shost->use_cmd_list) {
		BUG_ON(list_empty(&cmd->list));
		spin_lock_irqsave(&sdev->list_lock, flags);
		list_del_init(&cmd->list);
		spin_unlock_irqrestore(&sdev->list_lock, flags);
	}
600 601
}

L
Linus Torvalds 已提交
602 603 604
/*
 * Function:    scsi_release_buffers()
 *
605
 * Purpose:     Free resources allocate for a scsi_command.
L
Linus Torvalds 已提交
606 607 608 609 610 611 612 613 614 615
 *
 * Arguments:   cmd	- command that we are bailing.
 *
 * Lock status: Assumed that no lock is held upon entry.
 *
 * Returns:     Nothing
 *
 * Notes:       In the event that an upper level driver rejects a
 *		command, we must release resources allocated during
 *		the __init_io() function.  Primarily this would involve
616
 *		the scatter-gather table.
L
Linus Torvalds 已提交
617
 */
618
static void scsi_release_buffers(struct scsi_cmnd *cmd)
L
Linus Torvalds 已提交
619
{
620
	if (cmd->sdb.table.nents)
621
		sg_free_table_chained(&cmd->sdb.table, false);
622 623 624 625

	memset(&cmd->sdb, 0, sizeof(cmd->sdb));

	if (scsi_prot_sg_count(cmd))
626
		sg_free_table_chained(&cmd->prot_sdb->table, false);
L
Linus Torvalds 已提交
627 628
}

629 630 631 632
static void scsi_release_bidi_buffers(struct scsi_cmnd *cmd)
{
	struct scsi_data_buffer *bidi_sdb = cmd->request->next_rq->special;

633
	sg_free_table_chained(&bidi_sdb->table, false);
634 635 636 637
	kmem_cache_free(scsi_sdb_cache, bidi_sdb);
	cmd->request->next_rq->special = NULL;
}

638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655
static bool scsi_end_request(struct request *req, int error,
		unsigned int bytes, unsigned int bidi_bytes)
{
	struct scsi_cmnd *cmd = req->special;
	struct scsi_device *sdev = cmd->device;
	struct request_queue *q = sdev->request_queue;

	if (blk_update_request(req, error, bytes))
		return true;

	/* Bidi request must be completed as a whole */
	if (unlikely(bidi_bytes) &&
	    blk_update_request(req->next_rq, error, bidi_bytes))
		return true;

	if (blk_queue_add_random(q))
		add_disk_randomness(req->rq_disk);

656 657
	if (req->mq_ctx) {
		/*
658
		 * In the MQ case the command gets freed by __blk_mq_end_request,
659 660 661 662 663 664 665
		 * so we have to do all cleanup that depends on it earlier.
		 *
		 * We also can't kick the queues from irq context, so we
		 * will have to defer it to a workqueue.
		 */
		scsi_mq_uninit_cmd(cmd);

666
		__blk_mq_end_request(req, error);
667 668 669 670 671

		if (scsi_target(sdev)->single_lun ||
		    !list_empty(&sdev->host->starved_list))
			kblockd_schedule_work(&sdev->requeue_work);
		else
672
			blk_mq_run_hw_queues(q, true);
673 674 675
	} else {
		unsigned long flags;

676 677
		if (bidi_bytes)
			scsi_release_bidi_buffers(cmd);
678 679
		scsi_release_buffers(cmd);
		scsi_put_command(cmd);
680

681 682 683 684
		spin_lock_irqsave(q->queue_lock, flags);
		blk_finish_request(req, error);
		spin_unlock_irqrestore(q->queue_lock, flags);

C
Christoph Hellwig 已提交
685
		scsi_run_queue(q);
686
	}
687

C
Christoph Hellwig 已提交
688
	put_device(&sdev->sdev_gendev);
689 690 691
	return false;
}

692 693 694 695 696 697 698 699 700 701
/**
 * __scsi_error_from_host_byte - translate SCSI error code into errno
 * @cmd:	SCSI command (unused)
 * @result:	scsi error code
 *
 * Translate SCSI error code into standard UNIX errno.
 * Return values:
 * -ENOLINK	temporary transport failure
 * -EREMOTEIO	permanent target failure, do not retry
 * -EBADE	permanent nexus failure, retry on other path
702
 * -ENOSPC	No write space available
703
 * -ENODATA	Medium error
704 705
 * -EIO		unspecified I/O error
 */
706 707 708 709 710 711 712 713 714
static int __scsi_error_from_host_byte(struct scsi_cmnd *cmd, int result)
{
	int error = 0;

	switch(host_byte(result)) {
	case DID_TRANSPORT_FAILFAST:
		error = -ENOLINK;
		break;
	case DID_TARGET_FAILURE:
715
		set_host_byte(cmd, DID_OK);
716 717 718
		error = -EREMOTEIO;
		break;
	case DID_NEXUS_FAILURE:
719
		set_host_byte(cmd, DID_OK);
720 721
		error = -EBADE;
		break;
722 723 724 725
	case DID_ALLOC_FAILURE:
		set_host_byte(cmd, DID_OK);
		error = -ENOSPC;
		break;
726 727 728 729
	case DID_MEDIUM_ERROR:
		set_host_byte(cmd, DID_OK);
		error = -ENODATA;
		break;
730 731 732 733 734 735 736 737
	default:
		error = -EIO;
		break;
	}

	return error;
}

L
Linus Torvalds 已提交
738 739 740 741 742 743 744 745 746 747 748
/*
 * Function:    scsi_io_completion()
 *
 * Purpose:     Completion processing for block device I/O requests.
 *
 * Arguments:   cmd   - command that is finished.
 *
 * Lock status: Assumed that no lock is held upon entry.
 *
 * Returns:     Nothing
 *
749 750 751
 * Notes:       We will finish off the specified number of sectors.  If we
 *		are done, the command block will be released and the queue
 *		function will be goosed.  If we are not done then we have to
A
Alan Stern 已提交
752
 *		figure out what to do next:
L
Linus Torvalds 已提交
753
 *
A
Alan Stern 已提交
754 755 756 757 758
 *		a) We can call scsi_requeue_command().  The request
 *		   will be unprepared and put back on the queue.  Then
 *		   a new command will be created for it.  This should
 *		   be used if we made forward progress, or if we want
 *		   to switch from READ(10) to READ(6) for example.
L
Linus Torvalds 已提交
759
 *
760
 *		b) We can call __scsi_queue_insert().  The request will
A
Alan Stern 已提交
761 762 763
 *		   be put back on the queue and retried using the same
 *		   command as before, possibly after a delay.
 *
764
 *		c) We can call scsi_end_request() with -EIO to fail
A
Alan Stern 已提交
765
 *		   the remainder of the request.
L
Linus Torvalds 已提交
766
 */
767
void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
L
Linus Torvalds 已提交
768 769
{
	int result = cmd->result;
770
	struct request_queue *q = cmd->device->request_queue;
L
Linus Torvalds 已提交
771
	struct request *req = cmd->request;
772
	int error = 0;
L
Linus Torvalds 已提交
773
	struct scsi_sense_hdr sshdr;
774
	bool sense_valid = false;
775
	int sense_deferred = 0, level = 0;
A
Alan Stern 已提交
776 777
	enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY,
	      ACTION_DELAYED_RETRY} action;
778
	unsigned long wait_for = (cmd->allowed + 1) * req->timeout;
L
Linus Torvalds 已提交
779 780 781 782 783 784

	if (result) {
		sense_valid = scsi_command_normalize_sense(cmd, &sshdr);
		if (sense_valid)
			sense_deferred = scsi_sense_is_deferred(&sshdr);
	}
785

786
	if (blk_rq_is_passthrough(req)) {
L
Linus Torvalds 已提交
787
		if (result) {
788
			if (sense_valid) {
L
Linus Torvalds 已提交
789 790 791
				/*
				 * SG_IO wants current and deferred errors
				 */
792 793 794
				scsi_req(req)->sense_len =
					min(8 + cmd->sense_buffer[7],
					    SCSI_SENSE_BUFFERSIZE);
L
Linus Torvalds 已提交
795
			}
796
			if (!sense_deferred)
797
				error = __scsi_error_from_host_byte(cmd, result);
798
		}
799 800 801
		/*
		 * __scsi_error_from_host_byte may have reset the host_byte
		 */
802
		scsi_req(req)->result = cmd->result;
803
		scsi_req(req)->resid_len = scsi_get_resid(cmd);
804

805
		if (scsi_bidi_cmnd(cmd)) {
806 807 808 809
			/*
			 * Bidi commands Must be complete as a whole,
			 * both sides at once.
			 */
810
			scsi_req(req->next_rq)->resid_len = scsi_in(cmd)->resid;
811 812 813
			if (scsi_end_request(req, 0, blk_rq_bytes(req),
					blk_rq_bytes(req->next_rq)))
				BUG();
814 815
			return;
		}
816 817
	} else if (blk_rq_bytes(req) == 0 && result && !sense_deferred) {
		/*
818
		 * Flush commands do not transfers any data, and thus cannot use
819 820 821 822
		 * good_bytes != blk_rq_bytes(req) as the signal for an error.
		 * This sets the error explicitly for the problem case.
		 */
		error = __scsi_error_from_host_byte(cmd, result);
L
Linus Torvalds 已提交
823 824
	}

825
	/* no bidi support for !blk_rq_is_passthrough yet */
826
	BUG_ON(blk_bidi_rq(req));
B
Boaz Harrosh 已提交
827

L
Linus Torvalds 已提交
828 829 830 831
	/*
	 * Next deal with any sectors which we were able to correctly
	 * handle.
	 */
832 833 834
	SCSI_LOG_HLCOMPLETE(1, scmd_printk(KERN_INFO, cmd,
		"%u sectors total, %d bytes done.\n",
		blk_rq_sectors(req), good_bytes));
835

836
	/*
837 838
	 * Recovered errors need reporting, but they're always treated as
	 * success, so fiddle the result code here.  For passthrough requests
839
	 * we already took a copy of the original into sreq->result which
840 841
	 * is what gets returned to the user
	 */
842 843 844 845 846 847 848
	if (sense_valid && (sshdr.sense_key == RECOVERED_ERROR)) {
		/* if ATA PASS-THROUGH INFORMATION AVAILABLE skip
		 * print since caller wants ATA registers. Only occurs on
		 * SCSI ATA PASS_THROUGH commands when CK_COND=1
		 */
		if ((sshdr.asc == 0x0) && (sshdr.ascq == 0x1d))
			;
849
		else if (!(req->rq_flags & RQF_QUIET))
850
			scsi_print_sense(cmd);
851
		result = 0;
852
		/* for passthrough error may be set */
853 854 855 856
		error = 0;
	}

	/*
857 858 859
	 * special case: failed zero length commands always need to
	 * drop down into the retry code. Otherwise, if we finished
	 * all bytes in the request we are done now.
860
	 */
861 862
	if (!(blk_rq_bytes(req) == 0 && error) &&
	    !scsi_end_request(req, error, good_bytes, 0))
863
		return;
864 865 866 867 868

	/*
	 * Kill remainder if no retrys.
	 */
	if (error && scsi_noretry_cmd(cmd)) {
869 870 871
		if (scsi_end_request(req, error, blk_rq_bytes(req), 0))
			BUG();
		return;
872 873 874 875 876
	}

	/*
	 * If there had been no error, but we have leftover bytes in the
	 * requeues just queue the command up again.
877
	 */
878 879
	if (result == 0)
		goto requeue;
880

881
	error = __scsi_error_from_host_byte(cmd, result);
882

A
Alan Stern 已提交
883 884 885 886 887 888 889
	if (host_byte(result) == DID_RESET) {
		/* Third party bus reset or reset for error recovery
		 * reasons.  Just retry the command and see what
		 * happens.
		 */
		action = ACTION_RETRY;
	} else if (sense_valid && !sense_deferred) {
L
Linus Torvalds 已提交
890 891 892
		switch (sshdr.sense_key) {
		case UNIT_ATTENTION:
			if (cmd->device->removable) {
893
				/* Detected disc change.  Set a bit
L
Linus Torvalds 已提交
894 895 896
				 * and quietly refuse further access.
				 */
				cmd->device->changed = 1;
A
Alan Stern 已提交
897
				action = ACTION_FAIL;
L
Linus Torvalds 已提交
898
			} else {
899 900 901
				/* Must have been a power glitch, or a
				 * bus reset.  Could not have been a
				 * media change, so we just retry the
A
Alan Stern 已提交
902
				 * command and see what happens.
903
				 */
A
Alan Stern 已提交
904
				action = ACTION_RETRY;
L
Linus Torvalds 已提交
905 906 907
			}
			break;
		case ILLEGAL_REQUEST:
908 909 910 911 912 913 914 915
			/* If we had an ILLEGAL REQUEST returned, then
			 * we may have performed an unsupported
			 * command.  The only thing this should be
			 * would be a ten byte read where only a six
			 * byte read was supported.  Also, on a system
			 * where READ CAPACITY failed, we may have
			 * read past the end of the disk.
			 */
916 917
			if ((cmd->device->use_10_for_rw &&
			    sshdr.asc == 0x20 && sshdr.ascq == 0x00) &&
L
Linus Torvalds 已提交
918 919
			    (cmd->cmnd[0] == READ_10 ||
			     cmd->cmnd[0] == WRITE_10)) {
A
Alan Stern 已提交
920
				/* This will issue a new 6-byte command. */
L
Linus Torvalds 已提交
921
				cmd->device->use_10_for_rw = 0;
A
Alan Stern 已提交
922
				action = ACTION_REPREP;
923 924 925
			} else if (sshdr.asc == 0x10) /* DIX */ {
				action = ACTION_FAIL;
				error = -EILSEQ;
926
			/* INVALID COMMAND OPCODE or INVALID FIELD IN CDB */
927
			} else if (sshdr.asc == 0x20 || sshdr.asc == 0x24) {
928
				action = ACTION_FAIL;
929
				error = -EREMOTEIO;
A
Alan Stern 已提交
930 931 932
			} else
				action = ACTION_FAIL;
			break;
933
		case ABORTED_COMMAND:
934
			action = ACTION_FAIL;
935
			if (sshdr.asc == 0x10) /* DIF */
936
				error = -EILSEQ;
L
Linus Torvalds 已提交
937 938
			break;
		case NOT_READY:
939
			/* If the device is in the process of becoming
J
James Bottomley 已提交
940
			 * ready, or has a temporary blockage, retry.
L
Linus Torvalds 已提交
941
			 */
J
James Bottomley 已提交
942 943 944 945 946 947 948 949 950
			if (sshdr.asc == 0x04) {
				switch (sshdr.ascq) {
				case 0x01: /* becoming ready */
				case 0x04: /* format in progress */
				case 0x05: /* rebuild in progress */
				case 0x06: /* recalculation in progress */
				case 0x07: /* operation in progress */
				case 0x08: /* Long write in progress */
				case 0x09: /* self test in progress */
951
				case 0x14: /* space allocation in progress */
A
Alan Stern 已提交
952
					action = ACTION_DELAYED_RETRY;
J
James Bottomley 已提交
953
					break;
954 955 956
				default:
					action = ACTION_FAIL;
					break;
J
James Bottomley 已提交
957
				}
958
			} else
A
Alan Stern 已提交
959 960
				action = ACTION_FAIL;
			break;
L
Linus Torvalds 已提交
961
		case VOLUME_OVERFLOW:
962
			/* See SSC3rXX or current. */
A
Alan Stern 已提交
963 964
			action = ACTION_FAIL;
			break;
L
Linus Torvalds 已提交
965
		default:
A
Alan Stern 已提交
966
			action = ACTION_FAIL;
L
Linus Torvalds 已提交
967 968
			break;
		}
969
	} else
A
Alan Stern 已提交
970 971
		action = ACTION_FAIL;

972
	if (action != ACTION_FAIL &&
973
	    time_before(cmd->jiffies_at_alloc + wait_for, jiffies))
974 975
		action = ACTION_FAIL;

A
Alan Stern 已提交
976 977 978
	switch (action) {
	case ACTION_FAIL:
		/* Give up and fail the remainder of the request */
979
		if (!(req->rq_flags & RQF_QUIET)) {
980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997
			static DEFINE_RATELIMIT_STATE(_rs,
					DEFAULT_RATELIMIT_INTERVAL,
					DEFAULT_RATELIMIT_BURST);

			if (unlikely(scsi_logging_level))
				level = SCSI_LOG_LEVEL(SCSI_LOG_MLCOMPLETE_SHIFT,
						       SCSI_LOG_MLCOMPLETE_BITS);

			/*
			 * if logging is enabled the failure will be printed
			 * in scsi_log_completion(), so avoid duplicate messages
			 */
			if (!level && __ratelimit(&_rs)) {
				scsi_print_result(cmd, NULL, FAILED);
				if (driver_byte(result) & DRIVER_SENSE)
					scsi_print_sense(cmd);
				scsi_print_command(cmd);
			}
998
		}
999 1000
		if (!scsi_end_request(req, error, blk_rq_err_bytes(req), 0))
			return;
1001
		/*FALLTHRU*/
A
Alan Stern 已提交
1002
	case ACTION_REPREP:
1003
	requeue:
A
Alan Stern 已提交
1004 1005 1006
		/* Unprep the request and put it back at the head of the queue.
		 * A new command will be prepared and issued.
		 */
1007
		if (q->mq_ops) {
1008
			cmd->request->rq_flags &= ~RQF_DONTPREP;
1009 1010 1011 1012 1013 1014
			scsi_mq_uninit_cmd(cmd);
			scsi_mq_requeue_cmd(cmd);
		} else {
			scsi_release_buffers(cmd);
			scsi_requeue_command(q, cmd);
		}
A
Alan Stern 已提交
1015 1016 1017
		break;
	case ACTION_RETRY:
		/* Retry the same command immediately */
1018
		__scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY, 0);
A
Alan Stern 已提交
1019 1020 1021
		break;
	case ACTION_DELAYED_RETRY:
		/* Retry the same command after a delay */
1022
		__scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY, 0);
A
Alan Stern 已提交
1023
		break;
L
Linus Torvalds 已提交
1024 1025 1026
	}
}

1027
static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb)
L
Linus Torvalds 已提交
1028
{
1029
	int count;
L
Linus Torvalds 已提交
1030 1031

	/*
C
Christoph Hellwig 已提交
1032
	 * If sg table allocation fails, requeue request later.
L
Linus Torvalds 已提交
1033
	 */
1034 1035
	if (unlikely(sg_alloc_table_chained(&sdb->table,
			blk_rq_nr_phys_segments(req), sdb->table.sgl)))
L
Linus Torvalds 已提交
1036 1037 1038 1039 1040 1041
		return BLKPREP_DEFER;

	/* 
	 * Next, walk the list, and fill in the addresses and sizes of
	 * each segment.
	 */
B
Boaz Harrosh 已提交
1042 1043 1044
	count = blk_rq_map_sg(req->q, req, sdb->table.sgl);
	BUG_ON(count > sdb->table.nents);
	sdb->table.nents = count;
1045
	sdb->length = blk_rq_payload_bytes(req);
1046
	return BLKPREP_OK;
L
Linus Torvalds 已提交
1047
}
1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059

/*
 * Function:    scsi_init_io()
 *
 * Purpose:     SCSI I/O initialize function.
 *
 * Arguments:   cmd   - Command descriptor we wish to initialize
 *
 * Returns:     0 on success
 *		BLKPREP_DEFER if the failure is retryable
 *		BLKPREP_KILL if the failure is fatal
 */
1060
int scsi_init_io(struct scsi_cmnd *cmd)
1061
{
1062
	struct scsi_device *sdev = cmd->device;
1063
	struct request *rq = cmd->request;
1064
	bool is_mq = (rq->mq_ctx != NULL);
1065
	int error = BLKPREP_KILL;
1066

1067
	if (WARN_ON_ONCE(!blk_rq_nr_phys_segments(rq)))
1068
		goto err_exit;
1069

1070
	error = scsi_init_sgtable(rq, &cmd->sdb);
1071 1072 1073
	if (error)
		goto err_exit;

1074
	if (blk_bidi_rq(rq)) {
1075 1076 1077 1078 1079 1080 1081 1082 1083
		if (!rq->q->mq_ops) {
			struct scsi_data_buffer *bidi_sdb =
				kmem_cache_zalloc(scsi_sdb_cache, GFP_ATOMIC);
			if (!bidi_sdb) {
				error = BLKPREP_DEFER;
				goto err_exit;
			}

			rq->next_rq->special = bidi_sdb;
1084 1085
		}

1086
		error = scsi_init_sgtable(rq->next_rq, rq->next_rq->special);
1087 1088 1089 1090
		if (error)
			goto err_exit;
	}

1091
	if (blk_integrity_rq(rq)) {
1092 1093 1094
		struct scsi_data_buffer *prot_sdb = cmd->prot_sdb;
		int ivecs, count;

1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
		if (prot_sdb == NULL) {
			/*
			 * This can happen if someone (e.g. multipath)
			 * queues a command to a device on an adapter
			 * that does not support DIX.
			 */
			WARN_ON_ONCE(1);
			error = BLKPREP_KILL;
			goto err_exit;
		}

1106
		ivecs = blk_rq_count_integrity_sg(rq->q, rq->bio);
1107

1108
		if (sg_alloc_table_chained(&prot_sdb->table, ivecs,
1109
				prot_sdb->table.sgl)) {
1110 1111 1112 1113
			error = BLKPREP_DEFER;
			goto err_exit;
		}

1114
		count = blk_rq_map_integrity_sg(rq->q, rq->bio,
1115 1116
						prot_sdb->table.sgl);
		BUG_ON(unlikely(count > ivecs));
1117
		BUG_ON(unlikely(count > queue_max_integrity_segments(rq->q)));
1118 1119 1120 1121 1122

		cmd->prot_sdb = prot_sdb;
		cmd->prot_sdb->table.nents = count;
	}

1123
	return BLKPREP_OK;
1124
err_exit:
1125 1126 1127 1128 1129 1130 1131 1132
	if (is_mq) {
		scsi_mq_free_sgtables(cmd);
	} else {
		scsi_release_buffers(cmd);
		cmd->request->special = NULL;
		scsi_put_command(cmd);
		put_device(&sdev->sdev_gendev);
	}
1133 1134
	return error;
}
1135
EXPORT_SYMBOL(scsi_init_io);
L
Linus Torvalds 已提交
1136

1137
void scsi_init_command(struct scsi_device *dev, struct scsi_cmnd *cmd)
C
Christoph Hellwig 已提交
1138
{
1139 1140 1141
	void *buf = cmd->sense_buffer;
	void *prot = cmd->prot_sdb;
	unsigned long flags;
C
Christoph Hellwig 已提交
1142

1143 1144
	/* zero out the cmd, except for the embedded scsi_request */
	memset((char *)cmd + sizeof(cmd->req), 0,
1145
		sizeof(*cmd) - sizeof(cmd->req) + dev->host->hostt->cmd_size);
C
Christoph Hellwig 已提交
1146

1147 1148 1149 1150 1151
	cmd->device = dev;
	cmd->sense_buffer = buf;
	cmd->prot_sdb = prot;
	INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler);
	cmd->jiffies_at_alloc = jiffies;
1152

1153 1154 1155
	spin_lock_irqsave(&dev->list_lock, flags);
	list_add_tail(&cmd->list, &dev->cmd_list);
	spin_unlock_irqrestore(&dev->list_lock, flags);
C
Christoph Hellwig 已提交
1156 1157
}

1158
static int scsi_setup_scsi_cmnd(struct scsi_device *sdev, struct request *req)
J
James Bottomley 已提交
1159
{
1160
	struct scsi_cmnd *cmd = req->special;
C
Christoph Hellwig 已提交
1161 1162

	/*
1163
	 * Passthrough requests may transfer data, in which case they must
C
Christoph Hellwig 已提交
1164 1165 1166 1167 1168
	 * a bio attached to them.  Or they might contain a SCSI command
	 * that does not transfer data, in which case they may optionally
	 * submit a request without an attached bio.
	 */
	if (req->bio) {
1169
		int ret = scsi_init_io(cmd);
C
Christoph Hellwig 已提交
1170 1171 1172
		if (unlikely(ret))
			return ret;
	} else {
T
Tejun Heo 已提交
1173
		BUG_ON(blk_rq_bytes(req));
C
Christoph Hellwig 已提交
1174

B
Boaz Harrosh 已提交
1175
		memset(&cmd->sdb, 0, sizeof(cmd->sdb));
C
Christoph Hellwig 已提交
1176
	}
J
James Bottomley 已提交
1177

1178 1179
	cmd->cmd_len = scsi_req(req)->cmd_len;
	cmd->cmnd = scsi_req(req)->cmd;
T
Tejun Heo 已提交
1180
	cmd->transfersize = blk_rq_bytes(req);
1181
	cmd->allowed = scsi_req(req)->retries;
C
Christoph Hellwig 已提交
1182
	return BLKPREP_OK;
J
James Bottomley 已提交
1183 1184
}

C
Christoph Hellwig 已提交
1185
/*
1186
 * Setup a normal block command.  These are simple request from filesystems
1187
 * that still need to be translated to SCSI CDBs from the ULD.
C
Christoph Hellwig 已提交
1188
 */
1189
static int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req)
L
Linus Torvalds 已提交
1190
{
1191
	struct scsi_cmnd *cmd = req->special;
1192

1193 1194
	if (unlikely(sdev->handler && sdev->handler->prep_fn)) {
		int ret = sdev->handler->prep_fn(sdev, req);
1195 1196 1197 1198
		if (ret != BLKPREP_OK)
			return ret;
	}

1199
	cmd->cmnd = scsi_req(req)->cmd = scsi_req(req)->__cmd;
1200
	memset(cmd->cmnd, 0, BLK_MAX_CDB);
1201
	return scsi_cmd_to_driver(cmd)->init_command(cmd);
C
Christoph Hellwig 已提交
1202 1203
}

1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214
static int scsi_setup_cmnd(struct scsi_device *sdev, struct request *req)
{
	struct scsi_cmnd *cmd = req->special;

	if (!blk_rq_bytes(req))
		cmd->sc_data_direction = DMA_NONE;
	else if (rq_data_dir(req) == WRITE)
		cmd->sc_data_direction = DMA_TO_DEVICE;
	else
		cmd->sc_data_direction = DMA_FROM_DEVICE;

1215 1216 1217
	if (blk_rq_is_scsi(req))
		return scsi_setup_scsi_cmnd(sdev, req);
	else
1218 1219 1220
		return scsi_setup_fs_cmnd(sdev, req);
}

1221 1222
static int
scsi_prep_state_check(struct scsi_device *sdev, struct request *req)
C
Christoph Hellwig 已提交
1223 1224 1225
{
	int ret = BLKPREP_OK;

L
Linus Torvalds 已提交
1226
	/*
C
Christoph Hellwig 已提交
1227 1228
	 * If the device is not in running state we will reject some
	 * or all commands.
L
Linus Torvalds 已提交
1229
	 */
C
Christoph Hellwig 已提交
1230 1231 1232
	if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
		switch (sdev->sdev_state) {
		case SDEV_OFFLINE:
1233
		case SDEV_TRANSPORT_OFFLINE:
C
Christoph Hellwig 已提交
1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247
			/*
			 * If the device is offline we refuse to process any
			 * commands.  The device must be brought online
			 * before trying any recovery commands.
			 */
			sdev_printk(KERN_ERR, sdev,
				    "rejecting I/O to offline device\n");
			ret = BLKPREP_KILL;
			break;
		case SDEV_DEL:
			/*
			 * If the device is fully deleted, we refuse to
			 * process any commands as well.
			 */
1248
			sdev_printk(KERN_ERR, sdev,
C
Christoph Hellwig 已提交
1249 1250 1251 1252
				    "rejecting I/O to dead device\n");
			ret = BLKPREP_KILL;
			break;
		case SDEV_BLOCK:
1253
		case SDEV_CREATED_BLOCK:
1254 1255 1256
			ret = BLKPREP_DEFER;
			break;
		case SDEV_QUIESCE:
C
Christoph Hellwig 已提交
1257 1258 1259
			/*
			 * If the devices is blocked we defer normal commands.
			 */
1260
			if (!(req->rq_flags & RQF_PREEMPT))
C
Christoph Hellwig 已提交
1261 1262 1263 1264 1265 1266 1267 1268
				ret = BLKPREP_DEFER;
			break;
		default:
			/*
			 * For any other not fully online state we only allow
			 * special commands.  In particular any user initiated
			 * command is not allowed.
			 */
1269
			if (!(req->rq_flags & RQF_PREEMPT))
C
Christoph Hellwig 已提交
1270 1271
				ret = BLKPREP_KILL;
			break;
L
Linus Torvalds 已提交
1272 1273
		}
	}
1274 1275
	return ret;
}
L
Linus Torvalds 已提交
1276

1277 1278
static int
scsi_prep_return(struct request_queue *q, struct request *req, int ret)
1279 1280
{
	struct scsi_device *sdev = q->queuedata;
L
Linus Torvalds 已提交
1281

C
Christoph Hellwig 已提交
1282 1283
	switch (ret) {
	case BLKPREP_KILL:
1284
	case BLKPREP_INVALID:
1285
		scsi_req(req)->result = DID_NO_CONNECT << 16;
1286 1287 1288 1289 1290
		/* release the command and kill it */
		if (req->special) {
			struct scsi_cmnd *cmd = req->special;
			scsi_release_buffers(cmd);
			scsi_put_command(cmd);
1291
			put_device(&sdev->sdev_gendev);
1292 1293
			req->special = NULL;
		}
C
Christoph Hellwig 已提交
1294 1295
		break;
	case BLKPREP_DEFER:
L
Linus Torvalds 已提交
1296
		/*
1297
		 * If we defer, the blk_peek_request() returns NULL, but the
J
Jens Axboe 已提交
1298 1299
		 * queue must be restarted, so we schedule a callback to happen
		 * shortly.
L
Linus Torvalds 已提交
1300
		 */
1301
		if (atomic_read(&sdev->device_busy) == 0)
J
Jens Axboe 已提交
1302
			blk_delay_queue(q, SCSI_QUEUE_DELAY);
C
Christoph Hellwig 已提交
1303 1304
		break;
	default:
1305
		req->rq_flags |= RQF_DONTPREP;
L
Linus Torvalds 已提交
1306 1307
	}

C
Christoph Hellwig 已提交
1308
	return ret;
L
Linus Torvalds 已提交
1309
}
1310

1311
static int scsi_prep_fn(struct request_queue *q, struct request *req)
1312 1313
{
	struct scsi_device *sdev = q->queuedata;
1314
	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
1315 1316 1317 1318 1319 1320
	int ret;

	ret = scsi_prep_state_check(sdev, req);
	if (ret != BLKPREP_OK)
		goto out;

1321 1322 1323 1324 1325 1326 1327 1328 1329
	if (!req->special) {
		/* Bail if we can't get a reference to the device */
		if (unlikely(!get_device(&sdev->sdev_gendev))) {
			ret = BLKPREP_DEFER;
			goto out;
		}

		scsi_init_command(sdev, cmd);
		req->special = cmd;
1330
	}
1331

1332 1333 1334 1335
	cmd->tag = req->tag;
	cmd->request = req;
	cmd->prot_op = SCSI_PROT_NORMAL;

1336
	ret = scsi_setup_cmnd(sdev, req);
1337
out:
1338 1339
	return scsi_prep_return(q, req, ret);
}
1340 1341 1342

static void scsi_unprep_fn(struct request_queue *q, struct request *req)
{
1343
	scsi_uninit_cmd(req->special);
1344
}
L
Linus Torvalds 已提交
1345 1346 1347 1348 1349 1350 1351 1352 1353 1354

/*
 * scsi_dev_queue_ready: if we can send requests to sdev, return 1 else
 * return 0.
 *
 * Called with the queue_lock held.
 */
static inline int scsi_dev_queue_ready(struct request_queue *q,
				  struct scsi_device *sdev)
{
1355 1356 1357
	unsigned int busy;

	busy = atomic_inc_return(&sdev->device_busy) - 1;
1358
	if (atomic_read(&sdev->device_blocked)) {
1359 1360 1361
		if (busy)
			goto out_dec;

L
Linus Torvalds 已提交
1362 1363 1364
		/*
		 * unblock after device_blocked iterates to zero
		 */
1365
		if (atomic_dec_return(&sdev->device_blocked) > 0) {
1366 1367 1368 1369 1370
			/*
			 * For the MQ case we take care of this in the caller.
			 */
			if (!q->mq_ops)
				blk_delay_queue(q, SCSI_QUEUE_DELAY);
1371
			goto out_dec;
L
Linus Torvalds 已提交
1372
		}
1373 1374
		SCSI_LOG_MLQUEUE(3, sdev_printk(KERN_INFO, sdev,
				   "unblocking device at zero depth\n"));
L
Linus Torvalds 已提交
1375
	}
1376 1377 1378

	if (busy >= sdev->queue_depth)
		goto out_dec;
L
Linus Torvalds 已提交
1379 1380

	return 1;
1381 1382 1383
out_dec:
	atomic_dec(&sdev->device_busy);
	return 0;
L
Linus Torvalds 已提交
1384 1385
}

1386 1387 1388 1389 1390 1391 1392 1393
/*
 * scsi_target_queue_ready: checks if there we can send commands to target
 * @sdev: scsi device on starget to check.
 */
static inline int scsi_target_queue_ready(struct Scsi_Host *shost,
					   struct scsi_device *sdev)
{
	struct scsi_target *starget = scsi_target(sdev);
1394
	unsigned int busy;
1395 1396

	if (starget->single_lun) {
1397
		spin_lock_irq(shost->host_lock);
1398
		if (starget->starget_sdev_user &&
1399 1400 1401 1402
		    starget->starget_sdev_user != sdev) {
			spin_unlock_irq(shost->host_lock);
			return 0;
		}
1403
		starget->starget_sdev_user = sdev;
1404
		spin_unlock_irq(shost->host_lock);
1405 1406
	}

1407 1408 1409
	if (starget->can_queue <= 0)
		return 1;

1410
	busy = atomic_inc_return(&starget->target_busy) - 1;
1411
	if (atomic_read(&starget->target_blocked) > 0) {
1412 1413 1414
		if (busy)
			goto starved;

1415 1416 1417
		/*
		 * unblock after target_blocked iterates to zero
		 */
1418
		if (atomic_dec_return(&starget->target_blocked) > 0)
1419
			goto out_dec;
1420 1421 1422

		SCSI_LOG_MLQUEUE(3, starget_printk(KERN_INFO, starget,
				 "unblocking target at zero depth\n"));
1423 1424
	}

1425
	if (busy >= starget->can_queue)
1426
		goto starved;
1427

1428 1429 1430 1431 1432
	return 1;

starved:
	spin_lock_irq(shost->host_lock);
	list_move_tail(&sdev->starved_entry, &shost->starved_list);
1433
	spin_unlock_irq(shost->host_lock);
1434
out_dec:
1435 1436
	if (starget->can_queue > 0)
		atomic_dec(&starget->target_busy);
1437
	return 0;
1438 1439
}

L
Linus Torvalds 已提交
1440 1441 1442 1443 1444 1445 1446 1447 1448
/*
 * scsi_host_queue_ready: if we can send requests to shost, return 1 else
 * return 0. We must end up running the queue again whenever 0 is
 * returned, else IO can hang.
 */
static inline int scsi_host_queue_ready(struct request_queue *q,
				   struct Scsi_Host *shost,
				   struct scsi_device *sdev)
{
1449
	unsigned int busy;
1450

1451
	if (scsi_host_in_recovery(shost))
1452 1453 1454
		return 0;

	busy = atomic_inc_return(&shost->host_busy) - 1;
1455
	if (atomic_read(&shost->host_blocked) > 0) {
1456 1457 1458
		if (busy)
			goto starved;

L
Linus Torvalds 已提交
1459 1460 1461
		/*
		 * unblock after host_blocked iterates to zero
		 */
1462
		if (atomic_dec_return(&shost->host_blocked) > 0)
1463
			goto out_dec;
1464 1465 1466 1467

		SCSI_LOG_MLQUEUE(3,
			shost_printk(KERN_INFO, shost,
				     "unblocking host at zero depth\n"));
L
Linus Torvalds 已提交
1468
	}
1469 1470 1471 1472 1473

	if (shost->can_queue > 0 && busy >= shost->can_queue)
		goto starved;
	if (shost->host_self_blocked)
		goto starved;
L
Linus Torvalds 已提交
1474 1475

	/* We're OK to process the command, so we can't be starved */
1476 1477 1478 1479 1480 1481
	if (!list_empty(&sdev->starved_entry)) {
		spin_lock_irq(shost->host_lock);
		if (!list_empty(&sdev->starved_entry))
			list_del_init(&sdev->starved_entry);
		spin_unlock_irq(shost->host_lock);
	}
L
Linus Torvalds 已提交
1482

1483 1484 1485 1486 1487 1488
	return 1;

starved:
	spin_lock_irq(shost->host_lock);
	if (list_empty(&sdev->starved_entry))
		list_add_tail(&sdev->starved_entry, &shost->starved_list);
1489
	spin_unlock_irq(shost->host_lock);
1490 1491 1492
out_dec:
	atomic_dec(&shost->host_busy);
	return 0;
L
Linus Torvalds 已提交
1493 1494
}

1495 1496 1497 1498 1499 1500 1501 1502
/*
 * Busy state exporting function for request stacking drivers.
 *
 * For efficiency, no lock is taken to check the busy state of
 * shost/starget/sdev, since the returned value is not guaranteed and
 * may be changed after request stacking drivers call the function,
 * regardless of taking lock or not.
 *
1503 1504 1505
 * When scsi can't dispatch I/Os anymore and needs to kill I/Os scsi
 * needs to return 'not busy'. Otherwise, request stacking drivers
 * may hold requests forever.
1506 1507 1508 1509 1510 1511
 */
static int scsi_lld_busy(struct request_queue *q)
{
	struct scsi_device *sdev = q->queuedata;
	struct Scsi_Host *shost;

B
Bart Van Assche 已提交
1512
	if (blk_queue_dying(q))
1513 1514 1515 1516
		return 0;

	shost = sdev->host;

1517 1518 1519 1520 1521 1522 1523
	/*
	 * Ignore host/starget busy state.
	 * Since block layer does not have a concept of fairness across
	 * multiple queues, congestion of host/starget needs to be handled
	 * in SCSI layer.
	 */
	if (scsi_host_in_recovery(shost) || scsi_device_is_busy(sdev))
1524 1525 1526 1527 1528
		return 1;

	return 0;
}

L
Linus Torvalds 已提交
1529
/*
1530
 * Kill a request for a dead device
L
Linus Torvalds 已提交
1531
 */
1532
static void scsi_kill_request(struct request *req, struct request_queue *q)
L
Linus Torvalds 已提交
1533
{
1534
	struct scsi_cmnd *cmd = req->special;
1535 1536 1537
	struct scsi_device *sdev;
	struct scsi_target *starget;
	struct Scsi_Host *shost;
L
Linus Torvalds 已提交
1538

1539
	blk_start_request(req);
1540

1541 1542
	scmd_printk(KERN_INFO, cmd, "killing request\n");

1543 1544 1545
	sdev = cmd->device;
	starget = scsi_target(sdev);
	shost = sdev->host;
1546 1547 1548
	scsi_init_cmd_errh(cmd);
	cmd->result = DID_NO_CONNECT << 16;
	atomic_inc(&cmd->device->iorequest_cnt);
1549 1550 1551 1552 1553 1554

	/*
	 * SCSI request completion path will do scsi_device_unbusy(),
	 * bump busy counts.  To bump the counters, we need to dance
	 * with the locks as normal issue path does.
	 */
1555
	atomic_inc(&sdev->device_busy);
1556
	atomic_inc(&shost->host_busy);
1557 1558
	if (starget->can_queue > 0)
		atomic_inc(&starget->target_busy);
1559

J
Jens Axboe 已提交
1560
	blk_complete_request(req);
L
Linus Torvalds 已提交
1561 1562
}

1563 1564
static void scsi_softirq_done(struct request *rq)
{
J
Jens Axboe 已提交
1565 1566
	struct scsi_cmnd *cmd = rq->special;
	unsigned long wait_for = (cmd->allowed + 1) * rq->timeout;
1567 1568 1569 1570
	int disposition;

	INIT_LIST_HEAD(&cmd->eh_entry);

J
Jens Axboe 已提交
1571 1572 1573 1574
	atomic_inc(&cmd->device->iodone_cnt);
	if (cmd->result)
		atomic_inc(&cmd->device->ioerr_cnt);

1575 1576 1577 1578 1579 1580 1581 1582
	disposition = scsi_decide_disposition(cmd);
	if (disposition != SUCCESS &&
	    time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
		sdev_printk(KERN_ERR, cmd->device,
			    "timing out command, waited %lus\n",
			    wait_for/HZ);
		disposition = SUCCESS;
	}
1583

1584 1585 1586 1587 1588 1589 1590
	scsi_log_completion(cmd, disposition);

	switch (disposition) {
		case SUCCESS:
			scsi_finish_command(cmd);
			break;
		case NEEDS_RETRY:
1591
			scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);
1592 1593 1594 1595 1596
			break;
		case ADD_TO_MLQUEUE:
			scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
			break;
		default:
1597
			scsi_eh_scmd_add(cmd);
1598
			break;
1599 1600 1601
	}
}

1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682
/**
 * scsi_dispatch_command - Dispatch a command to the low-level driver.
 * @cmd: command block we are dispatching.
 *
 * Return: nonzero return request was rejected and device's queue needs to be
 * plugged.
 */
static int scsi_dispatch_cmd(struct scsi_cmnd *cmd)
{
	struct Scsi_Host *host = cmd->device->host;
	int rtn = 0;

	atomic_inc(&cmd->device->iorequest_cnt);

	/* check if the device is still usable */
	if (unlikely(cmd->device->sdev_state == SDEV_DEL)) {
		/* in SDEV_DEL we error all commands. DID_NO_CONNECT
		 * returns an immediate error upwards, and signals
		 * that the device is no longer present */
		cmd->result = DID_NO_CONNECT << 16;
		goto done;
	}

	/* Check to see if the scsi lld made this device blocked. */
	if (unlikely(scsi_device_blocked(cmd->device))) {
		/*
		 * in blocked state, the command is just put back on
		 * the device queue.  The suspend state has already
		 * blocked the queue so future requests should not
		 * occur until the device transitions out of the
		 * suspend state.
		 */
		SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,
			"queuecommand : device blocked\n"));
		return SCSI_MLQUEUE_DEVICE_BUSY;
	}

	/* Store the LUN value in cmnd, if needed. */
	if (cmd->device->lun_in_cdb)
		cmd->cmnd[1] = (cmd->cmnd[1] & 0x1f) |
			       (cmd->device->lun << 5 & 0xe0);

	scsi_log_send(cmd);

	/*
	 * Before we queue this command, check if the command
	 * length exceeds what the host adapter can handle.
	 */
	if (cmd->cmd_len > cmd->device->host->max_cmd_len) {
		SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,
			       "queuecommand : command too long. "
			       "cdb_size=%d host->max_cmd_len=%d\n",
			       cmd->cmd_len, cmd->device->host->max_cmd_len));
		cmd->result = (DID_ABORT << 16);
		goto done;
	}

	if (unlikely(host->shost_state == SHOST_DEL)) {
		cmd->result = (DID_NO_CONNECT << 16);
		goto done;

	}

	trace_scsi_dispatch_cmd_start(cmd);
	rtn = host->hostt->queuecommand(host, cmd);
	if (rtn) {
		trace_scsi_dispatch_cmd_error(cmd, rtn);
		if (rtn != SCSI_MLQUEUE_DEVICE_BUSY &&
		    rtn != SCSI_MLQUEUE_TARGET_BUSY)
			rtn = SCSI_MLQUEUE_HOST_BUSY;

		SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,
			"queuecommand : request rejected\n"));
	}

	return rtn;
 done:
	cmd->scsi_done(cmd);
	return 0;
}

1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699
/**
 * scsi_done - Invoke completion on finished SCSI command.
 * @cmd: The SCSI Command for which a low-level device driver (LLDD) gives
 * ownership back to SCSI Core -- i.e. the LLDD has finished with it.
 *
 * Description: This function is the mid-level's (SCSI Core) interrupt routine,
 * which regains ownership of the SCSI command (de facto) from a LLDD, and
 * calls blk_complete_request() for further processing.
 *
 * This function is interrupt context safe.
 */
static void scsi_done(struct scsi_cmnd *cmd)
{
	trace_scsi_dispatch_cmd_done(cmd);
	blk_complete_request(cmd->request);
}

L
Linus Torvalds 已提交
1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711
/*
 * Function:    scsi_request_fn()
 *
 * Purpose:     Main strategy routine for SCSI.
 *
 * Arguments:   q       - Pointer to actual queue.
 *
 * Returns:     Nothing
 *
 * Lock status: IO request lock assumed to be held when called.
 */
static void scsi_request_fn(struct request_queue *q)
1712 1713
	__releases(q->queue_lock)
	__acquires(q->queue_lock)
L
Linus Torvalds 已提交
1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724
{
	struct scsi_device *sdev = q->queuedata;
	struct Scsi_Host *shost;
	struct scsi_cmnd *cmd;
	struct request *req;

	/*
	 * To start with, we keep looping until the queue is empty, or until
	 * the host is no longer able to accept any more requests.
	 */
	shost = sdev->host;
J
Jens Axboe 已提交
1725
	for (;;) {
L
Linus Torvalds 已提交
1726 1727 1728
		int rtn;
		/*
		 * get next queueable request.  We do this early to make sure
1729
		 * that the request is fully prepared even if we cannot
L
Linus Torvalds 已提交
1730 1731
		 * accept it.
		 */
1732
		req = blk_peek_request(q);
1733
		if (!req)
L
Linus Torvalds 已提交
1734 1735 1736
			break;

		if (unlikely(!scsi_device_online(sdev))) {
1737 1738
			sdev_printk(KERN_ERR, sdev,
				    "rejecting I/O to offline device\n");
1739
			scsi_kill_request(req, q);
L
Linus Torvalds 已提交
1740 1741 1742
			continue;
		}

1743 1744
		if (!scsi_dev_queue_ready(q, sdev))
			break;
L
Linus Torvalds 已提交
1745 1746 1747 1748 1749

		/*
		 * Remove the request from the request list.
		 */
		if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
1750
			blk_start_request(req);
L
Linus Torvalds 已提交
1751

1752
		spin_unlock_irq(q->queue_lock);
1753 1754 1755 1756
		cmd = req->special;
		if (unlikely(cmd == NULL)) {
			printk(KERN_CRIT "impossible request in %s.\n"
					 "please mail a stack trace to "
1757
					 "linux-scsi@vger.kernel.org\n",
1758
					 __func__);
1759
			blk_dump_rq_flags(req, "foo");
1760 1761
			BUG();
		}
L
Linus Torvalds 已提交
1762

1763 1764 1765 1766 1767 1768 1769 1770
		/*
		 * We hit this when the driver is using a host wide
		 * tag map. For device level tag maps the queue_depth check
		 * in the device ready fn would prevent us from trying
		 * to allocate a tag. Since the map is a shared host resource
		 * we add the dev to the starved list so it eventually gets
		 * a run when a tag is freed.
		 */
1771
		if (blk_queue_tagged(q) && !(req->rq_flags & RQF_QUEUED)) {
1772
			spin_lock_irq(shost->host_lock);
1773 1774 1775
			if (list_empty(&sdev->starved_entry))
				list_add_tail(&sdev->starved_entry,
					      &shost->starved_list);
1776
			spin_unlock_irq(shost->host_lock);
1777 1778 1779
			goto not_ready;
		}

1780 1781 1782
		if (!scsi_target_queue_ready(shost, sdev))
			goto not_ready;

L
Linus Torvalds 已提交
1783
		if (!scsi_host_queue_ready(q, shost, sdev))
1784
			goto host_not_ready;
1785 1786 1787 1788 1789
	
		if (sdev->simple_tags)
			cmd->flags |= SCMD_TAGGED;
		else
			cmd->flags &= ~SCMD_TAGGED;
L
Linus Torvalds 已提交
1790 1791 1792 1793 1794 1795 1796 1797 1798 1799

		/*
		 * Finally, initialize any error handling parameters, and set up
		 * the timers for timeouts.
		 */
		scsi_init_cmd_errh(cmd);

		/*
		 * Dispatch the command to the low-level driver.
		 */
1800
		cmd->scsi_done = scsi_done;
L
Linus Torvalds 已提交
1801
		rtn = scsi_dispatch_cmd(cmd);
1802 1803 1804
		if (rtn) {
			scsi_queue_insert(cmd, rtn);
			spin_lock_irq(q->queue_lock);
J
Jens Axboe 已提交
1805
			goto out_delay;
1806 1807
		}
		spin_lock_irq(q->queue_lock);
L
Linus Torvalds 已提交
1808 1809
	}

1810
	return;
L
Linus Torvalds 已提交
1811

1812
 host_not_ready:
1813 1814
	if (scsi_target(sdev)->can_queue > 0)
		atomic_dec(&scsi_target(sdev)->target_busy);
1815
 not_ready:
L
Linus Torvalds 已提交
1816 1817 1818 1819 1820 1821 1822 1823 1824 1825
	/*
	 * lock q, handle tag, requeue req, and decrement device_busy. We
	 * must return with queue_lock held.
	 *
	 * Decrementing device_busy without checking it is OK, as all such
	 * cases (host limits or settings) should run the queue at some
	 * later time.
	 */
	spin_lock_irq(q->queue_lock);
	blk_requeue_request(q, req);
1826
	atomic_dec(&sdev->device_busy);
J
Jens Axboe 已提交
1827
out_delay:
G
Guenter Roeck 已提交
1828
	if (!atomic_read(&sdev->device_busy) && !scsi_device_blocked(sdev))
J
Jens Axboe 已提交
1829
		blk_delay_queue(q, SCSI_QUEUE_DELAY);
L
Linus Torvalds 已提交
1830 1831
}

1832 1833 1834 1835
static inline int prep_to_mq(int ret)
{
	switch (ret) {
	case BLKPREP_OK:
1836
		return BLK_MQ_RQ_QUEUE_OK;
1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851
	case BLKPREP_DEFER:
		return BLK_MQ_RQ_QUEUE_BUSY;
	default:
		return BLK_MQ_RQ_QUEUE_ERROR;
	}
}

static int scsi_mq_prep_fn(struct request *req)
{
	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
	struct scsi_device *sdev = req->q->queuedata;
	struct Scsi_Host *shost = sdev->host;
	unsigned char *sense_buf = cmd->sense_buffer;
	struct scatterlist *sg;

1852 1853 1854
	/* zero out the cmd, except for the embedded scsi_request */
	memset((char *)cmd + sizeof(cmd->req), 0,
		sizeof(*cmd) - sizeof(cmd->req));
1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869

	req->special = cmd;

	cmd->request = req;
	cmd->device = sdev;
	cmd->sense_buffer = sense_buf;

	cmd->tag = req->tag;

	cmd->prot_op = SCSI_PROT_NORMAL;

	INIT_LIST_HEAD(&cmd->list);
	INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler);
	cmd->jiffies_at_alloc = jiffies;

1870 1871 1872 1873 1874
	if (shost->use_cmd_list) {
		spin_lock_irq(&sdev->list_lock);
		list_add_tail(&cmd->list, &sdev->cmd_list);
		spin_unlock_irq(&sdev->list_lock);
	}
1875 1876 1877 1878 1879 1880

	sg = (void *)cmd + sizeof(struct scsi_cmnd) + shost->hostt->cmd_size;
	cmd->sdb.table.sgl = sg;

	if (scsi_host_get_prot(shost)) {
		cmd->prot_sdb = (void *)sg +
1881
			min_t(unsigned int,
1882
			      shost->sg_tablesize, SG_CHUNK_SIZE) *
1883
			sizeof(struct scatterlist);
1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900
		memset(cmd->prot_sdb, 0, sizeof(struct scsi_data_buffer));

		cmd->prot_sdb->table.sgl =
			(struct scatterlist *)(cmd->prot_sdb + 1);
	}

	if (blk_bidi_rq(req)) {
		struct request *next_rq = req->next_rq;
		struct scsi_data_buffer *bidi_sdb = blk_mq_rq_to_pdu(next_rq);

		memset(bidi_sdb, 0, sizeof(struct scsi_data_buffer));
		bidi_sdb->table.sgl =
			(struct scatterlist *)(bidi_sdb + 1);

		next_rq->special = bidi_sdb;
	}

1901 1902
	blk_mq_start_request(req);

1903 1904 1905 1906 1907 1908
	return scsi_setup_cmnd(sdev, req);
}

static void scsi_mq_done(struct scsi_cmnd *cmd)
{
	trace_scsi_dispatch_cmd_done(cmd);
1909
	blk_mq_complete_request(cmd->request);
1910 1911
}

1912 1913
static int scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
			 const struct blk_mq_queue_data *bd)
1914
{
1915
	struct request *req = bd->rq;
1916 1917 1918 1919 1920 1921 1922 1923
	struct request_queue *q = req->q;
	struct scsi_device *sdev = q->queuedata;
	struct Scsi_Host *shost = sdev->host;
	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
	int ret;
	int reason;

	ret = prep_to_mq(scsi_prep_state_check(sdev, req));
1924
	if (ret != BLK_MQ_RQ_QUEUE_OK)
1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937
		goto out;

	ret = BLK_MQ_RQ_QUEUE_BUSY;
	if (!get_device(&sdev->sdev_gendev))
		goto out;

	if (!scsi_dev_queue_ready(q, sdev))
		goto out_put_device;
	if (!scsi_target_queue_ready(shost, sdev))
		goto out_dec_device_busy;
	if (!scsi_host_queue_ready(q, shost, sdev))
		goto out_dec_target_busy;

1938
	if (!(req->rq_flags & RQF_DONTPREP)) {
1939
		ret = prep_to_mq(scsi_mq_prep_fn(req));
1940
		if (ret != BLK_MQ_RQ_QUEUE_OK)
1941
			goto out_dec_host_busy;
1942
		req->rq_flags |= RQF_DONTPREP;
1943 1944
	} else {
		blk_mq_start_request(req);
1945 1946
	}

1947 1948
	if (sdev->simple_tags)
		cmd->flags |= SCMD_TAGGED;
1949
	else
1950
		cmd->flags &= ~SCMD_TAGGED;
1951

1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977
	scsi_init_cmd_errh(cmd);
	cmd->scsi_done = scsi_mq_done;

	reason = scsi_dispatch_cmd(cmd);
	if (reason) {
		scsi_set_blocked(cmd, reason);
		ret = BLK_MQ_RQ_QUEUE_BUSY;
		goto out_dec_host_busy;
	}

	return BLK_MQ_RQ_QUEUE_OK;

out_dec_host_busy:
	atomic_dec(&shost->host_busy);
out_dec_target_busy:
	if (scsi_target(sdev)->can_queue > 0)
		atomic_dec(&scsi_target(sdev)->target_busy);
out_dec_device_busy:
	atomic_dec(&sdev->device_busy);
out_put_device:
	put_device(&sdev->sdev_gendev);
out:
	switch (ret) {
	case BLK_MQ_RQ_QUEUE_BUSY:
		if (atomic_read(&sdev->device_busy) == 0 &&
		    !scsi_device_blocked(sdev))
1978
			blk_mq_delay_run_hw_queue(hctx, SCSI_QUEUE_DELAY);
1979 1980 1981 1982 1983 1984 1985
		break;
	case BLK_MQ_RQ_QUEUE_ERROR:
		/*
		 * Make sure to release all allocated ressources when
		 * we hit an error, as we will never see this command
		 * again.
		 */
1986
		if (req->rq_flags & RQF_DONTPREP)
1987 1988 1989 1990 1991 1992 1993 1994
			scsi_mq_uninit_cmd(cmd);
		break;
	default:
		break;
	}
	return ret;
}

1995 1996 1997 1998 1999 2000 2001 2002
static enum blk_eh_timer_return scsi_timeout(struct request *req,
		bool reserved)
{
	if (reserved)
		return BLK_EH_RESET_TIMER;
	return scsi_times_out(req);
}

2003 2004
static int scsi_init_request(struct blk_mq_tag_set *set, struct request *rq,
		unsigned int hctx_idx, unsigned int numa_node)
2005
{
2006
	struct Scsi_Host *shost = set->driver_data;
2007 2008
	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);

2009 2010
	cmd->sense_buffer =
		scsi_alloc_sense_buffer(shost, GFP_KERNEL, numa_node);
2011 2012
	if (!cmd->sense_buffer)
		return -ENOMEM;
2013
	cmd->req.sense = cmd->sense_buffer;
2014 2015 2016
	return 0;
}

2017 2018
static void scsi_exit_request(struct blk_mq_tag_set *set, struct request *rq,
		unsigned int hctx_idx)
2019
{
2020
	struct Scsi_Host *shost = set->driver_data;
2021 2022
	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);

2023
	scsi_free_sense_buffer(shost, cmd->sense_buffer);
2024 2025
}

2026 2027 2028 2029 2030 2031 2032 2033 2034
static int scsi_map_queues(struct blk_mq_tag_set *set)
{
	struct Scsi_Host *shost = container_of(set, struct Scsi_Host, tag_set);

	if (shost->hostt->map_queues)
		return shost->hostt->map_queues(shost);
	return blk_mq_map_queues(set);
}

2035
static u64 scsi_calculate_bounce_limit(struct Scsi_Host *shost)
L
Linus Torvalds 已提交
2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050
{
	struct device *host_dev;
	u64 bounce_limit = 0xffffffff;

	if (shost->unchecked_isa_dma)
		return BLK_BOUNCE_ISA;
	/*
	 * Platforms with virtual-DMA translation
	 * hardware have no practical limit.
	 */
	if (!PCI_DMA_BUS_IS_PHYS)
		return BLK_BOUNCE_ANY;

	host_dev = scsi_get_device(shost);
	if (host_dev && host_dev->dma_mask)
2051
		bounce_limit = (u64)dma_max_pfn(host_dev) << PAGE_SHIFT;
L
Linus Torvalds 已提交
2052 2053 2054 2055

	return bounce_limit;
}

2056
void __scsi_init_queue(struct Scsi_Host *shost, struct request_queue *q)
L
Linus Torvalds 已提交
2057
{
2058
	struct device *dev = shost->dma_dev;
L
Linus Torvalds 已提交
2059

2060 2061 2062
	/*
	 * this limit is imposed by hardware restrictions
	 */
2063
	blk_queue_max_segments(q, min_t(unsigned short, shost->sg_tablesize,
2064
					SG_MAX_SEGMENTS));
2065

2066 2067 2068 2069 2070 2071 2072 2073
	if (scsi_host_prot_dma(shost)) {
		shost->sg_prot_tablesize =
			min_not_zero(shost->sg_prot_tablesize,
				     (unsigned short)SCSI_MAX_PROT_SG_SEGMENTS);
		BUG_ON(shost->sg_prot_tablesize < shost->sg_tablesize);
		blk_queue_max_integrity_segments(q, shost->sg_prot_tablesize);
	}

2074
	blk_queue_max_hw_sectors(q, shost->max_sectors);
L
Linus Torvalds 已提交
2075 2076
	blk_queue_bounce_limit(q, scsi_calculate_bounce_limit(shost));
	blk_queue_segment_boundary(q, shost->dma_boundary);
2077
	dma_set_seg_boundary(dev, shost->dma_boundary);
L
Linus Torvalds 已提交
2078

2079 2080
	blk_queue_max_segment_size(q, dma_get_max_seg_size(dev));

L
Linus Torvalds 已提交
2081
	if (!shost->use_clustering)
2082
		q->limits.cluster = 0;
2083 2084 2085 2086 2087 2088 2089

	/*
	 * set a reasonable default alignment on word boundaries: the
	 * host and device may alter it using
	 * blk_queue_update_dma_alignment() later.
	 */
	blk_queue_dma_alignment(q, 0x03);
2090
}
2091
EXPORT_SYMBOL_GPL(__scsi_init_queue);
2092

2093
static int scsi_init_rq(struct request_queue *q, struct request *rq, gfp_t gfp)
2094
{
2095 2096
	struct Scsi_Host *shost = q->rq_alloc_data;
	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
2097

2098 2099 2100 2101 2102
	memset(cmd, 0, sizeof(*cmd));

	cmd->sense_buffer = scsi_alloc_sense_buffer(shost, gfp, NUMA_NO_NODE);
	if (!cmd->sense_buffer)
		goto fail;
2103
	cmd->req.sense = cmd->sense_buffer;
2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126

	if (scsi_host_get_prot(shost) >= SHOST_DIX_TYPE0_PROTECTION) {
		cmd->prot_sdb = kmem_cache_zalloc(scsi_sdb_cache, gfp);
		if (!cmd->prot_sdb)
			goto fail_free_sense;
	}

	return 0;

fail_free_sense:
	scsi_free_sense_buffer(shost, cmd->sense_buffer);
fail:
	return -ENOMEM;
}

static void scsi_exit_rq(struct request_queue *q, struct request *rq)
{
	struct Scsi_Host *shost = q->rq_alloc_data;
	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);

	if (cmd->prot_sdb)
		kmem_cache_free(scsi_sdb_cache, cmd->prot_sdb);
	scsi_free_sense_buffer(shost, cmd->sense_buffer);
L
Linus Torvalds 已提交
2127
}
2128 2129 2130

struct request_queue *scsi_alloc_queue(struct scsi_device *sdev)
{
2131
	struct Scsi_Host *shost = sdev->host;
2132 2133
	struct request_queue *q;

2134
	q = blk_alloc_queue_node(GFP_KERNEL, NUMA_NO_NODE);
2135 2136
	if (!q)
		return NULL;
2137 2138 2139 2140 2141 2142 2143 2144 2145 2146
	q->cmd_size = sizeof(struct scsi_cmnd) + shost->hostt->cmd_size;
	q->rq_alloc_data = shost;
	q->request_fn = scsi_request_fn;
	q->init_rq_fn = scsi_init_rq;
	q->exit_rq_fn = scsi_exit_rq;

	if (blk_init_allocated_queue(q) < 0) {
		blk_cleanup_queue(q);
		return NULL;
	}
2147

2148
	__scsi_init_queue(shost, q);
2149
	blk_queue_prep_rq(q, scsi_prep_fn);
2150
	blk_queue_unprep_rq(q, scsi_unprep_fn);
2151
	blk_queue_softirq_done(q, scsi_softirq_done);
J
Jens Axboe 已提交
2152
	blk_queue_rq_timed_out(q, scsi_times_out);
2153
	blk_queue_lld_busy(q, scsi_lld_busy);
2154 2155
	return q;
}
L
Linus Torvalds 已提交
2156

2157
static const struct blk_mq_ops scsi_mq_ops = {
2158 2159
	.queue_rq	= scsi_queue_rq,
	.complete	= scsi_softirq_done,
2160
	.timeout	= scsi_timeout,
2161 2162 2163
#ifdef CONFIG_BLK_DEBUG_FS
	.show_rq	= scsi_show_rq,
#endif
2164 2165
	.init_request	= scsi_init_request,
	.exit_request	= scsi_exit_request,
2166
	.map_queues	= scsi_map_queues,
2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184
};

struct request_queue *scsi_mq_alloc_queue(struct scsi_device *sdev)
{
	sdev->request_queue = blk_mq_init_queue(&sdev->host->tag_set);
	if (IS_ERR(sdev->request_queue))
		return NULL;

	sdev->request_queue->queuedata = sdev;
	__scsi_init_queue(sdev->host, sdev->request_queue);
	return sdev->request_queue;
}

int scsi_mq_setup_tags(struct Scsi_Host *shost)
{
	unsigned int cmd_size, sgl_size, tbl_size;

	tbl_size = shost->sg_tablesize;
2185 2186
	if (tbl_size > SG_CHUNK_SIZE)
		tbl_size = SG_CHUNK_SIZE;
2187 2188 2189 2190 2191 2192 2193
	sgl_size = tbl_size * sizeof(struct scatterlist);
	cmd_size = sizeof(struct scsi_cmnd) + shost->hostt->cmd_size + sgl_size;
	if (scsi_host_get_prot(shost))
		cmd_size += sizeof(struct scsi_data_buffer) + sgl_size;

	memset(&shost->tag_set, 0, sizeof(shost->tag_set));
	shost->tag_set.ops = &scsi_mq_ops;
2194
	shost->tag_set.nr_hw_queues = shost->nr_hw_queues ? : 1;
2195 2196 2197 2198
	shost->tag_set.queue_depth = shost->can_queue;
	shost->tag_set.cmd_size = cmd_size;
	shost->tag_set.numa_node = NUMA_NO_NODE;
	shost->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE;
S
Shaohua Li 已提交
2199 2200
	shost->tag_set.flags |=
		BLK_ALLOC_POLICY_TO_MQ_FLAG(shost->hostt->tag_alloc_policy);
2201 2202 2203 2204 2205 2206 2207 2208 2209 2210
	shost->tag_set.driver_data = shost;

	return blk_mq_alloc_tag_set(&shost->tag_set);
}

void scsi_mq_destroy_tags(struct Scsi_Host *shost)
{
	blk_mq_free_tag_set(&shost->tag_set);
}

2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233
/**
 * scsi_device_from_queue - return sdev associated with a request_queue
 * @q: The request queue to return the sdev from
 *
 * Return the sdev associated with a request queue or NULL if the
 * request_queue does not reference a SCSI device.
 */
struct scsi_device *scsi_device_from_queue(struct request_queue *q)
{
	struct scsi_device *sdev = NULL;

	if (q->mq_ops) {
		if (q->mq_ops == &scsi_mq_ops)
			sdev = q->queuedata;
	} else if (q->request_fn == scsi_request_fn)
		sdev = q->queuedata;
	if (!sdev || !get_device(&sdev->sdev_gendev))
		sdev = NULL;

	return sdev;
}
EXPORT_SYMBOL_GPL(scsi_device_from_queue);

L
Linus Torvalds 已提交
2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284
/*
 * Function:    scsi_block_requests()
 *
 * Purpose:     Utility function used by low-level drivers to prevent further
 *		commands from being queued to the device.
 *
 * Arguments:   shost       - Host in question
 *
 * Returns:     Nothing
 *
 * Lock status: No locks are assumed held.
 *
 * Notes:       There is no timer nor any other means by which the requests
 *		get unblocked other than the low-level driver calling
 *		scsi_unblock_requests().
 */
void scsi_block_requests(struct Scsi_Host *shost)
{
	shost->host_self_blocked = 1;
}
EXPORT_SYMBOL(scsi_block_requests);

/*
 * Function:    scsi_unblock_requests()
 *
 * Purpose:     Utility function used by low-level drivers to allow further
 *		commands from being queued to the device.
 *
 * Arguments:   shost       - Host in question
 *
 * Returns:     Nothing
 *
 * Lock status: No locks are assumed held.
 *
 * Notes:       There is no timer nor any other means by which the requests
 *		get unblocked other than the low-level driver calling
 *		scsi_unblock_requests().
 *
 *		This is done as an API function so that changes to the
 *		internals of the scsi mid-layer won't require wholesale
 *		changes to drivers that use this feature.
 */
void scsi_unblock_requests(struct Scsi_Host *shost)
{
	shost->host_self_blocked = 0;
	scsi_run_host_queues(shost);
}
EXPORT_SYMBOL(scsi_unblock_requests);

int __init scsi_init_queue(void)
{
2285 2286 2287 2288 2289
	scsi_sdb_cache = kmem_cache_create("scsi_data_buffer",
					   sizeof(struct scsi_data_buffer),
					   0, 0, NULL);
	if (!scsi_sdb_cache) {
		printk(KERN_ERR "SCSI: can't init scsi sdb cache\n");
F
FUJITA Tomonori 已提交
2290
		return -ENOMEM;
2291 2292
	}

L
Linus Torvalds 已提交
2293 2294 2295 2296 2297
	return 0;
}

void scsi_exit_queue(void)
{
2298 2299
	kmem_cache_destroy(scsi_sense_cache);
	kmem_cache_destroy(scsi_sense_isadma_cache);
2300
	kmem_cache_destroy(scsi_sdb_cache);
L
Linus Torvalds 已提交
2301
}
2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313

/**
 *	scsi_mode_select - issue a mode select
 *	@sdev:	SCSI device to be queried
 *	@pf:	Page format bit (1 == standard, 0 == vendor specific)
 *	@sp:	Save page bit (0 == don't save, 1 == save)
 *	@modepage: mode page being requested
 *	@buffer: request buffer (may not be smaller than eight bytes)
 *	@len:	length of request buffer.
 *	@timeout: command timeout
 *	@retries: number of retries before failing
 *	@data: returns a structure abstracting the mode header data
2314
 *	@sshdr: place to put sense data (or NULL if no sense to be collected).
2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373
 *		must be SCSI_SENSE_BUFFERSIZE big.
 *
 *	Returns zero if successful; negative error number or scsi
 *	status on error
 *
 */
int
scsi_mode_select(struct scsi_device *sdev, int pf, int sp, int modepage,
		 unsigned char *buffer, int len, int timeout, int retries,
		 struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
{
	unsigned char cmd[10];
	unsigned char *real_buffer;
	int ret;

	memset(cmd, 0, sizeof(cmd));
	cmd[1] = (pf ? 0x10 : 0) | (sp ? 0x01 : 0);

	if (sdev->use_10_for_ms) {
		if (len > 65535)
			return -EINVAL;
		real_buffer = kmalloc(8 + len, GFP_KERNEL);
		if (!real_buffer)
			return -ENOMEM;
		memcpy(real_buffer + 8, buffer, len);
		len += 8;
		real_buffer[0] = 0;
		real_buffer[1] = 0;
		real_buffer[2] = data->medium_type;
		real_buffer[3] = data->device_specific;
		real_buffer[4] = data->longlba ? 0x01 : 0;
		real_buffer[5] = 0;
		real_buffer[6] = data->block_descriptor_length >> 8;
		real_buffer[7] = data->block_descriptor_length;

		cmd[0] = MODE_SELECT_10;
		cmd[7] = len >> 8;
		cmd[8] = len;
	} else {
		if (len > 255 || data->block_descriptor_length > 255 ||
		    data->longlba)
			return -EINVAL;

		real_buffer = kmalloc(4 + len, GFP_KERNEL);
		if (!real_buffer)
			return -ENOMEM;
		memcpy(real_buffer + 4, buffer, len);
		len += 4;
		real_buffer[0] = 0;
		real_buffer[1] = data->medium_type;
		real_buffer[2] = data->device_specific;
		real_buffer[3] = data->block_descriptor_length;
		

		cmd[0] = MODE_SELECT;
		cmd[4] = len;
	}

	ret = scsi_execute_req(sdev, cmd, DMA_TO_DEVICE, real_buffer, len,
2374
			       sshdr, timeout, retries, NULL);
2375 2376 2377 2378 2379
	kfree(real_buffer);
	return ret;
}
EXPORT_SYMBOL_GPL(scsi_mode_select);

L
Linus Torvalds 已提交
2380
/**
2381
 *	scsi_mode_sense - issue a mode sense, falling back from 10 to six bytes if necessary.
2382
 *	@sdev:	SCSI device to be queried
L
Linus Torvalds 已提交
2383 2384 2385 2386 2387 2388 2389
 *	@dbd:	set if mode sense will allow block descriptors to be returned
 *	@modepage: mode page being requested
 *	@buffer: request buffer (may not be smaller than eight bytes)
 *	@len:	length of request buffer.
 *	@timeout: command timeout
 *	@retries: number of retries before failing
 *	@data: returns a structure abstracting the mode header data
2390
 *	@sshdr: place to put sense data (or NULL if no sense to be collected).
2391
 *		must be SCSI_SENSE_BUFFERSIZE big.
L
Linus Torvalds 已提交
2392 2393 2394 2395
 *
 *	Returns zero if unsuccessful, or the header offset (either 4
 *	or 8 depending on whether a six or ten byte command was
 *	issued) if successful.
2396
 */
L
Linus Torvalds 已提交
2397
int
2398
scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
L
Linus Torvalds 已提交
2399
		  unsigned char *buffer, int len, int timeout, int retries,
2400 2401
		  struct scsi_mode_data *data, struct scsi_sense_hdr *sshdr)
{
L
Linus Torvalds 已提交
2402 2403 2404
	unsigned char cmd[12];
	int use_10_for_ms;
	int header_length;
2405
	int result, retry_count = retries;
2406
	struct scsi_sense_hdr my_sshdr;
L
Linus Torvalds 已提交
2407 2408 2409 2410 2411 2412

	memset(data, 0, sizeof(*data));
	memset(&cmd[0], 0, 12);
	cmd[1] = dbd & 0x18;	/* allows DBD and LLBA bits */
	cmd[2] = modepage;

2413 2414 2415 2416
	/* caller might not be interested in sense, but we need it */
	if (!sshdr)
		sshdr = &my_sshdr;

L
Linus Torvalds 已提交
2417
 retry:
2418
	use_10_for_ms = sdev->use_10_for_ms;
L
Linus Torvalds 已提交
2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437

	if (use_10_for_ms) {
		if (len < 8)
			len = 8;

		cmd[0] = MODE_SENSE_10;
		cmd[8] = len;
		header_length = 8;
	} else {
		if (len < 4)
			len = 4;

		cmd[0] = MODE_SENSE;
		cmd[4] = len;
		header_length = 4;
	}

	memset(buffer, 0, len);

2438
	result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len,
2439
				  sshdr, timeout, retries, NULL);
L
Linus Torvalds 已提交
2440 2441 2442 2443 2444 2445

	/* This code looks awful: what it's doing is making sure an
	 * ILLEGAL REQUEST sense return identifies the actual command
	 * byte as the problem.  MODE_SENSE commands can return
	 * ILLEGAL REQUEST if the code page isn't supported */

2446 2447
	if (use_10_for_ms && !scsi_status_is_good(result) &&
	    (driver_byte(result) & DRIVER_SENSE)) {
2448 2449 2450
		if (scsi_sense_valid(sshdr)) {
			if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
			    (sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
L
Linus Torvalds 已提交
2451 2452 2453
				/* 
				 * Invalid command operation code
				 */
2454
				sdev->use_10_for_ms = 0;
L
Linus Torvalds 已提交
2455 2456 2457 2458 2459
				goto retry;
			}
		}
	}

2460
	if(scsi_status_is_good(result)) {
2461 2462 2463 2464 2465 2466 2467 2468 2469 2470
		if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b &&
			     (modepage == 6 || modepage == 8))) {
			/* Initio breakage? */
			header_length = 0;
			data->length = 13;
			data->medium_type = 0;
			data->device_specific = 0;
			data->longlba = 0;
			data->block_descriptor_length = 0;
		} else if(use_10_for_ms) {
L
Linus Torvalds 已提交
2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482
			data->length = buffer[0]*256 + buffer[1] + 2;
			data->medium_type = buffer[2];
			data->device_specific = buffer[3];
			data->longlba = buffer[4] & 0x01;
			data->block_descriptor_length = buffer[6]*256
				+ buffer[7];
		} else {
			data->length = buffer[0] + 1;
			data->medium_type = buffer[1];
			data->device_specific = buffer[2];
			data->block_descriptor_length = buffer[3];
		}
2483
		data->header_length = header_length;
2484 2485 2486 2487 2488
	} else if ((status_byte(result) == CHECK_CONDITION) &&
		   scsi_sense_valid(sshdr) &&
		   sshdr->sense_key == UNIT_ATTENTION && retry_count) {
		retry_count--;
		goto retry;
L
Linus Torvalds 已提交
2489 2490
	}

2491
	return result;
L
Linus Torvalds 已提交
2492 2493 2494
}
EXPORT_SYMBOL(scsi_mode_sense);

2495 2496 2497 2498 2499
/**
 *	scsi_test_unit_ready - test if unit is ready
 *	@sdev:	scsi device to change the state of.
 *	@timeout: command timeout
 *	@retries: number of retries before failing
2500
 *	@sshdr: outpout pointer for decoded sense information.
2501 2502
 *
 *	Returns zero if unsuccessful or an error if TUR failed.  For
2503
 *	removable media, UNIT_ATTENTION sets ->changed flag.
2504
 **/
L
Linus Torvalds 已提交
2505
int
2506
scsi_test_unit_ready(struct scsi_device *sdev, int timeout, int retries,
2507
		     struct scsi_sense_hdr *sshdr)
L
Linus Torvalds 已提交
2508 2509 2510 2511 2512
{
	char cmd[] = {
		TEST_UNIT_READY, 0, 0, 0, 0, 0,
	};
	int result;
2513 2514 2515 2516

	/* try to eat the UNIT_ATTENTION if there are enough retries */
	do {
		result = scsi_execute_req(sdev, cmd, DMA_NONE, NULL, 0, sshdr,
2517
					  timeout, retries, NULL);
2518 2519 2520 2521 2522
		if (sdev->removable && scsi_sense_valid(sshdr) &&
		    sshdr->sense_key == UNIT_ATTENTION)
			sdev->changed = 1;
	} while (scsi_sense_valid(sshdr) &&
		 sshdr->sense_key == UNIT_ATTENTION && --retries);
2523

L
Linus Torvalds 已提交
2524 2525 2526 2527 2528
	return result;
}
EXPORT_SYMBOL(scsi_test_unit_ready);

/**
2529
 *	scsi_device_set_state - Take the given device through the device state model.
L
Linus Torvalds 已提交
2530 2531 2532 2533 2534
 *	@sdev:	scsi device to change the state of.
 *	@state:	state to change to.
 *
 *	Returns zero if unsuccessful or an error if the requested 
 *	transition is illegal.
2535
 */
L
Linus Torvalds 已提交
2536 2537 2538 2539 2540 2541 2542 2543 2544 2545
int
scsi_device_set_state(struct scsi_device *sdev, enum scsi_device_state state)
{
	enum scsi_device_state oldstate = sdev->sdev_state;

	if (state == oldstate)
		return 0;

	switch (state) {
	case SDEV_CREATED:
2546 2547 2548 2549 2550 2551 2552
		switch (oldstate) {
		case SDEV_CREATED_BLOCK:
			break;
		default:
			goto illegal;
		}
		break;
L
Linus Torvalds 已提交
2553 2554 2555 2556 2557
			
	case SDEV_RUNNING:
		switch (oldstate) {
		case SDEV_CREATED:
		case SDEV_OFFLINE:
2558
		case SDEV_TRANSPORT_OFFLINE:
L
Linus Torvalds 已提交
2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570
		case SDEV_QUIESCE:
		case SDEV_BLOCK:
			break;
		default:
			goto illegal;
		}
		break;

	case SDEV_QUIESCE:
		switch (oldstate) {
		case SDEV_RUNNING:
		case SDEV_OFFLINE:
2571
		case SDEV_TRANSPORT_OFFLINE:
L
Linus Torvalds 已提交
2572 2573 2574 2575 2576 2577 2578
			break;
		default:
			goto illegal;
		}
		break;

	case SDEV_OFFLINE:
2579
	case SDEV_TRANSPORT_OFFLINE:
L
Linus Torvalds 已提交
2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593
		switch (oldstate) {
		case SDEV_CREATED:
		case SDEV_RUNNING:
		case SDEV_QUIESCE:
		case SDEV_BLOCK:
			break;
		default:
			goto illegal;
		}
		break;

	case SDEV_BLOCK:
		switch (oldstate) {
		case SDEV_RUNNING:
2594 2595 2596 2597 2598 2599 2600 2601 2602 2603
		case SDEV_CREATED_BLOCK:
			break;
		default:
			goto illegal;
		}
		break;

	case SDEV_CREATED_BLOCK:
		switch (oldstate) {
		case SDEV_CREATED:
L
Linus Torvalds 已提交
2604 2605 2606 2607 2608 2609 2610 2611 2612 2613
			break;
		default:
			goto illegal;
		}
		break;

	case SDEV_CANCEL:
		switch (oldstate) {
		case SDEV_CREATED:
		case SDEV_RUNNING:
2614
		case SDEV_QUIESCE:
L
Linus Torvalds 已提交
2615
		case SDEV_OFFLINE:
2616
		case SDEV_TRANSPORT_OFFLINE:
L
Linus Torvalds 已提交
2617 2618 2619 2620 2621 2622 2623 2624 2625
		case SDEV_BLOCK:
			break;
		default:
			goto illegal;
		}
		break;

	case SDEV_DEL:
		switch (oldstate) {
2626 2627 2628
		case SDEV_CREATED:
		case SDEV_RUNNING:
		case SDEV_OFFLINE:
2629
		case SDEV_TRANSPORT_OFFLINE:
L
Linus Torvalds 已提交
2630
		case SDEV_CANCEL:
2631
		case SDEV_CREATED_BLOCK:
L
Linus Torvalds 已提交
2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642
			break;
		default:
			goto illegal;
		}
		break;

	}
	sdev->sdev_state = state;
	return 0;

 illegal:
2643
	SCSI_LOG_ERROR_RECOVERY(1,
2644
				sdev_printk(KERN_ERR, sdev,
2645
					    "Illegal state transition %s->%s",
2646 2647
					    scsi_device_state_name(oldstate),
					    scsi_device_state_name(state))
L
Linus Torvalds 已提交
2648 2649 2650 2651 2652
				);
	return -EINVAL;
}
EXPORT_SYMBOL(scsi_device_set_state);

2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668
/**
 * 	sdev_evt_emit - emit a single SCSI device uevent
 *	@sdev: associated SCSI device
 *	@evt: event to emit
 *
 *	Send a single uevent (scsi_event) to the associated scsi_device.
 */
static void scsi_evt_emit(struct scsi_device *sdev, struct scsi_event *evt)
{
	int idx = 0;
	char *envp[3];

	switch (evt->evt_type) {
	case SDEV_EVT_MEDIA_CHANGE:
		envp[idx++] = "SDEV_MEDIA_CHANGE=1";
		break;
2669
	case SDEV_EVT_INQUIRY_CHANGE_REPORTED:
H
Hannes Reinecke 已提交
2670
		scsi_rescan_device(&sdev->sdev_gendev);
2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684
		envp[idx++] = "SDEV_UA=INQUIRY_DATA_HAS_CHANGED";
		break;
	case SDEV_EVT_CAPACITY_CHANGE_REPORTED:
		envp[idx++] = "SDEV_UA=CAPACITY_DATA_HAS_CHANGED";
		break;
	case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED:
	       envp[idx++] = "SDEV_UA=THIN_PROVISIONING_SOFT_THRESHOLD_REACHED";
		break;
	case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED:
		envp[idx++] = "SDEV_UA=MODE_PARAMETERS_CHANGED";
		break;
	case SDEV_EVT_LUN_CHANGE_REPORTED:
		envp[idx++] = "SDEV_UA=REPORTED_LUNS_DATA_HAS_CHANGED";
		break;
2685 2686 2687
	case SDEV_EVT_ALUA_STATE_CHANGE_REPORTED:
		envp[idx++] = "SDEV_UA=ASYMMETRIC_ACCESS_STATE_CHANGED";
		break;
2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707
	default:
		/* do nothing */
		break;
	}

	envp[idx++] = NULL;

	kobject_uevent_env(&sdev->sdev_gendev.kobj, KOBJ_CHANGE, envp);
}

/**
 * 	sdev_evt_thread - send a uevent for each scsi event
 *	@work: work struct for scsi_device
 *
 *	Dispatch queued events to their associated scsi_device kobjects
 *	as uevents.
 */
void scsi_evt_thread(struct work_struct *work)
{
	struct scsi_device *sdev;
2708
	enum scsi_device_event evt_type;
2709 2710 2711 2712
	LIST_HEAD(event_list);

	sdev = container_of(work, struct scsi_device, event_work);

2713 2714 2715 2716
	for (evt_type = SDEV_EVT_FIRST; evt_type <= SDEV_EVT_LAST; evt_type++)
		if (test_and_clear_bit(evt_type, sdev->pending_events))
			sdev_evt_send_simple(sdev, evt_type, GFP_KERNEL);

2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748
	while (1) {
		struct scsi_event *evt;
		struct list_head *this, *tmp;
		unsigned long flags;

		spin_lock_irqsave(&sdev->list_lock, flags);
		list_splice_init(&sdev->event_list, &event_list);
		spin_unlock_irqrestore(&sdev->list_lock, flags);

		if (list_empty(&event_list))
			break;

		list_for_each_safe(this, tmp, &event_list) {
			evt = list_entry(this, struct scsi_event, node);
			list_del(&evt->node);
			scsi_evt_emit(sdev, evt);
			kfree(evt);
		}
	}
}

/**
 * 	sdev_evt_send - send asserted event to uevent thread
 *	@sdev: scsi_device event occurred on
 *	@evt: event to send
 *
 *	Assert scsi device event asynchronously.
 */
void sdev_evt_send(struct scsi_device *sdev, struct scsi_event *evt)
{
	unsigned long flags;

2749 2750 2751 2752
#if 0
	/* FIXME: currently this check eliminates all media change events
	 * for polled devices.  Need to update to discriminate between AN
	 * and polled events */
2753 2754 2755 2756
	if (!test_bit(evt->evt_type, sdev->supported_events)) {
		kfree(evt);
		return;
	}
2757
#endif
2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785

	spin_lock_irqsave(&sdev->list_lock, flags);
	list_add_tail(&evt->node, &sdev->event_list);
	schedule_work(&sdev->event_work);
	spin_unlock_irqrestore(&sdev->list_lock, flags);
}
EXPORT_SYMBOL_GPL(sdev_evt_send);

/**
 * 	sdev_evt_alloc - allocate a new scsi event
 *	@evt_type: type of event to allocate
 *	@gfpflags: GFP flags for allocation
 *
 *	Allocates and returns a new scsi_event.
 */
struct scsi_event *sdev_evt_alloc(enum scsi_device_event evt_type,
				  gfp_t gfpflags)
{
	struct scsi_event *evt = kzalloc(sizeof(struct scsi_event), gfpflags);
	if (!evt)
		return NULL;

	evt->evt_type = evt_type;
	INIT_LIST_HEAD(&evt->node);

	/* evt_type-specific initialization, if any */
	switch (evt_type) {
	case SDEV_EVT_MEDIA_CHANGE:
2786 2787 2788 2789 2790
	case SDEV_EVT_INQUIRY_CHANGE_REPORTED:
	case SDEV_EVT_CAPACITY_CHANGE_REPORTED:
	case SDEV_EVT_SOFT_THRESHOLD_REACHED_REPORTED:
	case SDEV_EVT_MODE_PARAMETER_CHANGE_REPORTED:
	case SDEV_EVT_LUN_CHANGE_REPORTED:
2791
	case SDEV_EVT_ALUA_STATE_CHANGE_REPORTED:
2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822
	default:
		/* do nothing */
		break;
	}

	return evt;
}
EXPORT_SYMBOL_GPL(sdev_evt_alloc);

/**
 * 	sdev_evt_send_simple - send asserted event to uevent thread
 *	@sdev: scsi_device event occurred on
 *	@evt_type: type of event to send
 *	@gfpflags: GFP flags for allocation
 *
 *	Assert scsi device event asynchronously, given an event type.
 */
void sdev_evt_send_simple(struct scsi_device *sdev,
			  enum scsi_device_event evt_type, gfp_t gfpflags)
{
	struct scsi_event *evt = sdev_evt_alloc(evt_type, gfpflags);
	if (!evt) {
		sdev_printk(KERN_ERR, sdev, "event %d eaten due to OOM\n",
			    evt_type);
		return;
	}

	sdev_evt_send(sdev, evt);
}
EXPORT_SYMBOL_GPL(sdev_evt_send_simple);

2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855
/**
 * scsi_request_fn_active() - number of kernel threads inside scsi_request_fn()
 * @sdev: SCSI device to count the number of scsi_request_fn() callers for.
 */
static int scsi_request_fn_active(struct scsi_device *sdev)
{
	struct request_queue *q = sdev->request_queue;
	int request_fn_active;

	WARN_ON_ONCE(sdev->host->use_blk_mq);

	spin_lock_irq(q->queue_lock);
	request_fn_active = q->request_fn_active;
	spin_unlock_irq(q->queue_lock);

	return request_fn_active;
}

/**
 * scsi_wait_for_queuecommand() - wait for ongoing queuecommand() calls
 * @sdev: SCSI device pointer.
 *
 * Wait until the ongoing shost->hostt->queuecommand() calls that are
 * invoked from scsi_request_fn() have finished.
 */
static void scsi_wait_for_queuecommand(struct scsi_device *sdev)
{
	WARN_ON_ONCE(sdev->host->use_blk_mq);

	while (scsi_request_fn_active(sdev))
		msleep(20);
}

L
Linus Torvalds 已提交
2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869
/**
 *	scsi_device_quiesce - Block user issued commands.
 *	@sdev:	scsi device to quiesce.
 *
 *	This works by trying to transition to the SDEV_QUIESCE state
 *	(which must be a legal transition).  When the device is in this
 *	state, only special requests will be accepted, all others will
 *	be deferred.  Since special requests may also be requeued requests,
 *	a successful return doesn't guarantee the device will be 
 *	totally quiescent.
 *
 *	Must be called with user context, may sleep.
 *
 *	Returns zero if unsuccessful or an error if not.
2870
 */
L
Linus Torvalds 已提交
2871 2872 2873 2874 2875 2876 2877 2878
int
scsi_device_quiesce(struct scsi_device *sdev)
{
	int err = scsi_device_set_state(sdev, SDEV_QUIESCE);
	if (err)
		return err;

	scsi_run_queue(sdev->request_queue);
2879
	while (atomic_read(&sdev->device_busy)) {
L
Linus Torvalds 已提交
2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894
		msleep_interruptible(200);
		scsi_run_queue(sdev->request_queue);
	}
	return 0;
}
EXPORT_SYMBOL(scsi_device_quiesce);

/**
 *	scsi_device_resume - Restart user issued commands to a quiesced device.
 *	@sdev:	scsi device to resume.
 *
 *	Moves the device from quiesced back to running and restarts the
 *	queues.
 *
 *	Must be called with user context, may sleep.
2895
 */
2896
void scsi_device_resume(struct scsi_device *sdev)
L
Linus Torvalds 已提交
2897
{
2898 2899 2900 2901 2902 2903
	/* check if the device state was mutated prior to resume, and if
	 * so assume the state is being managed elsewhere (for example
	 * device deleted during suspend)
	 */
	if (sdev->sdev_state != SDEV_QUIESCE ||
	    scsi_device_set_state(sdev, SDEV_RUNNING))
L
Linus Torvalds 已提交
2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935
		return;
	scsi_run_queue(sdev->request_queue);
}
EXPORT_SYMBOL(scsi_device_resume);

static void
device_quiesce_fn(struct scsi_device *sdev, void *data)
{
	scsi_device_quiesce(sdev);
}

void
scsi_target_quiesce(struct scsi_target *starget)
{
	starget_for_each_device(starget, NULL, device_quiesce_fn);
}
EXPORT_SYMBOL(scsi_target_quiesce);

static void
device_resume_fn(struct scsi_device *sdev, void *data)
{
	scsi_device_resume(sdev);
}

void
scsi_target_resume(struct scsi_target *starget)
{
	starget_for_each_device(starget, NULL, device_resume_fn);
}
EXPORT_SYMBOL(scsi_target_resume);

/**
2936
 * scsi_internal_device_block - internal function to put a device temporarily into the SDEV_BLOCK state
L
Linus Torvalds 已提交
2937
 * @sdev:	device to block
2938 2939
 * @wait:	Whether or not to wait until ongoing .queuecommand() /
 *		.queue_rq() calls have finished.
L
Linus Torvalds 已提交
2940 2941
 *
 * Block request made by scsi lld's to temporarily stop all
2942
 * scsi commands on the specified device. May sleep.
L
Linus Torvalds 已提交
2943 2944 2945 2946 2947 2948 2949 2950
 *
 * Returns zero if successful or error if not
 *
 * Notes:       
 *	This routine transitions the device to the SDEV_BLOCK state
 *	(which must be a legal transition).  When the device is in this
 *	state, all commands are deferred until the scsi lld reenables
 *	the device with scsi_device_unblock or device_block_tmo fires.
2951 2952 2953 2954
 *
 * To do: avoid that scsi_send_eh_cmnd() calls queuecommand() after
 * scsi_internal_device_block() has blocked a SCSI device and also
 * remove the rport mutex lock and unlock calls from srp_queuecommand().
2955
 */
L
Linus Torvalds 已提交
2956
int
2957
scsi_internal_device_block(struct scsi_device *sdev, bool wait)
L
Linus Torvalds 已提交
2958
{
2959
	struct request_queue *q = sdev->request_queue;
L
Linus Torvalds 已提交
2960 2961 2962 2963
	unsigned long flags;
	int err = 0;

	err = scsi_device_set_state(sdev, SDEV_BLOCK);
2964 2965 2966 2967 2968 2969
	if (err) {
		err = scsi_device_set_state(sdev, SDEV_CREATED_BLOCK);

		if (err)
			return err;
	}
L
Linus Torvalds 已提交
2970 2971 2972 2973 2974 2975

	/* 
	 * The device has transitioned to SDEV_BLOCK.  Stop the
	 * block layer from calling the midlayer with this device's
	 * request queue. 
	 */
2976
	if (q->mq_ops) {
2977 2978 2979 2980
		if (wait)
			blk_mq_quiesce_queue(q);
		else
			blk_mq_stop_hw_queues(q);
2981 2982 2983 2984
	} else {
		spin_lock_irqsave(q->queue_lock, flags);
		blk_stop_queue(q);
		spin_unlock_irqrestore(q->queue_lock, flags);
2985 2986
		if (wait)
			scsi_wait_for_queuecommand(sdev);
2987
	}
L
Linus Torvalds 已提交
2988 2989 2990 2991 2992 2993 2994 2995

	return 0;
}
EXPORT_SYMBOL_GPL(scsi_internal_device_block);
 
/**
 * scsi_internal_device_unblock - resume a device after a block request
 * @sdev:	device to resume
2996
 * @new_state:	state to set devices to after unblocking
L
Linus Torvalds 已提交
2997 2998 2999 3000 3001 3002 3003 3004 3005
 *
 * Called by scsi lld's or the midlayer to restart the device queue
 * for the previously suspended scsi device.  Called from interrupt or
 * normal process context.
 *
 * Returns zero if successful or error if not.
 *
 * Notes:       
 *	This routine transitions the device to the SDEV_RUNNING state
3006
 *	or to one of the offline states (which must be a legal transition)
3007
 *	allowing the midlayer to goose the queue for this device.
3008
 */
L
Linus Torvalds 已提交
3009
int
3010 3011
scsi_internal_device_unblock(struct scsi_device *sdev,
			     enum scsi_device_state new_state)
L
Linus Torvalds 已提交
3012
{
3013
	struct request_queue *q = sdev->request_queue; 
L
Linus Torvalds 已提交
3014
	unsigned long flags;
3015 3016 3017 3018

	/*
	 * Try to transition the scsi device to SDEV_RUNNING or one of the
	 * offlined states and goose the device queue if successful.
L
Linus Torvalds 已提交
3019
	 */
3020 3021
	if ((sdev->sdev_state == SDEV_BLOCK) ||
	    (sdev->sdev_state == SDEV_TRANSPORT_OFFLINE))
3022 3023 3024 3025 3026 3027 3028 3029
		sdev->sdev_state = new_state;
	else if (sdev->sdev_state == SDEV_CREATED_BLOCK) {
		if (new_state == SDEV_TRANSPORT_OFFLINE ||
		    new_state == SDEV_OFFLINE)
			sdev->sdev_state = new_state;
		else
			sdev->sdev_state = SDEV_CREATED;
	} else if (sdev->sdev_state != SDEV_CANCEL &&
3030
		 sdev->sdev_state != SDEV_OFFLINE)
3031
		return -EINVAL;
L
Linus Torvalds 已提交
3032

3033 3034 3035 3036 3037 3038 3039
	if (q->mq_ops) {
		blk_mq_start_stopped_hw_queues(q, false);
	} else {
		spin_lock_irqsave(q->queue_lock, flags);
		blk_start_queue(q);
		spin_unlock_irqrestore(q->queue_lock, flags);
	}
L
Linus Torvalds 已提交
3040 3041 3042 3043 3044 3045 3046 3047

	return 0;
}
EXPORT_SYMBOL_GPL(scsi_internal_device_unblock);

static void
device_block(struct scsi_device *sdev, void *data)
{
3048
	scsi_internal_device_block(sdev, true);
L
Linus Torvalds 已提交
3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073
}

static int
target_block(struct device *dev, void *data)
{
	if (scsi_is_target_device(dev))
		starget_for_each_device(to_scsi_target(dev), NULL,
					device_block);
	return 0;
}

void
scsi_target_block(struct device *dev)
{
	if (scsi_is_target_device(dev))
		starget_for_each_device(to_scsi_target(dev), NULL,
					device_block);
	else
		device_for_each_child(dev, NULL, target_block);
}
EXPORT_SYMBOL_GPL(scsi_target_block);

static void
device_unblock(struct scsi_device *sdev, void *data)
{
3074
	scsi_internal_device_unblock(sdev, *(enum scsi_device_state *)data);
L
Linus Torvalds 已提交
3075 3076 3077 3078 3079 3080
}

static int
target_unblock(struct device *dev, void *data)
{
	if (scsi_is_target_device(dev))
3081
		starget_for_each_device(to_scsi_target(dev), data,
L
Linus Torvalds 已提交
3082 3083 3084 3085 3086
					device_unblock);
	return 0;
}

void
3087
scsi_target_unblock(struct device *dev, enum scsi_device_state new_state)
L
Linus Torvalds 已提交
3088 3089
{
	if (scsi_is_target_device(dev))
3090
		starget_for_each_device(to_scsi_target(dev), &new_state,
L
Linus Torvalds 已提交
3091 3092
					device_unblock);
	else
3093
		device_for_each_child(dev, &new_state, target_unblock);
L
Linus Torvalds 已提交
3094 3095
}
EXPORT_SYMBOL_GPL(scsi_target_unblock);
3096 3097 3098

/**
 * scsi_kmap_atomic_sg - find and atomically map an sg-elemnt
3099
 * @sgl:	scatter-gather list
3100 3101 3102 3103 3104 3105
 * @sg_count:	number of segments in sg
 * @offset:	offset in bytes into sg, on return offset into the mapped area
 * @len:	bytes to map, on return number of bytes mapped
 *
 * Returns virtual address of the start of the mapped page
 */
J
Jens Axboe 已提交
3106
void *scsi_kmap_atomic_sg(struct scatterlist *sgl, int sg_count,
3107 3108 3109 3110
			  size_t *offset, size_t *len)
{
	int i;
	size_t sg_len = 0, len_complete = 0;
J
Jens Axboe 已提交
3111
	struct scatterlist *sg;
3112 3113
	struct page *page;

3114 3115
	WARN_ON(!irqs_disabled());

J
Jens Axboe 已提交
3116
	for_each_sg(sgl, sg, sg_count, i) {
3117
		len_complete = sg_len; /* Complete sg-entries */
J
Jens Axboe 已提交
3118
		sg_len += sg->length;
3119 3120 3121 3122 3123
		if (sg_len > *offset)
			break;
	}

	if (unlikely(i == sg_count)) {
3124 3125
		printk(KERN_ERR "%s: Bytes in sg: %zu, requested offset %zu, "
			"elements %d\n",
3126
		       __func__, sg_len, *offset, sg_count);
3127 3128 3129 3130 3131
		WARN_ON(1);
		return NULL;
	}

	/* Offset starting from the beginning of first page in this sg-entry */
J
Jens Axboe 已提交
3132
	*offset = *offset - len_complete + sg->offset;
3133 3134

	/* Assumption: contiguous pages can be accessed as "page + i" */
J
Jens Axboe 已提交
3135
	page = nth_page(sg_page(sg), (*offset >> PAGE_SHIFT));
3136 3137 3138 3139 3140 3141 3142
	*offset &= ~PAGE_MASK;

	/* Bytes in this sg-entry from *offset to the end of the page */
	sg_len = PAGE_SIZE - *offset;
	if (*len > sg_len)
		*len = sg_len;

3143
	return kmap_atomic(page);
3144 3145 3146 3147
}
EXPORT_SYMBOL(scsi_kmap_atomic_sg);

/**
3148
 * scsi_kunmap_atomic_sg - atomically unmap a virtual address, previously mapped with scsi_kmap_atomic_sg
3149 3150 3151 3152
 * @virt:	virtual address to be unmapped
 */
void scsi_kunmap_atomic_sg(void *virt)
{
3153
	kunmap_atomic(virt);
3154 3155
}
EXPORT_SYMBOL(scsi_kunmap_atomic_sg);
3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169

void sdev_disable_disk_events(struct scsi_device *sdev)
{
	atomic_inc(&sdev->disk_events_disable_depth);
}
EXPORT_SYMBOL(sdev_disable_disk_events);

void sdev_enable_disk_events(struct scsi_device *sdev)
{
	if (WARN_ON_ONCE(atomic_read(&sdev->disk_events_disable_depth) <= 0))
		return;
	atomic_dec(&sdev->disk_events_disable_depth);
}
EXPORT_SYMBOL(sdev_enable_disk_events);
H
Hannes Reinecke 已提交
3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208

/**
 * scsi_vpd_lun_id - return a unique device identification
 * @sdev: SCSI device
 * @id:   buffer for the identification
 * @id_len:  length of the buffer
 *
 * Copies a unique device identification into @id based
 * on the information in the VPD page 0x83 of the device.
 * The string will be formatted as a SCSI name string.
 *
 * Returns the length of the identification or error on failure.
 * If the identifier is longer than the supplied buffer the actual
 * identifier length is returned and the buffer is not zero-padded.
 */
int scsi_vpd_lun_id(struct scsi_device *sdev, char *id, size_t id_len)
{
	u8 cur_id_type = 0xff;
	u8 cur_id_size = 0;
	unsigned char *d, *cur_id_str;
	unsigned char __rcu *vpd_pg83;
	int id_size = -EINVAL;

	rcu_read_lock();
	vpd_pg83 = rcu_dereference(sdev->vpd_pg83);
	if (!vpd_pg83) {
		rcu_read_unlock();
		return -ENXIO;
	}

	/*
	 * Look for the correct descriptor.
	 * Order of preference for lun descriptor:
	 * - SCSI name string
	 * - NAA IEEE Registered Extended
	 * - EUI-64 based 16-byte
	 * - EUI-64 based 12-byte
	 * - NAA IEEE Registered
	 * - NAA IEEE Extended
3209
	 * - T10 Vendor ID
H
Hannes Reinecke 已提交
3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227
	 * as longer descriptors reduce the likelyhood
	 * of identification clashes.
	 */

	/* The id string must be at least 20 bytes + terminating NULL byte */
	if (id_len < 21) {
		rcu_read_unlock();
		return -EINVAL;
	}

	memset(id, 0, id_len);
	d = vpd_pg83 + 4;
	while (d < vpd_pg83 + sdev->vpd_pg83_len) {
		/* Skip designators not referring to the LUN */
		if ((d[1] & 0x30) != 0x00)
			goto next_desig;

		switch (d[1] & 0xf) {
3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242
		case 0x1:
			/* T10 Vendor ID */
			if (cur_id_size > d[3])
				break;
			/* Prefer anything */
			if (cur_id_type > 0x01 && cur_id_type != 0xff)
				break;
			cur_id_size = d[3];
			if (cur_id_size + 4 > id_len)
				cur_id_size = id_len - 4;
			cur_id_str = d + 4;
			cur_id_type = d[1] & 0xf;
			id_size = snprintf(id, id_len, "t10.%*pE",
					   cur_id_size, cur_id_str);
			break;
H
Hannes Reinecke 已提交
3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325
		case 0x2:
			/* EUI-64 */
			if (cur_id_size > d[3])
				break;
			/* Prefer NAA IEEE Registered Extended */
			if (cur_id_type == 0x3 &&
			    cur_id_size == d[3])
				break;
			cur_id_size = d[3];
			cur_id_str = d + 4;
			cur_id_type = d[1] & 0xf;
			switch (cur_id_size) {
			case 8:
				id_size = snprintf(id, id_len,
						   "eui.%8phN",
						   cur_id_str);
				break;
			case 12:
				id_size = snprintf(id, id_len,
						   "eui.%12phN",
						   cur_id_str);
				break;
			case 16:
				id_size = snprintf(id, id_len,
						   "eui.%16phN",
						   cur_id_str);
				break;
			default:
				cur_id_size = 0;
				break;
			}
			break;
		case 0x3:
			/* NAA */
			if (cur_id_size > d[3])
				break;
			cur_id_size = d[3];
			cur_id_str = d + 4;
			cur_id_type = d[1] & 0xf;
			switch (cur_id_size) {
			case 8:
				id_size = snprintf(id, id_len,
						   "naa.%8phN",
						   cur_id_str);
				break;
			case 16:
				id_size = snprintf(id, id_len,
						   "naa.%16phN",
						   cur_id_str);
				break;
			default:
				cur_id_size = 0;
				break;
			}
			break;
		case 0x8:
			/* SCSI name string */
			if (cur_id_size + 4 > d[3])
				break;
			/* Prefer others for truncated descriptor */
			if (cur_id_size && d[3] > id_len)
				break;
			cur_id_size = id_size = d[3];
			cur_id_str = d + 4;
			cur_id_type = d[1] & 0xf;
			if (cur_id_size >= id_len)
				cur_id_size = id_len - 1;
			memcpy(id, cur_id_str, cur_id_size);
			/* Decrease priority for truncated descriptor */
			if (cur_id_size != id_size)
				cur_id_size = 6;
			break;
		default:
			break;
		}
next_desig:
		d += d[3] + 4;
	}
	rcu_read_unlock();

	return id_size;
}
EXPORT_SYMBOL(scsi_vpd_lun_id);
H
Hannes Reinecke 已提交
3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372

/*
 * scsi_vpd_tpg_id - return a target port group identifier
 * @sdev: SCSI device
 *
 * Returns the Target Port Group identifier from the information
 * froom VPD page 0x83 of the device.
 *
 * Returns the identifier or error on failure.
 */
int scsi_vpd_tpg_id(struct scsi_device *sdev, int *rel_id)
{
	unsigned char *d;
	unsigned char __rcu *vpd_pg83;
	int group_id = -EAGAIN, rel_port = -1;

	rcu_read_lock();
	vpd_pg83 = rcu_dereference(sdev->vpd_pg83);
	if (!vpd_pg83) {
		rcu_read_unlock();
		return -ENXIO;
	}

	d = sdev->vpd_pg83 + 4;
	while (d < sdev->vpd_pg83 + sdev->vpd_pg83_len) {
		switch (d[1] & 0xf) {
		case 0x4:
			/* Relative target port */
			rel_port = get_unaligned_be16(&d[6]);
			break;
		case 0x5:
			/* Target port group */
			group_id = get_unaligned_be16(&d[6]);
			break;
		default:
			break;
		}
		d += d[3] + 4;
	}
	rcu_read_unlock();

	if (group_id >= 0 && rel_id && rel_port != -1)
		*rel_id = rel_port;

	return group_id;
}
EXPORT_SYMBOL(scsi_vpd_tpg_id);