blk-core.c 105.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5
/*
 * Copyright (C) 1991, 1992 Linus Torvalds
 * Copyright (C) 1994,      Karl Keyte: Added support for disk statistics
 * Elevator latency, (C) 2000  Andrea Arcangeli <andrea@suse.de> SuSE
 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
6 7
 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
 *	-  July2000
L
Linus Torvalds 已提交
8 9 10 11 12 13 14 15 16 17 18
 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
 */

/*
 * This handles all read/write requests to block devices
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
19
#include <linux/blk-mq.h>
L
Linus Torvalds 已提交
20 21 22 23 24 25 26 27 28
#include <linux/highmem.h>
#include <linux/mm.h>
#include <linux/kernel_stat.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/completion.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/writeback.h>
29
#include <linux/task_io_accounting_ops.h>
30
#include <linux/fault-inject.h>
31
#include <linux/list_sort.h>
T
Tejun Heo 已提交
32
#include <linux/delay.h>
33
#include <linux/ratelimit.h>
L
Lin Ming 已提交
34
#include <linux/pm_runtime.h>
35
#include <linux/blk-cgroup.h>
36
#include <linux/debugfs.h>
37
#include <linux/bpf.h>
38 39 40

#define CREATE_TRACE_POINTS
#include <trace/events/block.h>
L
Linus Torvalds 已提交
41

42
#include "blk.h"
43
#include "blk-mq.h"
44
#include "blk-mq-sched.h"
45
#include "blk-rq-qos.h"
46

47 48 49 50
#ifdef CONFIG_DEBUG_FS
struct dentry *blk_debugfs_root;
#endif

51
EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
52
EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
53
EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
K
Keith Busch 已提交
54
EXPORT_TRACEPOINT_SYMBOL_GPL(block_split);
N
NeilBrown 已提交
55
EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
56

57 58
DEFINE_IDA(blk_queue_ida);

L
Linus Torvalds 已提交
59 60 61
/*
 * For the allocated request tables
 */
62
struct kmem_cache *request_cachep;
L
Linus Torvalds 已提交
63 64 65 66

/*
 * For queue allocation
 */
67
struct kmem_cache *blk_requestq_cachep;
L
Linus Torvalds 已提交
68 69 70 71

/*
 * Controlling structure to kblockd
 */
72
static struct workqueue_struct *kblockd_workqueue;
L
Linus Torvalds 已提交
73

74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
/**
 * blk_queue_flag_set - atomically set a queue flag
 * @flag: flag to be set
 * @q: request queue
 */
void blk_queue_flag_set(unsigned int flag, struct request_queue *q)
{
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
	queue_flag_set(flag, q);
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_queue_flag_set);

/**
 * blk_queue_flag_clear - atomically clear a queue flag
 * @flag: flag to be cleared
 * @q: request queue
 */
void blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
{
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
	queue_flag_clear(flag, q);
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_queue_flag_clear);

/**
 * blk_queue_flag_test_and_set - atomically test and set a queue flag
 * @flag: flag to be set
 * @q: request queue
 *
 * Returns the previous value of @flag - 0 if the flag was not set and 1 if
 * the flag was already set.
 */
bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q)
{
	unsigned long flags;
	bool res;

	spin_lock_irqsave(q->queue_lock, flags);
	res = queue_flag_test_and_set(flag, q);
	spin_unlock_irqrestore(q->queue_lock, flags);

	return res;
}
EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_set);

/**
 * blk_queue_flag_test_and_clear - atomically test and clear a queue flag
 * @flag: flag to be cleared
 * @q: request queue
 *
 * Returns the previous value of @flag - 0 if the flag was not set and 1 if
 * the flag was set.
 */
bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q)
{
	unsigned long flags;
	bool res;

	spin_lock_irqsave(q->queue_lock, flags);
	res = queue_flag_test_and_clear(flag, q);
	spin_unlock_irqrestore(q->queue_lock, flags);

	return res;
}
EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_clear);

146 147 148 149 150
static void blk_clear_congested(struct request_list *rl, int sync)
{
#ifdef CONFIG_CGROUP_WRITEBACK
	clear_wb_congested(rl->blkg->wb_congested, sync);
#else
151 152 153 154 155
	/*
	 * If !CGROUP_WRITEBACK, all blkg's map to bdi->wb and we shouldn't
	 * flip its congestion state for events on other blkcgs.
	 */
	if (rl == &rl->q->root_rl)
156
		clear_wb_congested(rl->q->backing_dev_info->wb.congested, sync);
157 158 159 160 161 162 163 164
#endif
}

static void blk_set_congested(struct request_list *rl, int sync)
{
#ifdef CONFIG_CGROUP_WRITEBACK
	set_wb_congested(rl->blkg->wb_congested, sync);
#else
165 166
	/* see blk_clear_congested() */
	if (rl == &rl->q->root_rl)
167
		set_wb_congested(rl->q->backing_dev_info->wb.congested, sync);
168 169 170
#endif
}

171
void blk_queue_congestion_threshold(struct request_queue *q)
L
Linus Torvalds 已提交
172 173 174 175 176 177 178 179 180 181 182 183 184 185
{
	int nr;

	nr = q->nr_requests - (q->nr_requests / 8) + 1;
	if (nr > q->nr_requests)
		nr = q->nr_requests;
	q->nr_congestion_on = nr;

	nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
	if (nr < 1)
		nr = 1;
	q->nr_congestion_off = nr;
}

186
void blk_rq_init(struct request_queue *q, struct request *rq)
L
Linus Torvalds 已提交
187
{
188 189
	memset(rq, 0, sizeof(*rq));

L
Linus Torvalds 已提交
190
	INIT_LIST_HEAD(&rq->queuelist);
J
Jens Axboe 已提交
191
	INIT_LIST_HEAD(&rq->timeout_list);
192
	rq->cpu = -1;
J
Jens Axboe 已提交
193
	rq->q = q;
194
	rq->__sector = (sector_t) -1;
195 196
	INIT_HLIST_NODE(&rq->hash);
	RB_CLEAR_NODE(&rq->rb_node);
J
Jens Axboe 已提交
197
	rq->tag = -1;
198
	rq->internal_tag = -1;
199
	rq->start_time_ns = ktime_get_ns();
200
	rq->part = NULL;
201
	refcount_set(&rq->ref, 1);
L
Linus Torvalds 已提交
202
}
203
EXPORT_SYMBOL(blk_rq_init);
L
Linus Torvalds 已提交
204

205 206 207 208 209 210 211 212 213 214 215 216 217 218
static const struct {
	int		errno;
	const char	*name;
} blk_errors[] = {
	[BLK_STS_OK]		= { 0,		"" },
	[BLK_STS_NOTSUPP]	= { -EOPNOTSUPP, "operation not supported" },
	[BLK_STS_TIMEOUT]	= { -ETIMEDOUT,	"timeout" },
	[BLK_STS_NOSPC]		= { -ENOSPC,	"critical space allocation" },
	[BLK_STS_TRANSPORT]	= { -ENOLINK,	"recoverable transport" },
	[BLK_STS_TARGET]	= { -EREMOTEIO,	"critical target" },
	[BLK_STS_NEXUS]		= { -EBADE,	"critical nexus" },
	[BLK_STS_MEDIUM]	= { -ENODATA,	"critical medium" },
	[BLK_STS_PROTECTION]	= { -EILSEQ,	"protection" },
	[BLK_STS_RESOURCE]	= { -ENOMEM,	"kernel resource" },
219
	[BLK_STS_DEV_RESOURCE]	= { -EBUSY,	"device resource" },
220
	[BLK_STS_AGAIN]		= { -EAGAIN,	"nonblocking retry" },
221

222 223 224
	/* device mapper special case, should not leak out: */
	[BLK_STS_DM_REQUEUE]	= { -EREMCHG, "dm internal retry" },

225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
	/* everything else not covered above: */
	[BLK_STS_IOERR]		= { -EIO,	"I/O" },
};

blk_status_t errno_to_blk_status(int errno)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {
		if (blk_errors[i].errno == errno)
			return (__force blk_status_t)i;
	}

	return BLK_STS_IOERR;
}
EXPORT_SYMBOL_GPL(errno_to_blk_status);

int blk_status_to_errno(blk_status_t status)
{
	int idx = (__force int)status;

246
	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
247 248 249 250 251 252 253 254 255
		return -EIO;
	return blk_errors[idx].errno;
}
EXPORT_SYMBOL_GPL(blk_status_to_errno);

static void print_req_error(struct request *req, blk_status_t status)
{
	int idx = (__force int)status;

256
	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
257 258 259 260 261 262 263 264
		return;

	printk_ratelimited(KERN_ERR "%s: %s error, dev %s, sector %llu\n",
			   __func__, blk_errors[idx].name, req->rq_disk ?
			   req->rq_disk->disk_name : "?",
			   (unsigned long long)blk_rq_pos(req));
}

N
NeilBrown 已提交
265
static void req_bio_endio(struct request *rq, struct bio *bio,
266
			  unsigned int nbytes, blk_status_t error)
L
Linus Torvalds 已提交
267
{
268
	if (error)
269
		bio->bi_status = error;
270

271
	if (unlikely(rq->rq_flags & RQF_QUIET))
272
		bio_set_flag(bio, BIO_QUIET);
273

274
	bio_advance(bio, nbytes);
275

T
Tejun Heo 已提交
276
	/* don't actually finish bio if it's part of flush sequence */
277
	if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
278
		bio_endio(bio);
L
Linus Torvalds 已提交
279 280 281 282
}

void blk_dump_rq_flags(struct request *rq, char *msg)
{
283 284
	printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
		rq->rq_disk ? rq->rq_disk->disk_name : "?",
J
Jens Axboe 已提交
285
		(unsigned long long) rq->cmd_flags);
L
Linus Torvalds 已提交
286

287 288 289
	printk(KERN_INFO "  sector %llu, nr/cnr %u/%u\n",
	       (unsigned long long)blk_rq_pos(rq),
	       blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
290 291
	printk(KERN_INFO "  bio %p, biotail %p, len %u\n",
	       rq->bio, rq->biotail, blk_rq_bytes(rq));
L
Linus Torvalds 已提交
292 293 294
}
EXPORT_SYMBOL(blk_dump_rq_flags);

295
static void blk_delay_work(struct work_struct *work)
L
Linus Torvalds 已提交
296
{
297
	struct request_queue *q;
L
Linus Torvalds 已提交
298

299 300
	q = container_of(work, struct request_queue, delay_work.work);
	spin_lock_irq(q->queue_lock);
301
	__blk_run_queue(q);
302
	spin_unlock_irq(q->queue_lock);
L
Linus Torvalds 已提交
303 304 305
}

/**
306 307 308
 * blk_delay_queue - restart queueing after defined interval
 * @q:		The &struct request_queue in question
 * @msecs:	Delay in msecs
L
Linus Torvalds 已提交
309 310
 *
 * Description:
311 312
 *   Sometimes queueing needs to be postponed for a little while, to allow
 *   resources to come back. This function will make sure that queueing is
313
 *   restarted around the specified time.
314 315
 */
void blk_delay_queue(struct request_queue *q, unsigned long msecs)
316
{
317
	lockdep_assert_held(q->queue_lock);
318
	WARN_ON_ONCE(q->mq_ops);
319

320 321 322
	if (likely(!blk_queue_dead(q)))
		queue_delayed_work(kblockd_workqueue, &q->delay_work,
				   msecs_to_jiffies(msecs));
323
}
324
EXPORT_SYMBOL(blk_delay_queue);
325

J
Jens Axboe 已提交
326 327 328 329 330 331 332 333 334 335 336
/**
 * blk_start_queue_async - asynchronously restart a previously stopped queue
 * @q:    The &struct request_queue in question
 *
 * Description:
 *   blk_start_queue_async() will clear the stop flag on the queue, and
 *   ensure that the request_fn for the queue is run from an async
 *   context.
 **/
void blk_start_queue_async(struct request_queue *q)
{
337
	lockdep_assert_held(q->queue_lock);
338
	WARN_ON_ONCE(q->mq_ops);
339

J
Jens Axboe 已提交
340 341 342 343 344
	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
	blk_run_queue_async(q);
}
EXPORT_SYMBOL(blk_start_queue_async);

L
Linus Torvalds 已提交
345 346
/**
 * blk_start_queue - restart a previously stopped queue
347
 * @q:    The &struct request_queue in question
L
Linus Torvalds 已提交
348 349 350 351
 *
 * Description:
 *   blk_start_queue() will clear the stop flag on the queue, and call
 *   the request_fn for the queue if it was in a stopped state when
352
 *   entered. Also see blk_stop_queue().
L
Linus Torvalds 已提交
353
 **/
354
void blk_start_queue(struct request_queue *q)
L
Linus Torvalds 已提交
355
{
356
	lockdep_assert_held(q->queue_lock);
357
	WARN_ON_ONCE(q->mq_ops);
358

N
Nick Piggin 已提交
359
	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
360
	__blk_run_queue(q);
L
Linus Torvalds 已提交
361 362 363 364 365
}
EXPORT_SYMBOL(blk_start_queue);

/**
 * blk_stop_queue - stop a queue
366
 * @q:    The &struct request_queue in question
L
Linus Torvalds 已提交
367 368 369 370 371 372 373 374 375
 *
 * Description:
 *   The Linux block layer assumes that a block driver will consume all
 *   entries on the request queue when the request_fn strategy is called.
 *   Often this will not happen, because of hardware limitations (queue
 *   depth settings). If a device driver gets a 'queue full' response,
 *   or if it simply chooses not to queue more I/O at one point, it can
 *   call this function to prevent the request_fn from being called until
 *   the driver has signalled it's ready to go again. This happens by calling
376
 *   blk_start_queue() to restart queue operations.
L
Linus Torvalds 已提交
377
 **/
378
void blk_stop_queue(struct request_queue *q)
L
Linus Torvalds 已提交
379
{
380
	lockdep_assert_held(q->queue_lock);
381
	WARN_ON_ONCE(q->mq_ops);
382

383
	cancel_delayed_work(&q->delay_work);
N
Nick Piggin 已提交
384
	queue_flag_set(QUEUE_FLAG_STOPPED, q);
L
Linus Torvalds 已提交
385 386 387 388 389 390 391 392 393 394 395 396
}
EXPORT_SYMBOL(blk_stop_queue);

/**
 * blk_sync_queue - cancel any pending callbacks on a queue
 * @q: the queue
 *
 * Description:
 *     The block layer may perform asynchronous callback activity
 *     on a queue, such as calling the unplug function after a timeout.
 *     A block device may call blk_sync_queue to ensure that any
 *     such activity is cancelled, thus allowing it to release resources
397
 *     that the callbacks might use. The caller must already have made sure
L
Linus Torvalds 已提交
398 399 400
 *     that its ->make_request_fn will not re-add plugging prior to calling
 *     this function.
 *
401
 *     This function does not cancel any asynchronous activity arising
402
 *     out of elevator or throttling code. That would require elevator_exit()
403
 *     and blkcg_exit_queue() to be called with queue lock initialized.
404
 *
L
Linus Torvalds 已提交
405 406 407
 */
void blk_sync_queue(struct request_queue *q)
{
408
	del_timer_sync(&q->timeout);
409
	cancel_work_sync(&q->timeout_work);
410 411 412 413 414

	if (q->mq_ops) {
		struct blk_mq_hw_ctx *hctx;
		int i;

415
		queue_for_each_hw_ctx(q, hctx, i)
416
			cancel_delayed_work_sync(&hctx->run_work);
417 418 419
	} else {
		cancel_delayed_work_sync(&q->delay_work);
	}
L
Linus Torvalds 已提交
420 421 422
}
EXPORT_SYMBOL(blk_sync_queue);

423
/**
424
 * blk_set_pm_only - increment pm_only counter
425 426
 * @q: request queue pointer
 */
427
void blk_set_pm_only(struct request_queue *q)
428
{
429
	atomic_inc(&q->pm_only);
430
}
431
EXPORT_SYMBOL_GPL(blk_set_pm_only);
432

433
void blk_clear_pm_only(struct request_queue *q)
434
{
435 436 437 438 439 440
	int pm_only;

	pm_only = atomic_dec_return(&q->pm_only);
	WARN_ON_ONCE(pm_only < 0);
	if (pm_only == 0)
		wake_up_all(&q->mq_freeze_wq);
441
}
442
EXPORT_SYMBOL_GPL(blk_clear_pm_only);
443

444 445 446 447 448 449 450 451 452 453 454 455 456
/**
 * __blk_run_queue_uncond - run a queue whether or not it has been stopped
 * @q:	The queue to run
 *
 * Description:
 *    Invoke request handling on a queue if there are any pending requests.
 *    May be used to restart request handling after a request has completed.
 *    This variant runs the queue whether or not the queue has been
 *    stopped. Must be called with the queue lock held and interrupts
 *    disabled. See also @blk_run_queue.
 */
inline void __blk_run_queue_uncond(struct request_queue *q)
{
457
	lockdep_assert_held(q->queue_lock);
458
	WARN_ON_ONCE(q->mq_ops);
459

460 461 462
	if (unlikely(blk_queue_dead(q)))
		return;

463 464 465 466 467 468 469 470
	/*
	 * Some request_fn implementations, e.g. scsi_request_fn(), unlock
	 * the queue lock internally. As a result multiple threads may be
	 * running such a request function concurrently. Keep track of the
	 * number of active request_fn invocations such that blk_drain_queue()
	 * can wait until all these request_fn calls have finished.
	 */
	q->request_fn_active++;
471
	q->request_fn(q);
472
	q->request_fn_active--;
473
}
474
EXPORT_SYMBOL_GPL(__blk_run_queue_uncond);
475

L
Linus Torvalds 已提交
476
/**
477
 * __blk_run_queue - run a single device queue
L
Linus Torvalds 已提交
478
 * @q:	The queue to run
479 480
 *
 * Description:
481
 *    See @blk_run_queue.
L
Linus Torvalds 已提交
482
 */
483
void __blk_run_queue(struct request_queue *q)
L
Linus Torvalds 已提交
484
{
485
	lockdep_assert_held(q->queue_lock);
486
	WARN_ON_ONCE(q->mq_ops);
487

488 489 490
	if (unlikely(blk_queue_stopped(q)))
		return;

491
	__blk_run_queue_uncond(q);
N
Nick Piggin 已提交
492 493
}
EXPORT_SYMBOL(__blk_run_queue);
J
Jens Axboe 已提交
494

495 496 497 498 499 500
/**
 * blk_run_queue_async - run a single device queue in workqueue context
 * @q:	The queue to run
 *
 * Description:
 *    Tells kblockd to perform the equivalent of @blk_run_queue on behalf
501 502 503 504 505 506
 *    of us.
 *
 * Note:
 *    Since it is not allowed to run q->delay_work after blk_cleanup_queue()
 *    has canceled q->delay_work, callers must hold the queue lock to avoid
 *    race conditions between blk_cleanup_queue() and blk_run_queue_async().
507 508 509
 */
void blk_run_queue_async(struct request_queue *q)
{
510
	lockdep_assert_held(q->queue_lock);
511
	WARN_ON_ONCE(q->mq_ops);
512

513
	if (likely(!blk_queue_stopped(q) && !blk_queue_dead(q)))
514
		mod_delayed_work(kblockd_workqueue, &q->delay_work, 0);
515
}
516
EXPORT_SYMBOL(blk_run_queue_async);
517

N
Nick Piggin 已提交
518 519 520
/**
 * blk_run_queue - run a single device queue
 * @q: The queue to run
521 522 523
 *
 * Description:
 *    Invoke request handling on this queue, if it has pending work to do.
T
Tejun Heo 已提交
524
 *    May be used to restart queueing when a request has completed.
N
Nick Piggin 已提交
525 526 527 528 529
 */
void blk_run_queue(struct request_queue *q)
{
	unsigned long flags;

530 531
	WARN_ON_ONCE(q->mq_ops);

N
Nick Piggin 已提交
532
	spin_lock_irqsave(q->queue_lock, flags);
533
	__blk_run_queue(q);
L
Linus Torvalds 已提交
534 535 536 537
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_run_queue);

538
void blk_put_queue(struct request_queue *q)
539 540 541
{
	kobject_put(&q->kobj);
}
J
Jens Axboe 已提交
542
EXPORT_SYMBOL(blk_put_queue);
543

T
Tejun Heo 已提交
544
/**
545
 * __blk_drain_queue - drain requests from request_queue
T
Tejun Heo 已提交
546
 * @q: queue to drain
547
 * @drain_all: whether to drain all requests or only the ones w/ ELVPRIV
T
Tejun Heo 已提交
548
 *
549 550 551
 * Drain requests from @q.  If @drain_all is set, all requests are drained.
 * If not, only ELVPRIV requests are drained.  The caller is responsible
 * for ensuring that no new requests which need to be drained are queued.
T
Tejun Heo 已提交
552
 */
553 554 555
static void __blk_drain_queue(struct request_queue *q, bool drain_all)
	__releases(q->queue_lock)
	__acquires(q->queue_lock)
T
Tejun Heo 已提交
556
{
557 558
	int i;

559
	lockdep_assert_held(q->queue_lock);
560
	WARN_ON_ONCE(q->mq_ops);
561

T
Tejun Heo 已提交
562
	while (true) {
563
		bool drain = false;
T
Tejun Heo 已提交
564

565 566 567 568 569 570 571
		/*
		 * The caller might be trying to drain @q before its
		 * elevator is initialized.
		 */
		if (q->elevator)
			elv_drain_elevator(q);

572
		blkcg_drain_queue(q);
T
Tejun Heo 已提交
573

574 575
		/*
		 * This function might be called on a queue which failed
576 577 578 579
		 * driver init after queue creation or is not yet fully
		 * active yet.  Some drivers (e.g. fd and loop) get unhappy
		 * in such cases.  Kick queue iff dispatch queue has
		 * something on it and @q has request_fn set.
580
		 */
581
		if (!list_empty(&q->queue_head) && q->request_fn)
582
			__blk_run_queue(q);
583

584
		drain |= q->nr_rqs_elvpriv;
585
		drain |= q->request_fn_active;
586 587 588 589 590 591 592

		/*
		 * Unfortunately, requests are queued at and tracked from
		 * multiple places and there's no single counter which can
		 * be drained.  Check all the queues and counters.
		 */
		if (drain_all) {
593
			struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);
594 595
			drain |= !list_empty(&q->queue_head);
			for (i = 0; i < 2; i++) {
596
				drain |= q->nr_rqs[i];
597
				drain |= q->in_flight[i];
598 599
				if (fq)
				    drain |= !list_empty(&fq->flush_queue[i]);
600 601
			}
		}
T
Tejun Heo 已提交
602

603
		if (!drain)
T
Tejun Heo 已提交
604
			break;
605 606 607

		spin_unlock_irq(q->queue_lock);

T
Tejun Heo 已提交
608
		msleep(10);
609 610

		spin_lock_irq(q->queue_lock);
T
Tejun Heo 已提交
611
	}
612 613 614 615 616 617 618

	/*
	 * With queue marked dead, any woken up waiter will fail the
	 * allocation path, so the wakeup chaining is lost and we're
	 * left with hung waiters. We need to wake up those waiters.
	 */
	if (q->request_fn) {
619 620 621 622 623
		struct request_list *rl;

		blk_queue_for_each_rl(rl, q)
			for (i = 0; i < ARRAY_SIZE(rl->wait); i++)
				wake_up_all(&rl->wait[i]);
624
	}
T
Tejun Heo 已提交
625 626
}

627 628 629 630 631 632 633
void blk_drain_queue(struct request_queue *q)
{
	spin_lock_irq(q->queue_lock);
	__blk_drain_queue(q, true);
	spin_unlock_irq(q->queue_lock);
}

634 635 636 637 638 639
/**
 * blk_queue_bypass_start - enter queue bypass mode
 * @q: queue of interest
 *
 * In bypass mode, only the dispatch FIFO queue of @q is used.  This
 * function makes @q enter bypass mode and drains all requests which were
640
 * throttled or issued before.  On return, it's guaranteed that no request
641 642
 * is being throttled or has ELVPRIV set and blk_queue_bypass() %true
 * inside queue or RCU read lock.
643 644 645
 */
void blk_queue_bypass_start(struct request_queue *q)
{
646 647
	WARN_ON_ONCE(q->mq_ops);

648
	spin_lock_irq(q->queue_lock);
649
	q->bypass_depth++;
650 651 652
	queue_flag_set(QUEUE_FLAG_BYPASS, q);
	spin_unlock_irq(q->queue_lock);

653 654 655 656 657 658
	/*
	 * Queues start drained.  Skip actual draining till init is
	 * complete.  This avoids lenghty delays during queue init which
	 * can happen many times during boot.
	 */
	if (blk_queue_init_done(q)) {
659 660 661 662
		spin_lock_irq(q->queue_lock);
		__blk_drain_queue(q, false);
		spin_unlock_irq(q->queue_lock);

663 664 665
		/* ensure blk_queue_bypass() is %true inside RCU read lock */
		synchronize_rcu();
	}
666 667 668 669 670 671 672 673
}
EXPORT_SYMBOL_GPL(blk_queue_bypass_start);

/**
 * blk_queue_bypass_end - leave queue bypass mode
 * @q: queue of interest
 *
 * Leave bypass mode and restore the normal queueing behavior.
674 675 676
 *
 * Note: although blk_queue_bypass_start() is only called for blk-sq queues,
 * this function is called for both blk-sq and blk-mq queues.
677 678 679 680 681 682 683 684 685 686 687
 */
void blk_queue_bypass_end(struct request_queue *q)
{
	spin_lock_irq(q->queue_lock);
	if (!--q->bypass_depth)
		queue_flag_clear(QUEUE_FLAG_BYPASS, q);
	WARN_ON_ONCE(q->bypass_depth < 0);
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL_GPL(blk_queue_bypass_end);

688 689
void blk_set_queue_dying(struct request_queue *q)
{
690
	blk_queue_flag_set(QUEUE_FLAG_DYING, q);
691

692 693 694 695 696 697 698
	/*
	 * When queue DYING flag is set, we need to block new req
	 * entering queue, so we call blk_freeze_queue_start() to
	 * prevent I/O from crossing blk_queue_enter().
	 */
	blk_freeze_queue_start(q);

699 700 701 702 703
	if (q->mq_ops)
		blk_mq_wake_waiters(q);
	else {
		struct request_list *rl;

704
		spin_lock_irq(q->queue_lock);
705 706
		blk_queue_for_each_rl(rl, q) {
			if (rl->rq_pool) {
707 708
				wake_up_all(&rl->wait[BLK_RW_SYNC]);
				wake_up_all(&rl->wait[BLK_RW_ASYNC]);
709 710
			}
		}
711
		spin_unlock_irq(q->queue_lock);
712
	}
713 714 715

	/* Make blk_queue_enter() reexamine the DYING flag. */
	wake_up_all(&q->mq_freeze_wq);
716 717 718
}
EXPORT_SYMBOL_GPL(blk_set_queue_dying);

719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
/* Unconfigure the I/O scheduler and dissociate from the cgroup controller. */
void blk_exit_queue(struct request_queue *q)
{
	/*
	 * Since the I/O scheduler exit code may access cgroup information,
	 * perform I/O scheduler exit before disassociating from the block
	 * cgroup controller.
	 */
	if (q->elevator) {
		ioc_clear_queue(q);
		elevator_exit(q, q->elevator);
		q->elevator = NULL;
	}

	/*
	 * Remove all references to @q from the block cgroup controller before
	 * restoring @q->queue_lock to avoid that restoring this pointer causes
	 * e.g. blkcg_print_blkgs() to crash.
	 */
	blkcg_exit_queue(q);

	/*
	 * Since the cgroup code may dereference the @q->backing_dev_info
	 * pointer, only decrease its reference count after having removed the
	 * association with the block cgroup controller.
	 */
	bdi_put(q->backing_dev_info);
}

748 749 750 751
/**
 * blk_cleanup_queue - shutdown a request queue
 * @q: request queue to shutdown
 *
752 753
 * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and
 * put it.  All future requests will be failed immediately with -ENODEV.
754
 */
755
void blk_cleanup_queue(struct request_queue *q)
756
{
757
	spinlock_t *lock = q->queue_lock;
758

B
Bart Van Assche 已提交
759
	/* mark @q DYING, no new request or merges will be allowed afterwards */
760
	mutex_lock(&q->sysfs_lock);
761
	blk_set_queue_dying(q);
762
	spin_lock_irq(lock);
763

764
	/*
B
Bart Van Assche 已提交
765
	 * A dying queue is permanently in bypass mode till released.  Note
766 767 768 769 770 771 772
	 * that, unlike blk_queue_bypass_start(), we aren't performing
	 * synchronize_rcu() after entering bypass mode to avoid the delay
	 * as some drivers create and destroy a lot of queues while
	 * probing.  This is still safe because blk_release_queue() will be
	 * called only after the queue refcnt drops to zero and nothing,
	 * RCU or not, would be traversing the queue by then.
	 */
773 774 775
	q->bypass_depth++;
	queue_flag_set(QUEUE_FLAG_BYPASS, q);

776 777
	queue_flag_set(QUEUE_FLAG_NOMERGES, q);
	queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
B
Bart Van Assche 已提交
778
	queue_flag_set(QUEUE_FLAG_DYING, q);
779 780 781
	spin_unlock_irq(lock);
	mutex_unlock(&q->sysfs_lock);

782 783 784 785
	/*
	 * Drain all requests queued before DYING marking. Set DEAD flag to
	 * prevent that q->request_fn() gets invoked after draining finished.
	 */
786
	blk_freeze_queue(q);
787
	spin_lock_irq(lock);
788
	queue_flag_set(QUEUE_FLAG_DEAD, q);
789
	spin_unlock_irq(lock);
790

791 792 793 794
	/*
	 * make sure all in-progress dispatch are completed because
	 * blk_freeze_queue() can only complete all requests, and
	 * dispatch may still be in-progress since we dispatch requests
795 796
	 * from more than one contexts.
	 *
797 798
	 * We rely on driver to deal with the race in case that queue
	 * initialization isn't done.
799
	 */
800
	if (q->mq_ops && blk_queue_init_done(q))
801 802
		blk_mq_quiesce_queue(q);

803 804 805
	/* for synchronous bio-based driver finish in-flight integrity i/o */
	blk_flush_integrity();

806
	/* @q won't process any more request, flush async actions */
807
	del_timer_sync(&q->backing_dev_info->laptop_mode_wb_timer);
808 809
	blk_sync_queue(q);

810 811 812 813 814 815
	/*
	 * I/O scheduler exit is only safe after the sysfs scheduler attribute
	 * has been removed.
	 */
	WARN_ON_ONCE(q->kobj.state_in_sysfs);

816
	blk_exit_queue(q);
817

B
Bart Van Assche 已提交
818
	if (q->mq_ops)
819 820
		blk_mq_exit_queue(q);

821
	percpu_ref_exit(&q->q_usage_counter);
B
Bart Van Assche 已提交
822

823 824 825 826 827
	spin_lock_irq(lock);
	if (q->queue_lock != &q->__queue_lock)
		q->queue_lock = &q->__queue_lock;
	spin_unlock_irq(lock);

828
	/* @q is and will stay empty, shutdown and put */
829 830
	blk_put_queue(q);
}
L
Linus Torvalds 已提交
831 832
EXPORT_SYMBOL(blk_cleanup_queue);

833
/* Allocate memory local to the request queue */
834
static void *alloc_request_simple(gfp_t gfp_mask, void *data)
835
{
836 837 838
	struct request_queue *q = data;

	return kmem_cache_alloc_node(request_cachep, gfp_mask, q->node);
839 840
}

841
static void free_request_simple(void *element, void *data)
842 843 844 845
{
	kmem_cache_free(request_cachep, element);
}

846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868
static void *alloc_request_size(gfp_t gfp_mask, void *data)
{
	struct request_queue *q = data;
	struct request *rq;

	rq = kmalloc_node(sizeof(struct request) + q->cmd_size, gfp_mask,
			q->node);
	if (rq && q->init_rq_fn && q->init_rq_fn(q, rq, gfp_mask) < 0) {
		kfree(rq);
		rq = NULL;
	}
	return rq;
}

static void free_request_size(void *element, void *data)
{
	struct request_queue *q = data;

	if (q->exit_rq_fn)
		q->exit_rq_fn(q, element);
	kfree(element);
}

869 870
int blk_init_rl(struct request_list *rl, struct request_queue *q,
		gfp_t gfp_mask)
L
Linus Torvalds 已提交
871
{
S
Shaohua Li 已提交
872
	if (unlikely(rl->rq_pool) || q->mq_ops)
873 874
		return 0;

875
	rl->q = q;
876 877 878 879
	rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
	rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
	init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
	init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
L
Linus Torvalds 已提交
880

881 882 883 884 885 886 887 888 889
	if (q->cmd_size) {
		rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ,
				alloc_request_size, free_request_size,
				q, gfp_mask, q->node);
	} else {
		rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ,
				alloc_request_simple, free_request_simple,
				q, gfp_mask, q->node);
	}
L
Linus Torvalds 已提交
890 891 892
	if (!rl->rq_pool)
		return -ENOMEM;

893 894 895
	if (rl != &q->root_rl)
		WARN_ON_ONCE(!blk_get_queue(q));

L
Linus Torvalds 已提交
896 897 898
	return 0;
}

899
void blk_exit_rl(struct request_queue *q, struct request_list *rl)
900
{
901
	if (rl->rq_pool) {
902
		mempool_destroy(rl->rq_pool);
903 904 905
		if (rl != &q->root_rl)
			blk_put_queue(q);
	}
906 907
}

908
struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
L
Linus Torvalds 已提交
909
{
910
	return blk_alloc_queue_node(gfp_mask, NUMA_NO_NODE, NULL);
911 912
}
EXPORT_SYMBOL(blk_alloc_queue);
L
Linus Torvalds 已提交
913

914 915 916 917 918
/**
 * blk_queue_enter() - try to increase q->q_usage_counter
 * @q: request queue pointer
 * @flags: BLK_MQ_REQ_NOWAIT and/or BLK_MQ_REQ_PREEMPT
 */
919
int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags)
920
{
921
	const bool pm = flags & BLK_MQ_REQ_PREEMPT;
922

923
	while (true) {
924
		bool success = false;
925

926
		rcu_read_lock();
927 928
		if (percpu_ref_tryget_live(&q->q_usage_counter)) {
			/*
929 930 931
			 * The code that increments the pm_only counter is
			 * responsible for ensuring that that counter is
			 * globally visible before the queue is unfrozen.
932
			 */
933
			if (pm || !blk_queue_pm_only(q)) {
934 935 936 937 938
				success = true;
			} else {
				percpu_ref_put(&q->q_usage_counter);
			}
		}
939
		rcu_read_unlock();
940 941

		if (success)
942 943
			return 0;

944
		if (flags & BLK_MQ_REQ_NOWAIT)
945 946
			return -EBUSY;

947
		/*
948
		 * read pair of barrier in blk_freeze_queue_start(),
949
		 * we need to order reading __PERCPU_REF_DEAD flag of
950 951 952
		 * .q_usage_counter and reading .mq_freeze_depth or
		 * queue dying flag, otherwise the following wait may
		 * never return if the two reads are reordered.
953 954 955
		 */
		smp_rmb();

956 957
		wait_event(q->mq_freeze_wq,
			   (atomic_read(&q->mq_freeze_depth) == 0 &&
958
			    (pm || !blk_queue_pm_only(q))) ||
959
			   blk_queue_dying(q));
960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977
		if (blk_queue_dying(q))
			return -ENODEV;
	}
}

void blk_queue_exit(struct request_queue *q)
{
	percpu_ref_put(&q->q_usage_counter);
}

static void blk_queue_usage_counter_release(struct percpu_ref *ref)
{
	struct request_queue *q =
		container_of(ref, struct request_queue, q_usage_counter);

	wake_up_all(&q->mq_freeze_wq);
}

978
static void blk_rq_timed_out_timer(struct timer_list *t)
979
{
980
	struct request_queue *q = from_timer(q, t, timeout);
981 982 983 984

	kblockd_schedule_work(&q->timeout_work);
}

985 986 987 988
static void blk_timeout_work_dummy(struct work_struct *work)
{
}

989 990 991 992 993 994 995 996 997 998 999 1000 1001
/**
 * blk_alloc_queue_node - allocate a request queue
 * @gfp_mask: memory allocation flags
 * @node_id: NUMA node to allocate memory from
 * @lock: For legacy queues, pointer to a spinlock that will be used to e.g.
 *        serialize calls to the legacy .request_fn() callback. Ignored for
 *	  blk-mq request queues.
 *
 * Note: pass the queue lock as the third argument to this function instead of
 * setting the queue lock pointer explicitly to avoid triggering a sporadic
 * crash in the blkcg code. This function namely calls blkcg_init_queue() and
 * the queue lock pointer must be set before blkcg_init_queue() is called.
 */
1002 1003
struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
					   spinlock_t *lock)
1004
{
1005
	struct request_queue *q;
1006
	int ret;
1007

1008
	q = kmem_cache_alloc_node(blk_requestq_cachep,
1009
				gfp_mask | __GFP_ZERO, node_id);
L
Linus Torvalds 已提交
1010 1011 1012
	if (!q)
		return NULL;

1013 1014 1015 1016 1017
	INIT_LIST_HEAD(&q->queue_head);
	q->last_merge = NULL;
	q->end_sector = 0;
	q->boundary_rq = NULL;

1018
	q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
1019
	if (q->id < 0)
1020
		goto fail_q;
1021

1022 1023
	ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
	if (ret)
1024 1025
		goto fail_id;

1026 1027 1028 1029
	q->backing_dev_info = bdi_alloc_node(gfp_mask, node_id);
	if (!q->backing_dev_info)
		goto fail_split;

1030 1031 1032 1033
	q->stats = blk_alloc_queue_stats();
	if (!q->stats)
		goto fail_stats;

1034
	q->backing_dev_info->ra_pages =
1035
			(VM_MAX_READAHEAD * 1024) / PAGE_SIZE;
1036 1037
	q->backing_dev_info->capabilities = BDI_CAP_CGROUP_WRITEBACK;
	q->backing_dev_info->name = "block";
1038
	q->node = node_id;
1039

1040 1041 1042
	timer_setup(&q->backing_dev_info->laptop_mode_wb_timer,
		    laptop_mode_timer_fn, 0);
	timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);
1043
	INIT_WORK(&q->timeout_work, blk_timeout_work_dummy);
J
Jens Axboe 已提交
1044
	INIT_LIST_HEAD(&q->timeout_list);
1045
	INIT_LIST_HEAD(&q->icq_list);
1046
#ifdef CONFIG_BLK_CGROUP
1047
	INIT_LIST_HEAD(&q->blkg_list);
1048
#endif
1049
	INIT_DELAYED_WORK(&q->delay_work, blk_delay_work);
1050

1051
	kobject_init(&q->kobj, &blk_queue_ktype);
L
Linus Torvalds 已提交
1052

1053 1054 1055
#ifdef CONFIG_BLK_DEV_IO_TRACE
	mutex_init(&q->blk_trace_mutex);
#endif
1056
	mutex_init(&q->sysfs_lock);
1057
	spin_lock_init(&q->__queue_lock);
1058

1059 1060
	if (!q->mq_ops)
		q->queue_lock = lock ? : &q->__queue_lock;
1061

1062 1063 1064
	/*
	 * A queue starts its life with bypass turned on to avoid
	 * unnecessary bypass on/off overhead and nasty surprises during
1065 1066
	 * init.  The initial bypass will be finished when the queue is
	 * registered by blk_register_queue().
1067 1068
	 */
	q->bypass_depth = 1;
1069
	queue_flag_set_unlocked(QUEUE_FLAG_BYPASS, q);
1070

1071 1072
	init_waitqueue_head(&q->mq_freeze_wq);

1073 1074 1075 1076 1077 1078 1079
	/*
	 * Init percpu_ref in atomic mode so that it's faster to shutdown.
	 * See blk_register_queue() for details.
	 */
	if (percpu_ref_init(&q->q_usage_counter,
				blk_queue_usage_counter_release,
				PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
1080
		goto fail_bdi;
1081

1082 1083 1084
	if (blkcg_init_queue(q))
		goto fail_ref;

L
Linus Torvalds 已提交
1085
	return q;
1086

1087 1088
fail_ref:
	percpu_ref_exit(&q->q_usage_counter);
1089
fail_bdi:
1090 1091
	blk_free_queue_stats(q->stats);
fail_stats:
1092
	bdi_put(q->backing_dev_info);
1093
fail_split:
1094
	bioset_exit(&q->bio_split);
1095 1096 1097 1098 1099
fail_id:
	ida_simple_remove(&blk_queue_ida, q->id);
fail_q:
	kmem_cache_free(blk_requestq_cachep, q);
	return NULL;
L
Linus Torvalds 已提交
1100
}
1101
EXPORT_SYMBOL(blk_alloc_queue_node);
L
Linus Torvalds 已提交
1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124

/**
 * blk_init_queue  - prepare a request queue for use with a block device
 * @rfn:  The function to be called to process requests that have been
 *        placed on the queue.
 * @lock: Request queue spin lock
 *
 * Description:
 *    If a block device wishes to use the standard request handling procedures,
 *    which sorts requests and coalesces adjacent requests, then it must
 *    call blk_init_queue().  The function @rfn will be called when there
 *    are requests on the queue that need to be processed.  If the device
 *    supports plugging, then @rfn may not be called immediately when requests
 *    are available on the queue, but may be called at some time later instead.
 *    Plugged queues are generally unplugged when a buffer belonging to one
 *    of the requests on the queue is needed, or due to memory pressure.
 *
 *    @rfn is not required, or even expected, to remove all requests off the
 *    queue, but only as many as it can handle at a time.  If it does leave
 *    requests on the queue, it is responsible for arranging that the requests
 *    get dealt with eventually.
 *
 *    The queue spin lock must be held while manipulating the requests on the
1125 1126
 *    request queue; this lock will be taken also from interrupt context, so irq
 *    disabling is needed for it.
L
Linus Torvalds 已提交
1127
 *
1128
 *    Function returns a pointer to the initialized request queue, or %NULL if
L
Linus Torvalds 已提交
1129 1130 1131 1132 1133 1134
 *    it didn't succeed.
 *
 * Note:
 *    blk_init_queue() must be paired with a blk_cleanup_queue() call
 *    when the block device is deactivated (such as at module unload).
 **/
1135

1136
struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
L
Linus Torvalds 已提交
1137
{
1138
	return blk_init_queue_node(rfn, lock, NUMA_NO_NODE);
1139 1140 1141
}
EXPORT_SYMBOL(blk_init_queue);

1142
struct request_queue *
1143 1144
blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
{
1145
	struct request_queue *q;
L
Linus Torvalds 已提交
1146

1147
	q = blk_alloc_queue_node(GFP_KERNEL, node_id, lock);
1148
	if (!q)
1149 1150
		return NULL;

1151 1152 1153 1154 1155
	q->request_fn = rfn;
	if (blk_init_allocated_queue(q) < 0) {
		blk_cleanup_queue(q);
		return NULL;
	}
1156

1157
	return q;
1158 1159 1160
}
EXPORT_SYMBOL(blk_init_queue_node);

1161
static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio);
1162

L
Linus Torvalds 已提交
1163

1164 1165
int blk_init_allocated_queue(struct request_queue *q)
{
1166 1167
	WARN_ON_ONCE(q->mq_ops);

1168
	q->fq = blk_alloc_flush_queue(q, NUMA_NO_NODE, q->cmd_size);
1169
	if (!q->fq)
1170
		return -ENOMEM;
1171

1172 1173
	if (q->init_rq_fn && q->init_rq_fn(q, q->fq->flush_rq, GFP_KERNEL))
		goto out_free_flush_queue;
1174

1175
	if (blk_init_rl(&q->root_rl, q, GFP_KERNEL))
1176
		goto out_exit_flush_rq;
L
Linus Torvalds 已提交
1177

1178
	INIT_WORK(&q->timeout_work, blk_timeout_work);
1179
	q->queue_flags		|= QUEUE_FLAG_DEFAULT;
1180

1181 1182 1183
	/*
	 * This also sets hw/phys segments, boundary and size
	 */
1184
	blk_queue_make_request(q, blk_queue_bio);
L
Linus Torvalds 已提交
1185

1186 1187
	q->sg_reserved_size = INT_MAX;

1188
	if (elevator_init(q))
1189
		goto out_exit_flush_rq;
1190
	return 0;
1191

1192 1193 1194 1195
out_exit_flush_rq:
	if (q->exit_rq_fn)
		q->exit_rq_fn(q, q->fq->flush_rq);
out_free_flush_queue:
1196
	blk_free_flush_queue(q->fq);
1197
	q->fq = NULL;
1198
	return -ENOMEM;
L
Linus Torvalds 已提交
1199
}
1200
EXPORT_SYMBOL(blk_init_allocated_queue);
L
Linus Torvalds 已提交
1201

T
Tejun Heo 已提交
1202
bool blk_get_queue(struct request_queue *q)
L
Linus Torvalds 已提交
1203
{
B
Bart Van Assche 已提交
1204
	if (likely(!blk_queue_dying(q))) {
T
Tejun Heo 已提交
1205 1206
		__blk_get_queue(q);
		return true;
L
Linus Torvalds 已提交
1207 1208
	}

T
Tejun Heo 已提交
1209
	return false;
L
Linus Torvalds 已提交
1210
}
J
Jens Axboe 已提交
1211
EXPORT_SYMBOL(blk_get_queue);
L
Linus Torvalds 已提交
1212

1213
static inline void blk_free_request(struct request_list *rl, struct request *rq)
L
Linus Torvalds 已提交
1214
{
1215
	if (rq->rq_flags & RQF_ELVPRIV) {
1216
		elv_put_request(rl->q, rq);
1217
		if (rq->elv.icq)
1218
			put_io_context(rq->elv.icq->ioc);
1219 1220
	}

1221
	mempool_free(rq, rl->rq_pool);
L
Linus Torvalds 已提交
1222 1223 1224 1225 1226 1227
}

/*
 * ioc_batching returns true if the ioc is a valid batching request and
 * should be given priority access to a request.
 */
1228
static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
L
Linus Torvalds 已提交
1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248
{
	if (!ioc)
		return 0;

	/*
	 * Make sure the process is able to allocate at least 1 request
	 * even if the batch times out, otherwise we could theoretically
	 * lose wakeups.
	 */
	return ioc->nr_batch_requests == q->nr_batching ||
		(ioc->nr_batch_requests > 0
		&& time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
}

/*
 * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
 * will cause the process to be a "batcher" on all queues in the system. This
 * is the behaviour we want though - once it gets a wakeup it should be given
 * a nice run.
 */
1249
static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
L
Linus Torvalds 已提交
1250 1251 1252 1253 1254 1255 1256 1257
{
	if (!ioc || ioc_batching(q, ioc))
		return;

	ioc->nr_batch_requests = q->nr_batching;
	ioc->last_waited = jiffies;
}

1258
static void __freed_request(struct request_list *rl, int sync)
L
Linus Torvalds 已提交
1259
{
1260
	struct request_queue *q = rl->q;
L
Linus Torvalds 已提交
1261

1262 1263
	if (rl->count[sync] < queue_congestion_off_threshold(q))
		blk_clear_congested(rl, sync);
L
Linus Torvalds 已提交
1264

1265 1266 1267
	if (rl->count[sync] + 1 <= q->nr_requests) {
		if (waitqueue_active(&rl->wait[sync]))
			wake_up(&rl->wait[sync]);
L
Linus Torvalds 已提交
1268

1269
		blk_clear_rl_full(rl, sync);
L
Linus Torvalds 已提交
1270 1271 1272 1273 1274 1275 1276
	}
}

/*
 * A request has just been released.  Account for it, update the full and
 * congestion status, wake up any waiters.   Called under q->queue_lock.
 */
1277 1278
static void freed_request(struct request_list *rl, bool sync,
		req_flags_t rq_flags)
L
Linus Torvalds 已提交
1279
{
1280
	struct request_queue *q = rl->q;
L
Linus Torvalds 已提交
1281

1282
	q->nr_rqs[sync]--;
1283
	rl->count[sync]--;
1284
	if (rq_flags & RQF_ELVPRIV)
1285
		q->nr_rqs_elvpriv--;
L
Linus Torvalds 已提交
1286

1287
	__freed_request(rl, sync);
L
Linus Torvalds 已提交
1288

1289
	if (unlikely(rl->starved[sync ^ 1]))
1290
		__freed_request(rl, sync ^ 1);
L
Linus Torvalds 已提交
1291 1292
}

1293 1294 1295
int blk_update_nr_requests(struct request_queue *q, unsigned int nr)
{
	struct request_list *rl;
1296
	int on_thresh, off_thresh;
1297

1298 1299
	WARN_ON_ONCE(q->mq_ops);

1300 1301 1302
	spin_lock_irq(q->queue_lock);
	q->nr_requests = nr;
	blk_queue_congestion_threshold(q);
1303 1304
	on_thresh = queue_congestion_on_threshold(q);
	off_thresh = queue_congestion_off_threshold(q);
1305

1306 1307 1308 1309 1310
	blk_queue_for_each_rl(rl, q) {
		if (rl->count[BLK_RW_SYNC] >= on_thresh)
			blk_set_congested(rl, BLK_RW_SYNC);
		else if (rl->count[BLK_RW_SYNC] < off_thresh)
			blk_clear_congested(rl, BLK_RW_SYNC);
1311

1312 1313 1314 1315
		if (rl->count[BLK_RW_ASYNC] >= on_thresh)
			blk_set_congested(rl, BLK_RW_ASYNC);
		else if (rl->count[BLK_RW_ASYNC] < off_thresh)
			blk_clear_congested(rl, BLK_RW_ASYNC);
1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335

		if (rl->count[BLK_RW_SYNC] >= q->nr_requests) {
			blk_set_rl_full(rl, BLK_RW_SYNC);
		} else {
			blk_clear_rl_full(rl, BLK_RW_SYNC);
			wake_up(&rl->wait[BLK_RW_SYNC]);
		}

		if (rl->count[BLK_RW_ASYNC] >= q->nr_requests) {
			blk_set_rl_full(rl, BLK_RW_ASYNC);
		} else {
			blk_clear_rl_full(rl, BLK_RW_ASYNC);
			wake_up(&rl->wait[BLK_RW_ASYNC]);
		}
	}

	spin_unlock_irq(q->queue_lock);
	return 0;
}

1336
/**
T
Tejun Heo 已提交
1337
 * __get_request - get a free request
1338
 * @rl: request list to allocate from
1339
 * @op: operation and flags
1340
 * @bio: bio to allocate request for (can be %NULL)
1341
 * @flags: BLQ_MQ_REQ_* flags
1342
 * @gfp_mask: allocator flags
1343 1344 1345 1346
 *
 * Get a free request from @q.  This function may fail under memory
 * pressure or if @q is dead.
 *
1347
 * Must be called with @q->queue_lock held and,
1348 1349
 * Returns ERR_PTR on failure, with @q->queue_lock held.
 * Returns request pointer on success, with @q->queue_lock *not held*.
L
Linus Torvalds 已提交
1350
 */
1351
static struct request *__get_request(struct request_list *rl, unsigned int op,
1352
		struct bio *bio, blk_mq_req_flags_t flags, gfp_t gfp_mask)
L
Linus Torvalds 已提交
1353
{
1354
	struct request_queue *q = rl->q;
T
Tejun Heo 已提交
1355
	struct request *rq;
T
Tejun Heo 已提交
1356 1357
	struct elevator_type *et = q->elevator->type;
	struct io_context *ioc = rq_ioc(bio);
1358
	struct io_cq *icq = NULL;
1359
	const bool is_sync = op_is_sync(op);
1360
	int may_queue;
1361
	req_flags_t rq_flags = RQF_ALLOCED;
1362

1363 1364
	lockdep_assert_held(q->queue_lock);

B
Bart Van Assche 已提交
1365
	if (unlikely(blk_queue_dying(q)))
1366
		return ERR_PTR(-ENODEV);
1367

1368
	may_queue = elv_may_queue(q, op);
1369 1370 1371
	if (may_queue == ELV_MQUEUE_NO)
		goto rq_starved;

1372 1373
	if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
		if (rl->count[is_sync]+1 >= q->nr_requests) {
1374 1375 1376 1377 1378 1379
			/*
			 * The queue will fill after this allocation, so set
			 * it as full, and mark this process as "batching".
			 * This process will be allowed to complete a batch of
			 * requests, others will be blocked.
			 */
1380
			if (!blk_rl_full(rl, is_sync)) {
1381
				ioc_set_batching(q, ioc);
1382
				blk_set_rl_full(rl, is_sync);
1383 1384 1385 1386 1387 1388 1389 1390
			} else {
				if (may_queue != ELV_MQUEUE_MUST
						&& !ioc_batching(q, ioc)) {
					/*
					 * The queue is full and the allocating
					 * process is not a "batcher", and not
					 * exempted by the IO scheduler
					 */
1391
					return ERR_PTR(-ENOMEM);
1392 1393
				}
			}
L
Linus Torvalds 已提交
1394
		}
1395
		blk_set_congested(rl, is_sync);
L
Linus Torvalds 已提交
1396 1397
	}

1398 1399 1400 1401 1402
	/*
	 * Only allow batching queuers to allocate up to 50% over the defined
	 * limit of requests, otherwise we could have thousands of requests
	 * allocated with any setting of ->nr_requests
	 */
1403
	if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
1404
		return ERR_PTR(-ENOMEM);
H
Hugh Dickins 已提交
1405

1406
	q->nr_rqs[is_sync]++;
1407 1408
	rl->count[is_sync]++;
	rl->starved[is_sync] = 0;
T
Tejun Heo 已提交
1409

1410 1411
	/*
	 * Decide whether the new request will be managed by elevator.  If
1412
	 * so, mark @rq_flags and increment elvpriv.  Non-zero elvpriv will
1413 1414 1415 1416
	 * prevent the current elevator from being destroyed until the new
	 * request is freed.  This guarantees icq's won't be destroyed and
	 * makes creating new ones safe.
	 *
1417 1418 1419
	 * Flush requests do not use the elevator so skip initialization.
	 * This allows a request to share the flush and elevator data.
	 *
1420 1421 1422
	 * Also, lookup icq while holding queue_lock.  If it doesn't exist,
	 * it will be created after releasing queue_lock.
	 */
1423
	if (!op_is_flush(op) && !blk_queue_bypass(q)) {
1424
		rq_flags |= RQF_ELVPRIV;
1425
		q->nr_rqs_elvpriv++;
1426 1427
		if (et->icq_cache && ioc)
			icq = ioc_lookup_icq(ioc, q);
1428
	}
T
Tejun Heo 已提交
1429

1430
	if (blk_queue_io_stat(q))
1431
		rq_flags |= RQF_IO_STAT;
L
Linus Torvalds 已提交
1432 1433
	spin_unlock_irq(q->queue_lock);

1434
	/* allocate and init request */
1435
	rq = mempool_alloc(rl->rq_pool, gfp_mask);
1436
	if (!rq)
T
Tejun Heo 已提交
1437
		goto fail_alloc;
L
Linus Torvalds 已提交
1438

1439
	blk_rq_init(q, rq);
1440
	blk_rq_set_rl(rq, rl);
1441
	rq->cmd_flags = op;
1442
	rq->rq_flags = rq_flags;
1443 1444
	if (flags & BLK_MQ_REQ_PREEMPT)
		rq->rq_flags |= RQF_PREEMPT;
1445

1446
	/* init elvpriv */
1447
	if (rq_flags & RQF_ELVPRIV) {
1448
		if (unlikely(et->icq_cache && !icq)) {
T
Tejun Heo 已提交
1449 1450
			if (ioc)
				icq = ioc_create_icq(ioc, q, gfp_mask);
1451 1452
			if (!icq)
				goto fail_elvpriv;
1453
		}
1454 1455 1456 1457 1458 1459

		rq->elv.icq = icq;
		if (unlikely(elv_set_request(q, rq, bio, gfp_mask)))
			goto fail_elvpriv;

		/* @rq->elv.icq holds io_context until @rq is freed */
1460 1461 1462
		if (icq)
			get_io_context(icq->ioc);
	}
1463
out:
1464 1465 1466 1467 1468 1469
	/*
	 * ioc may be NULL here, and ioc_batching will be false. That's
	 * OK, if the queue is under the request limit then requests need
	 * not count toward the nr_batch_requests limit. There will always
	 * be some limit enforced by BLK_BATCH_TIME.
	 */
L
Linus Torvalds 已提交
1470 1471
	if (ioc_batching(q, ioc))
		ioc->nr_batch_requests--;
1472

1473
	trace_block_getrq(q, bio, op);
L
Linus Torvalds 已提交
1474
	return rq;
T
Tejun Heo 已提交
1475

1476 1477 1478 1479 1480 1481 1482
fail_elvpriv:
	/*
	 * elvpriv init failed.  ioc, icq and elvpriv aren't mempool backed
	 * and may fail indefinitely under memory pressure and thus
	 * shouldn't stall IO.  Treat this request as !elvpriv.  This will
	 * disturb iosched and blkcg but weird is bettern than dead.
	 */
1483
	printk_ratelimited(KERN_WARNING "%s: dev %s: request aux data allocation failed, iosched may be disturbed\n",
1484
			   __func__, dev_name(q->backing_dev_info->dev));
1485

1486
	rq->rq_flags &= ~RQF_ELVPRIV;
1487 1488 1489
	rq->elv.icq = NULL;

	spin_lock_irq(q->queue_lock);
1490
	q->nr_rqs_elvpriv--;
1491 1492 1493
	spin_unlock_irq(q->queue_lock);
	goto out;

T
Tejun Heo 已提交
1494 1495 1496 1497 1498 1499 1500 1501 1502
fail_alloc:
	/*
	 * Allocation failed presumably due to memory. Undo anything we
	 * might have messed up.
	 *
	 * Allocating task should really be put onto the front of the wait
	 * queue, but this is pretty rare.
	 */
	spin_lock_irq(q->queue_lock);
1503
	freed_request(rl, is_sync, rq_flags);
T
Tejun Heo 已提交
1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514

	/*
	 * in the very unlikely event that allocation failed and no
	 * requests for this direction was pending, mark us starved so that
	 * freeing of a request in the other direction will notice
	 * us. another possible fix would be to split the rq mempool into
	 * READ and WRITE
	 */
rq_starved:
	if (unlikely(rl->count[is_sync] == 0))
		rl->starved[is_sync] = 1;
1515
	return ERR_PTR(-ENOMEM);
L
Linus Torvalds 已提交
1516 1517
}

1518
/**
T
Tejun Heo 已提交
1519
 * get_request - get a free request
1520
 * @q: request_queue to allocate request from
1521
 * @op: operation and flags
1522
 * @bio: bio to allocate request for (can be %NULL)
1523
 * @flags: BLK_MQ_REQ_* flags.
1524
 * @gfp: allocator flags
1525
 *
1526
 * Get a free request from @q.  If %BLK_MQ_REQ_NOWAIT is set in @flags,
1527
 * this function keeps retrying under memory pressure and fails iff @q is dead.
N
Nick Piggin 已提交
1528
 *
1529
 * Must be called with @q->queue_lock held and,
1530 1531
 * Returns ERR_PTR on failure, with @q->queue_lock held.
 * Returns request pointer on success, with @q->queue_lock *not held*.
L
Linus Torvalds 已提交
1532
 */
1533
static struct request *get_request(struct request_queue *q, unsigned int op,
1534
		struct bio *bio, blk_mq_req_flags_t flags, gfp_t gfp)
L
Linus Torvalds 已提交
1535
{
1536
	const bool is_sync = op_is_sync(op);
T
Tejun Heo 已提交
1537
	DEFINE_WAIT(wait);
1538
	struct request_list *rl;
L
Linus Torvalds 已提交
1539
	struct request *rq;
1540

1541
	lockdep_assert_held(q->queue_lock);
1542
	WARN_ON_ONCE(q->mq_ops);
1543

1544
	rl = blk_get_rl(q, bio);	/* transferred to @rq on success */
T
Tejun Heo 已提交
1545
retry:
1546
	rq = __get_request(rl, op, bio, flags, gfp);
1547
	if (!IS_ERR(rq))
T
Tejun Heo 已提交
1548
		return rq;
L
Linus Torvalds 已提交
1549

1550 1551 1552 1553 1554
	if (op & REQ_NOWAIT) {
		blk_put_rl(rl);
		return ERR_PTR(-EAGAIN);
	}

1555
	if ((flags & BLK_MQ_REQ_NOWAIT) || unlikely(blk_queue_dying(q))) {
1556
		blk_put_rl(rl);
1557
		return rq;
1558
	}
L
Linus Torvalds 已提交
1559

T
Tejun Heo 已提交
1560 1561 1562
	/* wait on @rl and retry */
	prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
				  TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
1563

1564
	trace_block_sleeprq(q, bio, op);
L
Linus Torvalds 已提交
1565

T
Tejun Heo 已提交
1566 1567
	spin_unlock_irq(q->queue_lock);
	io_schedule();
N
Nick Piggin 已提交
1568

T
Tejun Heo 已提交
1569 1570 1571 1572 1573 1574
	/*
	 * After sleeping, we become a "batching" process and will be able
	 * to allocate at least one request, and up to a big batch of them
	 * for a small period time.  See ioc_batching, ioc_set_batching
	 */
	ioc_set_batching(q, current->io_context);
1575

T
Tejun Heo 已提交
1576 1577
	spin_lock_irq(q->queue_lock);
	finish_wait(&rl->wait[is_sync], &wait);
L
Linus Torvalds 已提交
1578

T
Tejun Heo 已提交
1579
	goto retry;
L
Linus Torvalds 已提交
1580 1581
}

1582
/* flags: BLK_MQ_REQ_PREEMPT and/or BLK_MQ_REQ_NOWAIT. */
1583
static struct request *blk_old_get_request(struct request_queue *q,
1584
				unsigned int op, blk_mq_req_flags_t flags)
L
Linus Torvalds 已提交
1585 1586
{
	struct request *rq;
1587
	gfp_t gfp_mask = flags & BLK_MQ_REQ_NOWAIT ? GFP_ATOMIC : GFP_NOIO;
1588
	int ret = 0;
L
Linus Torvalds 已提交
1589

1590 1591
	WARN_ON_ONCE(q->mq_ops);

T
Tejun Heo 已提交
1592 1593 1594
	/* create ioc upfront */
	create_io_context(gfp_mask, q->node);

1595
	ret = blk_queue_enter(q, flags);
1596 1597
	if (ret)
		return ERR_PTR(ret);
N
Nick Piggin 已提交
1598
	spin_lock_irq(q->queue_lock);
1599
	rq = get_request(q, op, NULL, flags, gfp_mask);
1600
	if (IS_ERR(rq)) {
1601
		spin_unlock_irq(q->queue_lock);
1602
		blk_queue_exit(q);
1603 1604
		return rq;
	}
L
Linus Torvalds 已提交
1605

1606 1607 1608 1609
	/* q->queue_lock is unlocked at this point */
	rq->__data_len = 0;
	rq->__sector = (sector_t) -1;
	rq->bio = rq->biotail = NULL;
L
Linus Torvalds 已提交
1610 1611
	return rq;
}
1612

1613
/**
1614
 * blk_get_request - allocate a request
1615 1616 1617 1618
 * @q: request queue to allocate a request for
 * @op: operation (REQ_OP_*) and REQ_* flags, e.g. REQ_SYNC.
 * @flags: BLK_MQ_REQ_* flags, e.g. BLK_MQ_REQ_NOWAIT.
 */
1619 1620
struct request *blk_get_request(struct request_queue *q, unsigned int op,
				blk_mq_req_flags_t flags)
1621
{
1622 1623
	struct request *req;

1624
	WARN_ON_ONCE(op & REQ_NOWAIT);
1625
	WARN_ON_ONCE(flags & ~(BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_PREEMPT));
1626

1627
	if (q->mq_ops) {
1628
		req = blk_mq_alloc_request(q, op, flags);
1629 1630 1631
		if (!IS_ERR(req) && q->mq_ops->initialize_rq_fn)
			q->mq_ops->initialize_rq_fn(req);
	} else {
1632
		req = blk_old_get_request(q, op, flags);
1633 1634 1635 1636 1637
		if (!IS_ERR(req) && q->initialize_rq_fn)
			q->initialize_rq_fn(req);
	}

	return req;
1638
}
L
Linus Torvalds 已提交
1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650
EXPORT_SYMBOL(blk_get_request);

/**
 * blk_requeue_request - put a request back on queue
 * @q:		request queue where request should be inserted
 * @rq:		request to be inserted
 *
 * Description:
 *    Drivers often keep queueing requests until the hardware cannot accept
 *    more, when that condition happens we need to put the request back
 *    on the queue. Must be called with queue lock held.
 */
1651
void blk_requeue_request(struct request_queue *q, struct request *rq)
L
Linus Torvalds 已提交
1652
{
1653
	lockdep_assert_held(q->queue_lock);
1654
	WARN_ON_ONCE(q->mq_ops);
1655

J
Jens Axboe 已提交
1656 1657
	blk_delete_timer(rq);
	blk_clear_rq_complete(rq);
1658
	trace_block_rq_requeue(q, rq);
1659
	rq_qos_requeue(q, rq);
1660

1661
	if (rq->rq_flags & RQF_QUEUED)
L
Linus Torvalds 已提交
1662 1663
		blk_queue_end_tag(q, rq);

1664 1665
	BUG_ON(blk_queued_rq(rq));

L
Linus Torvalds 已提交
1666 1667 1668 1669
	elv_requeue_request(q, rq);
}
EXPORT_SYMBOL(blk_requeue_request);

1670 1671 1672
static void add_acct_request(struct request_queue *q, struct request *rq,
			     int where)
{
1673
	blk_account_io_start(rq, true);
J
Jens Axboe 已提交
1674
	__elv_add_request(q, rq, where);
1675 1676
}

1677
static void part_round_stats_single(struct request_queue *q, int cpu,
1678 1679
				    struct hd_struct *part, unsigned long now,
				    unsigned int inflight)
T
Tejun Heo 已提交
1680
{
1681
	if (inflight) {
T
Tejun Heo 已提交
1682
		__part_stat_add(cpu, part, time_in_queue,
1683
				inflight * (now - part->stamp));
T
Tejun Heo 已提交
1684 1685 1686 1687 1688 1689
		__part_stat_add(cpu, part, io_ticks, (now - part->stamp));
	}
	part->stamp = now;
}

/**
1690
 * part_round_stats() - Round off the performance stats on a struct disk_stats.
1691
 * @q: target block queue
1692 1693
 * @cpu: cpu number for stats access
 * @part: target partition
L
Linus Torvalds 已提交
1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705
 *
 * The average IO queue length and utilisation statistics are maintained
 * by observing the current state of the queue length and the amount of
 * time it has been in this state for.
 *
 * Normally, that accounting is done on IO completion, but that can result
 * in more than a second's worth of IO being accounted for within any one
 * second, leading to >100% utilisation.  To deal with that, we call this
 * function to do a round-off before returning the results when reading
 * /proc/diskstats.  This accounts immediately for all queue usage up to
 * the current jiffies and restarts the counters again.
 */
1706
void part_round_stats(struct request_queue *q, int cpu, struct hd_struct *part)
1707
{
1708
	struct hd_struct *part2 = NULL;
1709
	unsigned long now = jiffies;
1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725
	unsigned int inflight[2];
	int stats = 0;

	if (part->stamp != now)
		stats |= 1;

	if (part->partno) {
		part2 = &part_to_disk(part)->part0;
		if (part2->stamp != now)
			stats |= 2;
	}

	if (!stats)
		return;

	part_in_flight(q, part, inflight);
1726

1727 1728 1729 1730
	if (stats & 2)
		part_round_stats_single(q, cpu, part2, now, inflight[1]);
	if (stats & 1)
		part_round_stats_single(q, cpu, part, now, inflight[0]);
1731
}
T
Tejun Heo 已提交
1732
EXPORT_SYMBOL_GPL(part_round_stats);
1733

1734
#ifdef CONFIG_PM
L
Lin Ming 已提交
1735 1736
static void blk_pm_put_request(struct request *rq)
{
1737
	if (rq->q->dev && !(rq->rq_flags & RQF_PM) && !--rq->q->nr_pending)
L
Lin Ming 已提交
1738 1739 1740 1741 1742 1743
		pm_runtime_mark_last_busy(rq->q->dev);
}
#else
static inline void blk_pm_put_request(struct request *rq) {}
#endif

1744
void __blk_put_request(struct request_queue *q, struct request *req)
L
Linus Torvalds 已提交
1745
{
1746 1747
	req_flags_t rq_flags = req->rq_flags;

L
Linus Torvalds 已提交
1748 1749 1750
	if (unlikely(!q))
		return;

1751 1752 1753 1754 1755
	if (q->mq_ops) {
		blk_mq_free_request(req);
		return;
	}

1756 1757
	lockdep_assert_held(q->queue_lock);

1758
	blk_req_zone_write_unlock(req);
L
Lin Ming 已提交
1759 1760
	blk_pm_put_request(req);

1761 1762
	elv_completed_request(q, req);

1763 1764 1765
	/* this is a bio leak */
	WARN_ON(req->bio != NULL);

1766
	rq_qos_done(q, req);
J
Jens Axboe 已提交
1767

L
Linus Torvalds 已提交
1768 1769 1770 1771
	/*
	 * Request may not have originated from ll_rw_blk. if not,
	 * it didn't come out of our reserved rq pools
	 */
1772
	if (rq_flags & RQF_ALLOCED) {
1773
		struct request_list *rl = blk_rq_rl(req);
1774
		bool sync = op_is_sync(req->cmd_flags);
L
Linus Torvalds 已提交
1775 1776

		BUG_ON(!list_empty(&req->queuelist));
1777
		BUG_ON(ELV_ON_HASH(req));
L
Linus Torvalds 已提交
1778

1779
		blk_free_request(rl, req);
1780
		freed_request(rl, sync, rq_flags);
1781
		blk_put_rl(rl);
1782
		blk_queue_exit(q);
L
Linus Torvalds 已提交
1783 1784
	}
}
1785 1786
EXPORT_SYMBOL_GPL(__blk_put_request);

L
Linus Torvalds 已提交
1787 1788
void blk_put_request(struct request *req)
{
1789
	struct request_queue *q = req->q;
1790

1791 1792 1793 1794 1795 1796 1797 1798 1799
	if (q->mq_ops)
		blk_mq_free_request(req);
	else {
		unsigned long flags;

		spin_lock_irqsave(q->queue_lock, flags);
		__blk_put_request(q, req);
		spin_unlock_irqrestore(q->queue_lock, flags);
	}
L
Linus Torvalds 已提交
1800 1801 1802
}
EXPORT_SYMBOL(blk_put_request);

1803 1804
bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
			    struct bio *bio)
1805
{
J
Jens Axboe 已提交
1806
	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
1807 1808 1809 1810

	if (!ll_back_merge_fn(q, req, bio))
		return false;

1811
	trace_block_bio_backmerge(q, req, bio);
1812 1813 1814 1815 1816 1817

	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
		blk_rq_set_mixed_merge(req);

	req->biotail->bi_next = bio;
	req->biotail = bio;
1818
	req->__data_len += bio->bi_iter.bi_size;
1819 1820
	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));

1821
	blk_account_io_start(req, false);
1822 1823 1824
	return true;
}

1825 1826
bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
			     struct bio *bio)
1827
{
J
Jens Axboe 已提交
1828
	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
1829 1830 1831 1832

	if (!ll_front_merge_fn(q, req, bio))
		return false;

1833
	trace_block_bio_frontmerge(q, req, bio);
1834 1835 1836 1837 1838 1839 1840

	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
		blk_rq_set_mixed_merge(req);

	bio->bi_next = req->bio;
	req->bio = bio;

1841 1842
	req->__sector = bio->bi_iter.bi_sector;
	req->__data_len += bio->bi_iter.bi_size;
1843 1844
	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));

1845
	blk_account_io_start(req, false);
1846 1847 1848
	return true;
}

1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872
bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
		struct bio *bio)
{
	unsigned short segments = blk_rq_nr_discard_segments(req);

	if (segments >= queue_max_discard_segments(q))
		goto no_merge;
	if (blk_rq_sectors(req) + bio_sectors(bio) >
	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))
		goto no_merge;

	req->biotail->bi_next = bio;
	req->biotail = bio;
	req->__data_len += bio->bi_iter.bi_size;
	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
	req->nr_phys_segments = segments + 1;

	blk_account_io_start(req, false);
	return true;
no_merge:
	req_set_nomerge(q, req);
	return false;
}

1873
/**
1874
 * blk_attempt_plug_merge - try to merge with %current's plugged list
1875 1876 1877
 * @q: request_queue new bio is being queued at
 * @bio: new bio being queued
 * @request_count: out parameter for number of traversed plugged requests
1878 1879 1880
 * @same_queue_rq: pointer to &struct request that gets filled in when
 * another request associated with @q is found on the plug list
 * (optional, may be %NULL)
1881 1882 1883 1884 1885
 *
 * Determine whether @bio being queued on @q can be merged with a request
 * on %current's plugged list.  Returns %true if merge was successful,
 * otherwise %false.
 *
1886 1887 1888 1889 1890 1891
 * Plugging coalesces IOs from the same issuer for the same purpose without
 * going through @q->queue_lock.  As such it's more of an issuing mechanism
 * than scheduling, and the request, while may have elvpriv data, is not
 * added on the elevator at this point.  In addition, we don't have
 * reliable access to the elevator outside queue lock.  Only check basic
 * merging parameters without querying the elevator.
1892 1893
 *
 * Caller must ensure !blk_queue_nomerges(q) beforehand.
1894
 */
1895
bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1896 1897
			    unsigned int *request_count,
			    struct request **same_queue_rq)
1898 1899 1900
{
	struct blk_plug *plug;
	struct request *rq;
S
Shaohua Li 已提交
1901
	struct list_head *plug_list;
1902

1903
	plug = current->plug;
1904
	if (!plug)
1905
		return false;
1906
	*request_count = 0;
1907

S
Shaohua Li 已提交
1908 1909 1910 1911 1912 1913
	if (q->mq_ops)
		plug_list = &plug->mq_list;
	else
		plug_list = &plug->list;

	list_for_each_entry_reverse(rq, plug_list, queuelist) {
1914
		bool merged = false;
1915

1916
		if (rq->q == q) {
1917
			(*request_count)++;
1918 1919 1920 1921 1922 1923 1924 1925
			/*
			 * Only blk-mq multiple hardware queues case checks the
			 * rq in the same queue, there should be only one such
			 * rq in a queue
			 **/
			if (same_queue_rq)
				*same_queue_rq = rq;
		}
1926

1927
		if (rq->q != q || !blk_rq_merge_ok(rq, bio))
1928 1929
			continue;

1930 1931 1932 1933 1934 1935 1936
		switch (blk_try_merge(rq, bio)) {
		case ELEVATOR_BACK_MERGE:
			merged = bio_attempt_back_merge(q, rq, bio);
			break;
		case ELEVATOR_FRONT_MERGE:
			merged = bio_attempt_front_merge(q, rq, bio);
			break;
1937 1938 1939
		case ELEVATOR_DISCARD_MERGE:
			merged = bio_attempt_discard_merge(q, rq, bio);
			break;
1940 1941
		default:
			break;
1942
		}
1943 1944 1945

		if (merged)
			return true;
1946
	}
1947 1948

	return false;
1949 1950
}

1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974
unsigned int blk_plug_queued_count(struct request_queue *q)
{
	struct blk_plug *plug;
	struct request *rq;
	struct list_head *plug_list;
	unsigned int ret = 0;

	plug = current->plug;
	if (!plug)
		goto out;

	if (q->mq_ops)
		plug_list = &plug->mq_list;
	else
		plug_list = &plug->list;

	list_for_each_entry(rq, plug_list, queuelist) {
		if (rq->q == q)
			ret++;
	}
out:
	return ret;
}

1975
void blk_init_request_from_bio(struct request *req, struct bio *bio)
1976
{
1977 1978
	struct io_context *ioc = rq_ioc(bio);

J
Jens Axboe 已提交
1979
	if (bio->bi_opf & REQ_RAHEAD)
1980
		req->cmd_flags |= REQ_FAILFAST_MASK;
J
Jens Axboe 已提交
1981

1982
	req->__sector = bio->bi_iter.bi_sector;
1983 1984
	if (ioprio_valid(bio_prio(bio)))
		req->ioprio = bio_prio(bio);
1985 1986 1987 1988
	else if (ioc)
		req->ioprio = ioc->ioprio;
	else
		req->ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0);
1989
	req->write_hint = bio->bi_write_hint;
1990
	blk_rq_bio_prep(req->q, req, bio);
1991
}
1992
EXPORT_SYMBOL_GPL(blk_init_request_from_bio);
1993

1994
static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
L
Linus Torvalds 已提交
1995
{
1996
	struct blk_plug *plug;
1997
	int where = ELEVATOR_INSERT_SORT;
1998
	struct request *req, *free;
1999
	unsigned int request_count = 0;
L
Linus Torvalds 已提交
2000 2001 2002 2003 2004 2005 2006 2007

	/*
	 * low level driver can indicate that it wants pages above a
	 * certain limit bounced to low memory (ie for highmem, or even
	 * ISA dma in theory)
	 */
	blk_queue_bounce(q, &bio);

2008
	blk_queue_split(q, &bio);
2009

2010
	if (!bio_integrity_prep(bio))
2011
		return BLK_QC_T_NONE;
2012

2013
	if (op_is_flush(bio->bi_opf)) {
2014
		spin_lock_irq(q->queue_lock);
2015
		where = ELEVATOR_INSERT_FLUSH;
2016 2017 2018
		goto get_rq;
	}

2019 2020 2021 2022
	/*
	 * Check if we can merge with the plugged list before grabbing
	 * any locks.
	 */
2023 2024
	if (!blk_queue_nomerges(q)) {
		if (blk_attempt_plug_merge(q, bio, &request_count, NULL))
2025
			return BLK_QC_T_NONE;
2026 2027
	} else
		request_count = blk_plug_queued_count(q);
L
Linus Torvalds 已提交
2028

2029
	spin_lock_irq(q->queue_lock);
2030

2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053
	switch (elv_merge(q, &req, bio)) {
	case ELEVATOR_BACK_MERGE:
		if (!bio_attempt_back_merge(q, req, bio))
			break;
		elv_bio_merged(q, req, bio);
		free = attempt_back_merge(q, req);
		if (free)
			__blk_put_request(q, free);
		else
			elv_merged_request(q, req, ELEVATOR_BACK_MERGE);
		goto out_unlock;
	case ELEVATOR_FRONT_MERGE:
		if (!bio_attempt_front_merge(q, req, bio))
			break;
		elv_bio_merged(q, req, bio);
		free = attempt_front_merge(q, req);
		if (free)
			__blk_put_request(q, free);
		else
			elv_merged_request(q, req, ELEVATOR_FRONT_MERGE);
		goto out_unlock;
	default:
		break;
L
Linus Torvalds 已提交
2054 2055
	}

2056
get_rq:
2057
	rq_qos_throttle(q, bio, q->queue_lock);
J
Jens Axboe 已提交
2058

L
Linus Torvalds 已提交
2059
	/*
2060
	 * Grab a free request. This is might sleep but can not fail.
N
Nick Piggin 已提交
2061
	 * Returns with the queue unlocked.
2062
	 */
2063
	blk_queue_enter_live(q);
2064
	req = get_request(q, bio->bi_opf, bio, 0, GFP_NOIO);
2065
	if (IS_ERR(req)) {
2066
		blk_queue_exit(q);
2067
		rq_qos_cleanup(q, bio);
2068 2069 2070 2071
		if (PTR_ERR(req) == -ENOMEM)
			bio->bi_status = BLK_STS_RESOURCE;
		else
			bio->bi_status = BLK_STS_IOERR;
2072
		bio_endio(bio);
2073 2074
		goto out_unlock;
	}
N
Nick Piggin 已提交
2075

2076
	rq_qos_track(q, req, bio);
J
Jens Axboe 已提交
2077

2078 2079 2080 2081 2082
	/*
	 * After dropping the lock and possibly sleeping here, our request
	 * may now be mergeable after it had proven unmergeable (above).
	 * We don't worry about that case for efficiency. It won't happen
	 * often, and the elevators are able to handle it.
L
Linus Torvalds 已提交
2083
	 */
2084
	blk_init_request_from_bio(req, bio);
L
Linus Torvalds 已提交
2085

2086
	if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags))
2087
		req->cpu = raw_smp_processor_id();
2088 2089

	plug = current->plug;
J
Jens Axboe 已提交
2090
	if (plug) {
J
Jens Axboe 已提交
2091 2092
		/*
		 * If this is the first request added after a plug, fire
2093
		 * of a plug trace.
2094 2095 2096
		 *
		 * @request_count may become stale because of schedule
		 * out, so check plug list again.
J
Jens Axboe 已提交
2097
		 */
2098
		if (!request_count || list_empty(&plug->list))
J
Jens Axboe 已提交
2099
			trace_block_plug(q);
2100
		else {
2101 2102 2103
			struct request *last = list_entry_rq(plug->list.prev);
			if (request_count >= BLK_MAX_REQUEST_COUNT ||
			    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE) {
2104
				blk_flush_plug_list(plug, false);
S
Shaohua Li 已提交
2105 2106
				trace_block_plug(q);
			}
2107 2108
		}
		list_add_tail(&req->queuelist, &plug->list);
2109
		blk_account_io_start(req, true);
2110 2111 2112
	} else {
		spin_lock_irq(q->queue_lock);
		add_acct_request(q, req, where);
2113
		__blk_run_queue(q);
2114 2115 2116
out_unlock:
		spin_unlock_irq(q->queue_lock);
	}
2117 2118

	return BLK_QC_T_NONE;
L
Linus Torvalds 已提交
2119 2120
}

2121
static void handle_bad_sector(struct bio *bio, sector_t maxsector)
L
Linus Torvalds 已提交
2122 2123 2124 2125
{
	char b[BDEVNAME_SIZE];

	printk(KERN_INFO "attempt to access beyond end of device\n");
2126
	printk(KERN_INFO "%s: rw=%d, want=%Lu, limit=%Lu\n",
2127
			bio_devname(bio, b), bio->bi_opf,
K
Kent Overstreet 已提交
2128
			(unsigned long long)bio_end_sector(bio),
2129
			(long long)maxsector);
L
Linus Torvalds 已提交
2130 2131
}

2132 2133 2134 2135 2136 2137 2138 2139 2140 2141
#ifdef CONFIG_FAIL_MAKE_REQUEST

static DECLARE_FAULT_ATTR(fail_make_request);

static int __init setup_fail_make_request(char *str)
{
	return setup_fault_attr(&fail_make_request, str);
}
__setup("fail_make_request=", setup_fail_make_request);

2142
static bool should_fail_request(struct hd_struct *part, unsigned int bytes)
2143
{
2144
	return part->make_it_fail && should_fail(&fail_make_request, bytes);
2145 2146 2147 2148
}

static int __init fail_make_request_debugfs(void)
{
2149 2150 2151
	struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
						NULL, &fail_make_request);

2152
	return PTR_ERR_OR_ZERO(dir);
2153 2154 2155 2156 2157 2158
}

late_initcall(fail_make_request_debugfs);

#else /* CONFIG_FAIL_MAKE_REQUEST */

2159 2160
static inline bool should_fail_request(struct hd_struct *part,
					unsigned int bytes)
2161
{
2162
	return false;
2163 2164 2165 2166
}

#endif /* CONFIG_FAIL_MAKE_REQUEST */

2167 2168
static inline bool bio_check_ro(struct bio *bio, struct hd_struct *part)
{
2169 2170
	const int op = bio_op(bio);

2171
	if (part->policy && op_is_write(op)) {
2172 2173
		char b[BDEVNAME_SIZE];

2174 2175 2176
		if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
			return false;

2177
		WARN_ONCE(1,
2178 2179 2180
		       "generic_make_request: Trying to write "
			"to read-only block-device %s (partno %d)\n",
			bio_devname(bio, b), part->partno);
2181 2182
		/* Older lvm-tools actually trigger this */
		return false;
2183 2184 2185 2186 2187
	}

	return false;
}

2188 2189 2190 2191 2192 2193 2194 2195
static noinline int should_fail_bio(struct bio *bio)
{
	if (should_fail_request(&bio->bi_disk->part0, bio->bi_iter.bi_size))
		return -EIO;
	return 0;
}
ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO);

2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213
/*
 * Check whether this bio extends beyond the end of the device or partition.
 * This may well happen - the kernel calls bread() without checking the size of
 * the device, e.g., when mounting a file system.
 */
static inline int bio_check_eod(struct bio *bio, sector_t maxsector)
{
	unsigned int nr_sectors = bio_sectors(bio);

	if (nr_sectors && maxsector &&
	    (nr_sectors > maxsector ||
	     bio->bi_iter.bi_sector > maxsector - nr_sectors)) {
		handle_bad_sector(bio, maxsector);
		return -EIO;
	}
	return 0;
}

2214 2215 2216 2217 2218 2219
/*
 * Remap block n of partition p to block n+start(p) of the disk.
 */
static inline int blk_partition_remap(struct bio *bio)
{
	struct hd_struct *p;
2220
	int ret = -EIO;
2221

2222 2223
	rcu_read_lock();
	p = __disk_get_part(bio->bi_disk, bio->bi_partno);
2224 2225 2226 2227 2228
	if (unlikely(!p))
		goto out;
	if (unlikely(should_fail_request(p, bio->bi_iter.bi_size)))
		goto out;
	if (unlikely(bio_check_ro(bio, p)))
2229 2230
		goto out;

2231 2232 2233 2234
	/*
	 * Zone reset does not include bi_size so bio_sectors() is always 0.
	 * Include a test for the reset op code and perform the remap if needed.
	 */
2235 2236 2237 2238 2239 2240 2241
	if (bio_sectors(bio) || bio_op(bio) == REQ_OP_ZONE_RESET) {
		if (bio_check_eod(bio, part_nr_sects_read(p)))
			goto out;
		bio->bi_iter.bi_sector += p->start_sect;
		trace_block_bio_remap(bio->bi_disk->queue, bio, part_devt(p),
				      bio->bi_iter.bi_sector - p->start_sect);
	}
2242
	bio->bi_partno = 0;
2243
	ret = 0;
2244 2245
out:
	rcu_read_unlock();
2246 2247 2248
	return ret;
}

2249 2250
static noinline_for_stack bool
generic_make_request_checks(struct bio *bio)
L
Linus Torvalds 已提交
2251
{
2252
	struct request_queue *q;
2253
	int nr_sectors = bio_sectors(bio);
2254
	blk_status_t status = BLK_STS_IOERR;
2255
	char b[BDEVNAME_SIZE];
L
Linus Torvalds 已提交
2256 2257 2258

	might_sleep();

2259
	q = bio->bi_disk->queue;
2260 2261 2262 2263
	if (unlikely(!q)) {
		printk(KERN_ERR
		       "generic_make_request: Trying to access "
			"nonexistent block-device %s (%Lu)\n",
2264
			bio_devname(bio, b), (long long)bio->bi_iter.bi_sector);
2265 2266
		goto end_io;
	}
2267

2268 2269 2270 2271 2272 2273 2274
	/*
	 * For a REQ_NOWAIT based request, return -EOPNOTSUPP
	 * if queue is not a request based queue.
	 */
	if ((bio->bi_opf & REQ_NOWAIT) && !queue_is_rq_based(q))
		goto not_supported;

2275
	if (should_fail_bio(bio))
2276
		goto end_io;
2277

2278 2279
	if (bio->bi_partno) {
		if (unlikely(blk_partition_remap(bio)))
2280 2281
			goto end_io;
	} else {
2282 2283 2284
		if (unlikely(bio_check_ro(bio, &bio->bi_disk->part0)))
			goto end_io;
		if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk))))
2285 2286
			goto end_io;
	}
2287

2288 2289 2290 2291 2292
	/*
	 * Filter flush bio's early so that make_request based
	 * drivers without flush support don't have to worry
	 * about them.
	 */
2293
	if (op_is_flush(bio->bi_opf) &&
J
Jens Axboe 已提交
2294
	    !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) {
J
Jens Axboe 已提交
2295
		bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA);
2296
		if (!nr_sectors) {
2297
			status = BLK_STS_OK;
2298 2299
			goto end_io;
		}
2300
	}
2301

2302 2303 2304 2305 2306 2307 2308 2309 2310 2311
	switch (bio_op(bio)) {
	case REQ_OP_DISCARD:
		if (!blk_queue_discard(q))
			goto not_supported;
		break;
	case REQ_OP_SECURE_ERASE:
		if (!blk_queue_secure_erase(q))
			goto not_supported;
		break;
	case REQ_OP_WRITE_SAME:
2312
		if (!q->limits.max_write_same_sectors)
2313
			goto not_supported;
2314
		break;
2315 2316
	case REQ_OP_ZONE_REPORT:
	case REQ_OP_ZONE_RESET:
2317
		if (!blk_queue_is_zoned(q))
2318
			goto not_supported;
2319
		break;
2320
	case REQ_OP_WRITE_ZEROES:
2321
		if (!q->limits.max_write_zeroes_sectors)
2322 2323
			goto not_supported;
		break;
2324 2325
	default:
		break;
2326
	}
2327

T
Tejun Heo 已提交
2328 2329 2330 2331 2332 2333 2334 2335
	/*
	 * Various block parts want %current->io_context and lazy ioc
	 * allocation ends up trading a lot of pain for a small amount of
	 * memory.  Just allocate it upfront.  This may fail and block
	 * layer knows how to live with it.
	 */
	create_io_context(GFP_ATOMIC, q->node);

2336 2337
	if (!blkcg_bio_issue_check(q, bio))
		return false;
2338

N
NeilBrown 已提交
2339 2340 2341 2342 2343 2344 2345
	if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
		trace_block_bio_queue(q, bio);
		/* Now that enqueuing has been traced, we need to trace
		 * completion as well.
		 */
		bio_set_flag(bio, BIO_TRACE_COMPLETION);
	}
2346
	return true;
2347

2348
not_supported:
2349
	status = BLK_STS_NOTSUPP;
2350
end_io:
2351
	bio->bi_status = status;
2352
	bio_endio(bio);
2353
	return false;
L
Linus Torvalds 已提交
2354 2355
}

2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378
/**
 * generic_make_request - hand a buffer to its device driver for I/O
 * @bio:  The bio describing the location in memory and on the device.
 *
 * generic_make_request() is used to make I/O requests of block
 * devices. It is passed a &struct bio, which describes the I/O that needs
 * to be done.
 *
 * generic_make_request() does not return any status.  The
 * success/failure status of the request, along with notification of
 * completion, is delivered asynchronously through the bio->bi_end_io
 * function described (one day) else where.
 *
 * The caller of generic_make_request must make sure that bi_io_vec
 * are set to describe the memory buffer, and that bi_dev and bi_sector are
 * set to describe the device address, and the
 * bi_end_io and optionally bi_private are set to describe how
 * completion notification should be signaled.
 *
 * generic_make_request and the drivers it calls may use bi_next if this
 * bio happens to be merged with someone else, and may resubmit the bio to
 * a lower device by calling into generic_make_request recursively, which
 * means the bio should NOT be touched after the call to ->make_request_fn.
2379
 */
2380
blk_qc_t generic_make_request(struct bio *bio)
2381
{
2382 2383 2384 2385 2386 2387 2388 2389
	/*
	 * bio_list_on_stack[0] contains bios submitted by the current
	 * make_request_fn.
	 * bio_list_on_stack[1] contains bios that were submitted before
	 * the current make_request_fn, but that haven't been processed
	 * yet.
	 */
	struct bio_list bio_list_on_stack[2];
2390 2391
	blk_mq_req_flags_t flags = 0;
	struct request_queue *q = bio->bi_disk->queue;
2392
	blk_qc_t ret = BLK_QC_T_NONE;
2393

2394 2395
	if (bio->bi_opf & REQ_NOWAIT)
		flags = BLK_MQ_REQ_NOWAIT;
2396 2397 2398
	if (bio_flagged(bio, BIO_QUEUE_ENTERED))
		blk_queue_enter_live(q);
	else if (blk_queue_enter(q, flags) < 0) {
2399 2400 2401 2402 2403 2404 2405
		if (!blk_queue_dying(q) && (bio->bi_opf & REQ_NOWAIT))
			bio_wouldblock_error(bio);
		else
			bio_io_error(bio);
		return ret;
	}

2406
	if (!generic_make_request_checks(bio))
2407
		goto out;
2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418

	/*
	 * We only want one ->make_request_fn to be active at a time, else
	 * stack usage with stacked devices could be a problem.  So use
	 * current->bio_list to keep a list of requests submited by a
	 * make_request_fn function.  current->bio_list is also used as a
	 * flag to say if generic_make_request is currently active in this
	 * task or not.  If it is NULL, then no make_request is active.  If
	 * it is non-NULL, then a make_request is active, and new requests
	 * should be added at the tail
	 */
2419
	if (current->bio_list) {
2420
		bio_list_add(&current->bio_list[0], bio);
2421
		goto out;
2422
	}
2423

2424 2425 2426 2427 2428
	/* following loop may be a bit non-obvious, and so deserves some
	 * explanation.
	 * Before entering the loop, bio->bi_next is NULL (as all callers
	 * ensure that) so we have a list with a single bio.
	 * We pretend that we have just taken it off a longer list, so
2429 2430
	 * we assign bio_list to a pointer to the bio_list_on_stack,
	 * thus initialising the bio_list of new bios to be
2431
	 * added.  ->make_request() may indeed add some more bios
2432 2433 2434
	 * through a recursive call to generic_make_request.  If it
	 * did, we find a non-NULL value in bio_list and re-enter the loop
	 * from the top.  In this case we really did just take the bio
2435
	 * of the top of the list (no pretending) and so remove it from
2436
	 * bio_list, and call into ->make_request() again.
2437 2438
	 */
	BUG_ON(bio->bi_next);
2439 2440
	bio_list_init(&bio_list_on_stack[0]);
	current->bio_list = bio_list_on_stack;
2441
	do {
2442 2443 2444 2445 2446 2447 2448 2449 2450
		bool enter_succeeded = true;

		if (unlikely(q != bio->bi_disk->queue)) {
			if (q)
				blk_queue_exit(q);
			q = bio->bi_disk->queue;
			flags = 0;
			if (bio->bi_opf & REQ_NOWAIT)
				flags = BLK_MQ_REQ_NOWAIT;
2451
			if (blk_queue_enter(q, flags) < 0)
2452 2453
				enter_succeeded = false;
		}
2454

2455
		if (enter_succeeded) {
2456 2457 2458
			struct bio_list lower, same;

			/* Create a fresh bio_list for all subordinate requests */
2459 2460
			bio_list_on_stack[1] = bio_list_on_stack[0];
			bio_list_init(&bio_list_on_stack[0]);
2461
			ret = q->make_request_fn(q, bio);
2462

2463 2464 2465 2466 2467
			/* sort new bios into those for a lower level
			 * and those for the same level
			 */
			bio_list_init(&lower);
			bio_list_init(&same);
2468
			while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
2469
				if (q == bio->bi_disk->queue)
2470 2471 2472 2473
					bio_list_add(&same, bio);
				else
					bio_list_add(&lower, bio);
			/* now assemble so we handle the lowest level first */
2474 2475 2476
			bio_list_merge(&bio_list_on_stack[0], &lower);
			bio_list_merge(&bio_list_on_stack[0], &same);
			bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]);
2477
		} else {
2478 2479 2480 2481 2482
			if (unlikely(!blk_queue_dying(q) &&
					(bio->bi_opf & REQ_NOWAIT)))
				bio_wouldblock_error(bio);
			else
				bio_io_error(bio);
2483
			q = NULL;
2484
		}
2485
		bio = bio_list_pop(&bio_list_on_stack[0]);
2486
	} while (bio);
2487
	current->bio_list = NULL; /* deactivate */
2488 2489

out:
2490 2491
	if (q)
		blk_queue_exit(q);
2492
	return ret;
2493
}
L
Linus Torvalds 已提交
2494 2495
EXPORT_SYMBOL(generic_make_request);

2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514
/**
 * direct_make_request - hand a buffer directly to its device driver for I/O
 * @bio:  The bio describing the location in memory and on the device.
 *
 * This function behaves like generic_make_request(), but does not protect
 * against recursion.  Must only be used if the called driver is known
 * to not call generic_make_request (or direct_make_request) again from
 * its make_request function.  (Calling direct_make_request again from
 * a workqueue is perfectly fine as that doesn't recurse).
 */
blk_qc_t direct_make_request(struct bio *bio)
{
	struct request_queue *q = bio->bi_disk->queue;
	bool nowait = bio->bi_opf & REQ_NOWAIT;
	blk_qc_t ret;

	if (!generic_make_request_checks(bio))
		return BLK_QC_T_NONE;

2515
	if (unlikely(blk_queue_enter(q, nowait ? BLK_MQ_REQ_NOWAIT : 0))) {
2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529
		if (nowait && !blk_queue_dying(q))
			bio->bi_status = BLK_STS_AGAIN;
		else
			bio->bi_status = BLK_STS_IOERR;
		bio_endio(bio);
		return BLK_QC_T_NONE;
	}

	ret = q->make_request_fn(q, bio);
	blk_queue_exit(q);
	return ret;
}
EXPORT_SYMBOL_GPL(direct_make_request);

L
Linus Torvalds 已提交
2530
/**
2531
 * submit_bio - submit a bio to the block device layer for I/O
L
Linus Torvalds 已提交
2532 2533 2534 2535
 * @bio: The &struct bio which describes the I/O
 *
 * submit_bio() is very similar in purpose to generic_make_request(), and
 * uses that function to do most of the work. Both are fairly rough
2536
 * interfaces; @bio must be presetup and ready for I/O.
L
Linus Torvalds 已提交
2537 2538
 *
 */
2539
blk_qc_t submit_bio(struct bio *bio)
L
Linus Torvalds 已提交
2540
{
2541 2542 2543 2544
	/*
	 * If it's a regular read/write or a barrier with data attached,
	 * go through the normal accounting stuff before submission.
	 */
2545
	if (bio_has_data(bio)) {
2546 2547
		unsigned int count;

2548
		if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
2549
			count = queue_logical_block_size(bio->bi_disk->queue) >> 9;
2550 2551 2552
		else
			count = bio_sectors(bio);

2553
		if (op_is_write(bio_op(bio))) {
2554 2555
			count_vm_events(PGPGOUT, count);
		} else {
2556
			task_io_account_read(bio->bi_iter.bi_size);
2557 2558 2559 2560 2561
			count_vm_events(PGPGIN, count);
		}

		if (unlikely(block_dump)) {
			char b[BDEVNAME_SIZE];
2562
			printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n",
2563
			current->comm, task_pid_nr(current),
2564
				op_is_write(bio_op(bio)) ? "WRITE" : "READ",
2565
				(unsigned long long)bio->bi_iter.bi_sector,
2566
				bio_devname(bio, b), count);
2567
		}
L
Linus Torvalds 已提交
2568 2569
	}

2570
	return generic_make_request(bio);
L
Linus Torvalds 已提交
2571 2572 2573
}
EXPORT_SYMBOL(submit_bio);

2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584
bool blk_poll(struct request_queue *q, blk_qc_t cookie)
{
	if (!q->poll_fn || !blk_qc_t_valid(cookie))
		return false;

	if (current->plug)
		blk_flush_plug_list(current->plug, false);
	return q->poll_fn(q, cookie);
}
EXPORT_SYMBOL_GPL(blk_poll);

2585
/**
2586 2587
 * blk_cloned_rq_check_limits - Helper function to check a cloned request
 *                              for new the queue limits
2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598
 * @q:  the queue
 * @rq: the request being checked
 *
 * Description:
 *    @rq may have been made based on weaker limitations of upper-level queues
 *    in request stacking drivers, and it may violate the limitation of @q.
 *    Since the block layer and the underlying device driver trust @rq
 *    after it is inserted to @q, it should be checked against @q before
 *    the insertion using this generic function.
 *
 *    Request stacking drivers like request-based dm may change the queue
2599 2600
 *    limits when retrying requests on other queues. Those requests need
 *    to be checked against the new queue limits again during dispatch.
2601
 */
2602 2603
static int blk_cloned_rq_check_limits(struct request_queue *q,
				      struct request *rq)
2604
{
2605
	if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, req_op(rq))) {
2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616
		printk(KERN_ERR "%s: over max size limit.\n", __func__);
		return -EIO;
	}

	/*
	 * queue's settings related to segment counting like q->bounce_pfn
	 * may differ from that of other stacking queues.
	 * Recalculate it to check the request correctly on this queue's
	 * limitation.
	 */
	blk_recalc_rq_segments(rq);
2617
	if (rq->nr_phys_segments > queue_max_segments(q)) {
2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629
		printk(KERN_ERR "%s: over max segments limit.\n", __func__);
		return -EIO;
	}

	return 0;
}

/**
 * blk_insert_cloned_request - Helper for stacking drivers to submit a request
 * @q:  the queue to submit the request
 * @rq: the request being queued
 */
2630
blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq)
2631 2632
{
	unsigned long flags;
2633
	int where = ELEVATOR_INSERT_BACK;
2634

2635
	if (blk_cloned_rq_check_limits(q, rq))
2636
		return BLK_STS_IOERR;
2637

2638 2639
	if (rq->rq_disk &&
	    should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
2640
		return BLK_STS_IOERR;
2641

2642 2643 2644
	if (q->mq_ops) {
		if (blk_queue_io_stat(q))
			blk_account_io_start(rq, true);
2645 2646 2647 2648 2649
		/*
		 * Since we have a scheduler attached on the top device,
		 * bypass a potential scheduler on the bottom device for
		 * insert.
		 */
2650
		return blk_mq_request_issue_directly(rq);
2651 2652
	}

2653
	spin_lock_irqsave(q->queue_lock, flags);
B
Bart Van Assche 已提交
2654
	if (unlikely(blk_queue_dying(q))) {
2655
		spin_unlock_irqrestore(q->queue_lock, flags);
2656
		return BLK_STS_IOERR;
2657
	}
2658 2659 2660 2661 2662 2663 2664

	/*
	 * Submitting request must be dequeued before calling this function
	 * because it will be linked to another request_queue
	 */
	BUG_ON(blk_queued_rq(rq));

2665
	if (op_is_flush(rq->cmd_flags))
2666 2667 2668
		where = ELEVATOR_INSERT_FLUSH;

	add_acct_request(q, rq, where);
J
Jeff Moyer 已提交
2669 2670
	if (where == ELEVATOR_INSERT_FLUSH)
		__blk_run_queue(q);
2671 2672
	spin_unlock_irqrestore(q->queue_lock, flags);

2673
	return BLK_STS_OK;
2674 2675 2676
}
EXPORT_SYMBOL_GPL(blk_insert_cloned_request);

2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695
/**
 * blk_rq_err_bytes - determine number of bytes till the next failure boundary
 * @rq: request to examine
 *
 * Description:
 *     A request could be merge of IOs which require different failure
 *     handling.  This function determines the number of bytes which
 *     can be failed from the beginning of the request without
 *     crossing into area which need to be retried further.
 *
 * Return:
 *     The number of bytes to fail.
 */
unsigned int blk_rq_err_bytes(const struct request *rq)
{
	unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
	unsigned int bytes = 0;
	struct bio *bio;

2696
	if (!(rq->rq_flags & RQF_MIXED_MERGE))
2697 2698 2699 2700 2701 2702 2703 2704 2705 2706
		return blk_rq_bytes(rq);

	/*
	 * Currently the only 'mixing' which can happen is between
	 * different fastfail types.  We can safely fail portions
	 * which have all the failfast bits that the first one has -
	 * the ones which are at least as eager to fail as the first
	 * one.
	 */
	for (bio = rq->bio; bio; bio = bio->bi_next) {
J
Jens Axboe 已提交
2707
		if ((bio->bi_opf & ff) != ff)
2708
			break;
2709
		bytes += bio->bi_iter.bi_size;
2710 2711 2712 2713 2714 2715 2716 2717
	}

	/* this could lead to infinite loop */
	BUG_ON(blk_rq_bytes(rq) && !bytes);
	return bytes;
}
EXPORT_SYMBOL_GPL(blk_rq_err_bytes);

2718
void blk_account_io_completion(struct request *req, unsigned int bytes)
2719
{
2720
	if (blk_do_io_stat(req)) {
2721
		const int sgrp = op_stat_group(req_op(req));
2722 2723 2724 2725
		struct hd_struct *part;
		int cpu;

		cpu = part_stat_lock();
2726
		part = req->part;
2727
		part_stat_add(cpu, part, sectors[sgrp], bytes >> 9);
2728 2729 2730 2731
		part_stat_unlock();
	}
}

2732
void blk_account_io_done(struct request *req, u64 now)
2733 2734
{
	/*
2735 2736 2737
	 * Account IO completion.  flush_rq isn't accounted as a
	 * normal IO on queueing nor completion.  Accounting the
	 * containing request is enough.
2738
	 */
2739
	if (blk_do_io_stat(req) && !(req->rq_flags & RQF_FLUSH_SEQ)) {
2740
		const int sgrp = op_stat_group(req_op(req));
2741 2742 2743 2744
		struct hd_struct *part;
		int cpu;

		cpu = part_stat_lock();
2745
		part = req->part;
2746

2747
		part_stat_inc(cpu, part, ios[sgrp]);
2748
		part_stat_add(cpu, part, nsecs[sgrp], now - req->start_time_ns);
2749
		part_round_stats(req->q, cpu, part);
2750
		part_dec_in_flight(req->q, part, rq_data_dir(req));
2751

2752
		hd_struct_put(part);
2753 2754 2755 2756
		part_stat_unlock();
	}
}

2757
#ifdef CONFIG_PM
L
Lin Ming 已提交
2758 2759 2760 2761
/*
 * Don't process normal requests when queue is suspended
 * or in the process of suspending/resuming
 */
2762
static bool blk_pm_allow_request(struct request *rq)
L
Lin Ming 已提交
2763
{
2764 2765 2766 2767 2768 2769
	switch (rq->q->rpm_status) {
	case RPM_RESUMING:
	case RPM_SUSPENDING:
		return rq->rq_flags & RQF_PM;
	case RPM_SUSPENDED:
		return false;
2770 2771
	default:
		return true;
2772
	}
L
Lin Ming 已提交
2773 2774
}
#else
2775
static bool blk_pm_allow_request(struct request *rq)
L
Lin Ming 已提交
2776
{
2777
	return true;
L
Lin Ming 已提交
2778 2779 2780
}
#endif

2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808
void blk_account_io_start(struct request *rq, bool new_io)
{
	struct hd_struct *part;
	int rw = rq_data_dir(rq);
	int cpu;

	if (!blk_do_io_stat(rq))
		return;

	cpu = part_stat_lock();

	if (!new_io) {
		part = rq->part;
		part_stat_inc(cpu, part, merges[rw]);
	} else {
		part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
		if (!hd_struct_try_get(part)) {
			/*
			 * The partition is already being removed,
			 * the request will be accounted on the disk only
			 *
			 * We take a reference on disk->part0 although that
			 * partition will never be deleted, so we can treat
			 * it as any other partition.
			 */
			part = &rq->rq_disk->part0;
			hd_struct_get(part);
		}
2809 2810
		part_round_stats(rq->q, cpu, part);
		part_inc_in_flight(rq->q, part, rw);
2811 2812 2813 2814 2815 2816
		rq->part = part;
	}

	part_stat_unlock();
}

2817 2818 2819 2820 2821 2822 2823 2824
static struct request *elv_next_request(struct request_queue *q)
{
	struct request *rq;
	struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);

	WARN_ON_ONCE(q->mq_ops);

	while (1) {
2825 2826 2827 2828 2829 2830
		list_for_each_entry(rq, &q->queue_head, queuelist) {
			if (blk_pm_allow_request(rq))
				return rq;

			if (rq->rq_flags & RQF_SOFTBARRIER)
				break;
2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858
		}

		/*
		 * Flush request is running and flush request isn't queueable
		 * in the drive, we can hold the queue till flush request is
		 * finished. Even we don't do this, driver can't dispatch next
		 * requests and will requeue them. And this can improve
		 * throughput too. For example, we have request flush1, write1,
		 * flush 2. flush1 is dispatched, then queue is hold, write1
		 * isn't inserted to queue. After flush1 is finished, flush2
		 * will be dispatched. Since disk cache is already clean,
		 * flush2 will be finished very soon, so looks like flush2 is
		 * folded to flush1.
		 * Since the queue is hold, a flag is set to indicate the queue
		 * should be restarted later. Please see flush_end_io() for
		 * details.
		 */
		if (fq->flush_pending_idx != fq->flush_running_idx &&
				!queue_flush_queueable(q)) {
			fq->flush_queue_delayed = 1;
			return NULL;
		}
		if (unlikely(blk_queue_bypass(q)) ||
		    !q->elevator->type->ops.sq.elevator_dispatch_fn(q, 0))
			return NULL;
	}
}

2859
/**
2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872
 * blk_peek_request - peek at the top of a request queue
 * @q: request queue to peek at
 *
 * Description:
 *     Return the request at the top of @q.  The returned request
 *     should be started using blk_start_request() before LLD starts
 *     processing it.
 *
 * Return:
 *     Pointer to the request at the top of @q if available.  Null
 *     otherwise.
 */
struct request *blk_peek_request(struct request_queue *q)
2873 2874 2875 2876
{
	struct request *rq;
	int ret;

2877
	lockdep_assert_held(q->queue_lock);
2878
	WARN_ON_ONCE(q->mq_ops);
2879

2880
	while ((rq = elv_next_request(q)) != NULL) {
2881
		if (!(rq->rq_flags & RQF_STARTED)) {
2882 2883 2884 2885 2886
			/*
			 * This is the first time the device driver
			 * sees this request (possibly after
			 * requeueing).  Notify IO scheduler.
			 */
2887
			if (rq->rq_flags & RQF_SORTED)
2888 2889 2890 2891 2892 2893 2894
				elv_activate_rq(q, rq);

			/*
			 * just mark as started even if we don't start
			 * it, a request that has been delayed should
			 * not be passed by new incoming requests
			 */
2895
			rq->rq_flags |= RQF_STARTED;
2896 2897 2898 2899 2900 2901 2902 2903
			trace_block_rq_issue(q, rq);
		}

		if (!q->boundary_rq || q->boundary_rq == rq) {
			q->end_sector = rq_end_sector(rq);
			q->boundary_rq = NULL;
		}

2904
		if (rq->rq_flags & RQF_DONTPREP)
2905 2906
			break;

2907
		if (q->dma_drain_size && blk_rq_bytes(rq)) {
2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926
			/*
			 * make sure space for the drain appears we
			 * know we can do this because max_hw_segments
			 * has been adjusted to be one fewer than the
			 * device can handle
			 */
			rq->nr_phys_segments++;
		}

		if (!q->prep_rq_fn)
			break;

		ret = q->prep_rq_fn(q, rq);
		if (ret == BLKPREP_OK) {
			break;
		} else if (ret == BLKPREP_DEFER) {
			/*
			 * the request may have been (partially) prepped.
			 * we need to keep this request in the front to
2927
			 * avoid resource deadlock.  RQF_STARTED will
2928 2929
			 * prevent other fs requests from passing this one.
			 */
2930
			if (q->dma_drain_size && blk_rq_bytes(rq) &&
2931
			    !(rq->rq_flags & RQF_DONTPREP)) {
2932 2933 2934 2935 2936 2937 2938 2939 2940
				/*
				 * remove the space for the drain we added
				 * so that we don't add it again
				 */
				--rq->nr_phys_segments;
			}

			rq = NULL;
			break;
2941
		} else if (ret == BLKPREP_KILL || ret == BLKPREP_INVALID) {
2942
			rq->rq_flags |= RQF_QUIET;
2943 2944 2945 2946 2947
			/*
			 * Mark this request as started so we don't trigger
			 * any debug logic in the end I/O path.
			 */
			blk_start_request(rq);
2948 2949
			__blk_end_request_all(rq, ret == BLKPREP_INVALID ?
					BLK_STS_TARGET : BLK_STS_IOERR);
2950 2951 2952 2953 2954 2955 2956 2957
		} else {
			printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
			break;
		}
	}

	return rq;
}
2958
EXPORT_SYMBOL(blk_peek_request);
2959

2960
static void blk_dequeue_request(struct request *rq)
2961
{
2962 2963
	struct request_queue *q = rq->q;

2964 2965 2966 2967 2968 2969 2970 2971 2972 2973
	BUG_ON(list_empty(&rq->queuelist));
	BUG_ON(ELV_ON_HASH(rq));

	list_del_init(&rq->queuelist);

	/*
	 * the time frame between a request being removed from the lists
	 * and to it is freed is accounted as io that is in progress at
	 * the driver side.
	 */
2974
	if (blk_account_rq(rq))
2975
		q->in_flight[rq_is_sync(rq)]++;
2976 2977
}

2978 2979 2980 2981 2982 2983 2984 2985 2986 2987
/**
 * blk_start_request - start request processing on the driver
 * @req: request to dequeue
 *
 * Description:
 *     Dequeue @req and start timeout timer on it.  This hands off the
 *     request to the driver.
 */
void blk_start_request(struct request *req)
{
2988
	lockdep_assert_held(req->q->queue_lock);
2989
	WARN_ON_ONCE(req->q->mq_ops);
2990

2991 2992
	blk_dequeue_request(req);

2993
	if (test_bit(QUEUE_FLAG_STATS, &req->q->queue_flags)) {
2994 2995 2996 2997
		req->io_start_time_ns = ktime_get_ns();
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
		req->throtl_size = blk_rq_sectors(req);
#endif
2998
		req->rq_flags |= RQF_STATS;
2999
		rq_qos_issue(req->q, req);
3000 3001
	}

3002
	BUG_ON(blk_rq_is_complete(req));
3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022
	blk_add_timer(req);
}
EXPORT_SYMBOL(blk_start_request);

/**
 * blk_fetch_request - fetch a request from a request queue
 * @q: request queue to fetch a request from
 *
 * Description:
 *     Return the request at the top of @q.  The request is started on
 *     return and LLD can start processing it immediately.
 *
 * Return:
 *     Pointer to the request at the top of @q if available.  Null
 *     otherwise.
 */
struct request *blk_fetch_request(struct request_queue *q)
{
	struct request *rq;

3023
	lockdep_assert_held(q->queue_lock);
3024
	WARN_ON_ONCE(q->mq_ops);
3025

3026 3027 3028 3029 3030 3031 3032
	rq = blk_peek_request(q);
	if (rq)
		blk_start_request(rq);
	return rq;
}
EXPORT_SYMBOL(blk_fetch_request);

3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053
/*
 * Steal bios from a request and add them to a bio list.
 * The request must not have been partially completed before.
 */
void blk_steal_bios(struct bio_list *list, struct request *rq)
{
	if (rq->bio) {
		if (list->tail)
			list->tail->bi_next = rq->bio;
		else
			list->head = rq->bio;
		list->tail = rq->biotail;

		rq->bio = NULL;
		rq->biotail = NULL;
	}

	rq->__data_len = 0;
}
EXPORT_SYMBOL_GPL(blk_steal_bios);

3054
/**
3055
 * blk_update_request - Special helper function for request stacking drivers
3056
 * @req:      the request being processed
3057
 * @error:    block status code
3058
 * @nr_bytes: number of bytes to complete @req
3059 3060
 *
 * Description:
3061 3062 3063
 *     Ends I/O on a number of bytes attached to @req, but doesn't complete
 *     the request structure even if @req doesn't have leftover.
 *     If @req has leftover, sets it up for the next range of segments.
3064 3065 3066 3067 3068 3069 3070
 *
 *     This special helper function is only for request stacking drivers
 *     (e.g. request-based dm) so that they can handle partial completion.
 *     Actual device drivers should use blk_end_request instead.
 *
 *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
 *     %false return from this function.
3071
 *
3072 3073 3074 3075
 * Note:
 *	The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in both
 *	blk_rq_bytes() and in blk_update_request().
 *
3076
 * Return:
3077 3078
 *     %false - this request doesn't have any more data
 *     %true  - this request has more data
3079
 **/
3080 3081
bool blk_update_request(struct request *req, blk_status_t error,
		unsigned int nr_bytes)
L
Linus Torvalds 已提交
3082
{
3083
	int total_bytes;
L
Linus Torvalds 已提交
3084

3085
	trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);
3086

3087 3088 3089
	if (!req->bio)
		return false;

3090 3091 3092
	if (unlikely(error && !blk_rq_is_passthrough(req) &&
		     !(req->rq_flags & RQF_QUIET)))
		print_req_error(req, error);
L
Linus Torvalds 已提交
3093

3094
	blk_account_io_completion(req, nr_bytes);
3095

3096 3097 3098
	total_bytes = 0;
	while (req->bio) {
		struct bio *bio = req->bio;
3099
		unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
L
Linus Torvalds 已提交
3100

3101
		if (bio_bytes == bio->bi_iter.bi_size)
L
Linus Torvalds 已提交
3102 3103
			req->bio = bio->bi_next;

N
NeilBrown 已提交
3104 3105
		/* Completion has already been traced */
		bio_clear_flag(bio, BIO_TRACE_COMPLETION);
3106
		req_bio_endio(req, bio, bio_bytes, error);
L
Linus Torvalds 已提交
3107

3108 3109
		total_bytes += bio_bytes;
		nr_bytes -= bio_bytes;
L
Linus Torvalds 已提交
3110

3111 3112
		if (!nr_bytes)
			break;
L
Linus Torvalds 已提交
3113 3114 3115 3116 3117
	}

	/*
	 * completely done
	 */
3118 3119 3120 3121 3122 3123
	if (!req->bio) {
		/*
		 * Reset counters so that the request stacking driver
		 * can find how many bytes remain in the request
		 * later.
		 */
3124
		req->__data_len = 0;
3125 3126
		return false;
	}
L
Linus Torvalds 已提交
3127

3128
	req->__data_len -= total_bytes;
3129 3130

	/* update sector only for requests with clear definition of sector */
3131
	if (!blk_rq_is_passthrough(req))
3132
		req->__sector += total_bytes >> 9;
3133

3134
	/* mixed attributes always follow the first bio */
3135
	if (req->rq_flags & RQF_MIXED_MERGE) {
3136
		req->cmd_flags &= ~REQ_FAILFAST_MASK;
J
Jens Axboe 已提交
3137
		req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
3138 3139
	}

3140 3141 3142 3143 3144 3145 3146 3147 3148
	if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
		/*
		 * If total number of sectors is less than the first segment
		 * size, something has gone terribly wrong.
		 */
		if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
			blk_dump_rq_flags(req, "request botched");
			req->__data_len = blk_rq_cur_bytes(req);
		}
3149

3150 3151 3152
		/* recalculate the number of segments */
		blk_recalc_rq_segments(req);
	}
3153

3154
	return true;
L
Linus Torvalds 已提交
3155
}
3156
EXPORT_SYMBOL_GPL(blk_update_request);
L
Linus Torvalds 已提交
3157

3158
static bool blk_update_bidi_request(struct request *rq, blk_status_t error,
3159 3160
				    unsigned int nr_bytes,
				    unsigned int bidi_bytes)
3161
{
3162 3163
	if (blk_update_request(rq, error, nr_bytes))
		return true;
3164

3165 3166 3167 3168
	/* Bidi request must be completed as a whole */
	if (unlikely(blk_bidi_rq(rq)) &&
	    blk_update_request(rq->next_rq, error, bidi_bytes))
		return true;
3169

3170 3171
	if (blk_queue_add_random(rq->q))
		add_disk_randomness(rq->rq_disk);
3172 3173

	return false;
L
Linus Torvalds 已提交
3174 3175
}

3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189
/**
 * blk_unprep_request - unprepare a request
 * @req:	the request
 *
 * This function makes a request ready for complete resubmission (or
 * completion).  It happens only after all error handling is complete,
 * so represents the appropriate moment to deallocate any resources
 * that were allocated to the request in the prep_rq_fn.  The queue
 * lock is held when calling this.
 */
void blk_unprep_request(struct request *req)
{
	struct request_queue *q = req->q;

3190
	req->rq_flags &= ~RQF_DONTPREP;
3191 3192 3193 3194 3195
	if (q->unprep_rq_fn)
		q->unprep_rq_fn(q, req);
}
EXPORT_SYMBOL_GPL(blk_unprep_request);

3196
void blk_finish_request(struct request *req, blk_status_t error)
L
Linus Torvalds 已提交
3197
{
3198
	struct request_queue *q = req->q;
3199
	u64 now = ktime_get_ns();
3200

3201
	lockdep_assert_held(req->q->queue_lock);
3202
	WARN_ON_ONCE(q->mq_ops);
3203

3204
	if (req->rq_flags & RQF_STATS)
3205
		blk_stat_add(req, now);
3206

3207
	if (req->rq_flags & RQF_QUEUED)
3208
		blk_queue_end_tag(q, req);
3209

3210
	BUG_ON(blk_queued_rq(req));
L
Linus Torvalds 已提交
3211

3212
	if (unlikely(laptop_mode) && !blk_rq_is_passthrough(req))
3213
		laptop_io_completion(req->q->backing_dev_info);
L
Linus Torvalds 已提交
3214

3215 3216
	blk_delete_timer(req);

3217
	if (req->rq_flags & RQF_DONTPREP)
3218 3219
		blk_unprep_request(req);

3220
	blk_account_io_done(req, now);
3221

J
Jens Axboe 已提交
3222
	if (req->end_io) {
3223
		rq_qos_done(q, req);
3224
		req->end_io(req, error);
J
Jens Axboe 已提交
3225
	} else {
3226 3227 3228
		if (blk_bidi_rq(req))
			__blk_put_request(req->next_rq->q, req->next_rq);

3229
		__blk_put_request(q, req);
3230
	}
L
Linus Torvalds 已提交
3231
}
3232
EXPORT_SYMBOL(blk_finish_request);
L
Linus Torvalds 已提交
3233

3234
/**
3235 3236
 * blk_end_bidi_request - Complete a bidi request
 * @rq:         the request to complete
3237
 * @error:      block status code
3238 3239
 * @nr_bytes:   number of bytes to complete @rq
 * @bidi_bytes: number of bytes to complete @rq->next_rq
3240 3241
 *
 * Description:
3242
 *     Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
3243 3244 3245
 *     Drivers that supports bidi can safely call this member for any
 *     type of request, bidi or uni.  In the later case @bidi_bytes is
 *     just ignored.
3246 3247
 *
 * Return:
3248 3249
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
3250
 **/
3251
static bool blk_end_bidi_request(struct request *rq, blk_status_t error,
K
Kiyoshi Ueda 已提交
3252 3253
				 unsigned int nr_bytes, unsigned int bidi_bytes)
{
3254
	struct request_queue *q = rq->q;
3255
	unsigned long flags;
K
Kiyoshi Ueda 已提交
3256

3257 3258
	WARN_ON_ONCE(q->mq_ops);

3259 3260
	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
		return true;
K
Kiyoshi Ueda 已提交
3261

3262
	spin_lock_irqsave(q->queue_lock, flags);
3263
	blk_finish_request(rq, error);
3264 3265
	spin_unlock_irqrestore(q->queue_lock, flags);

3266
	return false;
K
Kiyoshi Ueda 已提交
3267 3268
}

3269
/**
3270 3271
 * __blk_end_bidi_request - Complete a bidi request with queue lock held
 * @rq:         the request to complete
3272
 * @error:      block status code
3273 3274
 * @nr_bytes:   number of bytes to complete @rq
 * @bidi_bytes: number of bytes to complete @rq->next_rq
3275 3276
 *
 * Description:
3277 3278
 *     Identical to blk_end_bidi_request() except that queue lock is
 *     assumed to be locked on entry and remains so on return.
3279 3280
 *
 * Return:
3281 3282
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
3283
 **/
3284
static bool __blk_end_bidi_request(struct request *rq, blk_status_t error,
3285
				   unsigned int nr_bytes, unsigned int bidi_bytes)
3286
{
3287
	lockdep_assert_held(rq->q->queue_lock);
3288
	WARN_ON_ONCE(rq->q->mq_ops);
3289

3290 3291
	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
		return true;
3292

3293
	blk_finish_request(rq, error);
3294

3295
	return false;
3296
}
3297 3298 3299 3300

/**
 * blk_end_request - Helper function for drivers to complete the request.
 * @rq:       the request being processed
3301
 * @error:    block status code
3302 3303 3304 3305 3306 3307 3308
 * @nr_bytes: number of bytes to complete
 *
 * Description:
 *     Ends I/O on a number of bytes attached to @rq.
 *     If @rq has leftover, sets it up for the next range of segments.
 *
 * Return:
3309 3310
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
3311
 **/
3312 3313
bool blk_end_request(struct request *rq, blk_status_t error,
		unsigned int nr_bytes)
3314
{
3315
	WARN_ON_ONCE(rq->q->mq_ops);
3316
	return blk_end_bidi_request(rq, error, nr_bytes, 0);
3317
}
3318
EXPORT_SYMBOL(blk_end_request);
3319 3320

/**
3321 3322
 * blk_end_request_all - Helper function for drives to finish the request.
 * @rq: the request to finish
3323
 * @error: block status code
3324 3325
 *
 * Description:
3326 3327
 *     Completely finish @rq.
 */
3328
void blk_end_request_all(struct request *rq, blk_status_t error)
3329
{
3330 3331
	bool pending;
	unsigned int bidi_bytes = 0;
3332

3333 3334
	if (unlikely(blk_bidi_rq(rq)))
		bidi_bytes = blk_rq_bytes(rq->next_rq);
3335

3336 3337 3338
	pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
	BUG_ON(pending);
}
3339
EXPORT_SYMBOL(blk_end_request_all);
3340

3341
/**
3342 3343
 * __blk_end_request - Helper function for drivers to complete the request.
 * @rq:       the request being processed
3344
 * @error:    block status code
3345
 * @nr_bytes: number of bytes to complete
3346 3347
 *
 * Description:
3348
 *     Must be called with queue lock held unlike blk_end_request().
3349 3350
 *
 * Return:
3351 3352
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
3353
 **/
3354 3355
bool __blk_end_request(struct request *rq, blk_status_t error,
		unsigned int nr_bytes)
3356
{
3357
	lockdep_assert_held(rq->q->queue_lock);
3358
	WARN_ON_ONCE(rq->q->mq_ops);
3359

3360
	return __blk_end_bidi_request(rq, error, nr_bytes, 0);
3361
}
3362
EXPORT_SYMBOL(__blk_end_request);
3363

K
Kiyoshi Ueda 已提交
3364
/**
3365 3366
 * __blk_end_request_all - Helper function for drives to finish the request.
 * @rq: the request to finish
3367
 * @error:    block status code
K
Kiyoshi Ueda 已提交
3368 3369
 *
 * Description:
3370
 *     Completely finish @rq.  Must be called with queue lock held.
K
Kiyoshi Ueda 已提交
3371
 */
3372
void __blk_end_request_all(struct request *rq, blk_status_t error)
K
Kiyoshi Ueda 已提交
3373
{
3374 3375 3376
	bool pending;
	unsigned int bidi_bytes = 0;

3377
	lockdep_assert_held(rq->q->queue_lock);
3378
	WARN_ON_ONCE(rq->q->mq_ops);
3379

3380 3381 3382 3383 3384
	if (unlikely(blk_bidi_rq(rq)))
		bidi_bytes = blk_rq_bytes(rq->next_rq);

	pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
	BUG_ON(pending);
K
Kiyoshi Ueda 已提交
3385
}
3386
EXPORT_SYMBOL(__blk_end_request_all);
K
Kiyoshi Ueda 已提交
3387

3388
/**
3389 3390
 * __blk_end_request_cur - Helper function to finish the current request chunk.
 * @rq: the request to finish the current chunk for
3391
 * @error:    block status code
3392 3393
 *
 * Description:
3394 3395
 *     Complete the current consecutively mapped chunk from @rq.  Must
 *     be called with queue lock held.
3396 3397
 *
 * Return:
3398 3399 3400
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
 */
3401
bool __blk_end_request_cur(struct request *rq, blk_status_t error)
3402
{
3403
	return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
3404
}
3405
EXPORT_SYMBOL(__blk_end_request_cur);
3406

J
Jens Axboe 已提交
3407 3408
void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
		     struct bio *bio)
L
Linus Torvalds 已提交
3409
{
3410
	if (bio_has_data(bio))
D
David Woodhouse 已提交
3411
		rq->nr_phys_segments = bio_phys_segments(q, bio);
3412 3413
	else if (bio_op(bio) == REQ_OP_DISCARD)
		rq->nr_phys_segments = 1;
3414

3415
	rq->__data_len = bio->bi_iter.bi_size;
L
Linus Torvalds 已提交
3416 3417
	rq->bio = rq->biotail = bio;

3418 3419
	if (bio->bi_disk)
		rq->rq_disk = bio->bi_disk;
N
NeilBrown 已提交
3420
}
L
Linus Torvalds 已提交
3421

3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432
#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
/**
 * rq_flush_dcache_pages - Helper function to flush all pages in a request
 * @rq: the request to be flushed
 *
 * Description:
 *     Flush all pages in @rq.
 */
void rq_flush_dcache_pages(struct request *rq)
{
	struct req_iterator iter;
3433
	struct bio_vec bvec;
3434 3435

	rq_for_each_segment(bvec, rq, iter)
3436
		flush_dcache_page(bvec.bv_page);
3437 3438 3439 3440
}
EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
#endif

3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468
/**
 * blk_lld_busy - Check if underlying low-level drivers of a device are busy
 * @q : the queue of the device being checked
 *
 * Description:
 *    Check if underlying low-level drivers of a device are busy.
 *    If the drivers want to export their busy state, they must set own
 *    exporting function using blk_queue_lld_busy() first.
 *
 *    Basically, this function is used only by request stacking drivers
 *    to stop dispatching requests to underlying devices when underlying
 *    devices are busy.  This behavior helps more I/O merging on the queue
 *    of the request stacking driver and prevents I/O throughput regression
 *    on burst I/O load.
 *
 * Return:
 *    0 - Not busy (The request stacking driver should dispatch request)
 *    1 - Busy (The request stacking driver should stop dispatching request)
 */
int blk_lld_busy(struct request_queue *q)
{
	if (q->lld_busy_fn)
		return q->lld_busy_fn(q);

	return 0;
}
EXPORT_SYMBOL_GPL(blk_lld_busy);

3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492
/**
 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
 * @rq: the clone request to be cleaned up
 *
 * Description:
 *     Free all bios in @rq for a cloned request.
 */
void blk_rq_unprep_clone(struct request *rq)
{
	struct bio *bio;

	while ((bio = rq->bio) != NULL) {
		rq->bio = bio->bi_next;

		bio_put(bio);
	}
}
EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);

/*
 * Copy attributes of the original request to the clone request.
 * The actual data parts (e.g. ->cmd, ->sense) are not copied.
 */
static void __blk_rq_prep_clone(struct request *dst, struct request *src)
3493 3494 3495 3496
{
	dst->cpu = src->cpu;
	dst->__sector = blk_rq_pos(src);
	dst->__data_len = blk_rq_bytes(src);
3497 3498 3499 3500
	if (src->rq_flags & RQF_SPECIAL_PAYLOAD) {
		dst->rq_flags |= RQF_SPECIAL_PAYLOAD;
		dst->special_vec = src->special_vec;
	}
3501 3502 3503
	dst->nr_phys_segments = src->nr_phys_segments;
	dst->ioprio = src->ioprio;
	dst->extra_len = src->extra_len;
3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532
}

/**
 * blk_rq_prep_clone - Helper function to setup clone request
 * @rq: the request to be setup
 * @rq_src: original request to be cloned
 * @bs: bio_set that bios for clone are allocated from
 * @gfp_mask: memory allocation mask for bio
 * @bio_ctr: setup function to be called for each clone bio.
 *           Returns %0 for success, non %0 for failure.
 * @data: private data to be passed to @bio_ctr
 *
 * Description:
 *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
 *     The actual data parts of @rq_src (e.g. ->cmd, ->sense)
 *     are not copied, and copying such parts is the caller's responsibility.
 *     Also, pages which the original bios are pointing to are not copied
 *     and the cloned bios just point same pages.
 *     So cloned bios must be completed before original bios, which means
 *     the caller must complete @rq before @rq_src.
 */
int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
		      struct bio_set *bs, gfp_t gfp_mask,
		      int (*bio_ctr)(struct bio *, struct bio *, void *),
		      void *data)
{
	struct bio *bio, *bio_src;

	if (!bs)
3533
		bs = &fs_bio_set;
3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559

	__rq_for_each_bio(bio_src, rq_src) {
		bio = bio_clone_fast(bio_src, gfp_mask, bs);
		if (!bio)
			goto free_and_out;

		if (bio_ctr && bio_ctr(bio, bio_src, data))
			goto free_and_out;

		if (rq->bio) {
			rq->biotail->bi_next = bio;
			rq->biotail = bio;
		} else
			rq->bio = rq->biotail = bio;
	}

	__blk_rq_prep_clone(rq, rq_src);

	return 0;

free_and_out:
	if (bio)
		bio_put(bio);
	blk_rq_unprep_clone(rq);

	return -ENOMEM;
3560 3561 3562
}
EXPORT_SYMBOL_GPL(blk_rq_prep_clone);

3563
int kblockd_schedule_work(struct work_struct *work)
L
Linus Torvalds 已提交
3564 3565 3566 3567 3568
{
	return queue_work(kblockd_workqueue, work);
}
EXPORT_SYMBOL(kblockd_schedule_work);

3569 3570 3571 3572 3573 3574
int kblockd_schedule_work_on(int cpu, struct work_struct *work)
{
	return queue_work_on(cpu, kblockd_workqueue, work);
}
EXPORT_SYMBOL(kblockd_schedule_work_on);

3575 3576 3577 3578 3579 3580 3581
int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork,
				unsigned long delay)
{
	return mod_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
}
EXPORT_SYMBOL(kblockd_mod_delayed_work_on);

S
Suresh Jayaraman 已提交
3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595
/**
 * blk_start_plug - initialize blk_plug and track it inside the task_struct
 * @plug:	The &struct blk_plug that needs to be initialized
 *
 * Description:
 *   Tracking blk_plug inside the task_struct will help with auto-flushing the
 *   pending I/O should the task end up blocking between blk_start_plug() and
 *   blk_finish_plug(). This is important from a performance perspective, but
 *   also ensures that we don't deadlock. For instance, if the task is blocking
 *   for a memory allocation, memory reclaim could end up wanting to free a
 *   page belonging to that request that is currently residing in our private
 *   plug. By flushing the pending I/O when the process goes to sleep, we avoid
 *   this kind of deadlock.
 */
3596 3597 3598 3599
void blk_start_plug(struct blk_plug *plug)
{
	struct task_struct *tsk = current;

S
Shaohua Li 已提交
3600 3601 3602 3603 3604 3605
	/*
	 * If this is a nested plug, don't actually assign it.
	 */
	if (tsk->plug)
		return;

3606
	INIT_LIST_HEAD(&plug->list);
3607
	INIT_LIST_HEAD(&plug->mq_list);
3608
	INIT_LIST_HEAD(&plug->cb_list);
3609
	/*
S
Shaohua Li 已提交
3610 3611
	 * Store ordering should not be needed here, since a potential
	 * preempt will imply a full memory barrier
3612
	 */
S
Shaohua Li 已提交
3613
	tsk->plug = plug;
3614 3615 3616 3617 3618 3619 3620 3621
}
EXPORT_SYMBOL(blk_start_plug);

static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
{
	struct request *rqa = container_of(a, struct request, queuelist);
	struct request *rqb = container_of(b, struct request, queuelist);

3622 3623
	return !(rqa->q < rqb->q ||
		(rqa->q == rqb->q && blk_rq_pos(rqa) < blk_rq_pos(rqb)));
3624 3625
}

3626 3627 3628 3629 3630 3631
/*
 * If 'from_schedule' is true, then postpone the dispatch of requests
 * until a safe kblockd context. We due this to avoid accidental big
 * additional stack usage in driver dispatch, in places where the originally
 * plugger did not intend it.
 */
3632
static void queue_unplugged(struct request_queue *q, unsigned int depth,
3633
			    bool from_schedule)
3634
	__releases(q->queue_lock)
3635
{
3636 3637
	lockdep_assert_held(q->queue_lock);

3638
	trace_block_unplug(q, depth, !from_schedule);
3639

3640
	if (from_schedule)
3641
		blk_run_queue_async(q);
3642
	else
3643
		__blk_run_queue(q);
3644
	spin_unlock_irq(q->queue_lock);
3645 3646
}

3647
static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
3648 3649 3650
{
	LIST_HEAD(callbacks);

S
Shaohua Li 已提交
3651 3652
	while (!list_empty(&plug->cb_list)) {
		list_splice_init(&plug->cb_list, &callbacks);
3653

S
Shaohua Li 已提交
3654 3655
		while (!list_empty(&callbacks)) {
			struct blk_plug_cb *cb = list_first_entry(&callbacks,
3656 3657
							  struct blk_plug_cb,
							  list);
S
Shaohua Li 已提交
3658
			list_del(&cb->list);
3659
			cb->callback(cb, from_schedule);
S
Shaohua Li 已提交
3660
		}
3661 3662 3663
	}
}

3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688
struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
				      int size)
{
	struct blk_plug *plug = current->plug;
	struct blk_plug_cb *cb;

	if (!plug)
		return NULL;

	list_for_each_entry(cb, &plug->cb_list, list)
		if (cb->callback == unplug && cb->data == data)
			return cb;

	/* Not currently on the callback list */
	BUG_ON(size < sizeof(*cb));
	cb = kzalloc(size, GFP_ATOMIC);
	if (cb) {
		cb->data = data;
		cb->callback = unplug;
		list_add(&cb->list, &plug->cb_list);
	}
	return cb;
}
EXPORT_SYMBOL(blk_check_plugged);

3689
void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
3690 3691 3692
{
	struct request_queue *q;
	struct request *rq;
3693
	LIST_HEAD(list);
3694
	unsigned int depth;
3695

3696
	flush_plug_callbacks(plug, from_schedule);
3697 3698 3699 3700

	if (!list_empty(&plug->mq_list))
		blk_mq_flush_plug_list(plug, from_schedule);

3701 3702 3703
	if (list_empty(&plug->list))
		return;

3704 3705
	list_splice_init(&plug->list, &list);

3706
	list_sort(NULL, &list, plug_rq_cmp);
3707 3708

	q = NULL;
3709
	depth = 0;
3710

3711 3712
	while (!list_empty(&list)) {
		rq = list_entry_rq(list.next);
3713 3714 3715
		list_del_init(&rq->queuelist);
		BUG_ON(!rq->q);
		if (rq->q != q) {
3716 3717 3718 3719
			/*
			 * This drops the queue lock
			 */
			if (q)
3720
				queue_unplugged(q, depth, from_schedule);
3721
			q = rq->q;
3722
			depth = 0;
3723
			spin_lock_irq(q->queue_lock);
3724
		}
3725 3726 3727 3728

		/*
		 * Short-circuit if @q is dead
		 */
B
Bart Van Assche 已提交
3729
		if (unlikely(blk_queue_dying(q))) {
3730
			__blk_end_request_all(rq, BLK_STS_IOERR);
3731 3732 3733
			continue;
		}

3734 3735 3736
		/*
		 * rq is already accounted, so use raw insert
		 */
3737
		if (op_is_flush(rq->cmd_flags))
3738 3739 3740
			__elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH);
		else
			__elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE);
3741 3742

		depth++;
3743 3744
	}

3745 3746 3747 3748
	/*
	 * This drops the queue lock
	 */
	if (q)
3749
		queue_unplugged(q, depth, from_schedule);
3750 3751 3752 3753
}

void blk_finish_plug(struct blk_plug *plug)
{
S
Shaohua Li 已提交
3754 3755
	if (plug != current->plug)
		return;
3756
	blk_flush_plug_list(plug, false);
3757

S
Shaohua Li 已提交
3758
	current->plug = NULL;
3759
}
3760
EXPORT_SYMBOL(blk_finish_plug);
3761

3762
#ifdef CONFIG_PM
L
Lin Ming 已提交
3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785
/**
 * blk_pm_runtime_init - Block layer runtime PM initialization routine
 * @q: the queue of the device
 * @dev: the device the queue belongs to
 *
 * Description:
 *    Initialize runtime-PM-related fields for @q and start auto suspend for
 *    @dev. Drivers that want to take advantage of request-based runtime PM
 *    should call this function after @dev has been initialized, and its
 *    request queue @q has been allocated, and runtime PM for it can not happen
 *    yet(either due to disabled/forbidden or its usage_count > 0). In most
 *    cases, driver should call this function before any I/O has taken place.
 *
 *    This function takes care of setting up using auto suspend for the device,
 *    the autosuspend delay is set to -1 to make runtime suspend impossible
 *    until an updated value is either set by user or by driver. Drivers do
 *    not need to touch other autosuspend settings.
 *
 *    The block layer runtime PM is request based, so only works for drivers
 *    that use request as their IO unit instead of those directly use bio's.
 */
void blk_pm_runtime_init(struct request_queue *q, struct device *dev)
{
3786 3787 3788
	/* Don't enable runtime PM for blk-mq until it is ready */
	if (q->mq_ops) {
		pm_runtime_disable(dev);
3789
		return;
3790
	}
3791

L
Lin Ming 已提交
3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823
	q->dev = dev;
	q->rpm_status = RPM_ACTIVE;
	pm_runtime_set_autosuspend_delay(q->dev, -1);
	pm_runtime_use_autosuspend(q->dev);
}
EXPORT_SYMBOL(blk_pm_runtime_init);

/**
 * blk_pre_runtime_suspend - Pre runtime suspend check
 * @q: the queue of the device
 *
 * Description:
 *    This function will check if runtime suspend is allowed for the device
 *    by examining if there are any requests pending in the queue. If there
 *    are requests pending, the device can not be runtime suspended; otherwise,
 *    the queue's status will be updated to SUSPENDING and the driver can
 *    proceed to suspend the device.
 *
 *    For the not allowed case, we mark last busy for the device so that
 *    runtime PM core will try to autosuspend it some time later.
 *
 *    This function should be called near the start of the device's
 *    runtime_suspend callback.
 *
 * Return:
 *    0		- OK to runtime suspend the device
 *    -EBUSY	- Device should not be runtime suspended
 */
int blk_pre_runtime_suspend(struct request_queue *q)
{
	int ret = 0;

3824 3825 3826
	if (!q->dev)
		return ret;

L
Lin Ming 已提交
3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853
	spin_lock_irq(q->queue_lock);
	if (q->nr_pending) {
		ret = -EBUSY;
		pm_runtime_mark_last_busy(q->dev);
	} else {
		q->rpm_status = RPM_SUSPENDING;
	}
	spin_unlock_irq(q->queue_lock);
	return ret;
}
EXPORT_SYMBOL(blk_pre_runtime_suspend);

/**
 * blk_post_runtime_suspend - Post runtime suspend processing
 * @q: the queue of the device
 * @err: return value of the device's runtime_suspend function
 *
 * Description:
 *    Update the queue's runtime status according to the return value of the
 *    device's runtime suspend function and mark last busy for the device so
 *    that PM core will try to auto suspend the device at a later time.
 *
 *    This function should be called near the end of the device's
 *    runtime_suspend callback.
 */
void blk_post_runtime_suspend(struct request_queue *q, int err)
{
3854 3855 3856
	if (!q->dev)
		return;

L
Lin Ming 已提交
3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880
	spin_lock_irq(q->queue_lock);
	if (!err) {
		q->rpm_status = RPM_SUSPENDED;
	} else {
		q->rpm_status = RPM_ACTIVE;
		pm_runtime_mark_last_busy(q->dev);
	}
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL(blk_post_runtime_suspend);

/**
 * blk_pre_runtime_resume - Pre runtime resume processing
 * @q: the queue of the device
 *
 * Description:
 *    Update the queue's runtime status to RESUMING in preparation for the
 *    runtime resume of the device.
 *
 *    This function should be called near the start of the device's
 *    runtime_resume callback.
 */
void blk_pre_runtime_resume(struct request_queue *q)
{
3881 3882 3883
	if (!q->dev)
		return;

L
Lin Ming 已提交
3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905
	spin_lock_irq(q->queue_lock);
	q->rpm_status = RPM_RESUMING;
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL(blk_pre_runtime_resume);

/**
 * blk_post_runtime_resume - Post runtime resume processing
 * @q: the queue of the device
 * @err: return value of the device's runtime_resume function
 *
 * Description:
 *    Update the queue's runtime status according to the return value of the
 *    device's runtime_resume function. If it is successfully resumed, process
 *    the requests that are queued into the device's queue when it is resuming
 *    and then mark last busy and initiate autosuspend for it.
 *
 *    This function should be called near the end of the device's
 *    runtime_resume callback.
 */
void blk_post_runtime_resume(struct request_queue *q, int err)
{
3906 3907 3908
	if (!q->dev)
		return;

L
Lin Ming 已提交
3909 3910 3911 3912 3913
	spin_lock_irq(q->queue_lock);
	if (!err) {
		q->rpm_status = RPM_ACTIVE;
		__blk_run_queue(q);
		pm_runtime_mark_last_busy(q->dev);
3914
		pm_request_autosuspend(q->dev);
L
Lin Ming 已提交
3915 3916 3917 3918 3919 3920
	} else {
		q->rpm_status = RPM_SUSPENDED;
	}
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL(blk_post_runtime_resume);
3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944

/**
 * blk_set_runtime_active - Force runtime status of the queue to be active
 * @q: the queue of the device
 *
 * If the device is left runtime suspended during system suspend the resume
 * hook typically resumes the device and corrects runtime status
 * accordingly. However, that does not affect the queue runtime PM status
 * which is still "suspended". This prevents processing requests from the
 * queue.
 *
 * This function can be used in driver's resume hook to correct queue
 * runtime PM status and re-enable peeking requests from the queue. It
 * should be called before first request is added to the queue.
 */
void blk_set_runtime_active(struct request_queue *q)
{
	spin_lock_irq(q->queue_lock);
	q->rpm_status = RPM_ACTIVE;
	pm_runtime_mark_last_busy(q->dev);
	pm_request_autosuspend(q->dev);
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL(blk_set_runtime_active);
L
Lin Ming 已提交
3945 3946
#endif

L
Linus Torvalds 已提交
3947 3948
int __init blk_dev_init(void)
{
3949 3950
	BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS));
	BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
3951
			FIELD_SIZEOF(struct request, cmd_flags));
3952 3953
	BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
			FIELD_SIZEOF(struct bio, bi_opf));
3954

3955 3956
	/* used for unplugging and affects IO latency/throughput - HIGHPRI */
	kblockd_workqueue = alloc_workqueue("kblockd",
3957
					    WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
L
Linus Torvalds 已提交
3958 3959 3960 3961
	if (!kblockd_workqueue)
		panic("Failed to create kblockd\n");

	request_cachep = kmem_cache_create("blkdev_requests",
3962
			sizeof(struct request), 0, SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
3963

3964
	blk_requestq_cachep = kmem_cache_create("request_queue",
3965
			sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
3966

3967 3968 3969 3970
#ifdef CONFIG_DEBUG_FS
	blk_debugfs_root = debugfs_create_dir("block", NULL);
#endif

3971
	return 0;
L
Linus Torvalds 已提交
3972
}