blk-core.c 105.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5
/*
 * Copyright (C) 1991, 1992 Linus Torvalds
 * Copyright (C) 1994,      Karl Keyte: Added support for disk statistics
 * Elevator latency, (C) 2000  Andrea Arcangeli <andrea@suse.de> SuSE
 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
6 7
 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
 *	-  July2000
L
Linus Torvalds 已提交
8 9 10 11 12 13 14 15 16 17 18
 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
 */

/*
 * This handles all read/write requests to block devices
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
19
#include <linux/blk-mq.h>
L
Linus Torvalds 已提交
20 21 22 23 24 25 26 27 28
#include <linux/highmem.h>
#include <linux/mm.h>
#include <linux/kernel_stat.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/completion.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/writeback.h>
29
#include <linux/task_io_accounting_ops.h>
30
#include <linux/fault-inject.h>
31
#include <linux/list_sort.h>
T
Tejun Heo 已提交
32
#include <linux/delay.h>
33
#include <linux/ratelimit.h>
L
Lin Ming 已提交
34
#include <linux/pm_runtime.h>
35
#include <linux/blk-cgroup.h>
36
#include <linux/debugfs.h>
37
#include <linux/bpf.h>
38 39 40

#define CREATE_TRACE_POINTS
#include <trace/events/block.h>
L
Linus Torvalds 已提交
41

42
#include "blk.h"
43
#include "blk-mq.h"
44
#include "blk-mq-sched.h"
45
#include "blk-rq-qos.h"
46

47 48 49 50
#ifdef CONFIG_DEBUG_FS
struct dentry *blk_debugfs_root;
#endif

51
EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
52
EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
53
EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
K
Keith Busch 已提交
54
EXPORT_TRACEPOINT_SYMBOL_GPL(block_split);
N
NeilBrown 已提交
55
EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
56

57 58
DEFINE_IDA(blk_queue_ida);

L
Linus Torvalds 已提交
59 60 61
/*
 * For the allocated request tables
 */
62
struct kmem_cache *request_cachep;
L
Linus Torvalds 已提交
63 64 65 66

/*
 * For queue allocation
 */
67
struct kmem_cache *blk_requestq_cachep;
L
Linus Torvalds 已提交
68 69 70 71

/*
 * Controlling structure to kblockd
 */
72
static struct workqueue_struct *kblockd_workqueue;
L
Linus Torvalds 已提交
73

74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
/**
 * blk_queue_flag_set - atomically set a queue flag
 * @flag: flag to be set
 * @q: request queue
 */
void blk_queue_flag_set(unsigned int flag, struct request_queue *q)
{
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
	queue_flag_set(flag, q);
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_queue_flag_set);

/**
 * blk_queue_flag_clear - atomically clear a queue flag
 * @flag: flag to be cleared
 * @q: request queue
 */
void blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
{
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
	queue_flag_clear(flag, q);
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_queue_flag_clear);

/**
 * blk_queue_flag_test_and_set - atomically test and set a queue flag
 * @flag: flag to be set
 * @q: request queue
 *
 * Returns the previous value of @flag - 0 if the flag was not set and 1 if
 * the flag was already set.
 */
bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q)
{
	unsigned long flags;
	bool res;

	spin_lock_irqsave(q->queue_lock, flags);
	res = queue_flag_test_and_set(flag, q);
	spin_unlock_irqrestore(q->queue_lock, flags);

	return res;
}
EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_set);

/**
 * blk_queue_flag_test_and_clear - atomically test and clear a queue flag
 * @flag: flag to be cleared
 * @q: request queue
 *
 * Returns the previous value of @flag - 0 if the flag was not set and 1 if
 * the flag was set.
 */
bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q)
{
	unsigned long flags;
	bool res;

	spin_lock_irqsave(q->queue_lock, flags);
	res = queue_flag_test_and_clear(flag, q);
	spin_unlock_irqrestore(q->queue_lock, flags);

	return res;
}
EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_clear);

146 147 148 149 150
static void blk_clear_congested(struct request_list *rl, int sync)
{
#ifdef CONFIG_CGROUP_WRITEBACK
	clear_wb_congested(rl->blkg->wb_congested, sync);
#else
151 152 153 154 155
	/*
	 * If !CGROUP_WRITEBACK, all blkg's map to bdi->wb and we shouldn't
	 * flip its congestion state for events on other blkcgs.
	 */
	if (rl == &rl->q->root_rl)
156
		clear_wb_congested(rl->q->backing_dev_info->wb.congested, sync);
157 158 159 160 161 162 163 164
#endif
}

static void blk_set_congested(struct request_list *rl, int sync)
{
#ifdef CONFIG_CGROUP_WRITEBACK
	set_wb_congested(rl->blkg->wb_congested, sync);
#else
165 166
	/* see blk_clear_congested() */
	if (rl == &rl->q->root_rl)
167
		set_wb_congested(rl->q->backing_dev_info->wb.congested, sync);
168 169 170
#endif
}

171
void blk_queue_congestion_threshold(struct request_queue *q)
L
Linus Torvalds 已提交
172 173 174 175 176 177 178 179 180 181 182 183 184 185
{
	int nr;

	nr = q->nr_requests - (q->nr_requests / 8) + 1;
	if (nr > q->nr_requests)
		nr = q->nr_requests;
	q->nr_congestion_on = nr;

	nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
	if (nr < 1)
		nr = 1;
	q->nr_congestion_off = nr;
}

186
void blk_rq_init(struct request_queue *q, struct request *rq)
L
Linus Torvalds 已提交
187
{
188 189
	memset(rq, 0, sizeof(*rq));

L
Linus Torvalds 已提交
190
	INIT_LIST_HEAD(&rq->queuelist);
J
Jens Axboe 已提交
191
	INIT_LIST_HEAD(&rq->timeout_list);
192
	rq->cpu = -1;
J
Jens Axboe 已提交
193
	rq->q = q;
194
	rq->__sector = (sector_t) -1;
195 196
	INIT_HLIST_NODE(&rq->hash);
	RB_CLEAR_NODE(&rq->rb_node);
J
Jens Axboe 已提交
197
	rq->tag = -1;
198
	rq->internal_tag = -1;
199
	rq->start_time_ns = ktime_get_ns();
200
	rq->part = NULL;
L
Linus Torvalds 已提交
201
}
202
EXPORT_SYMBOL(blk_rq_init);
L
Linus Torvalds 已提交
203

204 205 206 207 208 209 210 211 212 213 214 215 216 217
static const struct {
	int		errno;
	const char	*name;
} blk_errors[] = {
	[BLK_STS_OK]		= { 0,		"" },
	[BLK_STS_NOTSUPP]	= { -EOPNOTSUPP, "operation not supported" },
	[BLK_STS_TIMEOUT]	= { -ETIMEDOUT,	"timeout" },
	[BLK_STS_NOSPC]		= { -ENOSPC,	"critical space allocation" },
	[BLK_STS_TRANSPORT]	= { -ENOLINK,	"recoverable transport" },
	[BLK_STS_TARGET]	= { -EREMOTEIO,	"critical target" },
	[BLK_STS_NEXUS]		= { -EBADE,	"critical nexus" },
	[BLK_STS_MEDIUM]	= { -ENODATA,	"critical medium" },
	[BLK_STS_PROTECTION]	= { -EILSEQ,	"protection" },
	[BLK_STS_RESOURCE]	= { -ENOMEM,	"kernel resource" },
218
	[BLK_STS_DEV_RESOURCE]	= { -EBUSY,	"device resource" },
219
	[BLK_STS_AGAIN]		= { -EAGAIN,	"nonblocking retry" },
220

221 222 223
	/* device mapper special case, should not leak out: */
	[BLK_STS_DM_REQUEUE]	= { -EREMCHG, "dm internal retry" },

224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
	/* everything else not covered above: */
	[BLK_STS_IOERR]		= { -EIO,	"I/O" },
};

blk_status_t errno_to_blk_status(int errno)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {
		if (blk_errors[i].errno == errno)
			return (__force blk_status_t)i;
	}

	return BLK_STS_IOERR;
}
EXPORT_SYMBOL_GPL(errno_to_blk_status);

int blk_status_to_errno(blk_status_t status)
{
	int idx = (__force int)status;

245
	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
246 247 248 249 250 251 252 253 254
		return -EIO;
	return blk_errors[idx].errno;
}
EXPORT_SYMBOL_GPL(blk_status_to_errno);

static void print_req_error(struct request *req, blk_status_t status)
{
	int idx = (__force int)status;

255
	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
256 257 258 259 260 261 262 263
		return;

	printk_ratelimited(KERN_ERR "%s: %s error, dev %s, sector %llu\n",
			   __func__, blk_errors[idx].name, req->rq_disk ?
			   req->rq_disk->disk_name : "?",
			   (unsigned long long)blk_rq_pos(req));
}

N
NeilBrown 已提交
264
static void req_bio_endio(struct request *rq, struct bio *bio,
265
			  unsigned int nbytes, blk_status_t error)
L
Linus Torvalds 已提交
266
{
267
	if (error)
268
		bio->bi_status = error;
269

270
	if (unlikely(rq->rq_flags & RQF_QUIET))
271
		bio_set_flag(bio, BIO_QUIET);
272

273
	bio_advance(bio, nbytes);
274

T
Tejun Heo 已提交
275
	/* don't actually finish bio if it's part of flush sequence */
276
	if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
277
		bio_endio(bio);
L
Linus Torvalds 已提交
278 279 280 281
}

void blk_dump_rq_flags(struct request *rq, char *msg)
{
282 283
	printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
		rq->rq_disk ? rq->rq_disk->disk_name : "?",
J
Jens Axboe 已提交
284
		(unsigned long long) rq->cmd_flags);
L
Linus Torvalds 已提交
285

286 287 288
	printk(KERN_INFO "  sector %llu, nr/cnr %u/%u\n",
	       (unsigned long long)blk_rq_pos(rq),
	       blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
289 290
	printk(KERN_INFO "  bio %p, biotail %p, len %u\n",
	       rq->bio, rq->biotail, blk_rq_bytes(rq));
L
Linus Torvalds 已提交
291 292 293
}
EXPORT_SYMBOL(blk_dump_rq_flags);

294
static void blk_delay_work(struct work_struct *work)
L
Linus Torvalds 已提交
295
{
296
	struct request_queue *q;
L
Linus Torvalds 已提交
297

298 299
	q = container_of(work, struct request_queue, delay_work.work);
	spin_lock_irq(q->queue_lock);
300
	__blk_run_queue(q);
301
	spin_unlock_irq(q->queue_lock);
L
Linus Torvalds 已提交
302 303 304
}

/**
305 306 307
 * blk_delay_queue - restart queueing after defined interval
 * @q:		The &struct request_queue in question
 * @msecs:	Delay in msecs
L
Linus Torvalds 已提交
308 309
 *
 * Description:
310 311
 *   Sometimes queueing needs to be postponed for a little while, to allow
 *   resources to come back. This function will make sure that queueing is
312
 *   restarted around the specified time.
313 314
 */
void blk_delay_queue(struct request_queue *q, unsigned long msecs)
315
{
316
	lockdep_assert_held(q->queue_lock);
317
	WARN_ON_ONCE(q->mq_ops);
318

319 320 321
	if (likely(!blk_queue_dead(q)))
		queue_delayed_work(kblockd_workqueue, &q->delay_work,
				   msecs_to_jiffies(msecs));
322
}
323
EXPORT_SYMBOL(blk_delay_queue);
324

J
Jens Axboe 已提交
325 326 327 328 329 330 331 332 333 334 335
/**
 * blk_start_queue_async - asynchronously restart a previously stopped queue
 * @q:    The &struct request_queue in question
 *
 * Description:
 *   blk_start_queue_async() will clear the stop flag on the queue, and
 *   ensure that the request_fn for the queue is run from an async
 *   context.
 **/
void blk_start_queue_async(struct request_queue *q)
{
336
	lockdep_assert_held(q->queue_lock);
337
	WARN_ON_ONCE(q->mq_ops);
338

J
Jens Axboe 已提交
339 340 341 342 343
	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
	blk_run_queue_async(q);
}
EXPORT_SYMBOL(blk_start_queue_async);

L
Linus Torvalds 已提交
344 345
/**
 * blk_start_queue - restart a previously stopped queue
346
 * @q:    The &struct request_queue in question
L
Linus Torvalds 已提交
347 348 349 350
 *
 * Description:
 *   blk_start_queue() will clear the stop flag on the queue, and call
 *   the request_fn for the queue if it was in a stopped state when
351
 *   entered. Also see blk_stop_queue().
L
Linus Torvalds 已提交
352
 **/
353
void blk_start_queue(struct request_queue *q)
L
Linus Torvalds 已提交
354
{
355
	lockdep_assert_held(q->queue_lock);
356
	WARN_ON_ONCE(q->mq_ops);
357

N
Nick Piggin 已提交
358
	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
359
	__blk_run_queue(q);
L
Linus Torvalds 已提交
360 361 362 363 364
}
EXPORT_SYMBOL(blk_start_queue);

/**
 * blk_stop_queue - stop a queue
365
 * @q:    The &struct request_queue in question
L
Linus Torvalds 已提交
366 367 368 369 370 371 372 373 374
 *
 * Description:
 *   The Linux block layer assumes that a block driver will consume all
 *   entries on the request queue when the request_fn strategy is called.
 *   Often this will not happen, because of hardware limitations (queue
 *   depth settings). If a device driver gets a 'queue full' response,
 *   or if it simply chooses not to queue more I/O at one point, it can
 *   call this function to prevent the request_fn from being called until
 *   the driver has signalled it's ready to go again. This happens by calling
375
 *   blk_start_queue() to restart queue operations.
L
Linus Torvalds 已提交
376
 **/
377
void blk_stop_queue(struct request_queue *q)
L
Linus Torvalds 已提交
378
{
379
	lockdep_assert_held(q->queue_lock);
380
	WARN_ON_ONCE(q->mq_ops);
381

382
	cancel_delayed_work(&q->delay_work);
N
Nick Piggin 已提交
383
	queue_flag_set(QUEUE_FLAG_STOPPED, q);
L
Linus Torvalds 已提交
384 385 386 387 388 389 390 391 392 393 394 395
}
EXPORT_SYMBOL(blk_stop_queue);

/**
 * blk_sync_queue - cancel any pending callbacks on a queue
 * @q: the queue
 *
 * Description:
 *     The block layer may perform asynchronous callback activity
 *     on a queue, such as calling the unplug function after a timeout.
 *     A block device may call blk_sync_queue to ensure that any
 *     such activity is cancelled, thus allowing it to release resources
396
 *     that the callbacks might use. The caller must already have made sure
L
Linus Torvalds 已提交
397 398 399
 *     that its ->make_request_fn will not re-add plugging prior to calling
 *     this function.
 *
400
 *     This function does not cancel any asynchronous activity arising
401
 *     out of elevator or throttling code. That would require elevator_exit()
402
 *     and blkcg_exit_queue() to be called with queue lock initialized.
403
 *
L
Linus Torvalds 已提交
404 405 406
 */
void blk_sync_queue(struct request_queue *q)
{
407
	del_timer_sync(&q->timeout);
408
	cancel_work_sync(&q->timeout_work);
409 410 411 412 413

	if (q->mq_ops) {
		struct blk_mq_hw_ctx *hctx;
		int i;

414
		queue_for_each_hw_ctx(q, hctx, i)
415
			cancel_delayed_work_sync(&hctx->run_work);
416 417 418
	} else {
		cancel_delayed_work_sync(&q->delay_work);
	}
L
Linus Torvalds 已提交
419 420 421
}
EXPORT_SYMBOL(blk_sync_queue);

422 423 424 425 426 427 428 429 430
/**
 * blk_set_preempt_only - set QUEUE_FLAG_PREEMPT_ONLY
 * @q: request queue pointer
 *
 * Returns the previous value of the PREEMPT_ONLY flag - 0 if the flag was not
 * set and 1 if the flag was already set.
 */
int blk_set_preempt_only(struct request_queue *q)
{
431
	return blk_queue_flag_test_and_set(QUEUE_FLAG_PREEMPT_ONLY, q);
432 433 434 435 436
}
EXPORT_SYMBOL_GPL(blk_set_preempt_only);

void blk_clear_preempt_only(struct request_queue *q)
{
437
	blk_queue_flag_clear(QUEUE_FLAG_PREEMPT_ONLY, q);
438
	wake_up_all(&q->mq_freeze_wq);
439 440 441
}
EXPORT_SYMBOL_GPL(blk_clear_preempt_only);

442 443 444 445 446 447 448 449 450 451 452 453 454
/**
 * __blk_run_queue_uncond - run a queue whether or not it has been stopped
 * @q:	The queue to run
 *
 * Description:
 *    Invoke request handling on a queue if there are any pending requests.
 *    May be used to restart request handling after a request has completed.
 *    This variant runs the queue whether or not the queue has been
 *    stopped. Must be called with the queue lock held and interrupts
 *    disabled. See also @blk_run_queue.
 */
inline void __blk_run_queue_uncond(struct request_queue *q)
{
455
	lockdep_assert_held(q->queue_lock);
456
	WARN_ON_ONCE(q->mq_ops);
457

458 459 460
	if (unlikely(blk_queue_dead(q)))
		return;

461 462 463 464 465 466 467 468
	/*
	 * Some request_fn implementations, e.g. scsi_request_fn(), unlock
	 * the queue lock internally. As a result multiple threads may be
	 * running such a request function concurrently. Keep track of the
	 * number of active request_fn invocations such that blk_drain_queue()
	 * can wait until all these request_fn calls have finished.
	 */
	q->request_fn_active++;
469
	q->request_fn(q);
470
	q->request_fn_active--;
471
}
472
EXPORT_SYMBOL_GPL(__blk_run_queue_uncond);
473

L
Linus Torvalds 已提交
474
/**
475
 * __blk_run_queue - run a single device queue
L
Linus Torvalds 已提交
476
 * @q:	The queue to run
477 478
 *
 * Description:
479
 *    See @blk_run_queue.
L
Linus Torvalds 已提交
480
 */
481
void __blk_run_queue(struct request_queue *q)
L
Linus Torvalds 已提交
482
{
483
	lockdep_assert_held(q->queue_lock);
484
	WARN_ON_ONCE(q->mq_ops);
485

486 487 488
	if (unlikely(blk_queue_stopped(q)))
		return;

489
	__blk_run_queue_uncond(q);
N
Nick Piggin 已提交
490 491
}
EXPORT_SYMBOL(__blk_run_queue);
J
Jens Axboe 已提交
492

493 494 495 496 497 498
/**
 * blk_run_queue_async - run a single device queue in workqueue context
 * @q:	The queue to run
 *
 * Description:
 *    Tells kblockd to perform the equivalent of @blk_run_queue on behalf
499 500 501 502 503 504
 *    of us.
 *
 * Note:
 *    Since it is not allowed to run q->delay_work after blk_cleanup_queue()
 *    has canceled q->delay_work, callers must hold the queue lock to avoid
 *    race conditions between blk_cleanup_queue() and blk_run_queue_async().
505 506 507
 */
void blk_run_queue_async(struct request_queue *q)
{
508
	lockdep_assert_held(q->queue_lock);
509
	WARN_ON_ONCE(q->mq_ops);
510

511
	if (likely(!blk_queue_stopped(q) && !blk_queue_dead(q)))
512
		mod_delayed_work(kblockd_workqueue, &q->delay_work, 0);
513
}
514
EXPORT_SYMBOL(blk_run_queue_async);
515

N
Nick Piggin 已提交
516 517 518
/**
 * blk_run_queue - run a single device queue
 * @q: The queue to run
519 520 521
 *
 * Description:
 *    Invoke request handling on this queue, if it has pending work to do.
T
Tejun Heo 已提交
522
 *    May be used to restart queueing when a request has completed.
N
Nick Piggin 已提交
523 524 525 526 527
 */
void blk_run_queue(struct request_queue *q)
{
	unsigned long flags;

528 529
	WARN_ON_ONCE(q->mq_ops);

N
Nick Piggin 已提交
530
	spin_lock_irqsave(q->queue_lock, flags);
531
	__blk_run_queue(q);
L
Linus Torvalds 已提交
532 533 534 535
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_run_queue);

536
void blk_put_queue(struct request_queue *q)
537 538 539
{
	kobject_put(&q->kobj);
}
J
Jens Axboe 已提交
540
EXPORT_SYMBOL(blk_put_queue);
541

T
Tejun Heo 已提交
542
/**
543
 * __blk_drain_queue - drain requests from request_queue
T
Tejun Heo 已提交
544
 * @q: queue to drain
545
 * @drain_all: whether to drain all requests or only the ones w/ ELVPRIV
T
Tejun Heo 已提交
546
 *
547 548 549
 * Drain requests from @q.  If @drain_all is set, all requests are drained.
 * If not, only ELVPRIV requests are drained.  The caller is responsible
 * for ensuring that no new requests which need to be drained are queued.
T
Tejun Heo 已提交
550
 */
551 552 553
static void __blk_drain_queue(struct request_queue *q, bool drain_all)
	__releases(q->queue_lock)
	__acquires(q->queue_lock)
T
Tejun Heo 已提交
554
{
555 556
	int i;

557
	lockdep_assert_held(q->queue_lock);
558
	WARN_ON_ONCE(q->mq_ops);
559

T
Tejun Heo 已提交
560
	while (true) {
561
		bool drain = false;
T
Tejun Heo 已提交
562

563 564 565 566 567 568 569
		/*
		 * The caller might be trying to drain @q before its
		 * elevator is initialized.
		 */
		if (q->elevator)
			elv_drain_elevator(q);

570
		blkcg_drain_queue(q);
T
Tejun Heo 已提交
571

572 573
		/*
		 * This function might be called on a queue which failed
574 575 576 577
		 * driver init after queue creation or is not yet fully
		 * active yet.  Some drivers (e.g. fd and loop) get unhappy
		 * in such cases.  Kick queue iff dispatch queue has
		 * something on it and @q has request_fn set.
578
		 */
579
		if (!list_empty(&q->queue_head) && q->request_fn)
580
			__blk_run_queue(q);
581

582
		drain |= q->nr_rqs_elvpriv;
583
		drain |= q->request_fn_active;
584 585 586 587 588 589 590

		/*
		 * Unfortunately, requests are queued at and tracked from
		 * multiple places and there's no single counter which can
		 * be drained.  Check all the queues and counters.
		 */
		if (drain_all) {
591
			struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);
592 593
			drain |= !list_empty(&q->queue_head);
			for (i = 0; i < 2; i++) {
594
				drain |= q->nr_rqs[i];
595
				drain |= q->in_flight[i];
596 597
				if (fq)
				    drain |= !list_empty(&fq->flush_queue[i]);
598 599
			}
		}
T
Tejun Heo 已提交
600

601
		if (!drain)
T
Tejun Heo 已提交
602
			break;
603 604 605

		spin_unlock_irq(q->queue_lock);

T
Tejun Heo 已提交
606
		msleep(10);
607 608

		spin_lock_irq(q->queue_lock);
T
Tejun Heo 已提交
609
	}
610 611 612 613 614 615 616

	/*
	 * With queue marked dead, any woken up waiter will fail the
	 * allocation path, so the wakeup chaining is lost and we're
	 * left with hung waiters. We need to wake up those waiters.
	 */
	if (q->request_fn) {
617 618 619 620 621
		struct request_list *rl;

		blk_queue_for_each_rl(rl, q)
			for (i = 0; i < ARRAY_SIZE(rl->wait); i++)
				wake_up_all(&rl->wait[i]);
622
	}
T
Tejun Heo 已提交
623 624
}

625 626 627 628 629 630 631
void blk_drain_queue(struct request_queue *q)
{
	spin_lock_irq(q->queue_lock);
	__blk_drain_queue(q, true);
	spin_unlock_irq(q->queue_lock);
}

632 633 634 635 636 637
/**
 * blk_queue_bypass_start - enter queue bypass mode
 * @q: queue of interest
 *
 * In bypass mode, only the dispatch FIFO queue of @q is used.  This
 * function makes @q enter bypass mode and drains all requests which were
638
 * throttled or issued before.  On return, it's guaranteed that no request
639 640
 * is being throttled or has ELVPRIV set and blk_queue_bypass() %true
 * inside queue or RCU read lock.
641 642 643
 */
void blk_queue_bypass_start(struct request_queue *q)
{
644 645
	WARN_ON_ONCE(q->mq_ops);

646
	spin_lock_irq(q->queue_lock);
647
	q->bypass_depth++;
648 649 650
	queue_flag_set(QUEUE_FLAG_BYPASS, q);
	spin_unlock_irq(q->queue_lock);

651 652 653 654 655 656
	/*
	 * Queues start drained.  Skip actual draining till init is
	 * complete.  This avoids lenghty delays during queue init which
	 * can happen many times during boot.
	 */
	if (blk_queue_init_done(q)) {
657 658 659 660
		spin_lock_irq(q->queue_lock);
		__blk_drain_queue(q, false);
		spin_unlock_irq(q->queue_lock);

661 662 663
		/* ensure blk_queue_bypass() is %true inside RCU read lock */
		synchronize_rcu();
	}
664 665 666 667 668 669 670 671
}
EXPORT_SYMBOL_GPL(blk_queue_bypass_start);

/**
 * blk_queue_bypass_end - leave queue bypass mode
 * @q: queue of interest
 *
 * Leave bypass mode and restore the normal queueing behavior.
672 673 674
 *
 * Note: although blk_queue_bypass_start() is only called for blk-sq queues,
 * this function is called for both blk-sq and blk-mq queues.
675 676 677 678 679 680 681 682 683 684 685
 */
void blk_queue_bypass_end(struct request_queue *q)
{
	spin_lock_irq(q->queue_lock);
	if (!--q->bypass_depth)
		queue_flag_clear(QUEUE_FLAG_BYPASS, q);
	WARN_ON_ONCE(q->bypass_depth < 0);
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL_GPL(blk_queue_bypass_end);

686 687
void blk_set_queue_dying(struct request_queue *q)
{
688
	blk_queue_flag_set(QUEUE_FLAG_DYING, q);
689

690 691 692 693 694 695 696
	/*
	 * When queue DYING flag is set, we need to block new req
	 * entering queue, so we call blk_freeze_queue_start() to
	 * prevent I/O from crossing blk_queue_enter().
	 */
	blk_freeze_queue_start(q);

697 698 699 700 701
	if (q->mq_ops)
		blk_mq_wake_waiters(q);
	else {
		struct request_list *rl;

702
		spin_lock_irq(q->queue_lock);
703 704
		blk_queue_for_each_rl(rl, q) {
			if (rl->rq_pool) {
705 706
				wake_up_all(&rl->wait[BLK_RW_SYNC]);
				wake_up_all(&rl->wait[BLK_RW_ASYNC]);
707 708
			}
		}
709
		spin_unlock_irq(q->queue_lock);
710
	}
711 712 713

	/* Make blk_queue_enter() reexamine the DYING flag. */
	wake_up_all(&q->mq_freeze_wq);
714 715 716
}
EXPORT_SYMBOL_GPL(blk_set_queue_dying);

717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745
/* Unconfigure the I/O scheduler and dissociate from the cgroup controller. */
void blk_exit_queue(struct request_queue *q)
{
	/*
	 * Since the I/O scheduler exit code may access cgroup information,
	 * perform I/O scheduler exit before disassociating from the block
	 * cgroup controller.
	 */
	if (q->elevator) {
		ioc_clear_queue(q);
		elevator_exit(q, q->elevator);
		q->elevator = NULL;
	}

	/*
	 * Remove all references to @q from the block cgroup controller before
	 * restoring @q->queue_lock to avoid that restoring this pointer causes
	 * e.g. blkcg_print_blkgs() to crash.
	 */
	blkcg_exit_queue(q);

	/*
	 * Since the cgroup code may dereference the @q->backing_dev_info
	 * pointer, only decrease its reference count after having removed the
	 * association with the block cgroup controller.
	 */
	bdi_put(q->backing_dev_info);
}

746 747 748 749
/**
 * blk_cleanup_queue - shutdown a request queue
 * @q: request queue to shutdown
 *
750 751
 * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and
 * put it.  All future requests will be failed immediately with -ENODEV.
752
 */
753
void blk_cleanup_queue(struct request_queue *q)
754
{
755
	spinlock_t *lock = q->queue_lock;
756

B
Bart Van Assche 已提交
757
	/* mark @q DYING, no new request or merges will be allowed afterwards */
758
	mutex_lock(&q->sysfs_lock);
759
	blk_set_queue_dying(q);
760
	spin_lock_irq(lock);
761

762
	/*
B
Bart Van Assche 已提交
763
	 * A dying queue is permanently in bypass mode till released.  Note
764 765 766 767 768 769 770
	 * that, unlike blk_queue_bypass_start(), we aren't performing
	 * synchronize_rcu() after entering bypass mode to avoid the delay
	 * as some drivers create and destroy a lot of queues while
	 * probing.  This is still safe because blk_release_queue() will be
	 * called only after the queue refcnt drops to zero and nothing,
	 * RCU or not, would be traversing the queue by then.
	 */
771 772 773
	q->bypass_depth++;
	queue_flag_set(QUEUE_FLAG_BYPASS, q);

774 775
	queue_flag_set(QUEUE_FLAG_NOMERGES, q);
	queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
B
Bart Van Assche 已提交
776
	queue_flag_set(QUEUE_FLAG_DYING, q);
777 778 779
	spin_unlock_irq(lock);
	mutex_unlock(&q->sysfs_lock);

780 781 782 783
	/*
	 * Drain all requests queued before DYING marking. Set DEAD flag to
	 * prevent that q->request_fn() gets invoked after draining finished.
	 */
784
	blk_freeze_queue(q);
785
	spin_lock_irq(lock);
786
	queue_flag_set(QUEUE_FLAG_DEAD, q);
787
	spin_unlock_irq(lock);
788

789 790 791 792
	/*
	 * make sure all in-progress dispatch are completed because
	 * blk_freeze_queue() can only complete all requests, and
	 * dispatch may still be in-progress since we dispatch requests
793 794
	 * from more than one contexts.
	 *
795 796
	 * We rely on driver to deal with the race in case that queue
	 * initialization isn't done.
797
	 */
798
	if (q->mq_ops && blk_queue_init_done(q))
799 800
		blk_mq_quiesce_queue(q);

801 802 803
	/* for synchronous bio-based driver finish in-flight integrity i/o */
	blk_flush_integrity();

804
	/* @q won't process any more request, flush async actions */
805
	del_timer_sync(&q->backing_dev_info->laptop_mode_wb_timer);
806 807
	blk_sync_queue(q);

808 809 810 811 812 813
	/*
	 * I/O scheduler exit is only safe after the sysfs scheduler attribute
	 * has been removed.
	 */
	WARN_ON_ONCE(q->kobj.state_in_sysfs);

814
	blk_exit_queue(q);
815

B
Bart Van Assche 已提交
816 817
	if (q->mq_ops)
		blk_mq_free_queue(q);
818
	percpu_ref_exit(&q->q_usage_counter);
B
Bart Van Assche 已提交
819

820 821 822 823 824
	spin_lock_irq(lock);
	if (q->queue_lock != &q->__queue_lock)
		q->queue_lock = &q->__queue_lock;
	spin_unlock_irq(lock);

825
	/* @q is and will stay empty, shutdown and put */
826 827
	blk_put_queue(q);
}
L
Linus Torvalds 已提交
828 829
EXPORT_SYMBOL(blk_cleanup_queue);

830
/* Allocate memory local to the request queue */
831
static void *alloc_request_simple(gfp_t gfp_mask, void *data)
832
{
833 834 835
	struct request_queue *q = data;

	return kmem_cache_alloc_node(request_cachep, gfp_mask, q->node);
836 837
}

838
static void free_request_simple(void *element, void *data)
839 840 841 842
{
	kmem_cache_free(request_cachep, element);
}

843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865
static void *alloc_request_size(gfp_t gfp_mask, void *data)
{
	struct request_queue *q = data;
	struct request *rq;

	rq = kmalloc_node(sizeof(struct request) + q->cmd_size, gfp_mask,
			q->node);
	if (rq && q->init_rq_fn && q->init_rq_fn(q, rq, gfp_mask) < 0) {
		kfree(rq);
		rq = NULL;
	}
	return rq;
}

static void free_request_size(void *element, void *data)
{
	struct request_queue *q = data;

	if (q->exit_rq_fn)
		q->exit_rq_fn(q, element);
	kfree(element);
}

866 867
int blk_init_rl(struct request_list *rl, struct request_queue *q,
		gfp_t gfp_mask)
L
Linus Torvalds 已提交
868
{
S
Shaohua Li 已提交
869
	if (unlikely(rl->rq_pool) || q->mq_ops)
870 871
		return 0;

872
	rl->q = q;
873 874 875 876
	rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
	rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
	init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
	init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
L
Linus Torvalds 已提交
877

878 879 880 881 882 883 884 885 886
	if (q->cmd_size) {
		rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ,
				alloc_request_size, free_request_size,
				q, gfp_mask, q->node);
	} else {
		rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ,
				alloc_request_simple, free_request_simple,
				q, gfp_mask, q->node);
	}
L
Linus Torvalds 已提交
887 888 889
	if (!rl->rq_pool)
		return -ENOMEM;

890 891 892
	if (rl != &q->root_rl)
		WARN_ON_ONCE(!blk_get_queue(q));

L
Linus Torvalds 已提交
893 894 895
	return 0;
}

896
void blk_exit_rl(struct request_queue *q, struct request_list *rl)
897
{
898
	if (rl->rq_pool) {
899
		mempool_destroy(rl->rq_pool);
900 901 902
		if (rl != &q->root_rl)
			blk_put_queue(q);
	}
903 904
}

905
struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
L
Linus Torvalds 已提交
906
{
907
	return blk_alloc_queue_node(gfp_mask, NUMA_NO_NODE, NULL);
908 909
}
EXPORT_SYMBOL(blk_alloc_queue);
L
Linus Torvalds 已提交
910

911 912 913 914 915
/**
 * blk_queue_enter() - try to increase q->q_usage_counter
 * @q: request queue pointer
 * @flags: BLK_MQ_REQ_NOWAIT and/or BLK_MQ_REQ_PREEMPT
 */
916
int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags)
917
{
918 919
	const bool preempt = flags & BLK_MQ_REQ_PREEMPT;

920
	while (true) {
921
		bool success = false;
922

923
		rcu_read_lock();
924 925 926 927 928 929 930 931 932 933 934 935
		if (percpu_ref_tryget_live(&q->q_usage_counter)) {
			/*
			 * The code that sets the PREEMPT_ONLY flag is
			 * responsible for ensuring that that flag is globally
			 * visible before the queue is unfrozen.
			 */
			if (preempt || !blk_queue_preempt_only(q)) {
				success = true;
			} else {
				percpu_ref_put(&q->q_usage_counter);
			}
		}
936
		rcu_read_unlock();
937 938

		if (success)
939 940
			return 0;

941
		if (flags & BLK_MQ_REQ_NOWAIT)
942 943
			return -EBUSY;

944
		/*
945
		 * read pair of barrier in blk_freeze_queue_start(),
946
		 * we need to order reading __PERCPU_REF_DEAD flag of
947 948 949
		 * .q_usage_counter and reading .mq_freeze_depth or
		 * queue dying flag, otherwise the following wait may
		 * never return if the two reads are reordered.
950 951 952
		 */
		smp_rmb();

953 954 955 956
		wait_event(q->mq_freeze_wq,
			   (atomic_read(&q->mq_freeze_depth) == 0 &&
			    (preempt || !blk_queue_preempt_only(q))) ||
			   blk_queue_dying(q));
957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974
		if (blk_queue_dying(q))
			return -ENODEV;
	}
}

void blk_queue_exit(struct request_queue *q)
{
	percpu_ref_put(&q->q_usage_counter);
}

static void blk_queue_usage_counter_release(struct percpu_ref *ref)
{
	struct request_queue *q =
		container_of(ref, struct request_queue, q_usage_counter);

	wake_up_all(&q->mq_freeze_wq);
}

975
static void blk_rq_timed_out_timer(struct timer_list *t)
976
{
977
	struct request_queue *q = from_timer(q, t, timeout);
978 979 980 981

	kblockd_schedule_work(&q->timeout_work);
}

982 983 984 985
static void blk_timeout_work_dummy(struct work_struct *work)
{
}

986 987 988 989 990 991 992 993 994 995 996 997 998
/**
 * blk_alloc_queue_node - allocate a request queue
 * @gfp_mask: memory allocation flags
 * @node_id: NUMA node to allocate memory from
 * @lock: For legacy queues, pointer to a spinlock that will be used to e.g.
 *        serialize calls to the legacy .request_fn() callback. Ignored for
 *	  blk-mq request queues.
 *
 * Note: pass the queue lock as the third argument to this function instead of
 * setting the queue lock pointer explicitly to avoid triggering a sporadic
 * crash in the blkcg code. This function namely calls blkcg_init_queue() and
 * the queue lock pointer must be set before blkcg_init_queue() is called.
 */
999 1000
struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
					   spinlock_t *lock)
1001
{
1002
	struct request_queue *q;
1003
	int ret;
1004

1005
	q = kmem_cache_alloc_node(blk_requestq_cachep,
1006
				gfp_mask | __GFP_ZERO, node_id);
L
Linus Torvalds 已提交
1007 1008 1009
	if (!q)
		return NULL;

1010 1011 1012 1013 1014
	INIT_LIST_HEAD(&q->queue_head);
	q->last_merge = NULL;
	q->end_sector = 0;
	q->boundary_rq = NULL;

1015
	q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
1016
	if (q->id < 0)
1017
		goto fail_q;
1018

1019 1020
	ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
	if (ret)
1021 1022
		goto fail_id;

1023 1024 1025 1026
	q->backing_dev_info = bdi_alloc_node(gfp_mask, node_id);
	if (!q->backing_dev_info)
		goto fail_split;

1027 1028 1029 1030
	q->stats = blk_alloc_queue_stats();
	if (!q->stats)
		goto fail_stats;

1031
	q->backing_dev_info->ra_pages =
1032
			(VM_MAX_READAHEAD * 1024) / PAGE_SIZE;
1033 1034
	q->backing_dev_info->capabilities = BDI_CAP_CGROUP_WRITEBACK;
	q->backing_dev_info->name = "block";
1035
	q->node = node_id;
1036

1037 1038 1039
	timer_setup(&q->backing_dev_info->laptop_mode_wb_timer,
		    laptop_mode_timer_fn, 0);
	timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);
1040
	INIT_WORK(&q->timeout_work, blk_timeout_work_dummy);
J
Jens Axboe 已提交
1041
	INIT_LIST_HEAD(&q->timeout_list);
1042
	INIT_LIST_HEAD(&q->icq_list);
1043
#ifdef CONFIG_BLK_CGROUP
1044
	INIT_LIST_HEAD(&q->blkg_list);
1045
#endif
1046
	INIT_DELAYED_WORK(&q->delay_work, blk_delay_work);
1047

1048
	kobject_init(&q->kobj, &blk_queue_ktype);
L
Linus Torvalds 已提交
1049

1050 1051 1052
#ifdef CONFIG_BLK_DEV_IO_TRACE
	mutex_init(&q->blk_trace_mutex);
#endif
1053
	mutex_init(&q->sysfs_lock);
1054
	spin_lock_init(&q->__queue_lock);
1055

1056 1057
	if (!q->mq_ops)
		q->queue_lock = lock ? : &q->__queue_lock;
1058

1059 1060 1061
	/*
	 * A queue starts its life with bypass turned on to avoid
	 * unnecessary bypass on/off overhead and nasty surprises during
1062 1063
	 * init.  The initial bypass will be finished when the queue is
	 * registered by blk_register_queue().
1064 1065
	 */
	q->bypass_depth = 1;
1066
	queue_flag_set_unlocked(QUEUE_FLAG_BYPASS, q);
1067

1068 1069
	init_waitqueue_head(&q->mq_freeze_wq);

1070 1071 1072 1073 1074 1075 1076
	/*
	 * Init percpu_ref in atomic mode so that it's faster to shutdown.
	 * See blk_register_queue() for details.
	 */
	if (percpu_ref_init(&q->q_usage_counter,
				blk_queue_usage_counter_release,
				PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
1077
		goto fail_bdi;
1078

1079 1080 1081
	if (blkcg_init_queue(q))
		goto fail_ref;

L
Linus Torvalds 已提交
1082
	return q;
1083

1084 1085
fail_ref:
	percpu_ref_exit(&q->q_usage_counter);
1086
fail_bdi:
1087 1088
	blk_free_queue_stats(q->stats);
fail_stats:
1089
	bdi_put(q->backing_dev_info);
1090
fail_split:
1091
	bioset_exit(&q->bio_split);
1092 1093 1094 1095 1096
fail_id:
	ida_simple_remove(&blk_queue_ida, q->id);
fail_q:
	kmem_cache_free(blk_requestq_cachep, q);
	return NULL;
L
Linus Torvalds 已提交
1097
}
1098
EXPORT_SYMBOL(blk_alloc_queue_node);
L
Linus Torvalds 已提交
1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121

/**
 * blk_init_queue  - prepare a request queue for use with a block device
 * @rfn:  The function to be called to process requests that have been
 *        placed on the queue.
 * @lock: Request queue spin lock
 *
 * Description:
 *    If a block device wishes to use the standard request handling procedures,
 *    which sorts requests and coalesces adjacent requests, then it must
 *    call blk_init_queue().  The function @rfn will be called when there
 *    are requests on the queue that need to be processed.  If the device
 *    supports plugging, then @rfn may not be called immediately when requests
 *    are available on the queue, but may be called at some time later instead.
 *    Plugged queues are generally unplugged when a buffer belonging to one
 *    of the requests on the queue is needed, or due to memory pressure.
 *
 *    @rfn is not required, or even expected, to remove all requests off the
 *    queue, but only as many as it can handle at a time.  If it does leave
 *    requests on the queue, it is responsible for arranging that the requests
 *    get dealt with eventually.
 *
 *    The queue spin lock must be held while manipulating the requests on the
1122 1123
 *    request queue; this lock will be taken also from interrupt context, so irq
 *    disabling is needed for it.
L
Linus Torvalds 已提交
1124
 *
1125
 *    Function returns a pointer to the initialized request queue, or %NULL if
L
Linus Torvalds 已提交
1126 1127 1128 1129 1130 1131
 *    it didn't succeed.
 *
 * Note:
 *    blk_init_queue() must be paired with a blk_cleanup_queue() call
 *    when the block device is deactivated (such as at module unload).
 **/
1132

1133
struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
L
Linus Torvalds 已提交
1134
{
1135
	return blk_init_queue_node(rfn, lock, NUMA_NO_NODE);
1136 1137 1138
}
EXPORT_SYMBOL(blk_init_queue);

1139
struct request_queue *
1140 1141
blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
{
1142
	struct request_queue *q;
L
Linus Torvalds 已提交
1143

1144
	q = blk_alloc_queue_node(GFP_KERNEL, node_id, lock);
1145
	if (!q)
1146 1147
		return NULL;

1148 1149 1150 1151 1152
	q->request_fn = rfn;
	if (blk_init_allocated_queue(q) < 0) {
		blk_cleanup_queue(q);
		return NULL;
	}
1153

1154
	return q;
1155 1156 1157
}
EXPORT_SYMBOL(blk_init_queue_node);

1158
static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio);
1159

L
Linus Torvalds 已提交
1160

1161 1162
int blk_init_allocated_queue(struct request_queue *q)
{
1163 1164
	WARN_ON_ONCE(q->mq_ops);

1165
	q->fq = blk_alloc_flush_queue(q, NUMA_NO_NODE, q->cmd_size);
1166
	if (!q->fq)
1167
		return -ENOMEM;
1168

1169 1170
	if (q->init_rq_fn && q->init_rq_fn(q, q->fq->flush_rq, GFP_KERNEL))
		goto out_free_flush_queue;
1171

1172
	if (blk_init_rl(&q->root_rl, q, GFP_KERNEL))
1173
		goto out_exit_flush_rq;
L
Linus Torvalds 已提交
1174

1175
	INIT_WORK(&q->timeout_work, blk_timeout_work);
1176
	q->queue_flags		|= QUEUE_FLAG_DEFAULT;
1177

1178 1179 1180
	/*
	 * This also sets hw/phys segments, boundary and size
	 */
1181
	blk_queue_make_request(q, blk_queue_bio);
L
Linus Torvalds 已提交
1182

1183 1184
	q->sg_reserved_size = INT_MAX;

1185
	if (elevator_init(q))
1186
		goto out_exit_flush_rq;
1187
	return 0;
1188

1189 1190 1191 1192
out_exit_flush_rq:
	if (q->exit_rq_fn)
		q->exit_rq_fn(q, q->fq->flush_rq);
out_free_flush_queue:
1193
	blk_free_flush_queue(q->fq);
1194
	q->fq = NULL;
1195
	return -ENOMEM;
L
Linus Torvalds 已提交
1196
}
1197
EXPORT_SYMBOL(blk_init_allocated_queue);
L
Linus Torvalds 已提交
1198

T
Tejun Heo 已提交
1199
bool blk_get_queue(struct request_queue *q)
L
Linus Torvalds 已提交
1200
{
B
Bart Van Assche 已提交
1201
	if (likely(!blk_queue_dying(q))) {
T
Tejun Heo 已提交
1202 1203
		__blk_get_queue(q);
		return true;
L
Linus Torvalds 已提交
1204 1205
	}

T
Tejun Heo 已提交
1206
	return false;
L
Linus Torvalds 已提交
1207
}
J
Jens Axboe 已提交
1208
EXPORT_SYMBOL(blk_get_queue);
L
Linus Torvalds 已提交
1209

1210
static inline void blk_free_request(struct request_list *rl, struct request *rq)
L
Linus Torvalds 已提交
1211
{
1212
	if (rq->rq_flags & RQF_ELVPRIV) {
1213
		elv_put_request(rl->q, rq);
1214
		if (rq->elv.icq)
1215
			put_io_context(rq->elv.icq->ioc);
1216 1217
	}

1218
	mempool_free(rq, rl->rq_pool);
L
Linus Torvalds 已提交
1219 1220 1221 1222 1223 1224
}

/*
 * ioc_batching returns true if the ioc is a valid batching request and
 * should be given priority access to a request.
 */
1225
static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
L
Linus Torvalds 已提交
1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245
{
	if (!ioc)
		return 0;

	/*
	 * Make sure the process is able to allocate at least 1 request
	 * even if the batch times out, otherwise we could theoretically
	 * lose wakeups.
	 */
	return ioc->nr_batch_requests == q->nr_batching ||
		(ioc->nr_batch_requests > 0
		&& time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
}

/*
 * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
 * will cause the process to be a "batcher" on all queues in the system. This
 * is the behaviour we want though - once it gets a wakeup it should be given
 * a nice run.
 */
1246
static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
L
Linus Torvalds 已提交
1247 1248 1249 1250 1251 1252 1253 1254
{
	if (!ioc || ioc_batching(q, ioc))
		return;

	ioc->nr_batch_requests = q->nr_batching;
	ioc->last_waited = jiffies;
}

1255
static void __freed_request(struct request_list *rl, int sync)
L
Linus Torvalds 已提交
1256
{
1257
	struct request_queue *q = rl->q;
L
Linus Torvalds 已提交
1258

1259 1260
	if (rl->count[sync] < queue_congestion_off_threshold(q))
		blk_clear_congested(rl, sync);
L
Linus Torvalds 已提交
1261

1262 1263 1264
	if (rl->count[sync] + 1 <= q->nr_requests) {
		if (waitqueue_active(&rl->wait[sync]))
			wake_up(&rl->wait[sync]);
L
Linus Torvalds 已提交
1265

1266
		blk_clear_rl_full(rl, sync);
L
Linus Torvalds 已提交
1267 1268 1269 1270 1271 1272 1273
	}
}

/*
 * A request has just been released.  Account for it, update the full and
 * congestion status, wake up any waiters.   Called under q->queue_lock.
 */
1274 1275
static void freed_request(struct request_list *rl, bool sync,
		req_flags_t rq_flags)
L
Linus Torvalds 已提交
1276
{
1277
	struct request_queue *q = rl->q;
L
Linus Torvalds 已提交
1278

1279
	q->nr_rqs[sync]--;
1280
	rl->count[sync]--;
1281
	if (rq_flags & RQF_ELVPRIV)
1282
		q->nr_rqs_elvpriv--;
L
Linus Torvalds 已提交
1283

1284
	__freed_request(rl, sync);
L
Linus Torvalds 已提交
1285

1286
	if (unlikely(rl->starved[sync ^ 1]))
1287
		__freed_request(rl, sync ^ 1);
L
Linus Torvalds 已提交
1288 1289
}

1290 1291 1292
int blk_update_nr_requests(struct request_queue *q, unsigned int nr)
{
	struct request_list *rl;
1293
	int on_thresh, off_thresh;
1294

1295 1296
	WARN_ON_ONCE(q->mq_ops);

1297 1298 1299
	spin_lock_irq(q->queue_lock);
	q->nr_requests = nr;
	blk_queue_congestion_threshold(q);
1300 1301
	on_thresh = queue_congestion_on_threshold(q);
	off_thresh = queue_congestion_off_threshold(q);
1302

1303 1304 1305 1306 1307
	blk_queue_for_each_rl(rl, q) {
		if (rl->count[BLK_RW_SYNC] >= on_thresh)
			blk_set_congested(rl, BLK_RW_SYNC);
		else if (rl->count[BLK_RW_SYNC] < off_thresh)
			blk_clear_congested(rl, BLK_RW_SYNC);
1308

1309 1310 1311 1312
		if (rl->count[BLK_RW_ASYNC] >= on_thresh)
			blk_set_congested(rl, BLK_RW_ASYNC);
		else if (rl->count[BLK_RW_ASYNC] < off_thresh)
			blk_clear_congested(rl, BLK_RW_ASYNC);
1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332

		if (rl->count[BLK_RW_SYNC] >= q->nr_requests) {
			blk_set_rl_full(rl, BLK_RW_SYNC);
		} else {
			blk_clear_rl_full(rl, BLK_RW_SYNC);
			wake_up(&rl->wait[BLK_RW_SYNC]);
		}

		if (rl->count[BLK_RW_ASYNC] >= q->nr_requests) {
			blk_set_rl_full(rl, BLK_RW_ASYNC);
		} else {
			blk_clear_rl_full(rl, BLK_RW_ASYNC);
			wake_up(&rl->wait[BLK_RW_ASYNC]);
		}
	}

	spin_unlock_irq(q->queue_lock);
	return 0;
}

1333
/**
T
Tejun Heo 已提交
1334
 * __get_request - get a free request
1335
 * @rl: request list to allocate from
1336
 * @op: operation and flags
1337
 * @bio: bio to allocate request for (can be %NULL)
1338
 * @flags: BLQ_MQ_REQ_* flags
1339
 * @gfp_mask: allocator flags
1340 1341 1342 1343
 *
 * Get a free request from @q.  This function may fail under memory
 * pressure or if @q is dead.
 *
1344
 * Must be called with @q->queue_lock held and,
1345 1346
 * Returns ERR_PTR on failure, with @q->queue_lock held.
 * Returns request pointer on success, with @q->queue_lock *not held*.
L
Linus Torvalds 已提交
1347
 */
1348
static struct request *__get_request(struct request_list *rl, unsigned int op,
1349
		struct bio *bio, blk_mq_req_flags_t flags, gfp_t gfp_mask)
L
Linus Torvalds 已提交
1350
{
1351
	struct request_queue *q = rl->q;
T
Tejun Heo 已提交
1352
	struct request *rq;
T
Tejun Heo 已提交
1353 1354
	struct elevator_type *et = q->elevator->type;
	struct io_context *ioc = rq_ioc(bio);
1355
	struct io_cq *icq = NULL;
1356
	const bool is_sync = op_is_sync(op);
1357
	int may_queue;
1358
	req_flags_t rq_flags = RQF_ALLOCED;
1359

1360 1361
	lockdep_assert_held(q->queue_lock);

B
Bart Van Assche 已提交
1362
	if (unlikely(blk_queue_dying(q)))
1363
		return ERR_PTR(-ENODEV);
1364

1365
	may_queue = elv_may_queue(q, op);
1366 1367 1368
	if (may_queue == ELV_MQUEUE_NO)
		goto rq_starved;

1369 1370
	if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
		if (rl->count[is_sync]+1 >= q->nr_requests) {
1371 1372 1373 1374 1375 1376
			/*
			 * The queue will fill after this allocation, so set
			 * it as full, and mark this process as "batching".
			 * This process will be allowed to complete a batch of
			 * requests, others will be blocked.
			 */
1377
			if (!blk_rl_full(rl, is_sync)) {
1378
				ioc_set_batching(q, ioc);
1379
				blk_set_rl_full(rl, is_sync);
1380 1381 1382 1383 1384 1385 1386 1387
			} else {
				if (may_queue != ELV_MQUEUE_MUST
						&& !ioc_batching(q, ioc)) {
					/*
					 * The queue is full and the allocating
					 * process is not a "batcher", and not
					 * exempted by the IO scheduler
					 */
1388
					return ERR_PTR(-ENOMEM);
1389 1390
				}
			}
L
Linus Torvalds 已提交
1391
		}
1392
		blk_set_congested(rl, is_sync);
L
Linus Torvalds 已提交
1393 1394
	}

1395 1396 1397 1398 1399
	/*
	 * Only allow batching queuers to allocate up to 50% over the defined
	 * limit of requests, otherwise we could have thousands of requests
	 * allocated with any setting of ->nr_requests
	 */
1400
	if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
1401
		return ERR_PTR(-ENOMEM);
H
Hugh Dickins 已提交
1402

1403
	q->nr_rqs[is_sync]++;
1404 1405
	rl->count[is_sync]++;
	rl->starved[is_sync] = 0;
T
Tejun Heo 已提交
1406

1407 1408
	/*
	 * Decide whether the new request will be managed by elevator.  If
1409
	 * so, mark @rq_flags and increment elvpriv.  Non-zero elvpriv will
1410 1411 1412 1413
	 * prevent the current elevator from being destroyed until the new
	 * request is freed.  This guarantees icq's won't be destroyed and
	 * makes creating new ones safe.
	 *
1414 1415 1416
	 * Flush requests do not use the elevator so skip initialization.
	 * This allows a request to share the flush and elevator data.
	 *
1417 1418 1419
	 * Also, lookup icq while holding queue_lock.  If it doesn't exist,
	 * it will be created after releasing queue_lock.
	 */
1420
	if (!op_is_flush(op) && !blk_queue_bypass(q)) {
1421
		rq_flags |= RQF_ELVPRIV;
1422
		q->nr_rqs_elvpriv++;
1423 1424
		if (et->icq_cache && ioc)
			icq = ioc_lookup_icq(ioc, q);
1425
	}
T
Tejun Heo 已提交
1426

1427
	if (blk_queue_io_stat(q))
1428
		rq_flags |= RQF_IO_STAT;
L
Linus Torvalds 已提交
1429 1430
	spin_unlock_irq(q->queue_lock);

1431
	/* allocate and init request */
1432
	rq = mempool_alloc(rl->rq_pool, gfp_mask);
1433
	if (!rq)
T
Tejun Heo 已提交
1434
		goto fail_alloc;
L
Linus Torvalds 已提交
1435

1436
	blk_rq_init(q, rq);
1437
	blk_rq_set_rl(rq, rl);
1438
	rq->cmd_flags = op;
1439
	rq->rq_flags = rq_flags;
1440 1441
	if (flags & BLK_MQ_REQ_PREEMPT)
		rq->rq_flags |= RQF_PREEMPT;
1442

1443
	/* init elvpriv */
1444
	if (rq_flags & RQF_ELVPRIV) {
1445
		if (unlikely(et->icq_cache && !icq)) {
T
Tejun Heo 已提交
1446 1447
			if (ioc)
				icq = ioc_create_icq(ioc, q, gfp_mask);
1448 1449
			if (!icq)
				goto fail_elvpriv;
1450
		}
1451 1452 1453 1454 1455 1456

		rq->elv.icq = icq;
		if (unlikely(elv_set_request(q, rq, bio, gfp_mask)))
			goto fail_elvpriv;

		/* @rq->elv.icq holds io_context until @rq is freed */
1457 1458 1459
		if (icq)
			get_io_context(icq->ioc);
	}
1460
out:
1461 1462 1463 1464 1465 1466
	/*
	 * ioc may be NULL here, and ioc_batching will be false. That's
	 * OK, if the queue is under the request limit then requests need
	 * not count toward the nr_batch_requests limit. There will always
	 * be some limit enforced by BLK_BATCH_TIME.
	 */
L
Linus Torvalds 已提交
1467 1468
	if (ioc_batching(q, ioc))
		ioc->nr_batch_requests--;
1469

1470
	trace_block_getrq(q, bio, op);
L
Linus Torvalds 已提交
1471
	return rq;
T
Tejun Heo 已提交
1472

1473 1474 1475 1476 1477 1478 1479
fail_elvpriv:
	/*
	 * elvpriv init failed.  ioc, icq and elvpriv aren't mempool backed
	 * and may fail indefinitely under memory pressure and thus
	 * shouldn't stall IO.  Treat this request as !elvpriv.  This will
	 * disturb iosched and blkcg but weird is bettern than dead.
	 */
1480
	printk_ratelimited(KERN_WARNING "%s: dev %s: request aux data allocation failed, iosched may be disturbed\n",
1481
			   __func__, dev_name(q->backing_dev_info->dev));
1482

1483
	rq->rq_flags &= ~RQF_ELVPRIV;
1484 1485 1486
	rq->elv.icq = NULL;

	spin_lock_irq(q->queue_lock);
1487
	q->nr_rqs_elvpriv--;
1488 1489 1490
	spin_unlock_irq(q->queue_lock);
	goto out;

T
Tejun Heo 已提交
1491 1492 1493 1494 1495 1496 1497 1498 1499
fail_alloc:
	/*
	 * Allocation failed presumably due to memory. Undo anything we
	 * might have messed up.
	 *
	 * Allocating task should really be put onto the front of the wait
	 * queue, but this is pretty rare.
	 */
	spin_lock_irq(q->queue_lock);
1500
	freed_request(rl, is_sync, rq_flags);
T
Tejun Heo 已提交
1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511

	/*
	 * in the very unlikely event that allocation failed and no
	 * requests for this direction was pending, mark us starved so that
	 * freeing of a request in the other direction will notice
	 * us. another possible fix would be to split the rq mempool into
	 * READ and WRITE
	 */
rq_starved:
	if (unlikely(rl->count[is_sync] == 0))
		rl->starved[is_sync] = 1;
1512
	return ERR_PTR(-ENOMEM);
L
Linus Torvalds 已提交
1513 1514
}

1515
/**
T
Tejun Heo 已提交
1516
 * get_request - get a free request
1517
 * @q: request_queue to allocate request from
1518
 * @op: operation and flags
1519
 * @bio: bio to allocate request for (can be %NULL)
1520
 * @flags: BLK_MQ_REQ_* flags.
1521
 * @gfp: allocator flags
1522
 *
1523
 * Get a free request from @q.  If %BLK_MQ_REQ_NOWAIT is set in @flags,
1524
 * this function keeps retrying under memory pressure and fails iff @q is dead.
N
Nick Piggin 已提交
1525
 *
1526
 * Must be called with @q->queue_lock held and,
1527 1528
 * Returns ERR_PTR on failure, with @q->queue_lock held.
 * Returns request pointer on success, with @q->queue_lock *not held*.
L
Linus Torvalds 已提交
1529
 */
1530
static struct request *get_request(struct request_queue *q, unsigned int op,
1531
		struct bio *bio, blk_mq_req_flags_t flags, gfp_t gfp)
L
Linus Torvalds 已提交
1532
{
1533
	const bool is_sync = op_is_sync(op);
T
Tejun Heo 已提交
1534
	DEFINE_WAIT(wait);
1535
	struct request_list *rl;
L
Linus Torvalds 已提交
1536
	struct request *rq;
1537

1538
	lockdep_assert_held(q->queue_lock);
1539
	WARN_ON_ONCE(q->mq_ops);
1540

1541
	rl = blk_get_rl(q, bio);	/* transferred to @rq on success */
T
Tejun Heo 已提交
1542
retry:
1543
	rq = __get_request(rl, op, bio, flags, gfp);
1544
	if (!IS_ERR(rq))
T
Tejun Heo 已提交
1545
		return rq;
L
Linus Torvalds 已提交
1546

1547 1548 1549 1550 1551
	if (op & REQ_NOWAIT) {
		blk_put_rl(rl);
		return ERR_PTR(-EAGAIN);
	}

1552
	if ((flags & BLK_MQ_REQ_NOWAIT) || unlikely(blk_queue_dying(q))) {
1553
		blk_put_rl(rl);
1554
		return rq;
1555
	}
L
Linus Torvalds 已提交
1556

T
Tejun Heo 已提交
1557 1558 1559
	/* wait on @rl and retry */
	prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
				  TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
1560

1561
	trace_block_sleeprq(q, bio, op);
L
Linus Torvalds 已提交
1562

T
Tejun Heo 已提交
1563 1564
	spin_unlock_irq(q->queue_lock);
	io_schedule();
N
Nick Piggin 已提交
1565

T
Tejun Heo 已提交
1566 1567 1568 1569 1570 1571
	/*
	 * After sleeping, we become a "batching" process and will be able
	 * to allocate at least one request, and up to a big batch of them
	 * for a small period time.  See ioc_batching, ioc_set_batching
	 */
	ioc_set_batching(q, current->io_context);
1572

T
Tejun Heo 已提交
1573 1574
	spin_lock_irq(q->queue_lock);
	finish_wait(&rl->wait[is_sync], &wait);
L
Linus Torvalds 已提交
1575

T
Tejun Heo 已提交
1576
	goto retry;
L
Linus Torvalds 已提交
1577 1578
}

1579
/* flags: BLK_MQ_REQ_PREEMPT and/or BLK_MQ_REQ_NOWAIT. */
1580
static struct request *blk_old_get_request(struct request_queue *q,
1581
				unsigned int op, blk_mq_req_flags_t flags)
L
Linus Torvalds 已提交
1582 1583
{
	struct request *rq;
1584
	gfp_t gfp_mask = flags & BLK_MQ_REQ_NOWAIT ? GFP_ATOMIC : GFP_NOIO;
1585
	int ret = 0;
L
Linus Torvalds 已提交
1586

1587 1588
	WARN_ON_ONCE(q->mq_ops);

T
Tejun Heo 已提交
1589 1590 1591
	/* create ioc upfront */
	create_io_context(gfp_mask, q->node);

1592
	ret = blk_queue_enter(q, flags);
1593 1594
	if (ret)
		return ERR_PTR(ret);
N
Nick Piggin 已提交
1595
	spin_lock_irq(q->queue_lock);
1596
	rq = get_request(q, op, NULL, flags, gfp_mask);
1597
	if (IS_ERR(rq)) {
1598
		spin_unlock_irq(q->queue_lock);
1599
		blk_queue_exit(q);
1600 1601
		return rq;
	}
L
Linus Torvalds 已提交
1602

1603 1604 1605 1606
	/* q->queue_lock is unlocked at this point */
	rq->__data_len = 0;
	rq->__sector = (sector_t) -1;
	rq->bio = rq->biotail = NULL;
L
Linus Torvalds 已提交
1607 1608
	return rq;
}
1609

1610
/**
1611
 * blk_get_request - allocate a request
1612 1613 1614 1615
 * @q: request queue to allocate a request for
 * @op: operation (REQ_OP_*) and REQ_* flags, e.g. REQ_SYNC.
 * @flags: BLK_MQ_REQ_* flags, e.g. BLK_MQ_REQ_NOWAIT.
 */
1616 1617
struct request *blk_get_request(struct request_queue *q, unsigned int op,
				blk_mq_req_flags_t flags)
1618
{
1619 1620
	struct request *req;

1621
	WARN_ON_ONCE(op & REQ_NOWAIT);
1622
	WARN_ON_ONCE(flags & ~(BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_PREEMPT));
1623

1624
	if (q->mq_ops) {
1625
		req = blk_mq_alloc_request(q, op, flags);
1626 1627 1628
		if (!IS_ERR(req) && q->mq_ops->initialize_rq_fn)
			q->mq_ops->initialize_rq_fn(req);
	} else {
1629
		req = blk_old_get_request(q, op, flags);
1630 1631 1632 1633 1634
		if (!IS_ERR(req) && q->initialize_rq_fn)
			q->initialize_rq_fn(req);
	}

	return req;
1635
}
L
Linus Torvalds 已提交
1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647
EXPORT_SYMBOL(blk_get_request);

/**
 * blk_requeue_request - put a request back on queue
 * @q:		request queue where request should be inserted
 * @rq:		request to be inserted
 *
 * Description:
 *    Drivers often keep queueing requests until the hardware cannot accept
 *    more, when that condition happens we need to put the request back
 *    on the queue. Must be called with queue lock held.
 */
1648
void blk_requeue_request(struct request_queue *q, struct request *rq)
L
Linus Torvalds 已提交
1649
{
1650
	lockdep_assert_held(q->queue_lock);
1651
	WARN_ON_ONCE(q->mq_ops);
1652

J
Jens Axboe 已提交
1653 1654
	blk_delete_timer(rq);
	blk_clear_rq_complete(rq);
1655
	trace_block_rq_requeue(q, rq);
1656
	rq_qos_requeue(q, rq);
1657

1658
	if (rq->rq_flags & RQF_QUEUED)
L
Linus Torvalds 已提交
1659 1660
		blk_queue_end_tag(q, rq);

1661 1662
	BUG_ON(blk_queued_rq(rq));

L
Linus Torvalds 已提交
1663 1664 1665 1666
	elv_requeue_request(q, rq);
}
EXPORT_SYMBOL(blk_requeue_request);

1667 1668 1669
static void add_acct_request(struct request_queue *q, struct request *rq,
			     int where)
{
1670
	blk_account_io_start(rq, true);
J
Jens Axboe 已提交
1671
	__elv_add_request(q, rq, where);
1672 1673
}

1674
static void part_round_stats_single(struct request_queue *q, int cpu,
1675 1676
				    struct hd_struct *part, unsigned long now,
				    unsigned int inflight)
T
Tejun Heo 已提交
1677
{
1678
	if (inflight) {
T
Tejun Heo 已提交
1679
		__part_stat_add(cpu, part, time_in_queue,
1680
				inflight * (now - part->stamp));
T
Tejun Heo 已提交
1681 1682 1683 1684 1685 1686
		__part_stat_add(cpu, part, io_ticks, (now - part->stamp));
	}
	part->stamp = now;
}

/**
1687
 * part_round_stats() - Round off the performance stats on a struct disk_stats.
1688
 * @q: target block queue
1689 1690
 * @cpu: cpu number for stats access
 * @part: target partition
L
Linus Torvalds 已提交
1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702
 *
 * The average IO queue length and utilisation statistics are maintained
 * by observing the current state of the queue length and the amount of
 * time it has been in this state for.
 *
 * Normally, that accounting is done on IO completion, but that can result
 * in more than a second's worth of IO being accounted for within any one
 * second, leading to >100% utilisation.  To deal with that, we call this
 * function to do a round-off before returning the results when reading
 * /proc/diskstats.  This accounts immediately for all queue usage up to
 * the current jiffies and restarts the counters again.
 */
1703
void part_round_stats(struct request_queue *q, int cpu, struct hd_struct *part)
1704
{
1705
	struct hd_struct *part2 = NULL;
1706
	unsigned long now = jiffies;
1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722
	unsigned int inflight[2];
	int stats = 0;

	if (part->stamp != now)
		stats |= 1;

	if (part->partno) {
		part2 = &part_to_disk(part)->part0;
		if (part2->stamp != now)
			stats |= 2;
	}

	if (!stats)
		return;

	part_in_flight(q, part, inflight);
1723

1724 1725 1726 1727
	if (stats & 2)
		part_round_stats_single(q, cpu, part2, now, inflight[1]);
	if (stats & 1)
		part_round_stats_single(q, cpu, part, now, inflight[0]);
1728
}
T
Tejun Heo 已提交
1729
EXPORT_SYMBOL_GPL(part_round_stats);
1730

1731
#ifdef CONFIG_PM
L
Lin Ming 已提交
1732 1733
static void blk_pm_put_request(struct request *rq)
{
1734
	if (rq->q->dev && !(rq->rq_flags & RQF_PM) && !--rq->q->nr_pending)
L
Lin Ming 已提交
1735 1736 1737 1738 1739 1740
		pm_runtime_mark_last_busy(rq->q->dev);
}
#else
static inline void blk_pm_put_request(struct request *rq) {}
#endif

1741
void __blk_put_request(struct request_queue *q, struct request *req)
L
Linus Torvalds 已提交
1742
{
1743 1744
	req_flags_t rq_flags = req->rq_flags;

L
Linus Torvalds 已提交
1745 1746 1747
	if (unlikely(!q))
		return;

1748 1749 1750 1751 1752
	if (q->mq_ops) {
		blk_mq_free_request(req);
		return;
	}

1753 1754
	lockdep_assert_held(q->queue_lock);

1755
	blk_req_zone_write_unlock(req);
L
Lin Ming 已提交
1756 1757
	blk_pm_put_request(req);

1758 1759
	elv_completed_request(q, req);

1760 1761 1762
	/* this is a bio leak */
	WARN_ON(req->bio != NULL);

1763
	rq_qos_done(q, req);
J
Jens Axboe 已提交
1764

L
Linus Torvalds 已提交
1765 1766 1767 1768
	/*
	 * Request may not have originated from ll_rw_blk. if not,
	 * it didn't come out of our reserved rq pools
	 */
1769
	if (rq_flags & RQF_ALLOCED) {
1770
		struct request_list *rl = blk_rq_rl(req);
1771
		bool sync = op_is_sync(req->cmd_flags);
L
Linus Torvalds 已提交
1772 1773

		BUG_ON(!list_empty(&req->queuelist));
1774
		BUG_ON(ELV_ON_HASH(req));
L
Linus Torvalds 已提交
1775

1776
		blk_free_request(rl, req);
1777
		freed_request(rl, sync, rq_flags);
1778
		blk_put_rl(rl);
1779
		blk_queue_exit(q);
L
Linus Torvalds 已提交
1780 1781
	}
}
1782 1783
EXPORT_SYMBOL_GPL(__blk_put_request);

L
Linus Torvalds 已提交
1784 1785
void blk_put_request(struct request *req)
{
1786
	struct request_queue *q = req->q;
1787

1788 1789 1790 1791 1792 1793 1794 1795 1796
	if (q->mq_ops)
		blk_mq_free_request(req);
	else {
		unsigned long flags;

		spin_lock_irqsave(q->queue_lock, flags);
		__blk_put_request(q, req);
		spin_unlock_irqrestore(q->queue_lock, flags);
	}
L
Linus Torvalds 已提交
1797 1798 1799
}
EXPORT_SYMBOL(blk_put_request);

1800 1801
bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
			    struct bio *bio)
1802
{
J
Jens Axboe 已提交
1803
	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
1804 1805 1806 1807

	if (!ll_back_merge_fn(q, req, bio))
		return false;

1808
	trace_block_bio_backmerge(q, req, bio);
1809 1810 1811 1812 1813 1814

	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
		blk_rq_set_mixed_merge(req);

	req->biotail->bi_next = bio;
	req->biotail = bio;
1815
	req->__data_len += bio->bi_iter.bi_size;
1816 1817
	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));

1818
	blk_account_io_start(req, false);
1819 1820 1821
	return true;
}

1822 1823
bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
			     struct bio *bio)
1824
{
J
Jens Axboe 已提交
1825
	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
1826 1827 1828 1829

	if (!ll_front_merge_fn(q, req, bio))
		return false;

1830
	trace_block_bio_frontmerge(q, req, bio);
1831 1832 1833 1834 1835 1836 1837

	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
		blk_rq_set_mixed_merge(req);

	bio->bi_next = req->bio;
	req->bio = bio;

1838 1839
	req->__sector = bio->bi_iter.bi_sector;
	req->__data_len += bio->bi_iter.bi_size;
1840 1841
	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));

1842
	blk_account_io_start(req, false);
1843 1844 1845
	return true;
}

1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869
bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
		struct bio *bio)
{
	unsigned short segments = blk_rq_nr_discard_segments(req);

	if (segments >= queue_max_discard_segments(q))
		goto no_merge;
	if (blk_rq_sectors(req) + bio_sectors(bio) >
	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))
		goto no_merge;

	req->biotail->bi_next = bio;
	req->biotail = bio;
	req->__data_len += bio->bi_iter.bi_size;
	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
	req->nr_phys_segments = segments + 1;

	blk_account_io_start(req, false);
	return true;
no_merge:
	req_set_nomerge(q, req);
	return false;
}

1870
/**
1871
 * blk_attempt_plug_merge - try to merge with %current's plugged list
1872 1873 1874
 * @q: request_queue new bio is being queued at
 * @bio: new bio being queued
 * @request_count: out parameter for number of traversed plugged requests
1875 1876 1877
 * @same_queue_rq: pointer to &struct request that gets filled in when
 * another request associated with @q is found on the plug list
 * (optional, may be %NULL)
1878 1879 1880 1881 1882
 *
 * Determine whether @bio being queued on @q can be merged with a request
 * on %current's plugged list.  Returns %true if merge was successful,
 * otherwise %false.
 *
1883 1884 1885 1886 1887 1888
 * Plugging coalesces IOs from the same issuer for the same purpose without
 * going through @q->queue_lock.  As such it's more of an issuing mechanism
 * than scheduling, and the request, while may have elvpriv data, is not
 * added on the elevator at this point.  In addition, we don't have
 * reliable access to the elevator outside queue lock.  Only check basic
 * merging parameters without querying the elevator.
1889 1890
 *
 * Caller must ensure !blk_queue_nomerges(q) beforehand.
1891
 */
1892
bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1893 1894
			    unsigned int *request_count,
			    struct request **same_queue_rq)
1895 1896 1897
{
	struct blk_plug *plug;
	struct request *rq;
S
Shaohua Li 已提交
1898
	struct list_head *plug_list;
1899

1900
	plug = current->plug;
1901
	if (!plug)
1902
		return false;
1903
	*request_count = 0;
1904

S
Shaohua Li 已提交
1905 1906 1907 1908 1909 1910
	if (q->mq_ops)
		plug_list = &plug->mq_list;
	else
		plug_list = &plug->list;

	list_for_each_entry_reverse(rq, plug_list, queuelist) {
1911
		bool merged = false;
1912

1913
		if (rq->q == q) {
1914
			(*request_count)++;
1915 1916 1917 1918 1919 1920 1921 1922
			/*
			 * Only blk-mq multiple hardware queues case checks the
			 * rq in the same queue, there should be only one such
			 * rq in a queue
			 **/
			if (same_queue_rq)
				*same_queue_rq = rq;
		}
1923

1924
		if (rq->q != q || !blk_rq_merge_ok(rq, bio))
1925 1926
			continue;

1927 1928 1929 1930 1931 1932 1933
		switch (blk_try_merge(rq, bio)) {
		case ELEVATOR_BACK_MERGE:
			merged = bio_attempt_back_merge(q, rq, bio);
			break;
		case ELEVATOR_FRONT_MERGE:
			merged = bio_attempt_front_merge(q, rq, bio);
			break;
1934 1935 1936
		case ELEVATOR_DISCARD_MERGE:
			merged = bio_attempt_discard_merge(q, rq, bio);
			break;
1937 1938
		default:
			break;
1939
		}
1940 1941 1942

		if (merged)
			return true;
1943
	}
1944 1945

	return false;
1946 1947
}

1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971
unsigned int blk_plug_queued_count(struct request_queue *q)
{
	struct blk_plug *plug;
	struct request *rq;
	struct list_head *plug_list;
	unsigned int ret = 0;

	plug = current->plug;
	if (!plug)
		goto out;

	if (q->mq_ops)
		plug_list = &plug->mq_list;
	else
		plug_list = &plug->list;

	list_for_each_entry(rq, plug_list, queuelist) {
		if (rq->q == q)
			ret++;
	}
out:
	return ret;
}

1972
void blk_init_request_from_bio(struct request *req, struct bio *bio)
1973
{
1974 1975
	struct io_context *ioc = rq_ioc(bio);

J
Jens Axboe 已提交
1976
	if (bio->bi_opf & REQ_RAHEAD)
1977
		req->cmd_flags |= REQ_FAILFAST_MASK;
J
Jens Axboe 已提交
1978

1979
	req->__sector = bio->bi_iter.bi_sector;
1980 1981
	if (ioprio_valid(bio_prio(bio)))
		req->ioprio = bio_prio(bio);
1982 1983 1984 1985
	else if (ioc)
		req->ioprio = ioc->ioprio;
	else
		req->ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0);
1986
	req->write_hint = bio->bi_write_hint;
1987
	blk_rq_bio_prep(req->q, req, bio);
1988
}
1989
EXPORT_SYMBOL_GPL(blk_init_request_from_bio);
1990

1991
static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
L
Linus Torvalds 已提交
1992
{
1993
	struct blk_plug *plug;
1994
	int where = ELEVATOR_INSERT_SORT;
1995
	struct request *req, *free;
1996
	unsigned int request_count = 0;
L
Linus Torvalds 已提交
1997 1998 1999 2000 2001 2002 2003 2004

	/*
	 * low level driver can indicate that it wants pages above a
	 * certain limit bounced to low memory (ie for highmem, or even
	 * ISA dma in theory)
	 */
	blk_queue_bounce(q, &bio);

2005
	blk_queue_split(q, &bio);
2006

2007
	if (!bio_integrity_prep(bio))
2008
		return BLK_QC_T_NONE;
2009

2010
	if (op_is_flush(bio->bi_opf)) {
2011
		spin_lock_irq(q->queue_lock);
2012
		where = ELEVATOR_INSERT_FLUSH;
2013 2014 2015
		goto get_rq;
	}

2016 2017 2018 2019
	/*
	 * Check if we can merge with the plugged list before grabbing
	 * any locks.
	 */
2020 2021
	if (!blk_queue_nomerges(q)) {
		if (blk_attempt_plug_merge(q, bio, &request_count, NULL))
2022
			return BLK_QC_T_NONE;
2023 2024
	} else
		request_count = blk_plug_queued_count(q);
L
Linus Torvalds 已提交
2025

2026
	spin_lock_irq(q->queue_lock);
2027

2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050
	switch (elv_merge(q, &req, bio)) {
	case ELEVATOR_BACK_MERGE:
		if (!bio_attempt_back_merge(q, req, bio))
			break;
		elv_bio_merged(q, req, bio);
		free = attempt_back_merge(q, req);
		if (free)
			__blk_put_request(q, free);
		else
			elv_merged_request(q, req, ELEVATOR_BACK_MERGE);
		goto out_unlock;
	case ELEVATOR_FRONT_MERGE:
		if (!bio_attempt_front_merge(q, req, bio))
			break;
		elv_bio_merged(q, req, bio);
		free = attempt_front_merge(q, req);
		if (free)
			__blk_put_request(q, free);
		else
			elv_merged_request(q, req, ELEVATOR_FRONT_MERGE);
		goto out_unlock;
	default:
		break;
L
Linus Torvalds 已提交
2051 2052
	}

2053
get_rq:
2054
	rq_qos_throttle(q, bio, q->queue_lock);
J
Jens Axboe 已提交
2055

L
Linus Torvalds 已提交
2056
	/*
2057
	 * Grab a free request. This is might sleep but can not fail.
N
Nick Piggin 已提交
2058
	 * Returns with the queue unlocked.
2059
	 */
2060
	blk_queue_enter_live(q);
2061
	req = get_request(q, bio->bi_opf, bio, 0, GFP_NOIO);
2062
	if (IS_ERR(req)) {
2063
		blk_queue_exit(q);
2064
		rq_qos_cleanup(q, bio);
2065 2066 2067 2068
		if (PTR_ERR(req) == -ENOMEM)
			bio->bi_status = BLK_STS_RESOURCE;
		else
			bio->bi_status = BLK_STS_IOERR;
2069
		bio_endio(bio);
2070 2071
		goto out_unlock;
	}
N
Nick Piggin 已提交
2072

2073
	rq_qos_track(q, req, bio);
J
Jens Axboe 已提交
2074

2075 2076 2077 2078 2079
	/*
	 * After dropping the lock and possibly sleeping here, our request
	 * may now be mergeable after it had proven unmergeable (above).
	 * We don't worry about that case for efficiency. It won't happen
	 * often, and the elevators are able to handle it.
L
Linus Torvalds 已提交
2080
	 */
2081
	blk_init_request_from_bio(req, bio);
L
Linus Torvalds 已提交
2082

2083
	if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags))
2084
		req->cpu = raw_smp_processor_id();
2085 2086

	plug = current->plug;
J
Jens Axboe 已提交
2087
	if (plug) {
J
Jens Axboe 已提交
2088 2089
		/*
		 * If this is the first request added after a plug, fire
2090
		 * of a plug trace.
2091 2092 2093
		 *
		 * @request_count may become stale because of schedule
		 * out, so check plug list again.
J
Jens Axboe 已提交
2094
		 */
2095
		if (!request_count || list_empty(&plug->list))
J
Jens Axboe 已提交
2096
			trace_block_plug(q);
2097
		else {
2098 2099 2100
			struct request *last = list_entry_rq(plug->list.prev);
			if (request_count >= BLK_MAX_REQUEST_COUNT ||
			    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE) {
2101
				blk_flush_plug_list(plug, false);
S
Shaohua Li 已提交
2102 2103
				trace_block_plug(q);
			}
2104 2105
		}
		list_add_tail(&req->queuelist, &plug->list);
2106
		blk_account_io_start(req, true);
2107 2108 2109
	} else {
		spin_lock_irq(q->queue_lock);
		add_acct_request(q, req, where);
2110
		__blk_run_queue(q);
2111 2112 2113
out_unlock:
		spin_unlock_irq(q->queue_lock);
	}
2114 2115

	return BLK_QC_T_NONE;
L
Linus Torvalds 已提交
2116 2117
}

2118
static void handle_bad_sector(struct bio *bio, sector_t maxsector)
L
Linus Torvalds 已提交
2119 2120 2121 2122
{
	char b[BDEVNAME_SIZE];

	printk(KERN_INFO "attempt to access beyond end of device\n");
2123
	printk(KERN_INFO "%s: rw=%d, want=%Lu, limit=%Lu\n",
2124
			bio_devname(bio, b), bio->bi_opf,
K
Kent Overstreet 已提交
2125
			(unsigned long long)bio_end_sector(bio),
2126
			(long long)maxsector);
L
Linus Torvalds 已提交
2127 2128
}

2129 2130 2131 2132 2133 2134 2135 2136 2137 2138
#ifdef CONFIG_FAIL_MAKE_REQUEST

static DECLARE_FAULT_ATTR(fail_make_request);

static int __init setup_fail_make_request(char *str)
{
	return setup_fault_attr(&fail_make_request, str);
}
__setup("fail_make_request=", setup_fail_make_request);

2139
static bool should_fail_request(struct hd_struct *part, unsigned int bytes)
2140
{
2141
	return part->make_it_fail && should_fail(&fail_make_request, bytes);
2142 2143 2144 2145
}

static int __init fail_make_request_debugfs(void)
{
2146 2147 2148
	struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
						NULL, &fail_make_request);

2149
	return PTR_ERR_OR_ZERO(dir);
2150 2151 2152 2153 2154 2155
}

late_initcall(fail_make_request_debugfs);

#else /* CONFIG_FAIL_MAKE_REQUEST */

2156 2157
static inline bool should_fail_request(struct hd_struct *part,
					unsigned int bytes)
2158
{
2159
	return false;
2160 2161 2162 2163
}

#endif /* CONFIG_FAIL_MAKE_REQUEST */

2164 2165
static inline bool bio_check_ro(struct bio *bio, struct hd_struct *part)
{
2166 2167
	const int op = bio_op(bio);

2168
	if (part->policy && op_is_write(op)) {
2169 2170
		char b[BDEVNAME_SIZE];

2171 2172 2173
		if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
			return false;

2174
		WARN_ONCE(1,
2175 2176 2177
		       "generic_make_request: Trying to write "
			"to read-only block-device %s (partno %d)\n",
			bio_devname(bio, b), part->partno);
2178 2179
		/* Older lvm-tools actually trigger this */
		return false;
2180 2181 2182 2183 2184
	}

	return false;
}

2185 2186 2187 2188 2189 2190 2191 2192
static noinline int should_fail_bio(struct bio *bio)
{
	if (should_fail_request(&bio->bi_disk->part0, bio->bi_iter.bi_size))
		return -EIO;
	return 0;
}
ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO);

2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210
/*
 * Check whether this bio extends beyond the end of the device or partition.
 * This may well happen - the kernel calls bread() without checking the size of
 * the device, e.g., when mounting a file system.
 */
static inline int bio_check_eod(struct bio *bio, sector_t maxsector)
{
	unsigned int nr_sectors = bio_sectors(bio);

	if (nr_sectors && maxsector &&
	    (nr_sectors > maxsector ||
	     bio->bi_iter.bi_sector > maxsector - nr_sectors)) {
		handle_bad_sector(bio, maxsector);
		return -EIO;
	}
	return 0;
}

2211 2212 2213 2214 2215 2216
/*
 * Remap block n of partition p to block n+start(p) of the disk.
 */
static inline int blk_partition_remap(struct bio *bio)
{
	struct hd_struct *p;
2217
	int ret = -EIO;
2218

2219 2220
	rcu_read_lock();
	p = __disk_get_part(bio->bi_disk, bio->bi_partno);
2221 2222 2223 2224 2225
	if (unlikely(!p))
		goto out;
	if (unlikely(should_fail_request(p, bio->bi_iter.bi_size)))
		goto out;
	if (unlikely(bio_check_ro(bio, p)))
2226 2227
		goto out;

2228 2229 2230 2231
	/*
	 * Zone reset does not include bi_size so bio_sectors() is always 0.
	 * Include a test for the reset op code and perform the remap if needed.
	 */
2232 2233 2234 2235 2236 2237 2238
	if (bio_sectors(bio) || bio_op(bio) == REQ_OP_ZONE_RESET) {
		if (bio_check_eod(bio, part_nr_sects_read(p)))
			goto out;
		bio->bi_iter.bi_sector += p->start_sect;
		trace_block_bio_remap(bio->bi_disk->queue, bio, part_devt(p),
				      bio->bi_iter.bi_sector - p->start_sect);
	}
2239
	bio->bi_partno = 0;
2240
	ret = 0;
2241 2242
out:
	rcu_read_unlock();
2243 2244 2245
	return ret;
}

2246 2247
static noinline_for_stack bool
generic_make_request_checks(struct bio *bio)
L
Linus Torvalds 已提交
2248
{
2249
	struct request_queue *q;
2250
	int nr_sectors = bio_sectors(bio);
2251
	blk_status_t status = BLK_STS_IOERR;
2252
	char b[BDEVNAME_SIZE];
L
Linus Torvalds 已提交
2253 2254 2255

	might_sleep();

2256
	q = bio->bi_disk->queue;
2257 2258 2259 2260
	if (unlikely(!q)) {
		printk(KERN_ERR
		       "generic_make_request: Trying to access "
			"nonexistent block-device %s (%Lu)\n",
2261
			bio_devname(bio, b), (long long)bio->bi_iter.bi_sector);
2262 2263
		goto end_io;
	}
2264

2265 2266 2267 2268 2269 2270 2271
	/*
	 * For a REQ_NOWAIT based request, return -EOPNOTSUPP
	 * if queue is not a request based queue.
	 */
	if ((bio->bi_opf & REQ_NOWAIT) && !queue_is_rq_based(q))
		goto not_supported;

2272
	if (should_fail_bio(bio))
2273
		goto end_io;
2274

2275 2276
	if (bio->bi_partno) {
		if (unlikely(blk_partition_remap(bio)))
2277 2278
			goto end_io;
	} else {
2279 2280 2281
		if (unlikely(bio_check_ro(bio, &bio->bi_disk->part0)))
			goto end_io;
		if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk))))
2282 2283
			goto end_io;
	}
2284

2285 2286 2287 2288 2289
	/*
	 * Filter flush bio's early so that make_request based
	 * drivers without flush support don't have to worry
	 * about them.
	 */
2290
	if (op_is_flush(bio->bi_opf) &&
J
Jens Axboe 已提交
2291
	    !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) {
J
Jens Axboe 已提交
2292
		bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA);
2293
		if (!nr_sectors) {
2294
			status = BLK_STS_OK;
2295 2296
			goto end_io;
		}
2297
	}
2298

2299 2300 2301 2302 2303 2304 2305 2306 2307 2308
	switch (bio_op(bio)) {
	case REQ_OP_DISCARD:
		if (!blk_queue_discard(q))
			goto not_supported;
		break;
	case REQ_OP_SECURE_ERASE:
		if (!blk_queue_secure_erase(q))
			goto not_supported;
		break;
	case REQ_OP_WRITE_SAME:
2309
		if (!q->limits.max_write_same_sectors)
2310
			goto not_supported;
2311
		break;
2312 2313
	case REQ_OP_ZONE_REPORT:
	case REQ_OP_ZONE_RESET:
2314
		if (!blk_queue_is_zoned(q))
2315
			goto not_supported;
2316
		break;
2317
	case REQ_OP_WRITE_ZEROES:
2318
		if (!q->limits.max_write_zeroes_sectors)
2319 2320
			goto not_supported;
		break;
2321 2322
	default:
		break;
2323
	}
2324

T
Tejun Heo 已提交
2325 2326 2327 2328 2329 2330 2331 2332
	/*
	 * Various block parts want %current->io_context and lazy ioc
	 * allocation ends up trading a lot of pain for a small amount of
	 * memory.  Just allocate it upfront.  This may fail and block
	 * layer knows how to live with it.
	 */
	create_io_context(GFP_ATOMIC, q->node);

2333 2334
	if (!blkcg_bio_issue_check(q, bio))
		return false;
2335

N
NeilBrown 已提交
2336 2337 2338 2339 2340 2341 2342
	if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
		trace_block_bio_queue(q, bio);
		/* Now that enqueuing has been traced, we need to trace
		 * completion as well.
		 */
		bio_set_flag(bio, BIO_TRACE_COMPLETION);
	}
2343
	return true;
2344

2345
not_supported:
2346
	status = BLK_STS_NOTSUPP;
2347
end_io:
2348
	bio->bi_status = status;
2349
	bio_endio(bio);
2350
	return false;
L
Linus Torvalds 已提交
2351 2352
}

2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375
/**
 * generic_make_request - hand a buffer to its device driver for I/O
 * @bio:  The bio describing the location in memory and on the device.
 *
 * generic_make_request() is used to make I/O requests of block
 * devices. It is passed a &struct bio, which describes the I/O that needs
 * to be done.
 *
 * generic_make_request() does not return any status.  The
 * success/failure status of the request, along with notification of
 * completion, is delivered asynchronously through the bio->bi_end_io
 * function described (one day) else where.
 *
 * The caller of generic_make_request must make sure that bi_io_vec
 * are set to describe the memory buffer, and that bi_dev and bi_sector are
 * set to describe the device address, and the
 * bi_end_io and optionally bi_private are set to describe how
 * completion notification should be signaled.
 *
 * generic_make_request and the drivers it calls may use bi_next if this
 * bio happens to be merged with someone else, and may resubmit the bio to
 * a lower device by calling into generic_make_request recursively, which
 * means the bio should NOT be touched after the call to ->make_request_fn.
2376
 */
2377
blk_qc_t generic_make_request(struct bio *bio)
2378
{
2379 2380 2381 2382 2383 2384 2385 2386
	/*
	 * bio_list_on_stack[0] contains bios submitted by the current
	 * make_request_fn.
	 * bio_list_on_stack[1] contains bios that were submitted before
	 * the current make_request_fn, but that haven't been processed
	 * yet.
	 */
	struct bio_list bio_list_on_stack[2];
2387 2388
	blk_mq_req_flags_t flags = 0;
	struct request_queue *q = bio->bi_disk->queue;
2389
	blk_qc_t ret = BLK_QC_T_NONE;
2390

2391 2392
	if (bio->bi_opf & REQ_NOWAIT)
		flags = BLK_MQ_REQ_NOWAIT;
2393 2394 2395
	if (bio_flagged(bio, BIO_QUEUE_ENTERED))
		blk_queue_enter_live(q);
	else if (blk_queue_enter(q, flags) < 0) {
2396 2397 2398 2399 2400 2401 2402
		if (!blk_queue_dying(q) && (bio->bi_opf & REQ_NOWAIT))
			bio_wouldblock_error(bio);
		else
			bio_io_error(bio);
		return ret;
	}

2403
	if (!generic_make_request_checks(bio))
2404
		goto out;
2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415

	/*
	 * We only want one ->make_request_fn to be active at a time, else
	 * stack usage with stacked devices could be a problem.  So use
	 * current->bio_list to keep a list of requests submited by a
	 * make_request_fn function.  current->bio_list is also used as a
	 * flag to say if generic_make_request is currently active in this
	 * task or not.  If it is NULL, then no make_request is active.  If
	 * it is non-NULL, then a make_request is active, and new requests
	 * should be added at the tail
	 */
2416
	if (current->bio_list) {
2417
		bio_list_add(&current->bio_list[0], bio);
2418
		goto out;
2419
	}
2420

2421 2422 2423 2424 2425
	/* following loop may be a bit non-obvious, and so deserves some
	 * explanation.
	 * Before entering the loop, bio->bi_next is NULL (as all callers
	 * ensure that) so we have a list with a single bio.
	 * We pretend that we have just taken it off a longer list, so
2426 2427
	 * we assign bio_list to a pointer to the bio_list_on_stack,
	 * thus initialising the bio_list of new bios to be
2428
	 * added.  ->make_request() may indeed add some more bios
2429 2430 2431
	 * through a recursive call to generic_make_request.  If it
	 * did, we find a non-NULL value in bio_list and re-enter the loop
	 * from the top.  In this case we really did just take the bio
2432
	 * of the top of the list (no pretending) and so remove it from
2433
	 * bio_list, and call into ->make_request() again.
2434 2435
	 */
	BUG_ON(bio->bi_next);
2436 2437
	bio_list_init(&bio_list_on_stack[0]);
	current->bio_list = bio_list_on_stack;
2438
	do {
2439 2440 2441 2442 2443 2444 2445 2446 2447
		bool enter_succeeded = true;

		if (unlikely(q != bio->bi_disk->queue)) {
			if (q)
				blk_queue_exit(q);
			q = bio->bi_disk->queue;
			flags = 0;
			if (bio->bi_opf & REQ_NOWAIT)
				flags = BLK_MQ_REQ_NOWAIT;
2448
			if (blk_queue_enter(q, flags) < 0)
2449 2450
				enter_succeeded = false;
		}
2451

2452
		if (enter_succeeded) {
2453 2454 2455
			struct bio_list lower, same;

			/* Create a fresh bio_list for all subordinate requests */
2456 2457
			bio_list_on_stack[1] = bio_list_on_stack[0];
			bio_list_init(&bio_list_on_stack[0]);
2458
			ret = q->make_request_fn(q, bio);
2459

2460 2461 2462 2463 2464
			/* sort new bios into those for a lower level
			 * and those for the same level
			 */
			bio_list_init(&lower);
			bio_list_init(&same);
2465
			while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
2466
				if (q == bio->bi_disk->queue)
2467 2468 2469 2470
					bio_list_add(&same, bio);
				else
					bio_list_add(&lower, bio);
			/* now assemble so we handle the lowest level first */
2471 2472 2473
			bio_list_merge(&bio_list_on_stack[0], &lower);
			bio_list_merge(&bio_list_on_stack[0], &same);
			bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]);
2474
		} else {
2475 2476 2477 2478 2479
			if (unlikely(!blk_queue_dying(q) &&
					(bio->bi_opf & REQ_NOWAIT)))
				bio_wouldblock_error(bio);
			else
				bio_io_error(bio);
2480
			q = NULL;
2481
		}
2482
		bio = bio_list_pop(&bio_list_on_stack[0]);
2483
	} while (bio);
2484
	current->bio_list = NULL; /* deactivate */
2485 2486

out:
2487 2488
	if (q)
		blk_queue_exit(q);
2489
	return ret;
2490
}
L
Linus Torvalds 已提交
2491 2492
EXPORT_SYMBOL(generic_make_request);

2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511
/**
 * direct_make_request - hand a buffer directly to its device driver for I/O
 * @bio:  The bio describing the location in memory and on the device.
 *
 * This function behaves like generic_make_request(), but does not protect
 * against recursion.  Must only be used if the called driver is known
 * to not call generic_make_request (or direct_make_request) again from
 * its make_request function.  (Calling direct_make_request again from
 * a workqueue is perfectly fine as that doesn't recurse).
 */
blk_qc_t direct_make_request(struct bio *bio)
{
	struct request_queue *q = bio->bi_disk->queue;
	bool nowait = bio->bi_opf & REQ_NOWAIT;
	blk_qc_t ret;

	if (!generic_make_request_checks(bio))
		return BLK_QC_T_NONE;

2512
	if (unlikely(blk_queue_enter(q, nowait ? BLK_MQ_REQ_NOWAIT : 0))) {
2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526
		if (nowait && !blk_queue_dying(q))
			bio->bi_status = BLK_STS_AGAIN;
		else
			bio->bi_status = BLK_STS_IOERR;
		bio_endio(bio);
		return BLK_QC_T_NONE;
	}

	ret = q->make_request_fn(q, bio);
	blk_queue_exit(q);
	return ret;
}
EXPORT_SYMBOL_GPL(direct_make_request);

L
Linus Torvalds 已提交
2527
/**
2528
 * submit_bio - submit a bio to the block device layer for I/O
L
Linus Torvalds 已提交
2529 2530 2531 2532
 * @bio: The &struct bio which describes the I/O
 *
 * submit_bio() is very similar in purpose to generic_make_request(), and
 * uses that function to do most of the work. Both are fairly rough
2533
 * interfaces; @bio must be presetup and ready for I/O.
L
Linus Torvalds 已提交
2534 2535
 *
 */
2536
blk_qc_t submit_bio(struct bio *bio)
L
Linus Torvalds 已提交
2537
{
2538 2539 2540 2541
	/*
	 * If it's a regular read/write or a barrier with data attached,
	 * go through the normal accounting stuff before submission.
	 */
2542
	if (bio_has_data(bio)) {
2543 2544
		unsigned int count;

2545
		if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
2546
			count = queue_logical_block_size(bio->bi_disk->queue) >> 9;
2547 2548 2549
		else
			count = bio_sectors(bio);

2550
		if (op_is_write(bio_op(bio))) {
2551 2552
			count_vm_events(PGPGOUT, count);
		} else {
2553
			task_io_account_read(bio->bi_iter.bi_size);
2554 2555 2556 2557 2558
			count_vm_events(PGPGIN, count);
		}

		if (unlikely(block_dump)) {
			char b[BDEVNAME_SIZE];
2559
			printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n",
2560
			current->comm, task_pid_nr(current),
2561
				op_is_write(bio_op(bio)) ? "WRITE" : "READ",
2562
				(unsigned long long)bio->bi_iter.bi_sector,
2563
				bio_devname(bio, b), count);
2564
		}
L
Linus Torvalds 已提交
2565 2566
	}

2567
	return generic_make_request(bio);
L
Linus Torvalds 已提交
2568 2569 2570
}
EXPORT_SYMBOL(submit_bio);

2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581
bool blk_poll(struct request_queue *q, blk_qc_t cookie)
{
	if (!q->poll_fn || !blk_qc_t_valid(cookie))
		return false;

	if (current->plug)
		blk_flush_plug_list(current->plug, false);
	return q->poll_fn(q, cookie);
}
EXPORT_SYMBOL_GPL(blk_poll);

2582
/**
2583 2584
 * blk_cloned_rq_check_limits - Helper function to check a cloned request
 *                              for new the queue limits
2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595
 * @q:  the queue
 * @rq: the request being checked
 *
 * Description:
 *    @rq may have been made based on weaker limitations of upper-level queues
 *    in request stacking drivers, and it may violate the limitation of @q.
 *    Since the block layer and the underlying device driver trust @rq
 *    after it is inserted to @q, it should be checked against @q before
 *    the insertion using this generic function.
 *
 *    Request stacking drivers like request-based dm may change the queue
2596 2597
 *    limits when retrying requests on other queues. Those requests need
 *    to be checked against the new queue limits again during dispatch.
2598
 */
2599 2600
static int blk_cloned_rq_check_limits(struct request_queue *q,
				      struct request *rq)
2601
{
2602
	if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, req_op(rq))) {
2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613
		printk(KERN_ERR "%s: over max size limit.\n", __func__);
		return -EIO;
	}

	/*
	 * queue's settings related to segment counting like q->bounce_pfn
	 * may differ from that of other stacking queues.
	 * Recalculate it to check the request correctly on this queue's
	 * limitation.
	 */
	blk_recalc_rq_segments(rq);
2614
	if (rq->nr_phys_segments > queue_max_segments(q)) {
2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626
		printk(KERN_ERR "%s: over max segments limit.\n", __func__);
		return -EIO;
	}

	return 0;
}

/**
 * blk_insert_cloned_request - Helper for stacking drivers to submit a request
 * @q:  the queue to submit the request
 * @rq: the request being queued
 */
2627
blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq)
2628 2629
{
	unsigned long flags;
2630
	int where = ELEVATOR_INSERT_BACK;
2631

2632
	if (blk_cloned_rq_check_limits(q, rq))
2633
		return BLK_STS_IOERR;
2634

2635 2636
	if (rq->rq_disk &&
	    should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
2637
		return BLK_STS_IOERR;
2638

2639 2640 2641
	if (q->mq_ops) {
		if (blk_queue_io_stat(q))
			blk_account_io_start(rq, true);
2642 2643 2644 2645 2646
		/*
		 * Since we have a scheduler attached on the top device,
		 * bypass a potential scheduler on the bottom device for
		 * insert.
		 */
2647
		return blk_mq_request_issue_directly(rq);
2648 2649
	}

2650
	spin_lock_irqsave(q->queue_lock, flags);
B
Bart Van Assche 已提交
2651
	if (unlikely(blk_queue_dying(q))) {
2652
		spin_unlock_irqrestore(q->queue_lock, flags);
2653
		return BLK_STS_IOERR;
2654
	}
2655 2656 2657 2658 2659 2660 2661

	/*
	 * Submitting request must be dequeued before calling this function
	 * because it will be linked to another request_queue
	 */
	BUG_ON(blk_queued_rq(rq));

2662
	if (op_is_flush(rq->cmd_flags))
2663 2664 2665
		where = ELEVATOR_INSERT_FLUSH;

	add_acct_request(q, rq, where);
J
Jeff Moyer 已提交
2666 2667
	if (where == ELEVATOR_INSERT_FLUSH)
		__blk_run_queue(q);
2668 2669
	spin_unlock_irqrestore(q->queue_lock, flags);

2670
	return BLK_STS_OK;
2671 2672 2673
}
EXPORT_SYMBOL_GPL(blk_insert_cloned_request);

2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692
/**
 * blk_rq_err_bytes - determine number of bytes till the next failure boundary
 * @rq: request to examine
 *
 * Description:
 *     A request could be merge of IOs which require different failure
 *     handling.  This function determines the number of bytes which
 *     can be failed from the beginning of the request without
 *     crossing into area which need to be retried further.
 *
 * Return:
 *     The number of bytes to fail.
 */
unsigned int blk_rq_err_bytes(const struct request *rq)
{
	unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
	unsigned int bytes = 0;
	struct bio *bio;

2693
	if (!(rq->rq_flags & RQF_MIXED_MERGE))
2694 2695 2696 2697 2698 2699 2700 2701 2702 2703
		return blk_rq_bytes(rq);

	/*
	 * Currently the only 'mixing' which can happen is between
	 * different fastfail types.  We can safely fail portions
	 * which have all the failfast bits that the first one has -
	 * the ones which are at least as eager to fail as the first
	 * one.
	 */
	for (bio = rq->bio; bio; bio = bio->bi_next) {
J
Jens Axboe 已提交
2704
		if ((bio->bi_opf & ff) != ff)
2705
			break;
2706
		bytes += bio->bi_iter.bi_size;
2707 2708 2709 2710 2711 2712 2713 2714
	}

	/* this could lead to infinite loop */
	BUG_ON(blk_rq_bytes(rq) && !bytes);
	return bytes;
}
EXPORT_SYMBOL_GPL(blk_rq_err_bytes);

2715
void blk_account_io_completion(struct request *req, unsigned int bytes)
2716
{
2717
	if (blk_do_io_stat(req)) {
2718
		const int sgrp = op_stat_group(req_op(req));
2719 2720 2721 2722
		struct hd_struct *part;
		int cpu;

		cpu = part_stat_lock();
2723
		part = req->part;
2724
		part_stat_add(cpu, part, sectors[sgrp], bytes >> 9);
2725 2726 2727 2728
		part_stat_unlock();
	}
}

2729
void blk_account_io_done(struct request *req, u64 now)
2730 2731
{
	/*
2732 2733 2734
	 * Account IO completion.  flush_rq isn't accounted as a
	 * normal IO on queueing nor completion.  Accounting the
	 * containing request is enough.
2735
	 */
2736
	if (blk_do_io_stat(req) && !(req->rq_flags & RQF_FLUSH_SEQ)) {
2737
		const int sgrp = op_stat_group(req_op(req));
2738 2739 2740 2741
		struct hd_struct *part;
		int cpu;

		cpu = part_stat_lock();
2742
		part = req->part;
2743

2744
		part_stat_inc(cpu, part, ios[sgrp]);
2745
		part_stat_add(cpu, part, nsecs[sgrp], now - req->start_time_ns);
2746
		part_round_stats(req->q, cpu, part);
2747
		part_dec_in_flight(req->q, part, rq_data_dir(req));
2748

2749
		hd_struct_put(part);
2750 2751 2752 2753
		part_stat_unlock();
	}
}

2754
#ifdef CONFIG_PM
L
Lin Ming 已提交
2755 2756 2757 2758
/*
 * Don't process normal requests when queue is suspended
 * or in the process of suspending/resuming
 */
2759
static bool blk_pm_allow_request(struct request *rq)
L
Lin Ming 已提交
2760
{
2761 2762 2763 2764 2765 2766
	switch (rq->q->rpm_status) {
	case RPM_RESUMING:
	case RPM_SUSPENDING:
		return rq->rq_flags & RQF_PM;
	case RPM_SUSPENDED:
		return false;
2767 2768
	default:
		return true;
2769
	}
L
Lin Ming 已提交
2770 2771
}
#else
2772
static bool blk_pm_allow_request(struct request *rq)
L
Lin Ming 已提交
2773
{
2774
	return true;
L
Lin Ming 已提交
2775 2776 2777
}
#endif

2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805
void blk_account_io_start(struct request *rq, bool new_io)
{
	struct hd_struct *part;
	int rw = rq_data_dir(rq);
	int cpu;

	if (!blk_do_io_stat(rq))
		return;

	cpu = part_stat_lock();

	if (!new_io) {
		part = rq->part;
		part_stat_inc(cpu, part, merges[rw]);
	} else {
		part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
		if (!hd_struct_try_get(part)) {
			/*
			 * The partition is already being removed,
			 * the request will be accounted on the disk only
			 *
			 * We take a reference on disk->part0 although that
			 * partition will never be deleted, so we can treat
			 * it as any other partition.
			 */
			part = &rq->rq_disk->part0;
			hd_struct_get(part);
		}
2806 2807
		part_round_stats(rq->q, cpu, part);
		part_inc_in_flight(rq->q, part, rw);
2808 2809 2810 2811 2812 2813
		rq->part = part;
	}

	part_stat_unlock();
}

2814 2815 2816 2817 2818 2819 2820 2821
static struct request *elv_next_request(struct request_queue *q)
{
	struct request *rq;
	struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);

	WARN_ON_ONCE(q->mq_ops);

	while (1) {
2822 2823 2824 2825 2826 2827
		list_for_each_entry(rq, &q->queue_head, queuelist) {
			if (blk_pm_allow_request(rq))
				return rq;

			if (rq->rq_flags & RQF_SOFTBARRIER)
				break;
2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855
		}

		/*
		 * Flush request is running and flush request isn't queueable
		 * in the drive, we can hold the queue till flush request is
		 * finished. Even we don't do this, driver can't dispatch next
		 * requests and will requeue them. And this can improve
		 * throughput too. For example, we have request flush1, write1,
		 * flush 2. flush1 is dispatched, then queue is hold, write1
		 * isn't inserted to queue. After flush1 is finished, flush2
		 * will be dispatched. Since disk cache is already clean,
		 * flush2 will be finished very soon, so looks like flush2 is
		 * folded to flush1.
		 * Since the queue is hold, a flag is set to indicate the queue
		 * should be restarted later. Please see flush_end_io() for
		 * details.
		 */
		if (fq->flush_pending_idx != fq->flush_running_idx &&
				!queue_flush_queueable(q)) {
			fq->flush_queue_delayed = 1;
			return NULL;
		}
		if (unlikely(blk_queue_bypass(q)) ||
		    !q->elevator->type->ops.sq.elevator_dispatch_fn(q, 0))
			return NULL;
	}
}

2856
/**
2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869
 * blk_peek_request - peek at the top of a request queue
 * @q: request queue to peek at
 *
 * Description:
 *     Return the request at the top of @q.  The returned request
 *     should be started using blk_start_request() before LLD starts
 *     processing it.
 *
 * Return:
 *     Pointer to the request at the top of @q if available.  Null
 *     otherwise.
 */
struct request *blk_peek_request(struct request_queue *q)
2870 2871 2872 2873
{
	struct request *rq;
	int ret;

2874
	lockdep_assert_held(q->queue_lock);
2875
	WARN_ON_ONCE(q->mq_ops);
2876

2877
	while ((rq = elv_next_request(q)) != NULL) {
2878
		if (!(rq->rq_flags & RQF_STARTED)) {
2879 2880 2881 2882 2883
			/*
			 * This is the first time the device driver
			 * sees this request (possibly after
			 * requeueing).  Notify IO scheduler.
			 */
2884
			if (rq->rq_flags & RQF_SORTED)
2885 2886 2887 2888 2889 2890 2891
				elv_activate_rq(q, rq);

			/*
			 * just mark as started even if we don't start
			 * it, a request that has been delayed should
			 * not be passed by new incoming requests
			 */
2892
			rq->rq_flags |= RQF_STARTED;
2893 2894 2895 2896 2897 2898 2899 2900
			trace_block_rq_issue(q, rq);
		}

		if (!q->boundary_rq || q->boundary_rq == rq) {
			q->end_sector = rq_end_sector(rq);
			q->boundary_rq = NULL;
		}

2901
		if (rq->rq_flags & RQF_DONTPREP)
2902 2903
			break;

2904
		if (q->dma_drain_size && blk_rq_bytes(rq)) {
2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923
			/*
			 * make sure space for the drain appears we
			 * know we can do this because max_hw_segments
			 * has been adjusted to be one fewer than the
			 * device can handle
			 */
			rq->nr_phys_segments++;
		}

		if (!q->prep_rq_fn)
			break;

		ret = q->prep_rq_fn(q, rq);
		if (ret == BLKPREP_OK) {
			break;
		} else if (ret == BLKPREP_DEFER) {
			/*
			 * the request may have been (partially) prepped.
			 * we need to keep this request in the front to
2924
			 * avoid resource deadlock.  RQF_STARTED will
2925 2926
			 * prevent other fs requests from passing this one.
			 */
2927
			if (q->dma_drain_size && blk_rq_bytes(rq) &&
2928
			    !(rq->rq_flags & RQF_DONTPREP)) {
2929 2930 2931 2932 2933 2934 2935 2936 2937
				/*
				 * remove the space for the drain we added
				 * so that we don't add it again
				 */
				--rq->nr_phys_segments;
			}

			rq = NULL;
			break;
2938
		} else if (ret == BLKPREP_KILL || ret == BLKPREP_INVALID) {
2939
			rq->rq_flags |= RQF_QUIET;
2940 2941 2942 2943 2944
			/*
			 * Mark this request as started so we don't trigger
			 * any debug logic in the end I/O path.
			 */
			blk_start_request(rq);
2945 2946
			__blk_end_request_all(rq, ret == BLKPREP_INVALID ?
					BLK_STS_TARGET : BLK_STS_IOERR);
2947 2948 2949 2950 2951 2952 2953 2954
		} else {
			printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
			break;
		}
	}

	return rq;
}
2955
EXPORT_SYMBOL(blk_peek_request);
2956

2957
static void blk_dequeue_request(struct request *rq)
2958
{
2959 2960
	struct request_queue *q = rq->q;

2961 2962 2963 2964 2965 2966 2967 2968 2969 2970
	BUG_ON(list_empty(&rq->queuelist));
	BUG_ON(ELV_ON_HASH(rq));

	list_del_init(&rq->queuelist);

	/*
	 * the time frame between a request being removed from the lists
	 * and to it is freed is accounted as io that is in progress at
	 * the driver side.
	 */
2971
	if (blk_account_rq(rq))
2972
		q->in_flight[rq_is_sync(rq)]++;
2973 2974
}

2975 2976 2977 2978 2979 2980 2981 2982 2983 2984
/**
 * blk_start_request - start request processing on the driver
 * @req: request to dequeue
 *
 * Description:
 *     Dequeue @req and start timeout timer on it.  This hands off the
 *     request to the driver.
 */
void blk_start_request(struct request *req)
{
2985
	lockdep_assert_held(req->q->queue_lock);
2986
	WARN_ON_ONCE(req->q->mq_ops);
2987

2988 2989
	blk_dequeue_request(req);

2990
	if (test_bit(QUEUE_FLAG_STATS, &req->q->queue_flags)) {
2991 2992 2993 2994
		req->io_start_time_ns = ktime_get_ns();
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
		req->throtl_size = blk_rq_sectors(req);
#endif
2995
		req->rq_flags |= RQF_STATS;
2996
		rq_qos_issue(req->q, req);
2997 2998
	}

2999
	BUG_ON(blk_rq_is_complete(req));
3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019
	blk_add_timer(req);
}
EXPORT_SYMBOL(blk_start_request);

/**
 * blk_fetch_request - fetch a request from a request queue
 * @q: request queue to fetch a request from
 *
 * Description:
 *     Return the request at the top of @q.  The request is started on
 *     return and LLD can start processing it immediately.
 *
 * Return:
 *     Pointer to the request at the top of @q if available.  Null
 *     otherwise.
 */
struct request *blk_fetch_request(struct request_queue *q)
{
	struct request *rq;

3020
	lockdep_assert_held(q->queue_lock);
3021
	WARN_ON_ONCE(q->mq_ops);
3022

3023 3024 3025 3026 3027 3028 3029
	rq = blk_peek_request(q);
	if (rq)
		blk_start_request(rq);
	return rq;
}
EXPORT_SYMBOL(blk_fetch_request);

3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050
/*
 * Steal bios from a request and add them to a bio list.
 * The request must not have been partially completed before.
 */
void blk_steal_bios(struct bio_list *list, struct request *rq)
{
	if (rq->bio) {
		if (list->tail)
			list->tail->bi_next = rq->bio;
		else
			list->head = rq->bio;
		list->tail = rq->biotail;

		rq->bio = NULL;
		rq->biotail = NULL;
	}

	rq->__data_len = 0;
}
EXPORT_SYMBOL_GPL(blk_steal_bios);

3051
/**
3052
 * blk_update_request - Special helper function for request stacking drivers
3053
 * @req:      the request being processed
3054
 * @error:    block status code
3055
 * @nr_bytes: number of bytes to complete @req
3056 3057
 *
 * Description:
3058 3059 3060
 *     Ends I/O on a number of bytes attached to @req, but doesn't complete
 *     the request structure even if @req doesn't have leftover.
 *     If @req has leftover, sets it up for the next range of segments.
3061 3062 3063 3064 3065 3066 3067
 *
 *     This special helper function is only for request stacking drivers
 *     (e.g. request-based dm) so that they can handle partial completion.
 *     Actual device drivers should use blk_end_request instead.
 *
 *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
 *     %false return from this function.
3068
 *
3069 3070 3071 3072
 * Note:
 *	The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in both
 *	blk_rq_bytes() and in blk_update_request().
 *
3073
 * Return:
3074 3075
 *     %false - this request doesn't have any more data
 *     %true  - this request has more data
3076
 **/
3077 3078
bool blk_update_request(struct request *req, blk_status_t error,
		unsigned int nr_bytes)
L
Linus Torvalds 已提交
3079
{
3080
	int total_bytes;
L
Linus Torvalds 已提交
3081

3082
	trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);
3083

3084 3085 3086
	if (!req->bio)
		return false;

3087 3088 3089
	if (unlikely(error && !blk_rq_is_passthrough(req) &&
		     !(req->rq_flags & RQF_QUIET)))
		print_req_error(req, error);
L
Linus Torvalds 已提交
3090

3091
	blk_account_io_completion(req, nr_bytes);
3092

3093 3094 3095
	total_bytes = 0;
	while (req->bio) {
		struct bio *bio = req->bio;
3096
		unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
L
Linus Torvalds 已提交
3097

3098
		if (bio_bytes == bio->bi_iter.bi_size)
L
Linus Torvalds 已提交
3099 3100
			req->bio = bio->bi_next;

N
NeilBrown 已提交
3101 3102
		/* Completion has already been traced */
		bio_clear_flag(bio, BIO_TRACE_COMPLETION);
3103
		req_bio_endio(req, bio, bio_bytes, error);
L
Linus Torvalds 已提交
3104

3105 3106
		total_bytes += bio_bytes;
		nr_bytes -= bio_bytes;
L
Linus Torvalds 已提交
3107

3108 3109
		if (!nr_bytes)
			break;
L
Linus Torvalds 已提交
3110 3111 3112 3113 3114
	}

	/*
	 * completely done
	 */
3115 3116 3117 3118 3119 3120
	if (!req->bio) {
		/*
		 * Reset counters so that the request stacking driver
		 * can find how many bytes remain in the request
		 * later.
		 */
3121
		req->__data_len = 0;
3122 3123
		return false;
	}
L
Linus Torvalds 已提交
3124

3125
	req->__data_len -= total_bytes;
3126 3127

	/* update sector only for requests with clear definition of sector */
3128
	if (!blk_rq_is_passthrough(req))
3129
		req->__sector += total_bytes >> 9;
3130

3131
	/* mixed attributes always follow the first bio */
3132
	if (req->rq_flags & RQF_MIXED_MERGE) {
3133
		req->cmd_flags &= ~REQ_FAILFAST_MASK;
J
Jens Axboe 已提交
3134
		req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
3135 3136
	}

3137 3138 3139 3140 3141 3142 3143 3144 3145
	if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
		/*
		 * If total number of sectors is less than the first segment
		 * size, something has gone terribly wrong.
		 */
		if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
			blk_dump_rq_flags(req, "request botched");
			req->__data_len = blk_rq_cur_bytes(req);
		}
3146

3147 3148 3149
		/* recalculate the number of segments */
		blk_recalc_rq_segments(req);
	}
3150

3151
	return true;
L
Linus Torvalds 已提交
3152
}
3153
EXPORT_SYMBOL_GPL(blk_update_request);
L
Linus Torvalds 已提交
3154

3155
static bool blk_update_bidi_request(struct request *rq, blk_status_t error,
3156 3157
				    unsigned int nr_bytes,
				    unsigned int bidi_bytes)
3158
{
3159 3160
	if (blk_update_request(rq, error, nr_bytes))
		return true;
3161

3162 3163 3164 3165
	/* Bidi request must be completed as a whole */
	if (unlikely(blk_bidi_rq(rq)) &&
	    blk_update_request(rq->next_rq, error, bidi_bytes))
		return true;
3166

3167 3168
	if (blk_queue_add_random(rq->q))
		add_disk_randomness(rq->rq_disk);
3169 3170

	return false;
L
Linus Torvalds 已提交
3171 3172
}

3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186
/**
 * blk_unprep_request - unprepare a request
 * @req:	the request
 *
 * This function makes a request ready for complete resubmission (or
 * completion).  It happens only after all error handling is complete,
 * so represents the appropriate moment to deallocate any resources
 * that were allocated to the request in the prep_rq_fn.  The queue
 * lock is held when calling this.
 */
void blk_unprep_request(struct request *req)
{
	struct request_queue *q = req->q;

3187
	req->rq_flags &= ~RQF_DONTPREP;
3188 3189 3190 3191 3192
	if (q->unprep_rq_fn)
		q->unprep_rq_fn(q, req);
}
EXPORT_SYMBOL_GPL(blk_unprep_request);

3193
void blk_finish_request(struct request *req, blk_status_t error)
L
Linus Torvalds 已提交
3194
{
3195
	struct request_queue *q = req->q;
3196
	u64 now = ktime_get_ns();
3197

3198
	lockdep_assert_held(req->q->queue_lock);
3199
	WARN_ON_ONCE(q->mq_ops);
3200

3201
	if (req->rq_flags & RQF_STATS)
3202
		blk_stat_add(req, now);
3203

3204
	if (req->rq_flags & RQF_QUEUED)
3205
		blk_queue_end_tag(q, req);
3206

3207
	BUG_ON(blk_queued_rq(req));
L
Linus Torvalds 已提交
3208

3209
	if (unlikely(laptop_mode) && !blk_rq_is_passthrough(req))
3210
		laptop_io_completion(req->q->backing_dev_info);
L
Linus Torvalds 已提交
3211

3212 3213
	blk_delete_timer(req);

3214
	if (req->rq_flags & RQF_DONTPREP)
3215 3216
		blk_unprep_request(req);

3217
	blk_account_io_done(req, now);
3218

J
Jens Axboe 已提交
3219
	if (req->end_io) {
3220
		rq_qos_done(q, req);
3221
		req->end_io(req, error);
J
Jens Axboe 已提交
3222
	} else {
3223 3224 3225
		if (blk_bidi_rq(req))
			__blk_put_request(req->next_rq->q, req->next_rq);

3226
		__blk_put_request(q, req);
3227
	}
L
Linus Torvalds 已提交
3228
}
3229
EXPORT_SYMBOL(blk_finish_request);
L
Linus Torvalds 已提交
3230

3231
/**
3232 3233
 * blk_end_bidi_request - Complete a bidi request
 * @rq:         the request to complete
3234
 * @error:      block status code
3235 3236
 * @nr_bytes:   number of bytes to complete @rq
 * @bidi_bytes: number of bytes to complete @rq->next_rq
3237 3238
 *
 * Description:
3239
 *     Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
3240 3241 3242
 *     Drivers that supports bidi can safely call this member for any
 *     type of request, bidi or uni.  In the later case @bidi_bytes is
 *     just ignored.
3243 3244
 *
 * Return:
3245 3246
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
3247
 **/
3248
static bool blk_end_bidi_request(struct request *rq, blk_status_t error,
K
Kiyoshi Ueda 已提交
3249 3250
				 unsigned int nr_bytes, unsigned int bidi_bytes)
{
3251
	struct request_queue *q = rq->q;
3252
	unsigned long flags;
K
Kiyoshi Ueda 已提交
3253

3254 3255
	WARN_ON_ONCE(q->mq_ops);

3256 3257
	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
		return true;
K
Kiyoshi Ueda 已提交
3258

3259
	spin_lock_irqsave(q->queue_lock, flags);
3260
	blk_finish_request(rq, error);
3261 3262
	spin_unlock_irqrestore(q->queue_lock, flags);

3263
	return false;
K
Kiyoshi Ueda 已提交
3264 3265
}

3266
/**
3267 3268
 * __blk_end_bidi_request - Complete a bidi request with queue lock held
 * @rq:         the request to complete
3269
 * @error:      block status code
3270 3271
 * @nr_bytes:   number of bytes to complete @rq
 * @bidi_bytes: number of bytes to complete @rq->next_rq
3272 3273
 *
 * Description:
3274 3275
 *     Identical to blk_end_bidi_request() except that queue lock is
 *     assumed to be locked on entry and remains so on return.
3276 3277
 *
 * Return:
3278 3279
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
3280
 **/
3281
static bool __blk_end_bidi_request(struct request *rq, blk_status_t error,
3282
				   unsigned int nr_bytes, unsigned int bidi_bytes)
3283
{
3284
	lockdep_assert_held(rq->q->queue_lock);
3285
	WARN_ON_ONCE(rq->q->mq_ops);
3286

3287 3288
	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
		return true;
3289

3290
	blk_finish_request(rq, error);
3291

3292
	return false;
3293
}
3294 3295 3296 3297

/**
 * blk_end_request - Helper function for drivers to complete the request.
 * @rq:       the request being processed
3298
 * @error:    block status code
3299 3300 3301 3302 3303 3304 3305
 * @nr_bytes: number of bytes to complete
 *
 * Description:
 *     Ends I/O on a number of bytes attached to @rq.
 *     If @rq has leftover, sets it up for the next range of segments.
 *
 * Return:
3306 3307
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
3308
 **/
3309 3310
bool blk_end_request(struct request *rq, blk_status_t error,
		unsigned int nr_bytes)
3311
{
3312
	WARN_ON_ONCE(rq->q->mq_ops);
3313
	return blk_end_bidi_request(rq, error, nr_bytes, 0);
3314
}
3315
EXPORT_SYMBOL(blk_end_request);
3316 3317

/**
3318 3319
 * blk_end_request_all - Helper function for drives to finish the request.
 * @rq: the request to finish
3320
 * @error: block status code
3321 3322
 *
 * Description:
3323 3324
 *     Completely finish @rq.
 */
3325
void blk_end_request_all(struct request *rq, blk_status_t error)
3326
{
3327 3328
	bool pending;
	unsigned int bidi_bytes = 0;
3329

3330 3331
	if (unlikely(blk_bidi_rq(rq)))
		bidi_bytes = blk_rq_bytes(rq->next_rq);
3332

3333 3334 3335
	pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
	BUG_ON(pending);
}
3336
EXPORT_SYMBOL(blk_end_request_all);
3337

3338
/**
3339 3340
 * __blk_end_request - Helper function for drivers to complete the request.
 * @rq:       the request being processed
3341
 * @error:    block status code
3342
 * @nr_bytes: number of bytes to complete
3343 3344
 *
 * Description:
3345
 *     Must be called with queue lock held unlike blk_end_request().
3346 3347
 *
 * Return:
3348 3349
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
3350
 **/
3351 3352
bool __blk_end_request(struct request *rq, blk_status_t error,
		unsigned int nr_bytes)
3353
{
3354
	lockdep_assert_held(rq->q->queue_lock);
3355
	WARN_ON_ONCE(rq->q->mq_ops);
3356

3357
	return __blk_end_bidi_request(rq, error, nr_bytes, 0);
3358
}
3359
EXPORT_SYMBOL(__blk_end_request);
3360

K
Kiyoshi Ueda 已提交
3361
/**
3362 3363
 * __blk_end_request_all - Helper function for drives to finish the request.
 * @rq: the request to finish
3364
 * @error:    block status code
K
Kiyoshi Ueda 已提交
3365 3366
 *
 * Description:
3367
 *     Completely finish @rq.  Must be called with queue lock held.
K
Kiyoshi Ueda 已提交
3368
 */
3369
void __blk_end_request_all(struct request *rq, blk_status_t error)
K
Kiyoshi Ueda 已提交
3370
{
3371 3372 3373
	bool pending;
	unsigned int bidi_bytes = 0;

3374
	lockdep_assert_held(rq->q->queue_lock);
3375
	WARN_ON_ONCE(rq->q->mq_ops);
3376

3377 3378 3379 3380 3381
	if (unlikely(blk_bidi_rq(rq)))
		bidi_bytes = blk_rq_bytes(rq->next_rq);

	pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
	BUG_ON(pending);
K
Kiyoshi Ueda 已提交
3382
}
3383
EXPORT_SYMBOL(__blk_end_request_all);
K
Kiyoshi Ueda 已提交
3384

3385
/**
3386 3387
 * __blk_end_request_cur - Helper function to finish the current request chunk.
 * @rq: the request to finish the current chunk for
3388
 * @error:    block status code
3389 3390
 *
 * Description:
3391 3392
 *     Complete the current consecutively mapped chunk from @rq.  Must
 *     be called with queue lock held.
3393 3394
 *
 * Return:
3395 3396 3397
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
 */
3398
bool __blk_end_request_cur(struct request *rq, blk_status_t error)
3399
{
3400
	return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
3401
}
3402
EXPORT_SYMBOL(__blk_end_request_cur);
3403

J
Jens Axboe 已提交
3404 3405
void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
		     struct bio *bio)
L
Linus Torvalds 已提交
3406
{
3407
	if (bio_has_data(bio))
D
David Woodhouse 已提交
3408
		rq->nr_phys_segments = bio_phys_segments(q, bio);
3409 3410
	else if (bio_op(bio) == REQ_OP_DISCARD)
		rq->nr_phys_segments = 1;
3411

3412
	rq->__data_len = bio->bi_iter.bi_size;
L
Linus Torvalds 已提交
3413 3414
	rq->bio = rq->biotail = bio;

3415 3416
	if (bio->bi_disk)
		rq->rq_disk = bio->bi_disk;
N
NeilBrown 已提交
3417
}
L
Linus Torvalds 已提交
3418

3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429
#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
/**
 * rq_flush_dcache_pages - Helper function to flush all pages in a request
 * @rq: the request to be flushed
 *
 * Description:
 *     Flush all pages in @rq.
 */
void rq_flush_dcache_pages(struct request *rq)
{
	struct req_iterator iter;
3430
	struct bio_vec bvec;
3431 3432

	rq_for_each_segment(bvec, rq, iter)
3433
		flush_dcache_page(bvec.bv_page);
3434 3435 3436 3437
}
EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
#endif

3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465
/**
 * blk_lld_busy - Check if underlying low-level drivers of a device are busy
 * @q : the queue of the device being checked
 *
 * Description:
 *    Check if underlying low-level drivers of a device are busy.
 *    If the drivers want to export their busy state, they must set own
 *    exporting function using blk_queue_lld_busy() first.
 *
 *    Basically, this function is used only by request stacking drivers
 *    to stop dispatching requests to underlying devices when underlying
 *    devices are busy.  This behavior helps more I/O merging on the queue
 *    of the request stacking driver and prevents I/O throughput regression
 *    on burst I/O load.
 *
 * Return:
 *    0 - Not busy (The request stacking driver should dispatch request)
 *    1 - Busy (The request stacking driver should stop dispatching request)
 */
int blk_lld_busy(struct request_queue *q)
{
	if (q->lld_busy_fn)
		return q->lld_busy_fn(q);

	return 0;
}
EXPORT_SYMBOL_GPL(blk_lld_busy);

3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489
/**
 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
 * @rq: the clone request to be cleaned up
 *
 * Description:
 *     Free all bios in @rq for a cloned request.
 */
void blk_rq_unprep_clone(struct request *rq)
{
	struct bio *bio;

	while ((bio = rq->bio) != NULL) {
		rq->bio = bio->bi_next;

		bio_put(bio);
	}
}
EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);

/*
 * Copy attributes of the original request to the clone request.
 * The actual data parts (e.g. ->cmd, ->sense) are not copied.
 */
static void __blk_rq_prep_clone(struct request *dst, struct request *src)
3490 3491 3492 3493
{
	dst->cpu = src->cpu;
	dst->__sector = blk_rq_pos(src);
	dst->__data_len = blk_rq_bytes(src);
3494 3495 3496 3497
	if (src->rq_flags & RQF_SPECIAL_PAYLOAD) {
		dst->rq_flags |= RQF_SPECIAL_PAYLOAD;
		dst->special_vec = src->special_vec;
	}
3498 3499 3500
	dst->nr_phys_segments = src->nr_phys_segments;
	dst->ioprio = src->ioprio;
	dst->extra_len = src->extra_len;
3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529
}

/**
 * blk_rq_prep_clone - Helper function to setup clone request
 * @rq: the request to be setup
 * @rq_src: original request to be cloned
 * @bs: bio_set that bios for clone are allocated from
 * @gfp_mask: memory allocation mask for bio
 * @bio_ctr: setup function to be called for each clone bio.
 *           Returns %0 for success, non %0 for failure.
 * @data: private data to be passed to @bio_ctr
 *
 * Description:
 *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
 *     The actual data parts of @rq_src (e.g. ->cmd, ->sense)
 *     are not copied, and copying such parts is the caller's responsibility.
 *     Also, pages which the original bios are pointing to are not copied
 *     and the cloned bios just point same pages.
 *     So cloned bios must be completed before original bios, which means
 *     the caller must complete @rq before @rq_src.
 */
int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
		      struct bio_set *bs, gfp_t gfp_mask,
		      int (*bio_ctr)(struct bio *, struct bio *, void *),
		      void *data)
{
	struct bio *bio, *bio_src;

	if (!bs)
3530
		bs = &fs_bio_set;
3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556

	__rq_for_each_bio(bio_src, rq_src) {
		bio = bio_clone_fast(bio_src, gfp_mask, bs);
		if (!bio)
			goto free_and_out;

		if (bio_ctr && bio_ctr(bio, bio_src, data))
			goto free_and_out;

		if (rq->bio) {
			rq->biotail->bi_next = bio;
			rq->biotail = bio;
		} else
			rq->bio = rq->biotail = bio;
	}

	__blk_rq_prep_clone(rq, rq_src);

	return 0;

free_and_out:
	if (bio)
		bio_put(bio);
	blk_rq_unprep_clone(rq);

	return -ENOMEM;
3557 3558 3559
}
EXPORT_SYMBOL_GPL(blk_rq_prep_clone);

3560
int kblockd_schedule_work(struct work_struct *work)
L
Linus Torvalds 已提交
3561 3562 3563 3564 3565
{
	return queue_work(kblockd_workqueue, work);
}
EXPORT_SYMBOL(kblockd_schedule_work);

3566 3567 3568 3569 3570 3571
int kblockd_schedule_work_on(int cpu, struct work_struct *work)
{
	return queue_work_on(cpu, kblockd_workqueue, work);
}
EXPORT_SYMBOL(kblockd_schedule_work_on);

3572 3573 3574 3575 3576 3577 3578
int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork,
				unsigned long delay)
{
	return mod_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
}
EXPORT_SYMBOL(kblockd_mod_delayed_work_on);

S
Suresh Jayaraman 已提交
3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592
/**
 * blk_start_plug - initialize blk_plug and track it inside the task_struct
 * @plug:	The &struct blk_plug that needs to be initialized
 *
 * Description:
 *   Tracking blk_plug inside the task_struct will help with auto-flushing the
 *   pending I/O should the task end up blocking between blk_start_plug() and
 *   blk_finish_plug(). This is important from a performance perspective, but
 *   also ensures that we don't deadlock. For instance, if the task is blocking
 *   for a memory allocation, memory reclaim could end up wanting to free a
 *   page belonging to that request that is currently residing in our private
 *   plug. By flushing the pending I/O when the process goes to sleep, we avoid
 *   this kind of deadlock.
 */
3593 3594 3595 3596
void blk_start_plug(struct blk_plug *plug)
{
	struct task_struct *tsk = current;

S
Shaohua Li 已提交
3597 3598 3599 3600 3601 3602
	/*
	 * If this is a nested plug, don't actually assign it.
	 */
	if (tsk->plug)
		return;

3603
	INIT_LIST_HEAD(&plug->list);
3604
	INIT_LIST_HEAD(&plug->mq_list);
3605
	INIT_LIST_HEAD(&plug->cb_list);
3606
	/*
S
Shaohua Li 已提交
3607 3608
	 * Store ordering should not be needed here, since a potential
	 * preempt will imply a full memory barrier
3609
	 */
S
Shaohua Li 已提交
3610
	tsk->plug = plug;
3611 3612 3613 3614 3615 3616 3617 3618
}
EXPORT_SYMBOL(blk_start_plug);

static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
{
	struct request *rqa = container_of(a, struct request, queuelist);
	struct request *rqb = container_of(b, struct request, queuelist);

3619 3620
	return !(rqa->q < rqb->q ||
		(rqa->q == rqb->q && blk_rq_pos(rqa) < blk_rq_pos(rqb)));
3621 3622
}

3623 3624 3625 3626 3627 3628
/*
 * If 'from_schedule' is true, then postpone the dispatch of requests
 * until a safe kblockd context. We due this to avoid accidental big
 * additional stack usage in driver dispatch, in places where the originally
 * plugger did not intend it.
 */
3629
static void queue_unplugged(struct request_queue *q, unsigned int depth,
3630
			    bool from_schedule)
3631
	__releases(q->queue_lock)
3632
{
3633 3634
	lockdep_assert_held(q->queue_lock);

3635
	trace_block_unplug(q, depth, !from_schedule);
3636

3637
	if (from_schedule)
3638
		blk_run_queue_async(q);
3639
	else
3640
		__blk_run_queue(q);
3641
	spin_unlock_irq(q->queue_lock);
3642 3643
}

3644
static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
3645 3646 3647
{
	LIST_HEAD(callbacks);

S
Shaohua Li 已提交
3648 3649
	while (!list_empty(&plug->cb_list)) {
		list_splice_init(&plug->cb_list, &callbacks);
3650

S
Shaohua Li 已提交
3651 3652
		while (!list_empty(&callbacks)) {
			struct blk_plug_cb *cb = list_first_entry(&callbacks,
3653 3654
							  struct blk_plug_cb,
							  list);
S
Shaohua Li 已提交
3655
			list_del(&cb->list);
3656
			cb->callback(cb, from_schedule);
S
Shaohua Li 已提交
3657
		}
3658 3659 3660
	}
}

3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685
struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
				      int size)
{
	struct blk_plug *plug = current->plug;
	struct blk_plug_cb *cb;

	if (!plug)
		return NULL;

	list_for_each_entry(cb, &plug->cb_list, list)
		if (cb->callback == unplug && cb->data == data)
			return cb;

	/* Not currently on the callback list */
	BUG_ON(size < sizeof(*cb));
	cb = kzalloc(size, GFP_ATOMIC);
	if (cb) {
		cb->data = data;
		cb->callback = unplug;
		list_add(&cb->list, &plug->cb_list);
	}
	return cb;
}
EXPORT_SYMBOL(blk_check_plugged);

3686
void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
3687 3688 3689
{
	struct request_queue *q;
	struct request *rq;
3690
	LIST_HEAD(list);
3691
	unsigned int depth;
3692

3693
	flush_plug_callbacks(plug, from_schedule);
3694 3695 3696 3697

	if (!list_empty(&plug->mq_list))
		blk_mq_flush_plug_list(plug, from_schedule);

3698 3699 3700
	if (list_empty(&plug->list))
		return;

3701 3702
	list_splice_init(&plug->list, &list);

3703
	list_sort(NULL, &list, plug_rq_cmp);
3704 3705

	q = NULL;
3706
	depth = 0;
3707

3708 3709
	while (!list_empty(&list)) {
		rq = list_entry_rq(list.next);
3710 3711 3712
		list_del_init(&rq->queuelist);
		BUG_ON(!rq->q);
		if (rq->q != q) {
3713 3714 3715 3716
			/*
			 * This drops the queue lock
			 */
			if (q)
3717
				queue_unplugged(q, depth, from_schedule);
3718
			q = rq->q;
3719
			depth = 0;
3720
			spin_lock_irq(q->queue_lock);
3721
		}
3722 3723 3724 3725

		/*
		 * Short-circuit if @q is dead
		 */
B
Bart Van Assche 已提交
3726
		if (unlikely(blk_queue_dying(q))) {
3727
			__blk_end_request_all(rq, BLK_STS_IOERR);
3728 3729 3730
			continue;
		}

3731 3732 3733
		/*
		 * rq is already accounted, so use raw insert
		 */
3734
		if (op_is_flush(rq->cmd_flags))
3735 3736 3737
			__elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH);
		else
			__elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE);
3738 3739

		depth++;
3740 3741
	}

3742 3743 3744 3745
	/*
	 * This drops the queue lock
	 */
	if (q)
3746
		queue_unplugged(q, depth, from_schedule);
3747 3748 3749 3750
}

void blk_finish_plug(struct blk_plug *plug)
{
S
Shaohua Li 已提交
3751 3752
	if (plug != current->plug)
		return;
3753
	blk_flush_plug_list(plug, false);
3754

S
Shaohua Li 已提交
3755
	current->plug = NULL;
3756
}
3757
EXPORT_SYMBOL(blk_finish_plug);
3758

3759
#ifdef CONFIG_PM
L
Lin Ming 已提交
3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782
/**
 * blk_pm_runtime_init - Block layer runtime PM initialization routine
 * @q: the queue of the device
 * @dev: the device the queue belongs to
 *
 * Description:
 *    Initialize runtime-PM-related fields for @q and start auto suspend for
 *    @dev. Drivers that want to take advantage of request-based runtime PM
 *    should call this function after @dev has been initialized, and its
 *    request queue @q has been allocated, and runtime PM for it can not happen
 *    yet(either due to disabled/forbidden or its usage_count > 0). In most
 *    cases, driver should call this function before any I/O has taken place.
 *
 *    This function takes care of setting up using auto suspend for the device,
 *    the autosuspend delay is set to -1 to make runtime suspend impossible
 *    until an updated value is either set by user or by driver. Drivers do
 *    not need to touch other autosuspend settings.
 *
 *    The block layer runtime PM is request based, so only works for drivers
 *    that use request as their IO unit instead of those directly use bio's.
 */
void blk_pm_runtime_init(struct request_queue *q, struct device *dev)
{
3783 3784 3785
	/* Don't enable runtime PM for blk-mq until it is ready */
	if (q->mq_ops) {
		pm_runtime_disable(dev);
3786
		return;
3787
	}
3788

L
Lin Ming 已提交
3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820
	q->dev = dev;
	q->rpm_status = RPM_ACTIVE;
	pm_runtime_set_autosuspend_delay(q->dev, -1);
	pm_runtime_use_autosuspend(q->dev);
}
EXPORT_SYMBOL(blk_pm_runtime_init);

/**
 * blk_pre_runtime_suspend - Pre runtime suspend check
 * @q: the queue of the device
 *
 * Description:
 *    This function will check if runtime suspend is allowed for the device
 *    by examining if there are any requests pending in the queue. If there
 *    are requests pending, the device can not be runtime suspended; otherwise,
 *    the queue's status will be updated to SUSPENDING and the driver can
 *    proceed to suspend the device.
 *
 *    For the not allowed case, we mark last busy for the device so that
 *    runtime PM core will try to autosuspend it some time later.
 *
 *    This function should be called near the start of the device's
 *    runtime_suspend callback.
 *
 * Return:
 *    0		- OK to runtime suspend the device
 *    -EBUSY	- Device should not be runtime suspended
 */
int blk_pre_runtime_suspend(struct request_queue *q)
{
	int ret = 0;

3821 3822 3823
	if (!q->dev)
		return ret;

L
Lin Ming 已提交
3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850
	spin_lock_irq(q->queue_lock);
	if (q->nr_pending) {
		ret = -EBUSY;
		pm_runtime_mark_last_busy(q->dev);
	} else {
		q->rpm_status = RPM_SUSPENDING;
	}
	spin_unlock_irq(q->queue_lock);
	return ret;
}
EXPORT_SYMBOL(blk_pre_runtime_suspend);

/**
 * blk_post_runtime_suspend - Post runtime suspend processing
 * @q: the queue of the device
 * @err: return value of the device's runtime_suspend function
 *
 * Description:
 *    Update the queue's runtime status according to the return value of the
 *    device's runtime suspend function and mark last busy for the device so
 *    that PM core will try to auto suspend the device at a later time.
 *
 *    This function should be called near the end of the device's
 *    runtime_suspend callback.
 */
void blk_post_runtime_suspend(struct request_queue *q, int err)
{
3851 3852 3853
	if (!q->dev)
		return;

L
Lin Ming 已提交
3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877
	spin_lock_irq(q->queue_lock);
	if (!err) {
		q->rpm_status = RPM_SUSPENDED;
	} else {
		q->rpm_status = RPM_ACTIVE;
		pm_runtime_mark_last_busy(q->dev);
	}
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL(blk_post_runtime_suspend);

/**
 * blk_pre_runtime_resume - Pre runtime resume processing
 * @q: the queue of the device
 *
 * Description:
 *    Update the queue's runtime status to RESUMING in preparation for the
 *    runtime resume of the device.
 *
 *    This function should be called near the start of the device's
 *    runtime_resume callback.
 */
void blk_pre_runtime_resume(struct request_queue *q)
{
3878 3879 3880
	if (!q->dev)
		return;

L
Lin Ming 已提交
3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902
	spin_lock_irq(q->queue_lock);
	q->rpm_status = RPM_RESUMING;
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL(blk_pre_runtime_resume);

/**
 * blk_post_runtime_resume - Post runtime resume processing
 * @q: the queue of the device
 * @err: return value of the device's runtime_resume function
 *
 * Description:
 *    Update the queue's runtime status according to the return value of the
 *    device's runtime_resume function. If it is successfully resumed, process
 *    the requests that are queued into the device's queue when it is resuming
 *    and then mark last busy and initiate autosuspend for it.
 *
 *    This function should be called near the end of the device's
 *    runtime_resume callback.
 */
void blk_post_runtime_resume(struct request_queue *q, int err)
{
3903 3904 3905
	if (!q->dev)
		return;

L
Lin Ming 已提交
3906 3907 3908 3909 3910
	spin_lock_irq(q->queue_lock);
	if (!err) {
		q->rpm_status = RPM_ACTIVE;
		__blk_run_queue(q);
		pm_runtime_mark_last_busy(q->dev);
3911
		pm_request_autosuspend(q->dev);
L
Lin Ming 已提交
3912 3913 3914 3915 3916 3917
	} else {
		q->rpm_status = RPM_SUSPENDED;
	}
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL(blk_post_runtime_resume);
3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941

/**
 * blk_set_runtime_active - Force runtime status of the queue to be active
 * @q: the queue of the device
 *
 * If the device is left runtime suspended during system suspend the resume
 * hook typically resumes the device and corrects runtime status
 * accordingly. However, that does not affect the queue runtime PM status
 * which is still "suspended". This prevents processing requests from the
 * queue.
 *
 * This function can be used in driver's resume hook to correct queue
 * runtime PM status and re-enable peeking requests from the queue. It
 * should be called before first request is added to the queue.
 */
void blk_set_runtime_active(struct request_queue *q)
{
	spin_lock_irq(q->queue_lock);
	q->rpm_status = RPM_ACTIVE;
	pm_runtime_mark_last_busy(q->dev);
	pm_request_autosuspend(q->dev);
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL(blk_set_runtime_active);
L
Lin Ming 已提交
3942 3943
#endif

L
Linus Torvalds 已提交
3944 3945
int __init blk_dev_init(void)
{
3946 3947
	BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS));
	BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
3948
			FIELD_SIZEOF(struct request, cmd_flags));
3949 3950
	BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
			FIELD_SIZEOF(struct bio, bi_opf));
3951

3952 3953
	/* used for unplugging and affects IO latency/throughput - HIGHPRI */
	kblockd_workqueue = alloc_workqueue("kblockd",
3954
					    WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
L
Linus Torvalds 已提交
3955 3956 3957 3958
	if (!kblockd_workqueue)
		panic("Failed to create kblockd\n");

	request_cachep = kmem_cache_create("blkdev_requests",
3959
			sizeof(struct request), 0, SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
3960

3961
	blk_requestq_cachep = kmem_cache_create("request_queue",
3962
			sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
3963

3964 3965 3966 3967
#ifdef CONFIG_DEBUG_FS
	blk_debugfs_root = debugfs_create_dir("block", NULL);
#endif

3968
	return 0;
L
Linus Torvalds 已提交
3969
}