blk-core.c 105.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5
/*
 * Copyright (C) 1991, 1992 Linus Torvalds
 * Copyright (C) 1994,      Karl Keyte: Added support for disk statistics
 * Elevator latency, (C) 2000  Andrea Arcangeli <andrea@suse.de> SuSE
 * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de>
6 7
 * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au>
 *	-  July2000
L
Linus Torvalds 已提交
8 9 10 11 12 13 14 15 16 17 18
 * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001
 */

/*
 * This handles all read/write requests to block devices
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
19
#include <linux/blk-mq.h>
L
Linus Torvalds 已提交
20 21 22 23 24 25 26 27 28
#include <linux/highmem.h>
#include <linux/mm.h>
#include <linux/kernel_stat.h>
#include <linux/string.h>
#include <linux/init.h>
#include <linux/completion.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/writeback.h>
29
#include <linux/task_io_accounting_ops.h>
30
#include <linux/fault-inject.h>
31
#include <linux/list_sort.h>
T
Tejun Heo 已提交
32
#include <linux/delay.h>
33
#include <linux/ratelimit.h>
L
Lin Ming 已提交
34
#include <linux/pm_runtime.h>
35
#include <linux/blk-cgroup.h>
36
#include <linux/debugfs.h>
37
#include <linux/bpf.h>
38 39 40

#define CREATE_TRACE_POINTS
#include <trace/events/block.h>
L
Linus Torvalds 已提交
41

42
#include "blk.h"
43
#include "blk-mq.h"
44
#include "blk-mq-sched.h"
45
#include "blk-rq-qos.h"
46

47 48 49 50
#ifdef CONFIG_DEBUG_FS
struct dentry *blk_debugfs_root;
#endif

51
EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
52
EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
53
EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
K
Keith Busch 已提交
54
EXPORT_TRACEPOINT_SYMBOL_GPL(block_split);
N
NeilBrown 已提交
55
EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
56

57 58
DEFINE_IDA(blk_queue_ida);

L
Linus Torvalds 已提交
59 60 61
/*
 * For the allocated request tables
 */
62
struct kmem_cache *request_cachep;
L
Linus Torvalds 已提交
63 64 65 66

/*
 * For queue allocation
 */
67
struct kmem_cache *blk_requestq_cachep;
L
Linus Torvalds 已提交
68 69 70 71

/*
 * Controlling structure to kblockd
 */
72
static struct workqueue_struct *kblockd_workqueue;
L
Linus Torvalds 已提交
73

74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
/**
 * blk_queue_flag_set - atomically set a queue flag
 * @flag: flag to be set
 * @q: request queue
 */
void blk_queue_flag_set(unsigned int flag, struct request_queue *q)
{
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
	queue_flag_set(flag, q);
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_queue_flag_set);

/**
 * blk_queue_flag_clear - atomically clear a queue flag
 * @flag: flag to be cleared
 * @q: request queue
 */
void blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
{
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
	queue_flag_clear(flag, q);
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_queue_flag_clear);

/**
 * blk_queue_flag_test_and_set - atomically test and set a queue flag
 * @flag: flag to be set
 * @q: request queue
 *
 * Returns the previous value of @flag - 0 if the flag was not set and 1 if
 * the flag was already set.
 */
bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q)
{
	unsigned long flags;
	bool res;

	spin_lock_irqsave(q->queue_lock, flags);
	res = queue_flag_test_and_set(flag, q);
	spin_unlock_irqrestore(q->queue_lock, flags);

	return res;
}
EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_set);

/**
 * blk_queue_flag_test_and_clear - atomically test and clear a queue flag
 * @flag: flag to be cleared
 * @q: request queue
 *
 * Returns the previous value of @flag - 0 if the flag was not set and 1 if
 * the flag was set.
 */
bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q)
{
	unsigned long flags;
	bool res;

	spin_lock_irqsave(q->queue_lock, flags);
	res = queue_flag_test_and_clear(flag, q);
	spin_unlock_irqrestore(q->queue_lock, flags);

	return res;
}
EXPORT_SYMBOL_GPL(blk_queue_flag_test_and_clear);

146 147 148 149 150
static void blk_clear_congested(struct request_list *rl, int sync)
{
#ifdef CONFIG_CGROUP_WRITEBACK
	clear_wb_congested(rl->blkg->wb_congested, sync);
#else
151 152 153 154 155
	/*
	 * If !CGROUP_WRITEBACK, all blkg's map to bdi->wb and we shouldn't
	 * flip its congestion state for events on other blkcgs.
	 */
	if (rl == &rl->q->root_rl)
156
		clear_wb_congested(rl->q->backing_dev_info->wb.congested, sync);
157 158 159 160 161 162 163 164
#endif
}

static void blk_set_congested(struct request_list *rl, int sync)
{
#ifdef CONFIG_CGROUP_WRITEBACK
	set_wb_congested(rl->blkg->wb_congested, sync);
#else
165 166
	/* see blk_clear_congested() */
	if (rl == &rl->q->root_rl)
167
		set_wb_congested(rl->q->backing_dev_info->wb.congested, sync);
168 169 170
#endif
}

171
void blk_queue_congestion_threshold(struct request_queue *q)
L
Linus Torvalds 已提交
172 173 174 175 176 177 178 179 180 181 182 183 184 185
{
	int nr;

	nr = q->nr_requests - (q->nr_requests / 8) + 1;
	if (nr > q->nr_requests)
		nr = q->nr_requests;
	q->nr_congestion_on = nr;

	nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
	if (nr < 1)
		nr = 1;
	q->nr_congestion_off = nr;
}

186
void blk_rq_init(struct request_queue *q, struct request *rq)
L
Linus Torvalds 已提交
187
{
188 189
	memset(rq, 0, sizeof(*rq));

L
Linus Torvalds 已提交
190
	INIT_LIST_HEAD(&rq->queuelist);
J
Jens Axboe 已提交
191
	INIT_LIST_HEAD(&rq->timeout_list);
192
	rq->cpu = -1;
J
Jens Axboe 已提交
193
	rq->q = q;
194
	rq->__sector = (sector_t) -1;
195 196
	INIT_HLIST_NODE(&rq->hash);
	RB_CLEAR_NODE(&rq->rb_node);
J
Jens Axboe 已提交
197
	rq->tag = -1;
198
	rq->internal_tag = -1;
199
	rq->start_time_ns = ktime_get_ns();
200
	rq->part = NULL;
L
Linus Torvalds 已提交
201
}
202
EXPORT_SYMBOL(blk_rq_init);
L
Linus Torvalds 已提交
203

204 205 206 207 208 209 210 211 212 213 214 215 216 217
static const struct {
	int		errno;
	const char	*name;
} blk_errors[] = {
	[BLK_STS_OK]		= { 0,		"" },
	[BLK_STS_NOTSUPP]	= { -EOPNOTSUPP, "operation not supported" },
	[BLK_STS_TIMEOUT]	= { -ETIMEDOUT,	"timeout" },
	[BLK_STS_NOSPC]		= { -ENOSPC,	"critical space allocation" },
	[BLK_STS_TRANSPORT]	= { -ENOLINK,	"recoverable transport" },
	[BLK_STS_TARGET]	= { -EREMOTEIO,	"critical target" },
	[BLK_STS_NEXUS]		= { -EBADE,	"critical nexus" },
	[BLK_STS_MEDIUM]	= { -ENODATA,	"critical medium" },
	[BLK_STS_PROTECTION]	= { -EILSEQ,	"protection" },
	[BLK_STS_RESOURCE]	= { -ENOMEM,	"kernel resource" },
218
	[BLK_STS_DEV_RESOURCE]	= { -EBUSY,	"device resource" },
219
	[BLK_STS_AGAIN]		= { -EAGAIN,	"nonblocking retry" },
220

221 222 223
	/* device mapper special case, should not leak out: */
	[BLK_STS_DM_REQUEUE]	= { -EREMCHG, "dm internal retry" },

224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
	/* everything else not covered above: */
	[BLK_STS_IOERR]		= { -EIO,	"I/O" },
};

blk_status_t errno_to_blk_status(int errno)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {
		if (blk_errors[i].errno == errno)
			return (__force blk_status_t)i;
	}

	return BLK_STS_IOERR;
}
EXPORT_SYMBOL_GPL(errno_to_blk_status);

int blk_status_to_errno(blk_status_t status)
{
	int idx = (__force int)status;

245
	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
246 247 248 249 250 251 252 253 254
		return -EIO;
	return blk_errors[idx].errno;
}
EXPORT_SYMBOL_GPL(blk_status_to_errno);

static void print_req_error(struct request *req, blk_status_t status)
{
	int idx = (__force int)status;

255
	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(blk_errors)))
256 257 258 259 260 261 262 263
		return;

	printk_ratelimited(KERN_ERR "%s: %s error, dev %s, sector %llu\n",
			   __func__, blk_errors[idx].name, req->rq_disk ?
			   req->rq_disk->disk_name : "?",
			   (unsigned long long)blk_rq_pos(req));
}

N
NeilBrown 已提交
264
static void req_bio_endio(struct request *rq, struct bio *bio,
265
			  unsigned int nbytes, blk_status_t error)
L
Linus Torvalds 已提交
266
{
267
	if (error)
268
		bio->bi_status = error;
269

270
	if (unlikely(rq->rq_flags & RQF_QUIET))
271
		bio_set_flag(bio, BIO_QUIET);
272

273
	bio_advance(bio, nbytes);
274

T
Tejun Heo 已提交
275
	/* don't actually finish bio if it's part of flush sequence */
276
	if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
277
		bio_endio(bio);
L
Linus Torvalds 已提交
278 279 280 281
}

void blk_dump_rq_flags(struct request *rq, char *msg)
{
282 283
	printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
		rq->rq_disk ? rq->rq_disk->disk_name : "?",
J
Jens Axboe 已提交
284
		(unsigned long long) rq->cmd_flags);
L
Linus Torvalds 已提交
285

286 287 288
	printk(KERN_INFO "  sector %llu, nr/cnr %u/%u\n",
	       (unsigned long long)blk_rq_pos(rq),
	       blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
289 290
	printk(KERN_INFO "  bio %p, biotail %p, len %u\n",
	       rq->bio, rq->biotail, blk_rq_bytes(rq));
L
Linus Torvalds 已提交
291 292 293
}
EXPORT_SYMBOL(blk_dump_rq_flags);

294
static void blk_delay_work(struct work_struct *work)
L
Linus Torvalds 已提交
295
{
296
	struct request_queue *q;
L
Linus Torvalds 已提交
297

298 299
	q = container_of(work, struct request_queue, delay_work.work);
	spin_lock_irq(q->queue_lock);
300
	__blk_run_queue(q);
301
	spin_unlock_irq(q->queue_lock);
L
Linus Torvalds 已提交
302 303 304
}

/**
305 306 307
 * blk_delay_queue - restart queueing after defined interval
 * @q:		The &struct request_queue in question
 * @msecs:	Delay in msecs
L
Linus Torvalds 已提交
308 309
 *
 * Description:
310 311
 *   Sometimes queueing needs to be postponed for a little while, to allow
 *   resources to come back. This function will make sure that queueing is
312
 *   restarted around the specified time.
313 314
 */
void blk_delay_queue(struct request_queue *q, unsigned long msecs)
315
{
316
	lockdep_assert_held(q->queue_lock);
317
	WARN_ON_ONCE(q->mq_ops);
318

319 320 321
	if (likely(!blk_queue_dead(q)))
		queue_delayed_work(kblockd_workqueue, &q->delay_work,
				   msecs_to_jiffies(msecs));
322
}
323
EXPORT_SYMBOL(blk_delay_queue);
324

J
Jens Axboe 已提交
325 326 327 328 329 330 331 332 333 334 335
/**
 * blk_start_queue_async - asynchronously restart a previously stopped queue
 * @q:    The &struct request_queue in question
 *
 * Description:
 *   blk_start_queue_async() will clear the stop flag on the queue, and
 *   ensure that the request_fn for the queue is run from an async
 *   context.
 **/
void blk_start_queue_async(struct request_queue *q)
{
336
	lockdep_assert_held(q->queue_lock);
337
	WARN_ON_ONCE(q->mq_ops);
338

J
Jens Axboe 已提交
339 340 341 342 343
	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
	blk_run_queue_async(q);
}
EXPORT_SYMBOL(blk_start_queue_async);

L
Linus Torvalds 已提交
344 345
/**
 * blk_start_queue - restart a previously stopped queue
346
 * @q:    The &struct request_queue in question
L
Linus Torvalds 已提交
347 348 349 350
 *
 * Description:
 *   blk_start_queue() will clear the stop flag on the queue, and call
 *   the request_fn for the queue if it was in a stopped state when
351
 *   entered. Also see blk_stop_queue().
L
Linus Torvalds 已提交
352
 **/
353
void blk_start_queue(struct request_queue *q)
L
Linus Torvalds 已提交
354
{
355
	lockdep_assert_held(q->queue_lock);
356
	WARN_ON_ONCE(q->mq_ops);
357

N
Nick Piggin 已提交
358
	queue_flag_clear(QUEUE_FLAG_STOPPED, q);
359
	__blk_run_queue(q);
L
Linus Torvalds 已提交
360 361 362 363 364
}
EXPORT_SYMBOL(blk_start_queue);

/**
 * blk_stop_queue - stop a queue
365
 * @q:    The &struct request_queue in question
L
Linus Torvalds 已提交
366 367 368 369 370 371 372 373 374
 *
 * Description:
 *   The Linux block layer assumes that a block driver will consume all
 *   entries on the request queue when the request_fn strategy is called.
 *   Often this will not happen, because of hardware limitations (queue
 *   depth settings). If a device driver gets a 'queue full' response,
 *   or if it simply chooses not to queue more I/O at one point, it can
 *   call this function to prevent the request_fn from being called until
 *   the driver has signalled it's ready to go again. This happens by calling
375
 *   blk_start_queue() to restart queue operations.
L
Linus Torvalds 已提交
376
 **/
377
void blk_stop_queue(struct request_queue *q)
L
Linus Torvalds 已提交
378
{
379
	lockdep_assert_held(q->queue_lock);
380
	WARN_ON_ONCE(q->mq_ops);
381

382
	cancel_delayed_work(&q->delay_work);
N
Nick Piggin 已提交
383
	queue_flag_set(QUEUE_FLAG_STOPPED, q);
L
Linus Torvalds 已提交
384 385 386 387 388 389 390 391 392 393 394 395
}
EXPORT_SYMBOL(blk_stop_queue);

/**
 * blk_sync_queue - cancel any pending callbacks on a queue
 * @q: the queue
 *
 * Description:
 *     The block layer may perform asynchronous callback activity
 *     on a queue, such as calling the unplug function after a timeout.
 *     A block device may call blk_sync_queue to ensure that any
 *     such activity is cancelled, thus allowing it to release resources
396
 *     that the callbacks might use. The caller must already have made sure
L
Linus Torvalds 已提交
397 398 399
 *     that its ->make_request_fn will not re-add plugging prior to calling
 *     this function.
 *
400
 *     This function does not cancel any asynchronous activity arising
401
 *     out of elevator or throttling code. That would require elevator_exit()
402
 *     and blkcg_exit_queue() to be called with queue lock initialized.
403
 *
L
Linus Torvalds 已提交
404 405 406
 */
void blk_sync_queue(struct request_queue *q)
{
407
	del_timer_sync(&q->timeout);
408
	cancel_work_sync(&q->timeout_work);
409 410 411 412 413

	if (q->mq_ops) {
		struct blk_mq_hw_ctx *hctx;
		int i;

414
		cancel_delayed_work_sync(&q->requeue_work);
415
		queue_for_each_hw_ctx(q, hctx, i)
416
			cancel_delayed_work_sync(&hctx->run_work);
417 418 419
	} else {
		cancel_delayed_work_sync(&q->delay_work);
	}
L
Linus Torvalds 已提交
420 421 422
}
EXPORT_SYMBOL(blk_sync_queue);

423 424 425 426 427 428 429 430 431
/**
 * blk_set_preempt_only - set QUEUE_FLAG_PREEMPT_ONLY
 * @q: request queue pointer
 *
 * Returns the previous value of the PREEMPT_ONLY flag - 0 if the flag was not
 * set and 1 if the flag was already set.
 */
int blk_set_preempt_only(struct request_queue *q)
{
432
	return blk_queue_flag_test_and_set(QUEUE_FLAG_PREEMPT_ONLY, q);
433 434 435 436 437
}
EXPORT_SYMBOL_GPL(blk_set_preempt_only);

void blk_clear_preempt_only(struct request_queue *q)
{
438
	blk_queue_flag_clear(QUEUE_FLAG_PREEMPT_ONLY, q);
439
	wake_up_all(&q->mq_freeze_wq);
440 441 442
}
EXPORT_SYMBOL_GPL(blk_clear_preempt_only);

443 444 445 446 447 448 449 450 451 452 453 454 455
/**
 * __blk_run_queue_uncond - run a queue whether or not it has been stopped
 * @q:	The queue to run
 *
 * Description:
 *    Invoke request handling on a queue if there are any pending requests.
 *    May be used to restart request handling after a request has completed.
 *    This variant runs the queue whether or not the queue has been
 *    stopped. Must be called with the queue lock held and interrupts
 *    disabled. See also @blk_run_queue.
 */
inline void __blk_run_queue_uncond(struct request_queue *q)
{
456
	lockdep_assert_held(q->queue_lock);
457
	WARN_ON_ONCE(q->mq_ops);
458

459 460 461
	if (unlikely(blk_queue_dead(q)))
		return;

462 463 464 465 466 467 468 469
	/*
	 * Some request_fn implementations, e.g. scsi_request_fn(), unlock
	 * the queue lock internally. As a result multiple threads may be
	 * running such a request function concurrently. Keep track of the
	 * number of active request_fn invocations such that blk_drain_queue()
	 * can wait until all these request_fn calls have finished.
	 */
	q->request_fn_active++;
470
	q->request_fn(q);
471
	q->request_fn_active--;
472
}
473
EXPORT_SYMBOL_GPL(__blk_run_queue_uncond);
474

L
Linus Torvalds 已提交
475
/**
476
 * __blk_run_queue - run a single device queue
L
Linus Torvalds 已提交
477
 * @q:	The queue to run
478 479
 *
 * Description:
480
 *    See @blk_run_queue.
L
Linus Torvalds 已提交
481
 */
482
void __blk_run_queue(struct request_queue *q)
L
Linus Torvalds 已提交
483
{
484
	lockdep_assert_held(q->queue_lock);
485
	WARN_ON_ONCE(q->mq_ops);
486

487 488 489
	if (unlikely(blk_queue_stopped(q)))
		return;

490
	__blk_run_queue_uncond(q);
N
Nick Piggin 已提交
491 492
}
EXPORT_SYMBOL(__blk_run_queue);
J
Jens Axboe 已提交
493

494 495 496 497 498 499
/**
 * blk_run_queue_async - run a single device queue in workqueue context
 * @q:	The queue to run
 *
 * Description:
 *    Tells kblockd to perform the equivalent of @blk_run_queue on behalf
500 501 502 503 504 505
 *    of us.
 *
 * Note:
 *    Since it is not allowed to run q->delay_work after blk_cleanup_queue()
 *    has canceled q->delay_work, callers must hold the queue lock to avoid
 *    race conditions between blk_cleanup_queue() and blk_run_queue_async().
506 507 508
 */
void blk_run_queue_async(struct request_queue *q)
{
509
	lockdep_assert_held(q->queue_lock);
510
	WARN_ON_ONCE(q->mq_ops);
511

512
	if (likely(!blk_queue_stopped(q) && !blk_queue_dead(q)))
513
		mod_delayed_work(kblockd_workqueue, &q->delay_work, 0);
514
}
515
EXPORT_SYMBOL(blk_run_queue_async);
516

N
Nick Piggin 已提交
517 518 519
/**
 * blk_run_queue - run a single device queue
 * @q: The queue to run
520 521 522
 *
 * Description:
 *    Invoke request handling on this queue, if it has pending work to do.
T
Tejun Heo 已提交
523
 *    May be used to restart queueing when a request has completed.
N
Nick Piggin 已提交
524 525 526 527 528
 */
void blk_run_queue(struct request_queue *q)
{
	unsigned long flags;

529 530
	WARN_ON_ONCE(q->mq_ops);

N
Nick Piggin 已提交
531
	spin_lock_irqsave(q->queue_lock, flags);
532
	__blk_run_queue(q);
L
Linus Torvalds 已提交
533 534 535 536
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL(blk_run_queue);

537
void blk_put_queue(struct request_queue *q)
538 539 540
{
	kobject_put(&q->kobj);
}
J
Jens Axboe 已提交
541
EXPORT_SYMBOL(blk_put_queue);
542

T
Tejun Heo 已提交
543
/**
544
 * __blk_drain_queue - drain requests from request_queue
T
Tejun Heo 已提交
545
 * @q: queue to drain
546
 * @drain_all: whether to drain all requests or only the ones w/ ELVPRIV
T
Tejun Heo 已提交
547
 *
548 549 550
 * Drain requests from @q.  If @drain_all is set, all requests are drained.
 * If not, only ELVPRIV requests are drained.  The caller is responsible
 * for ensuring that no new requests which need to be drained are queued.
T
Tejun Heo 已提交
551
 */
552 553 554
static void __blk_drain_queue(struct request_queue *q, bool drain_all)
	__releases(q->queue_lock)
	__acquires(q->queue_lock)
T
Tejun Heo 已提交
555
{
556 557
	int i;

558
	lockdep_assert_held(q->queue_lock);
559
	WARN_ON_ONCE(q->mq_ops);
560

T
Tejun Heo 已提交
561
	while (true) {
562
		bool drain = false;
T
Tejun Heo 已提交
563

564 565 566 567 568 569 570
		/*
		 * The caller might be trying to drain @q before its
		 * elevator is initialized.
		 */
		if (q->elevator)
			elv_drain_elevator(q);

571
		blkcg_drain_queue(q);
T
Tejun Heo 已提交
572

573 574
		/*
		 * This function might be called on a queue which failed
575 576 577 578
		 * driver init after queue creation or is not yet fully
		 * active yet.  Some drivers (e.g. fd and loop) get unhappy
		 * in such cases.  Kick queue iff dispatch queue has
		 * something on it and @q has request_fn set.
579
		 */
580
		if (!list_empty(&q->queue_head) && q->request_fn)
581
			__blk_run_queue(q);
582

583
		drain |= q->nr_rqs_elvpriv;
584
		drain |= q->request_fn_active;
585 586 587 588 589 590 591

		/*
		 * Unfortunately, requests are queued at and tracked from
		 * multiple places and there's no single counter which can
		 * be drained.  Check all the queues and counters.
		 */
		if (drain_all) {
592
			struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);
593 594
			drain |= !list_empty(&q->queue_head);
			for (i = 0; i < 2; i++) {
595
				drain |= q->nr_rqs[i];
596
				drain |= q->in_flight[i];
597 598
				if (fq)
				    drain |= !list_empty(&fq->flush_queue[i]);
599 600
			}
		}
T
Tejun Heo 已提交
601

602
		if (!drain)
T
Tejun Heo 已提交
603
			break;
604 605 606

		spin_unlock_irq(q->queue_lock);

T
Tejun Heo 已提交
607
		msleep(10);
608 609

		spin_lock_irq(q->queue_lock);
T
Tejun Heo 已提交
610
	}
611 612 613 614 615 616 617

	/*
	 * With queue marked dead, any woken up waiter will fail the
	 * allocation path, so the wakeup chaining is lost and we're
	 * left with hung waiters. We need to wake up those waiters.
	 */
	if (q->request_fn) {
618 619 620 621 622
		struct request_list *rl;

		blk_queue_for_each_rl(rl, q)
			for (i = 0; i < ARRAY_SIZE(rl->wait); i++)
				wake_up_all(&rl->wait[i]);
623
	}
T
Tejun Heo 已提交
624 625
}

626 627 628 629 630 631 632
void blk_drain_queue(struct request_queue *q)
{
	spin_lock_irq(q->queue_lock);
	__blk_drain_queue(q, true);
	spin_unlock_irq(q->queue_lock);
}

633 634 635 636 637 638
/**
 * blk_queue_bypass_start - enter queue bypass mode
 * @q: queue of interest
 *
 * In bypass mode, only the dispatch FIFO queue of @q is used.  This
 * function makes @q enter bypass mode and drains all requests which were
639
 * throttled or issued before.  On return, it's guaranteed that no request
640 641
 * is being throttled or has ELVPRIV set and blk_queue_bypass() %true
 * inside queue or RCU read lock.
642 643 644
 */
void blk_queue_bypass_start(struct request_queue *q)
{
645 646
	WARN_ON_ONCE(q->mq_ops);

647
	spin_lock_irq(q->queue_lock);
648
	q->bypass_depth++;
649 650 651
	queue_flag_set(QUEUE_FLAG_BYPASS, q);
	spin_unlock_irq(q->queue_lock);

652 653 654 655 656 657
	/*
	 * Queues start drained.  Skip actual draining till init is
	 * complete.  This avoids lenghty delays during queue init which
	 * can happen many times during boot.
	 */
	if (blk_queue_init_done(q)) {
658 659 660 661
		spin_lock_irq(q->queue_lock);
		__blk_drain_queue(q, false);
		spin_unlock_irq(q->queue_lock);

662 663 664
		/* ensure blk_queue_bypass() is %true inside RCU read lock */
		synchronize_rcu();
	}
665 666 667 668 669 670 671 672
}
EXPORT_SYMBOL_GPL(blk_queue_bypass_start);

/**
 * blk_queue_bypass_end - leave queue bypass mode
 * @q: queue of interest
 *
 * Leave bypass mode and restore the normal queueing behavior.
673 674 675
 *
 * Note: although blk_queue_bypass_start() is only called for blk-sq queues,
 * this function is called for both blk-sq and blk-mq queues.
676 677 678 679 680 681 682 683 684 685 686
 */
void blk_queue_bypass_end(struct request_queue *q)
{
	spin_lock_irq(q->queue_lock);
	if (!--q->bypass_depth)
		queue_flag_clear(QUEUE_FLAG_BYPASS, q);
	WARN_ON_ONCE(q->bypass_depth < 0);
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL_GPL(blk_queue_bypass_end);

687 688
void blk_set_queue_dying(struct request_queue *q)
{
689
	blk_queue_flag_set(QUEUE_FLAG_DYING, q);
690

691 692 693 694 695 696 697
	/*
	 * When queue DYING flag is set, we need to block new req
	 * entering queue, so we call blk_freeze_queue_start() to
	 * prevent I/O from crossing blk_queue_enter().
	 */
	blk_freeze_queue_start(q);

698 699 700 701 702
	if (q->mq_ops)
		blk_mq_wake_waiters(q);
	else {
		struct request_list *rl;

703
		spin_lock_irq(q->queue_lock);
704 705
		blk_queue_for_each_rl(rl, q) {
			if (rl->rq_pool) {
706 707
				wake_up_all(&rl->wait[BLK_RW_SYNC]);
				wake_up_all(&rl->wait[BLK_RW_ASYNC]);
708 709
			}
		}
710
		spin_unlock_irq(q->queue_lock);
711
	}
712 713 714

	/* Make blk_queue_enter() reexamine the DYING flag. */
	wake_up_all(&q->mq_freeze_wq);
715 716 717
}
EXPORT_SYMBOL_GPL(blk_set_queue_dying);

718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746
/* Unconfigure the I/O scheduler and dissociate from the cgroup controller. */
void blk_exit_queue(struct request_queue *q)
{
	/*
	 * Since the I/O scheduler exit code may access cgroup information,
	 * perform I/O scheduler exit before disassociating from the block
	 * cgroup controller.
	 */
	if (q->elevator) {
		ioc_clear_queue(q);
		elevator_exit(q, q->elevator);
		q->elevator = NULL;
	}

	/*
	 * Remove all references to @q from the block cgroup controller before
	 * restoring @q->queue_lock to avoid that restoring this pointer causes
	 * e.g. blkcg_print_blkgs() to crash.
	 */
	blkcg_exit_queue(q);

	/*
	 * Since the cgroup code may dereference the @q->backing_dev_info
	 * pointer, only decrease its reference count after having removed the
	 * association with the block cgroup controller.
	 */
	bdi_put(q->backing_dev_info);
}

747 748 749 750
/**
 * blk_cleanup_queue - shutdown a request queue
 * @q: request queue to shutdown
 *
751 752
 * Mark @q DYING, drain all pending requests, mark @q DEAD, destroy and
 * put it.  All future requests will be failed immediately with -ENODEV.
753
 */
754
void blk_cleanup_queue(struct request_queue *q)
755
{
756
	spinlock_t *lock = q->queue_lock;
757

B
Bart Van Assche 已提交
758
	/* mark @q DYING, no new request or merges will be allowed afterwards */
759
	mutex_lock(&q->sysfs_lock);
760
	blk_set_queue_dying(q);
761
	spin_lock_irq(lock);
762

763
	/*
B
Bart Van Assche 已提交
764
	 * A dying queue is permanently in bypass mode till released.  Note
765 766 767 768 769 770 771
	 * that, unlike blk_queue_bypass_start(), we aren't performing
	 * synchronize_rcu() after entering bypass mode to avoid the delay
	 * as some drivers create and destroy a lot of queues while
	 * probing.  This is still safe because blk_release_queue() will be
	 * called only after the queue refcnt drops to zero and nothing,
	 * RCU or not, would be traversing the queue by then.
	 */
772 773 774
	q->bypass_depth++;
	queue_flag_set(QUEUE_FLAG_BYPASS, q);

775 776
	queue_flag_set(QUEUE_FLAG_NOMERGES, q);
	queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
B
Bart Van Assche 已提交
777
	queue_flag_set(QUEUE_FLAG_DYING, q);
778 779 780
	spin_unlock_irq(lock);
	mutex_unlock(&q->sysfs_lock);

781 782 783 784
	/*
	 * Drain all requests queued before DYING marking. Set DEAD flag to
	 * prevent that q->request_fn() gets invoked after draining finished.
	 */
785
	blk_freeze_queue(q);
786
	spin_lock_irq(lock);
787
	queue_flag_set(QUEUE_FLAG_DEAD, q);
788
	spin_unlock_irq(lock);
789

790 791 792 793
	/*
	 * make sure all in-progress dispatch are completed because
	 * blk_freeze_queue() can only complete all requests, and
	 * dispatch may still be in-progress since we dispatch requests
794 795 796 797 798
	 * from more than one contexts.
	 *
	 * No need to quiesce queue if it isn't initialized yet since
	 * blk_freeze_queue() should be enough for cases of passthrough
	 * request.
799
	 */
800
	if (q->mq_ops && blk_queue_init_done(q))
801 802
		blk_mq_quiesce_queue(q);

803 804 805
	/* for synchronous bio-based driver finish in-flight integrity i/o */
	blk_flush_integrity();

806
	/* @q won't process any more request, flush async actions */
807
	del_timer_sync(&q->backing_dev_info->laptop_mode_wb_timer);
808 809
	blk_sync_queue(q);

810 811 812 813 814 815
	/*
	 * I/O scheduler exit is only safe after the sysfs scheduler attribute
	 * has been removed.
	 */
	WARN_ON_ONCE(q->kobj.state_in_sysfs);

816
	blk_exit_queue(q);
817

B
Bart Van Assche 已提交
818 819
	if (q->mq_ops)
		blk_mq_free_queue(q);
820
	percpu_ref_exit(&q->q_usage_counter);
B
Bart Van Assche 已提交
821

822 823 824 825 826
	spin_lock_irq(lock);
	if (q->queue_lock != &q->__queue_lock)
		q->queue_lock = &q->__queue_lock;
	spin_unlock_irq(lock);

827
	/* @q is and will stay empty, shutdown and put */
828 829
	blk_put_queue(q);
}
L
Linus Torvalds 已提交
830 831
EXPORT_SYMBOL(blk_cleanup_queue);

832
/* Allocate memory local to the request queue */
833
static void *alloc_request_simple(gfp_t gfp_mask, void *data)
834
{
835 836 837
	struct request_queue *q = data;

	return kmem_cache_alloc_node(request_cachep, gfp_mask, q->node);
838 839
}

840
static void free_request_simple(void *element, void *data)
841 842 843 844
{
	kmem_cache_free(request_cachep, element);
}

845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867
static void *alloc_request_size(gfp_t gfp_mask, void *data)
{
	struct request_queue *q = data;
	struct request *rq;

	rq = kmalloc_node(sizeof(struct request) + q->cmd_size, gfp_mask,
			q->node);
	if (rq && q->init_rq_fn && q->init_rq_fn(q, rq, gfp_mask) < 0) {
		kfree(rq);
		rq = NULL;
	}
	return rq;
}

static void free_request_size(void *element, void *data)
{
	struct request_queue *q = data;

	if (q->exit_rq_fn)
		q->exit_rq_fn(q, element);
	kfree(element);
}

868 869
int blk_init_rl(struct request_list *rl, struct request_queue *q,
		gfp_t gfp_mask)
L
Linus Torvalds 已提交
870
{
S
Shaohua Li 已提交
871
	if (unlikely(rl->rq_pool) || q->mq_ops)
872 873
		return 0;

874
	rl->q = q;
875 876 877 878
	rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
	rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
	init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
	init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
L
Linus Torvalds 已提交
879

880 881 882 883 884 885 886 887 888
	if (q->cmd_size) {
		rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ,
				alloc_request_size, free_request_size,
				q, gfp_mask, q->node);
	} else {
		rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ,
				alloc_request_simple, free_request_simple,
				q, gfp_mask, q->node);
	}
L
Linus Torvalds 已提交
889 890 891
	if (!rl->rq_pool)
		return -ENOMEM;

892 893 894
	if (rl != &q->root_rl)
		WARN_ON_ONCE(!blk_get_queue(q));

L
Linus Torvalds 已提交
895 896 897
	return 0;
}

898
void blk_exit_rl(struct request_queue *q, struct request_list *rl)
899
{
900
	if (rl->rq_pool) {
901
		mempool_destroy(rl->rq_pool);
902 903 904
		if (rl != &q->root_rl)
			blk_put_queue(q);
	}
905 906
}

907
struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
L
Linus Torvalds 已提交
908
{
909
	return blk_alloc_queue_node(gfp_mask, NUMA_NO_NODE, NULL);
910 911
}
EXPORT_SYMBOL(blk_alloc_queue);
L
Linus Torvalds 已提交
912

913 914 915 916 917
/**
 * blk_queue_enter() - try to increase q->q_usage_counter
 * @q: request queue pointer
 * @flags: BLK_MQ_REQ_NOWAIT and/or BLK_MQ_REQ_PREEMPT
 */
918
int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags)
919
{
920 921
	const bool preempt = flags & BLK_MQ_REQ_PREEMPT;

922
	while (true) {
923
		bool success = false;
924

925
		rcu_read_lock();
926 927 928 929 930 931 932 933 934 935 936 937
		if (percpu_ref_tryget_live(&q->q_usage_counter)) {
			/*
			 * The code that sets the PREEMPT_ONLY flag is
			 * responsible for ensuring that that flag is globally
			 * visible before the queue is unfrozen.
			 */
			if (preempt || !blk_queue_preempt_only(q)) {
				success = true;
			} else {
				percpu_ref_put(&q->q_usage_counter);
			}
		}
938
		rcu_read_unlock();
939 940

		if (success)
941 942
			return 0;

943
		if (flags & BLK_MQ_REQ_NOWAIT)
944 945
			return -EBUSY;

946
		/*
947
		 * read pair of barrier in blk_freeze_queue_start(),
948
		 * we need to order reading __PERCPU_REF_DEAD flag of
949 950 951
		 * .q_usage_counter and reading .mq_freeze_depth or
		 * queue dying flag, otherwise the following wait may
		 * never return if the two reads are reordered.
952 953 954
		 */
		smp_rmb();

955 956 957 958
		wait_event(q->mq_freeze_wq,
			   (atomic_read(&q->mq_freeze_depth) == 0 &&
			    (preempt || !blk_queue_preempt_only(q))) ||
			   blk_queue_dying(q));
959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976
		if (blk_queue_dying(q))
			return -ENODEV;
	}
}

void blk_queue_exit(struct request_queue *q)
{
	percpu_ref_put(&q->q_usage_counter);
}

static void blk_queue_usage_counter_release(struct percpu_ref *ref)
{
	struct request_queue *q =
		container_of(ref, struct request_queue, q_usage_counter);

	wake_up_all(&q->mq_freeze_wq);
}

977
static void blk_rq_timed_out_timer(struct timer_list *t)
978
{
979
	struct request_queue *q = from_timer(q, t, timeout);
980 981 982 983

	kblockd_schedule_work(&q->timeout_work);
}

984 985 986 987 988 989 990 991 992 993 994 995 996
/**
 * blk_alloc_queue_node - allocate a request queue
 * @gfp_mask: memory allocation flags
 * @node_id: NUMA node to allocate memory from
 * @lock: For legacy queues, pointer to a spinlock that will be used to e.g.
 *        serialize calls to the legacy .request_fn() callback. Ignored for
 *	  blk-mq request queues.
 *
 * Note: pass the queue lock as the third argument to this function instead of
 * setting the queue lock pointer explicitly to avoid triggering a sporadic
 * crash in the blkcg code. This function namely calls blkcg_init_queue() and
 * the queue lock pointer must be set before blkcg_init_queue() is called.
 */
997 998
struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id,
					   spinlock_t *lock)
999
{
1000
	struct request_queue *q;
1001
	int ret;
1002

1003
	q = kmem_cache_alloc_node(blk_requestq_cachep,
1004
				gfp_mask | __GFP_ZERO, node_id);
L
Linus Torvalds 已提交
1005 1006 1007
	if (!q)
		return NULL;

1008 1009 1010 1011 1012
	INIT_LIST_HEAD(&q->queue_head);
	q->last_merge = NULL;
	q->end_sector = 0;
	q->boundary_rq = NULL;

1013
	q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
1014
	if (q->id < 0)
1015
		goto fail_q;
1016

1017 1018
	ret = bioset_init(&q->bio_split, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS);
	if (ret)
1019 1020
		goto fail_id;

1021 1022 1023 1024
	q->backing_dev_info = bdi_alloc_node(gfp_mask, node_id);
	if (!q->backing_dev_info)
		goto fail_split;

1025 1026 1027 1028
	q->stats = blk_alloc_queue_stats();
	if (!q->stats)
		goto fail_stats;

1029
	q->backing_dev_info->ra_pages =
1030
			(VM_MAX_READAHEAD * 1024) / PAGE_SIZE;
1031 1032
	q->backing_dev_info->capabilities = BDI_CAP_CGROUP_WRITEBACK;
	q->backing_dev_info->name = "block";
1033
	q->node = node_id;
1034

1035 1036 1037
	timer_setup(&q->backing_dev_info->laptop_mode_wb_timer,
		    laptop_mode_timer_fn, 0);
	timer_setup(&q->timeout, blk_rq_timed_out_timer, 0);
1038
	INIT_WORK(&q->timeout_work, NULL);
J
Jens Axboe 已提交
1039
	INIT_LIST_HEAD(&q->timeout_list);
1040
	INIT_LIST_HEAD(&q->icq_list);
1041
#ifdef CONFIG_BLK_CGROUP
1042
	INIT_LIST_HEAD(&q->blkg_list);
1043
#endif
1044
	INIT_DELAYED_WORK(&q->delay_work, blk_delay_work);
1045

1046
	kobject_init(&q->kobj, &blk_queue_ktype);
L
Linus Torvalds 已提交
1047

1048 1049 1050
#ifdef CONFIG_BLK_DEV_IO_TRACE
	mutex_init(&q->blk_trace_mutex);
#endif
1051
	mutex_init(&q->sysfs_lock);
1052
	spin_lock_init(&q->__queue_lock);
1053

1054 1055
	if (!q->mq_ops)
		q->queue_lock = lock ? : &q->__queue_lock;
1056

1057 1058 1059
	/*
	 * A queue starts its life with bypass turned on to avoid
	 * unnecessary bypass on/off overhead and nasty surprises during
1060 1061
	 * init.  The initial bypass will be finished when the queue is
	 * registered by blk_register_queue().
1062 1063
	 */
	q->bypass_depth = 1;
1064
	queue_flag_set_unlocked(QUEUE_FLAG_BYPASS, q);
1065

1066 1067
	init_waitqueue_head(&q->mq_freeze_wq);

1068 1069 1070 1071 1072 1073 1074
	/*
	 * Init percpu_ref in atomic mode so that it's faster to shutdown.
	 * See blk_register_queue() for details.
	 */
	if (percpu_ref_init(&q->q_usage_counter,
				blk_queue_usage_counter_release,
				PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
1075
		goto fail_bdi;
1076

1077 1078 1079
	if (blkcg_init_queue(q))
		goto fail_ref;

L
Linus Torvalds 已提交
1080
	return q;
1081

1082 1083
fail_ref:
	percpu_ref_exit(&q->q_usage_counter);
1084
fail_bdi:
1085 1086
	blk_free_queue_stats(q->stats);
fail_stats:
1087
	bdi_put(q->backing_dev_info);
1088
fail_split:
1089
	bioset_exit(&q->bio_split);
1090 1091 1092 1093 1094
fail_id:
	ida_simple_remove(&blk_queue_ida, q->id);
fail_q:
	kmem_cache_free(blk_requestq_cachep, q);
	return NULL;
L
Linus Torvalds 已提交
1095
}
1096
EXPORT_SYMBOL(blk_alloc_queue_node);
L
Linus Torvalds 已提交
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119

/**
 * blk_init_queue  - prepare a request queue for use with a block device
 * @rfn:  The function to be called to process requests that have been
 *        placed on the queue.
 * @lock: Request queue spin lock
 *
 * Description:
 *    If a block device wishes to use the standard request handling procedures,
 *    which sorts requests and coalesces adjacent requests, then it must
 *    call blk_init_queue().  The function @rfn will be called when there
 *    are requests on the queue that need to be processed.  If the device
 *    supports plugging, then @rfn may not be called immediately when requests
 *    are available on the queue, but may be called at some time later instead.
 *    Plugged queues are generally unplugged when a buffer belonging to one
 *    of the requests on the queue is needed, or due to memory pressure.
 *
 *    @rfn is not required, or even expected, to remove all requests off the
 *    queue, but only as many as it can handle at a time.  If it does leave
 *    requests on the queue, it is responsible for arranging that the requests
 *    get dealt with eventually.
 *
 *    The queue spin lock must be held while manipulating the requests on the
1120 1121
 *    request queue; this lock will be taken also from interrupt context, so irq
 *    disabling is needed for it.
L
Linus Torvalds 已提交
1122
 *
1123
 *    Function returns a pointer to the initialized request queue, or %NULL if
L
Linus Torvalds 已提交
1124 1125 1126 1127 1128 1129
 *    it didn't succeed.
 *
 * Note:
 *    blk_init_queue() must be paired with a blk_cleanup_queue() call
 *    when the block device is deactivated (such as at module unload).
 **/
1130

1131
struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
L
Linus Torvalds 已提交
1132
{
1133
	return blk_init_queue_node(rfn, lock, NUMA_NO_NODE);
1134 1135 1136
}
EXPORT_SYMBOL(blk_init_queue);

1137
struct request_queue *
1138 1139
blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
{
1140
	struct request_queue *q;
L
Linus Torvalds 已提交
1141

1142
	q = blk_alloc_queue_node(GFP_KERNEL, node_id, lock);
1143
	if (!q)
1144 1145
		return NULL;

1146 1147 1148 1149 1150
	q->request_fn = rfn;
	if (blk_init_allocated_queue(q) < 0) {
		blk_cleanup_queue(q);
		return NULL;
	}
1151

1152
	return q;
1153 1154 1155
}
EXPORT_SYMBOL(blk_init_queue_node);

1156
static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio);
1157

L
Linus Torvalds 已提交
1158

1159 1160
int blk_init_allocated_queue(struct request_queue *q)
{
1161 1162
	WARN_ON_ONCE(q->mq_ops);

1163
	q->fq = blk_alloc_flush_queue(q, NUMA_NO_NODE, q->cmd_size);
1164
	if (!q->fq)
1165
		return -ENOMEM;
1166

1167 1168
	if (q->init_rq_fn && q->init_rq_fn(q, q->fq->flush_rq, GFP_KERNEL))
		goto out_free_flush_queue;
1169

1170
	if (blk_init_rl(&q->root_rl, q, GFP_KERNEL))
1171
		goto out_exit_flush_rq;
L
Linus Torvalds 已提交
1172

1173
	INIT_WORK(&q->timeout_work, blk_timeout_work);
1174
	q->queue_flags		|= QUEUE_FLAG_DEFAULT;
1175

1176 1177 1178
	/*
	 * This also sets hw/phys segments, boundary and size
	 */
1179
	blk_queue_make_request(q, blk_queue_bio);
L
Linus Torvalds 已提交
1180

1181 1182
	q->sg_reserved_size = INT_MAX;

1183
	if (elevator_init(q))
1184
		goto out_exit_flush_rq;
1185
	return 0;
1186

1187 1188 1189 1190
out_exit_flush_rq:
	if (q->exit_rq_fn)
		q->exit_rq_fn(q, q->fq->flush_rq);
out_free_flush_queue:
1191
	blk_free_flush_queue(q->fq);
1192
	q->fq = NULL;
1193
	return -ENOMEM;
L
Linus Torvalds 已提交
1194
}
1195
EXPORT_SYMBOL(blk_init_allocated_queue);
L
Linus Torvalds 已提交
1196

T
Tejun Heo 已提交
1197
bool blk_get_queue(struct request_queue *q)
L
Linus Torvalds 已提交
1198
{
B
Bart Van Assche 已提交
1199
	if (likely(!blk_queue_dying(q))) {
T
Tejun Heo 已提交
1200 1201
		__blk_get_queue(q);
		return true;
L
Linus Torvalds 已提交
1202 1203
	}

T
Tejun Heo 已提交
1204
	return false;
L
Linus Torvalds 已提交
1205
}
J
Jens Axboe 已提交
1206
EXPORT_SYMBOL(blk_get_queue);
L
Linus Torvalds 已提交
1207

1208
static inline void blk_free_request(struct request_list *rl, struct request *rq)
L
Linus Torvalds 已提交
1209
{
1210
	if (rq->rq_flags & RQF_ELVPRIV) {
1211
		elv_put_request(rl->q, rq);
1212
		if (rq->elv.icq)
1213
			put_io_context(rq->elv.icq->ioc);
1214 1215
	}

1216
	mempool_free(rq, rl->rq_pool);
L
Linus Torvalds 已提交
1217 1218 1219 1220 1221 1222
}

/*
 * ioc_batching returns true if the ioc is a valid batching request and
 * should be given priority access to a request.
 */
1223
static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
L
Linus Torvalds 已提交
1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
{
	if (!ioc)
		return 0;

	/*
	 * Make sure the process is able to allocate at least 1 request
	 * even if the batch times out, otherwise we could theoretically
	 * lose wakeups.
	 */
	return ioc->nr_batch_requests == q->nr_batching ||
		(ioc->nr_batch_requests > 0
		&& time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
}

/*
 * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This
 * will cause the process to be a "batcher" on all queues in the system. This
 * is the behaviour we want though - once it gets a wakeup it should be given
 * a nice run.
 */
1244
static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
L
Linus Torvalds 已提交
1245 1246 1247 1248 1249 1250 1251 1252
{
	if (!ioc || ioc_batching(q, ioc))
		return;

	ioc->nr_batch_requests = q->nr_batching;
	ioc->last_waited = jiffies;
}

1253
static void __freed_request(struct request_list *rl, int sync)
L
Linus Torvalds 已提交
1254
{
1255
	struct request_queue *q = rl->q;
L
Linus Torvalds 已提交
1256

1257 1258
	if (rl->count[sync] < queue_congestion_off_threshold(q))
		blk_clear_congested(rl, sync);
L
Linus Torvalds 已提交
1259

1260 1261 1262
	if (rl->count[sync] + 1 <= q->nr_requests) {
		if (waitqueue_active(&rl->wait[sync]))
			wake_up(&rl->wait[sync]);
L
Linus Torvalds 已提交
1263

1264
		blk_clear_rl_full(rl, sync);
L
Linus Torvalds 已提交
1265 1266 1267 1268 1269 1270 1271
	}
}

/*
 * A request has just been released.  Account for it, update the full and
 * congestion status, wake up any waiters.   Called under q->queue_lock.
 */
1272 1273
static void freed_request(struct request_list *rl, bool sync,
		req_flags_t rq_flags)
L
Linus Torvalds 已提交
1274
{
1275
	struct request_queue *q = rl->q;
L
Linus Torvalds 已提交
1276

1277
	q->nr_rqs[sync]--;
1278
	rl->count[sync]--;
1279
	if (rq_flags & RQF_ELVPRIV)
1280
		q->nr_rqs_elvpriv--;
L
Linus Torvalds 已提交
1281

1282
	__freed_request(rl, sync);
L
Linus Torvalds 已提交
1283

1284
	if (unlikely(rl->starved[sync ^ 1]))
1285
		__freed_request(rl, sync ^ 1);
L
Linus Torvalds 已提交
1286 1287
}

1288 1289 1290
int blk_update_nr_requests(struct request_queue *q, unsigned int nr)
{
	struct request_list *rl;
1291
	int on_thresh, off_thresh;
1292

1293 1294
	WARN_ON_ONCE(q->mq_ops);

1295 1296 1297
	spin_lock_irq(q->queue_lock);
	q->nr_requests = nr;
	blk_queue_congestion_threshold(q);
1298 1299
	on_thresh = queue_congestion_on_threshold(q);
	off_thresh = queue_congestion_off_threshold(q);
1300

1301 1302 1303 1304 1305
	blk_queue_for_each_rl(rl, q) {
		if (rl->count[BLK_RW_SYNC] >= on_thresh)
			blk_set_congested(rl, BLK_RW_SYNC);
		else if (rl->count[BLK_RW_SYNC] < off_thresh)
			blk_clear_congested(rl, BLK_RW_SYNC);
1306

1307 1308 1309 1310
		if (rl->count[BLK_RW_ASYNC] >= on_thresh)
			blk_set_congested(rl, BLK_RW_ASYNC);
		else if (rl->count[BLK_RW_ASYNC] < off_thresh)
			blk_clear_congested(rl, BLK_RW_ASYNC);
1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330

		if (rl->count[BLK_RW_SYNC] >= q->nr_requests) {
			blk_set_rl_full(rl, BLK_RW_SYNC);
		} else {
			blk_clear_rl_full(rl, BLK_RW_SYNC);
			wake_up(&rl->wait[BLK_RW_SYNC]);
		}

		if (rl->count[BLK_RW_ASYNC] >= q->nr_requests) {
			blk_set_rl_full(rl, BLK_RW_ASYNC);
		} else {
			blk_clear_rl_full(rl, BLK_RW_ASYNC);
			wake_up(&rl->wait[BLK_RW_ASYNC]);
		}
	}

	spin_unlock_irq(q->queue_lock);
	return 0;
}

1331
/**
T
Tejun Heo 已提交
1332
 * __get_request - get a free request
1333
 * @rl: request list to allocate from
1334
 * @op: operation and flags
1335
 * @bio: bio to allocate request for (can be %NULL)
1336
 * @flags: BLQ_MQ_REQ_* flags
1337
 * @gfp_mask: allocator flags
1338 1339 1340 1341
 *
 * Get a free request from @q.  This function may fail under memory
 * pressure or if @q is dead.
 *
1342
 * Must be called with @q->queue_lock held and,
1343 1344
 * Returns ERR_PTR on failure, with @q->queue_lock held.
 * Returns request pointer on success, with @q->queue_lock *not held*.
L
Linus Torvalds 已提交
1345
 */
1346
static struct request *__get_request(struct request_list *rl, unsigned int op,
1347
		struct bio *bio, blk_mq_req_flags_t flags, gfp_t gfp_mask)
L
Linus Torvalds 已提交
1348
{
1349
	struct request_queue *q = rl->q;
T
Tejun Heo 已提交
1350
	struct request *rq;
T
Tejun Heo 已提交
1351 1352
	struct elevator_type *et = q->elevator->type;
	struct io_context *ioc = rq_ioc(bio);
1353
	struct io_cq *icq = NULL;
1354
	const bool is_sync = op_is_sync(op);
1355
	int may_queue;
1356
	req_flags_t rq_flags = RQF_ALLOCED;
1357

1358 1359
	lockdep_assert_held(q->queue_lock);

B
Bart Van Assche 已提交
1360
	if (unlikely(blk_queue_dying(q)))
1361
		return ERR_PTR(-ENODEV);
1362

1363
	may_queue = elv_may_queue(q, op);
1364 1365 1366
	if (may_queue == ELV_MQUEUE_NO)
		goto rq_starved;

1367 1368
	if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
		if (rl->count[is_sync]+1 >= q->nr_requests) {
1369 1370 1371 1372 1373 1374
			/*
			 * The queue will fill after this allocation, so set
			 * it as full, and mark this process as "batching".
			 * This process will be allowed to complete a batch of
			 * requests, others will be blocked.
			 */
1375
			if (!blk_rl_full(rl, is_sync)) {
1376
				ioc_set_batching(q, ioc);
1377
				blk_set_rl_full(rl, is_sync);
1378 1379 1380 1381 1382 1383 1384 1385
			} else {
				if (may_queue != ELV_MQUEUE_MUST
						&& !ioc_batching(q, ioc)) {
					/*
					 * The queue is full and the allocating
					 * process is not a "batcher", and not
					 * exempted by the IO scheduler
					 */
1386
					return ERR_PTR(-ENOMEM);
1387 1388
				}
			}
L
Linus Torvalds 已提交
1389
		}
1390
		blk_set_congested(rl, is_sync);
L
Linus Torvalds 已提交
1391 1392
	}

1393 1394 1395 1396 1397
	/*
	 * Only allow batching queuers to allocate up to 50% over the defined
	 * limit of requests, otherwise we could have thousands of requests
	 * allocated with any setting of ->nr_requests
	 */
1398
	if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
1399
		return ERR_PTR(-ENOMEM);
H
Hugh Dickins 已提交
1400

1401
	q->nr_rqs[is_sync]++;
1402 1403
	rl->count[is_sync]++;
	rl->starved[is_sync] = 0;
T
Tejun Heo 已提交
1404

1405 1406
	/*
	 * Decide whether the new request will be managed by elevator.  If
1407
	 * so, mark @rq_flags and increment elvpriv.  Non-zero elvpriv will
1408 1409 1410 1411
	 * prevent the current elevator from being destroyed until the new
	 * request is freed.  This guarantees icq's won't be destroyed and
	 * makes creating new ones safe.
	 *
1412 1413 1414
	 * Flush requests do not use the elevator so skip initialization.
	 * This allows a request to share the flush and elevator data.
	 *
1415 1416 1417
	 * Also, lookup icq while holding queue_lock.  If it doesn't exist,
	 * it will be created after releasing queue_lock.
	 */
1418
	if (!op_is_flush(op) && !blk_queue_bypass(q)) {
1419
		rq_flags |= RQF_ELVPRIV;
1420
		q->nr_rqs_elvpriv++;
1421 1422
		if (et->icq_cache && ioc)
			icq = ioc_lookup_icq(ioc, q);
1423
	}
T
Tejun Heo 已提交
1424

1425
	if (blk_queue_io_stat(q))
1426
		rq_flags |= RQF_IO_STAT;
L
Linus Torvalds 已提交
1427 1428
	spin_unlock_irq(q->queue_lock);

1429
	/* allocate and init request */
1430
	rq = mempool_alloc(rl->rq_pool, gfp_mask);
1431
	if (!rq)
T
Tejun Heo 已提交
1432
		goto fail_alloc;
L
Linus Torvalds 已提交
1433

1434
	blk_rq_init(q, rq);
1435
	blk_rq_set_rl(rq, rl);
1436
	rq->cmd_flags = op;
1437
	rq->rq_flags = rq_flags;
1438 1439
	if (flags & BLK_MQ_REQ_PREEMPT)
		rq->rq_flags |= RQF_PREEMPT;
1440

1441
	/* init elvpriv */
1442
	if (rq_flags & RQF_ELVPRIV) {
1443
		if (unlikely(et->icq_cache && !icq)) {
T
Tejun Heo 已提交
1444 1445
			if (ioc)
				icq = ioc_create_icq(ioc, q, gfp_mask);
1446 1447
			if (!icq)
				goto fail_elvpriv;
1448
		}
1449 1450 1451 1452 1453 1454

		rq->elv.icq = icq;
		if (unlikely(elv_set_request(q, rq, bio, gfp_mask)))
			goto fail_elvpriv;

		/* @rq->elv.icq holds io_context until @rq is freed */
1455 1456 1457
		if (icq)
			get_io_context(icq->ioc);
	}
1458
out:
1459 1460 1461 1462 1463 1464
	/*
	 * ioc may be NULL here, and ioc_batching will be false. That's
	 * OK, if the queue is under the request limit then requests need
	 * not count toward the nr_batch_requests limit. There will always
	 * be some limit enforced by BLK_BATCH_TIME.
	 */
L
Linus Torvalds 已提交
1465 1466
	if (ioc_batching(q, ioc))
		ioc->nr_batch_requests--;
1467

1468
	trace_block_getrq(q, bio, op);
L
Linus Torvalds 已提交
1469
	return rq;
T
Tejun Heo 已提交
1470

1471 1472 1473 1474 1475 1476 1477
fail_elvpriv:
	/*
	 * elvpriv init failed.  ioc, icq and elvpriv aren't mempool backed
	 * and may fail indefinitely under memory pressure and thus
	 * shouldn't stall IO.  Treat this request as !elvpriv.  This will
	 * disturb iosched and blkcg but weird is bettern than dead.
	 */
1478
	printk_ratelimited(KERN_WARNING "%s: dev %s: request aux data allocation failed, iosched may be disturbed\n",
1479
			   __func__, dev_name(q->backing_dev_info->dev));
1480

1481
	rq->rq_flags &= ~RQF_ELVPRIV;
1482 1483 1484
	rq->elv.icq = NULL;

	spin_lock_irq(q->queue_lock);
1485
	q->nr_rqs_elvpriv--;
1486 1487 1488
	spin_unlock_irq(q->queue_lock);
	goto out;

T
Tejun Heo 已提交
1489 1490 1491 1492 1493 1494 1495 1496 1497
fail_alloc:
	/*
	 * Allocation failed presumably due to memory. Undo anything we
	 * might have messed up.
	 *
	 * Allocating task should really be put onto the front of the wait
	 * queue, but this is pretty rare.
	 */
	spin_lock_irq(q->queue_lock);
1498
	freed_request(rl, is_sync, rq_flags);
T
Tejun Heo 已提交
1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509

	/*
	 * in the very unlikely event that allocation failed and no
	 * requests for this direction was pending, mark us starved so that
	 * freeing of a request in the other direction will notice
	 * us. another possible fix would be to split the rq mempool into
	 * READ and WRITE
	 */
rq_starved:
	if (unlikely(rl->count[is_sync] == 0))
		rl->starved[is_sync] = 1;
1510
	return ERR_PTR(-ENOMEM);
L
Linus Torvalds 已提交
1511 1512
}

1513
/**
T
Tejun Heo 已提交
1514
 * get_request - get a free request
1515
 * @q: request_queue to allocate request from
1516
 * @op: operation and flags
1517
 * @bio: bio to allocate request for (can be %NULL)
1518
 * @flags: BLK_MQ_REQ_* flags.
1519
 * @gfp: allocator flags
1520
 *
1521
 * Get a free request from @q.  If %BLK_MQ_REQ_NOWAIT is set in @flags,
1522
 * this function keeps retrying under memory pressure and fails iff @q is dead.
N
Nick Piggin 已提交
1523
 *
1524
 * Must be called with @q->queue_lock held and,
1525 1526
 * Returns ERR_PTR on failure, with @q->queue_lock held.
 * Returns request pointer on success, with @q->queue_lock *not held*.
L
Linus Torvalds 已提交
1527
 */
1528
static struct request *get_request(struct request_queue *q, unsigned int op,
1529
		struct bio *bio, blk_mq_req_flags_t flags, gfp_t gfp)
L
Linus Torvalds 已提交
1530
{
1531
	const bool is_sync = op_is_sync(op);
T
Tejun Heo 已提交
1532
	DEFINE_WAIT(wait);
1533
	struct request_list *rl;
L
Linus Torvalds 已提交
1534
	struct request *rq;
1535

1536
	lockdep_assert_held(q->queue_lock);
1537
	WARN_ON_ONCE(q->mq_ops);
1538

1539
	rl = blk_get_rl(q, bio);	/* transferred to @rq on success */
T
Tejun Heo 已提交
1540
retry:
1541
	rq = __get_request(rl, op, bio, flags, gfp);
1542
	if (!IS_ERR(rq))
T
Tejun Heo 已提交
1543
		return rq;
L
Linus Torvalds 已提交
1544

1545 1546 1547 1548 1549
	if (op & REQ_NOWAIT) {
		blk_put_rl(rl);
		return ERR_PTR(-EAGAIN);
	}

1550
	if ((flags & BLK_MQ_REQ_NOWAIT) || unlikely(blk_queue_dying(q))) {
1551
		blk_put_rl(rl);
1552
		return rq;
1553
	}
L
Linus Torvalds 已提交
1554

T
Tejun Heo 已提交
1555 1556 1557
	/* wait on @rl and retry */
	prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
				  TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
1558

1559
	trace_block_sleeprq(q, bio, op);
L
Linus Torvalds 已提交
1560

T
Tejun Heo 已提交
1561 1562
	spin_unlock_irq(q->queue_lock);
	io_schedule();
N
Nick Piggin 已提交
1563

T
Tejun Heo 已提交
1564 1565 1566 1567 1568 1569
	/*
	 * After sleeping, we become a "batching" process and will be able
	 * to allocate at least one request, and up to a big batch of them
	 * for a small period time.  See ioc_batching, ioc_set_batching
	 */
	ioc_set_batching(q, current->io_context);
1570

T
Tejun Heo 已提交
1571 1572
	spin_lock_irq(q->queue_lock);
	finish_wait(&rl->wait[is_sync], &wait);
L
Linus Torvalds 已提交
1573

T
Tejun Heo 已提交
1574
	goto retry;
L
Linus Torvalds 已提交
1575 1576
}

1577
/* flags: BLK_MQ_REQ_PREEMPT and/or BLK_MQ_REQ_NOWAIT. */
1578
static struct request *blk_old_get_request(struct request_queue *q,
1579
				unsigned int op, blk_mq_req_flags_t flags)
L
Linus Torvalds 已提交
1580 1581
{
	struct request *rq;
1582
	gfp_t gfp_mask = flags & BLK_MQ_REQ_NOWAIT ? GFP_ATOMIC : GFP_NOIO;
1583
	int ret = 0;
L
Linus Torvalds 已提交
1584

1585 1586
	WARN_ON_ONCE(q->mq_ops);

T
Tejun Heo 已提交
1587 1588 1589
	/* create ioc upfront */
	create_io_context(gfp_mask, q->node);

1590
	ret = blk_queue_enter(q, flags);
1591 1592
	if (ret)
		return ERR_PTR(ret);
N
Nick Piggin 已提交
1593
	spin_lock_irq(q->queue_lock);
1594
	rq = get_request(q, op, NULL, flags, gfp_mask);
1595
	if (IS_ERR(rq)) {
1596
		spin_unlock_irq(q->queue_lock);
1597
		blk_queue_exit(q);
1598 1599
		return rq;
	}
L
Linus Torvalds 已提交
1600

1601 1602 1603 1604
	/* q->queue_lock is unlocked at this point */
	rq->__data_len = 0;
	rq->__sector = (sector_t) -1;
	rq->bio = rq->biotail = NULL;
L
Linus Torvalds 已提交
1605 1606
	return rq;
}
1607

1608
/**
1609
 * blk_get_request - allocate a request
1610 1611 1612 1613
 * @q: request queue to allocate a request for
 * @op: operation (REQ_OP_*) and REQ_* flags, e.g. REQ_SYNC.
 * @flags: BLK_MQ_REQ_* flags, e.g. BLK_MQ_REQ_NOWAIT.
 */
1614 1615
struct request *blk_get_request(struct request_queue *q, unsigned int op,
				blk_mq_req_flags_t flags)
1616
{
1617 1618
	struct request *req;

1619
	WARN_ON_ONCE(op & REQ_NOWAIT);
1620
	WARN_ON_ONCE(flags & ~(BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_PREEMPT));
1621

1622
	if (q->mq_ops) {
1623
		req = blk_mq_alloc_request(q, op, flags);
1624 1625 1626
		if (!IS_ERR(req) && q->mq_ops->initialize_rq_fn)
			q->mq_ops->initialize_rq_fn(req);
	} else {
1627
		req = blk_old_get_request(q, op, flags);
1628 1629 1630 1631 1632
		if (!IS_ERR(req) && q->initialize_rq_fn)
			q->initialize_rq_fn(req);
	}

	return req;
1633
}
L
Linus Torvalds 已提交
1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645
EXPORT_SYMBOL(blk_get_request);

/**
 * blk_requeue_request - put a request back on queue
 * @q:		request queue where request should be inserted
 * @rq:		request to be inserted
 *
 * Description:
 *    Drivers often keep queueing requests until the hardware cannot accept
 *    more, when that condition happens we need to put the request back
 *    on the queue. Must be called with queue lock held.
 */
1646
void blk_requeue_request(struct request_queue *q, struct request *rq)
L
Linus Torvalds 已提交
1647
{
1648
	lockdep_assert_held(q->queue_lock);
1649
	WARN_ON_ONCE(q->mq_ops);
1650

J
Jens Axboe 已提交
1651 1652
	blk_delete_timer(rq);
	blk_clear_rq_complete(rq);
1653
	trace_block_rq_requeue(q, rq);
1654
	rq_qos_requeue(q, rq);
1655

1656
	if (rq->rq_flags & RQF_QUEUED)
L
Linus Torvalds 已提交
1657 1658
		blk_queue_end_tag(q, rq);

1659 1660
	BUG_ON(blk_queued_rq(rq));

L
Linus Torvalds 已提交
1661 1662 1663 1664
	elv_requeue_request(q, rq);
}
EXPORT_SYMBOL(blk_requeue_request);

1665 1666 1667
static void add_acct_request(struct request_queue *q, struct request *rq,
			     int where)
{
1668
	blk_account_io_start(rq, true);
J
Jens Axboe 已提交
1669
	__elv_add_request(q, rq, where);
1670 1671
}

1672
static void part_round_stats_single(struct request_queue *q, int cpu,
1673 1674
				    struct hd_struct *part, unsigned long now,
				    unsigned int inflight)
T
Tejun Heo 已提交
1675
{
1676
	if (inflight) {
T
Tejun Heo 已提交
1677
		__part_stat_add(cpu, part, time_in_queue,
1678
				inflight * (now - part->stamp));
T
Tejun Heo 已提交
1679 1680 1681 1682 1683 1684
		__part_stat_add(cpu, part, io_ticks, (now - part->stamp));
	}
	part->stamp = now;
}

/**
1685
 * part_round_stats() - Round off the performance stats on a struct disk_stats.
1686
 * @q: target block queue
1687 1688
 * @cpu: cpu number for stats access
 * @part: target partition
L
Linus Torvalds 已提交
1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700
 *
 * The average IO queue length and utilisation statistics are maintained
 * by observing the current state of the queue length and the amount of
 * time it has been in this state for.
 *
 * Normally, that accounting is done on IO completion, but that can result
 * in more than a second's worth of IO being accounted for within any one
 * second, leading to >100% utilisation.  To deal with that, we call this
 * function to do a round-off before returning the results when reading
 * /proc/diskstats.  This accounts immediately for all queue usage up to
 * the current jiffies and restarts the counters again.
 */
1701
void part_round_stats(struct request_queue *q, int cpu, struct hd_struct *part)
1702
{
1703
	struct hd_struct *part2 = NULL;
1704
	unsigned long now = jiffies;
1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720
	unsigned int inflight[2];
	int stats = 0;

	if (part->stamp != now)
		stats |= 1;

	if (part->partno) {
		part2 = &part_to_disk(part)->part0;
		if (part2->stamp != now)
			stats |= 2;
	}

	if (!stats)
		return;

	part_in_flight(q, part, inflight);
1721

1722 1723 1724 1725
	if (stats & 2)
		part_round_stats_single(q, cpu, part2, now, inflight[1]);
	if (stats & 1)
		part_round_stats_single(q, cpu, part, now, inflight[0]);
1726
}
T
Tejun Heo 已提交
1727
EXPORT_SYMBOL_GPL(part_round_stats);
1728

1729
#ifdef CONFIG_PM
L
Lin Ming 已提交
1730 1731
static void blk_pm_put_request(struct request *rq)
{
1732
	if (rq->q->dev && !(rq->rq_flags & RQF_PM) && !--rq->q->nr_pending)
L
Lin Ming 已提交
1733 1734 1735 1736 1737 1738
		pm_runtime_mark_last_busy(rq->q->dev);
}
#else
static inline void blk_pm_put_request(struct request *rq) {}
#endif

1739
void __blk_put_request(struct request_queue *q, struct request *req)
L
Linus Torvalds 已提交
1740
{
1741 1742
	req_flags_t rq_flags = req->rq_flags;

L
Linus Torvalds 已提交
1743 1744 1745
	if (unlikely(!q))
		return;

1746 1747 1748 1749 1750
	if (q->mq_ops) {
		blk_mq_free_request(req);
		return;
	}

1751 1752
	lockdep_assert_held(q->queue_lock);

1753
	blk_req_zone_write_unlock(req);
L
Lin Ming 已提交
1754 1755
	blk_pm_put_request(req);

1756 1757
	elv_completed_request(q, req);

1758 1759 1760
	/* this is a bio leak */
	WARN_ON(req->bio != NULL);

1761
	rq_qos_done(q, req);
J
Jens Axboe 已提交
1762

L
Linus Torvalds 已提交
1763 1764 1765 1766
	/*
	 * Request may not have originated from ll_rw_blk. if not,
	 * it didn't come out of our reserved rq pools
	 */
1767
	if (rq_flags & RQF_ALLOCED) {
1768
		struct request_list *rl = blk_rq_rl(req);
1769
		bool sync = op_is_sync(req->cmd_flags);
L
Linus Torvalds 已提交
1770 1771

		BUG_ON(!list_empty(&req->queuelist));
1772
		BUG_ON(ELV_ON_HASH(req));
L
Linus Torvalds 已提交
1773

1774
		blk_free_request(rl, req);
1775
		freed_request(rl, sync, rq_flags);
1776
		blk_put_rl(rl);
1777
		blk_queue_exit(q);
L
Linus Torvalds 已提交
1778 1779
	}
}
1780 1781
EXPORT_SYMBOL_GPL(__blk_put_request);

L
Linus Torvalds 已提交
1782 1783
void blk_put_request(struct request *req)
{
1784
	struct request_queue *q = req->q;
1785

1786 1787 1788 1789 1790 1791 1792 1793 1794
	if (q->mq_ops)
		blk_mq_free_request(req);
	else {
		unsigned long flags;

		spin_lock_irqsave(q->queue_lock, flags);
		__blk_put_request(q, req);
		spin_unlock_irqrestore(q->queue_lock, flags);
	}
L
Linus Torvalds 已提交
1795 1796 1797
}
EXPORT_SYMBOL(blk_put_request);

1798 1799
bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
			    struct bio *bio)
1800
{
J
Jens Axboe 已提交
1801
	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
1802 1803 1804 1805

	if (!ll_back_merge_fn(q, req, bio))
		return false;

1806
	trace_block_bio_backmerge(q, req, bio);
1807 1808 1809 1810 1811 1812

	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
		blk_rq_set_mixed_merge(req);

	req->biotail->bi_next = bio;
	req->biotail = bio;
1813
	req->__data_len += bio->bi_iter.bi_size;
1814 1815
	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));

1816
	blk_account_io_start(req, false);
1817 1818 1819
	return true;
}

1820 1821
bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
			     struct bio *bio)
1822
{
J
Jens Axboe 已提交
1823
	const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
1824 1825 1826 1827

	if (!ll_front_merge_fn(q, req, bio))
		return false;

1828
	trace_block_bio_frontmerge(q, req, bio);
1829 1830 1831 1832 1833 1834 1835

	if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
		blk_rq_set_mixed_merge(req);

	bio->bi_next = req->bio;
	req->bio = bio;

1836 1837
	req->__sector = bio->bi_iter.bi_sector;
	req->__data_len += bio->bi_iter.bi_size;
1838 1839
	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));

1840
	blk_account_io_start(req, false);
1841 1842 1843
	return true;
}

1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867
bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
		struct bio *bio)
{
	unsigned short segments = blk_rq_nr_discard_segments(req);

	if (segments >= queue_max_discard_segments(q))
		goto no_merge;
	if (blk_rq_sectors(req) + bio_sectors(bio) >
	    blk_rq_get_max_sectors(req, blk_rq_pos(req)))
		goto no_merge;

	req->biotail->bi_next = bio;
	req->biotail = bio;
	req->__data_len += bio->bi_iter.bi_size;
	req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
	req->nr_phys_segments = segments + 1;

	blk_account_io_start(req, false);
	return true;
no_merge:
	req_set_nomerge(q, req);
	return false;
}

1868
/**
1869
 * blk_attempt_plug_merge - try to merge with %current's plugged list
1870 1871 1872
 * @q: request_queue new bio is being queued at
 * @bio: new bio being queued
 * @request_count: out parameter for number of traversed plugged requests
1873 1874 1875
 * @same_queue_rq: pointer to &struct request that gets filled in when
 * another request associated with @q is found on the plug list
 * (optional, may be %NULL)
1876 1877 1878 1879 1880
 *
 * Determine whether @bio being queued on @q can be merged with a request
 * on %current's plugged list.  Returns %true if merge was successful,
 * otherwise %false.
 *
1881 1882 1883 1884 1885 1886
 * Plugging coalesces IOs from the same issuer for the same purpose without
 * going through @q->queue_lock.  As such it's more of an issuing mechanism
 * than scheduling, and the request, while may have elvpriv data, is not
 * added on the elevator at this point.  In addition, we don't have
 * reliable access to the elevator outside queue lock.  Only check basic
 * merging parameters without querying the elevator.
1887 1888
 *
 * Caller must ensure !blk_queue_nomerges(q) beforehand.
1889
 */
1890
bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
1891 1892
			    unsigned int *request_count,
			    struct request **same_queue_rq)
1893 1894 1895
{
	struct blk_plug *plug;
	struct request *rq;
S
Shaohua Li 已提交
1896
	struct list_head *plug_list;
1897

1898
	plug = current->plug;
1899
	if (!plug)
1900
		return false;
1901
	*request_count = 0;
1902

S
Shaohua Li 已提交
1903 1904 1905 1906 1907 1908
	if (q->mq_ops)
		plug_list = &plug->mq_list;
	else
		plug_list = &plug->list;

	list_for_each_entry_reverse(rq, plug_list, queuelist) {
1909
		bool merged = false;
1910

1911
		if (rq->q == q) {
1912
			(*request_count)++;
1913 1914 1915 1916 1917 1918 1919 1920
			/*
			 * Only blk-mq multiple hardware queues case checks the
			 * rq in the same queue, there should be only one such
			 * rq in a queue
			 **/
			if (same_queue_rq)
				*same_queue_rq = rq;
		}
1921

1922
		if (rq->q != q || !blk_rq_merge_ok(rq, bio))
1923 1924
			continue;

1925 1926 1927 1928 1929 1930 1931
		switch (blk_try_merge(rq, bio)) {
		case ELEVATOR_BACK_MERGE:
			merged = bio_attempt_back_merge(q, rq, bio);
			break;
		case ELEVATOR_FRONT_MERGE:
			merged = bio_attempt_front_merge(q, rq, bio);
			break;
1932 1933 1934
		case ELEVATOR_DISCARD_MERGE:
			merged = bio_attempt_discard_merge(q, rq, bio);
			break;
1935 1936
		default:
			break;
1937
		}
1938 1939 1940

		if (merged)
			return true;
1941
	}
1942 1943

	return false;
1944 1945
}

1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969
unsigned int blk_plug_queued_count(struct request_queue *q)
{
	struct blk_plug *plug;
	struct request *rq;
	struct list_head *plug_list;
	unsigned int ret = 0;

	plug = current->plug;
	if (!plug)
		goto out;

	if (q->mq_ops)
		plug_list = &plug->mq_list;
	else
		plug_list = &plug->list;

	list_for_each_entry(rq, plug_list, queuelist) {
		if (rq->q == q)
			ret++;
	}
out:
	return ret;
}

1970
void blk_init_request_from_bio(struct request *req, struct bio *bio)
1971
{
1972 1973
	struct io_context *ioc = rq_ioc(bio);

J
Jens Axboe 已提交
1974
	if (bio->bi_opf & REQ_RAHEAD)
1975
		req->cmd_flags |= REQ_FAILFAST_MASK;
J
Jens Axboe 已提交
1976

1977
	req->__sector = bio->bi_iter.bi_sector;
1978 1979
	if (ioprio_valid(bio_prio(bio)))
		req->ioprio = bio_prio(bio);
1980 1981 1982 1983
	else if (ioc)
		req->ioprio = ioc->ioprio;
	else
		req->ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0);
1984
	req->write_hint = bio->bi_write_hint;
1985
	blk_rq_bio_prep(req->q, req, bio);
1986
}
1987
EXPORT_SYMBOL_GPL(blk_init_request_from_bio);
1988

1989
static blk_qc_t blk_queue_bio(struct request_queue *q, struct bio *bio)
L
Linus Torvalds 已提交
1990
{
1991
	struct blk_plug *plug;
1992
	int where = ELEVATOR_INSERT_SORT;
1993
	struct request *req, *free;
1994
	unsigned int request_count = 0;
L
Linus Torvalds 已提交
1995 1996 1997 1998 1999 2000 2001 2002

	/*
	 * low level driver can indicate that it wants pages above a
	 * certain limit bounced to low memory (ie for highmem, or even
	 * ISA dma in theory)
	 */
	blk_queue_bounce(q, &bio);

2003
	blk_queue_split(q, &bio);
2004

2005
	if (!bio_integrity_prep(bio))
2006
		return BLK_QC_T_NONE;
2007

2008
	if (op_is_flush(bio->bi_opf)) {
2009
		spin_lock_irq(q->queue_lock);
2010
		where = ELEVATOR_INSERT_FLUSH;
2011 2012 2013
		goto get_rq;
	}

2014 2015 2016 2017
	/*
	 * Check if we can merge with the plugged list before grabbing
	 * any locks.
	 */
2018 2019
	if (!blk_queue_nomerges(q)) {
		if (blk_attempt_plug_merge(q, bio, &request_count, NULL))
2020
			return BLK_QC_T_NONE;
2021 2022
	} else
		request_count = blk_plug_queued_count(q);
L
Linus Torvalds 已提交
2023

2024
	spin_lock_irq(q->queue_lock);
2025

2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048
	switch (elv_merge(q, &req, bio)) {
	case ELEVATOR_BACK_MERGE:
		if (!bio_attempt_back_merge(q, req, bio))
			break;
		elv_bio_merged(q, req, bio);
		free = attempt_back_merge(q, req);
		if (free)
			__blk_put_request(q, free);
		else
			elv_merged_request(q, req, ELEVATOR_BACK_MERGE);
		goto out_unlock;
	case ELEVATOR_FRONT_MERGE:
		if (!bio_attempt_front_merge(q, req, bio))
			break;
		elv_bio_merged(q, req, bio);
		free = attempt_front_merge(q, req);
		if (free)
			__blk_put_request(q, free);
		else
			elv_merged_request(q, req, ELEVATOR_FRONT_MERGE);
		goto out_unlock;
	default:
		break;
L
Linus Torvalds 已提交
2049 2050
	}

2051
get_rq:
2052
	rq_qos_throttle(q, bio, q->queue_lock);
J
Jens Axboe 已提交
2053

L
Linus Torvalds 已提交
2054
	/*
2055
	 * Grab a free request. This is might sleep but can not fail.
N
Nick Piggin 已提交
2056
	 * Returns with the queue unlocked.
2057
	 */
2058
	blk_queue_enter_live(q);
2059
	req = get_request(q, bio->bi_opf, bio, 0, GFP_NOIO);
2060
	if (IS_ERR(req)) {
2061
		blk_queue_exit(q);
2062
		rq_qos_cleanup(q, bio);
2063 2064 2065 2066
		if (PTR_ERR(req) == -ENOMEM)
			bio->bi_status = BLK_STS_RESOURCE;
		else
			bio->bi_status = BLK_STS_IOERR;
2067
		bio_endio(bio);
2068 2069
		goto out_unlock;
	}
N
Nick Piggin 已提交
2070

2071
	rq_qos_track(q, req, bio);
J
Jens Axboe 已提交
2072

2073 2074 2075 2076 2077
	/*
	 * After dropping the lock and possibly sleeping here, our request
	 * may now be mergeable after it had proven unmergeable (above).
	 * We don't worry about that case for efficiency. It won't happen
	 * often, and the elevators are able to handle it.
L
Linus Torvalds 已提交
2078
	 */
2079
	blk_init_request_from_bio(req, bio);
L
Linus Torvalds 已提交
2080

2081
	if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags))
2082
		req->cpu = raw_smp_processor_id();
2083 2084

	plug = current->plug;
J
Jens Axboe 已提交
2085
	if (plug) {
J
Jens Axboe 已提交
2086 2087
		/*
		 * If this is the first request added after a plug, fire
2088
		 * of a plug trace.
2089 2090 2091
		 *
		 * @request_count may become stale because of schedule
		 * out, so check plug list again.
J
Jens Axboe 已提交
2092
		 */
2093
		if (!request_count || list_empty(&plug->list))
J
Jens Axboe 已提交
2094
			trace_block_plug(q);
2095
		else {
2096 2097 2098
			struct request *last = list_entry_rq(plug->list.prev);
			if (request_count >= BLK_MAX_REQUEST_COUNT ||
			    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE) {
2099
				blk_flush_plug_list(plug, false);
S
Shaohua Li 已提交
2100 2101
				trace_block_plug(q);
			}
2102 2103
		}
		list_add_tail(&req->queuelist, &plug->list);
2104
		blk_account_io_start(req, true);
2105 2106 2107
	} else {
		spin_lock_irq(q->queue_lock);
		add_acct_request(q, req, where);
2108
		__blk_run_queue(q);
2109 2110 2111
out_unlock:
		spin_unlock_irq(q->queue_lock);
	}
2112 2113

	return BLK_QC_T_NONE;
L
Linus Torvalds 已提交
2114 2115
}

2116
static void handle_bad_sector(struct bio *bio, sector_t maxsector)
L
Linus Torvalds 已提交
2117 2118 2119 2120
{
	char b[BDEVNAME_SIZE];

	printk(KERN_INFO "attempt to access beyond end of device\n");
2121
	printk(KERN_INFO "%s: rw=%d, want=%Lu, limit=%Lu\n",
2122
			bio_devname(bio, b), bio->bi_opf,
K
Kent Overstreet 已提交
2123
			(unsigned long long)bio_end_sector(bio),
2124
			(long long)maxsector);
L
Linus Torvalds 已提交
2125 2126
}

2127 2128 2129 2130 2131 2132 2133 2134 2135 2136
#ifdef CONFIG_FAIL_MAKE_REQUEST

static DECLARE_FAULT_ATTR(fail_make_request);

static int __init setup_fail_make_request(char *str)
{
	return setup_fault_attr(&fail_make_request, str);
}
__setup("fail_make_request=", setup_fail_make_request);

2137
static bool should_fail_request(struct hd_struct *part, unsigned int bytes)
2138
{
2139
	return part->make_it_fail && should_fail(&fail_make_request, bytes);
2140 2141 2142 2143
}

static int __init fail_make_request_debugfs(void)
{
2144 2145 2146
	struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
						NULL, &fail_make_request);

2147
	return PTR_ERR_OR_ZERO(dir);
2148 2149 2150 2151 2152 2153
}

late_initcall(fail_make_request_debugfs);

#else /* CONFIG_FAIL_MAKE_REQUEST */

2154 2155
static inline bool should_fail_request(struct hd_struct *part,
					unsigned int bytes)
2156
{
2157
	return false;
2158 2159 2160 2161
}

#endif /* CONFIG_FAIL_MAKE_REQUEST */

2162 2163
static inline bool bio_check_ro(struct bio *bio, struct hd_struct *part)
{
2164 2165
	const int op = bio_op(bio);

2166
	if (part->policy && op_is_write(op)) {
2167 2168
		char b[BDEVNAME_SIZE];

2169 2170 2171
		if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
			return false;

2172
		WARN_ONCE(1,
2173 2174 2175
		       "generic_make_request: Trying to write "
			"to read-only block-device %s (partno %d)\n",
			bio_devname(bio, b), part->partno);
2176 2177
		/* Older lvm-tools actually trigger this */
		return false;
2178 2179 2180 2181 2182
	}

	return false;
}

2183 2184 2185 2186 2187 2188 2189 2190
static noinline int should_fail_bio(struct bio *bio)
{
	if (should_fail_request(&bio->bi_disk->part0, bio->bi_iter.bi_size))
		return -EIO;
	return 0;
}
ALLOW_ERROR_INJECTION(should_fail_bio, ERRNO);

2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208
/*
 * Check whether this bio extends beyond the end of the device or partition.
 * This may well happen - the kernel calls bread() without checking the size of
 * the device, e.g., when mounting a file system.
 */
static inline int bio_check_eod(struct bio *bio, sector_t maxsector)
{
	unsigned int nr_sectors = bio_sectors(bio);

	if (nr_sectors && maxsector &&
	    (nr_sectors > maxsector ||
	     bio->bi_iter.bi_sector > maxsector - nr_sectors)) {
		handle_bad_sector(bio, maxsector);
		return -EIO;
	}
	return 0;
}

2209 2210 2211 2212 2213 2214
/*
 * Remap block n of partition p to block n+start(p) of the disk.
 */
static inline int blk_partition_remap(struct bio *bio)
{
	struct hd_struct *p;
2215
	int ret = -EIO;
2216

2217 2218
	rcu_read_lock();
	p = __disk_get_part(bio->bi_disk, bio->bi_partno);
2219 2220 2221 2222 2223
	if (unlikely(!p))
		goto out;
	if (unlikely(should_fail_request(p, bio->bi_iter.bi_size)))
		goto out;
	if (unlikely(bio_check_ro(bio, p)))
2224 2225
		goto out;

2226 2227 2228 2229
	/*
	 * Zone reset does not include bi_size so bio_sectors() is always 0.
	 * Include a test for the reset op code and perform the remap if needed.
	 */
2230 2231 2232 2233 2234 2235 2236
	if (bio_sectors(bio) || bio_op(bio) == REQ_OP_ZONE_RESET) {
		if (bio_check_eod(bio, part_nr_sects_read(p)))
			goto out;
		bio->bi_iter.bi_sector += p->start_sect;
		trace_block_bio_remap(bio->bi_disk->queue, bio, part_devt(p),
				      bio->bi_iter.bi_sector - p->start_sect);
	}
2237
	bio->bi_partno = 0;
2238
	ret = 0;
2239 2240
out:
	rcu_read_unlock();
2241 2242 2243
	return ret;
}

2244 2245
static noinline_for_stack bool
generic_make_request_checks(struct bio *bio)
L
Linus Torvalds 已提交
2246
{
2247
	struct request_queue *q;
2248
	int nr_sectors = bio_sectors(bio);
2249
	blk_status_t status = BLK_STS_IOERR;
2250
	char b[BDEVNAME_SIZE];
L
Linus Torvalds 已提交
2251 2252 2253

	might_sleep();

2254
	q = bio->bi_disk->queue;
2255 2256 2257 2258
	if (unlikely(!q)) {
		printk(KERN_ERR
		       "generic_make_request: Trying to access "
			"nonexistent block-device %s (%Lu)\n",
2259
			bio_devname(bio, b), (long long)bio->bi_iter.bi_sector);
2260 2261
		goto end_io;
	}
2262

2263 2264 2265 2266 2267 2268 2269
	/*
	 * For a REQ_NOWAIT based request, return -EOPNOTSUPP
	 * if queue is not a request based queue.
	 */
	if ((bio->bi_opf & REQ_NOWAIT) && !queue_is_rq_based(q))
		goto not_supported;

2270
	if (should_fail_bio(bio))
2271
		goto end_io;
2272

2273 2274
	if (bio->bi_partno) {
		if (unlikely(blk_partition_remap(bio)))
2275 2276
			goto end_io;
	} else {
2277 2278 2279
		if (unlikely(bio_check_ro(bio, &bio->bi_disk->part0)))
			goto end_io;
		if (unlikely(bio_check_eod(bio, get_capacity(bio->bi_disk))))
2280 2281
			goto end_io;
	}
2282

2283 2284 2285 2286 2287
	/*
	 * Filter flush bio's early so that make_request based
	 * drivers without flush support don't have to worry
	 * about them.
	 */
2288
	if (op_is_flush(bio->bi_opf) &&
J
Jens Axboe 已提交
2289
	    !test_bit(QUEUE_FLAG_WC, &q->queue_flags)) {
J
Jens Axboe 已提交
2290
		bio->bi_opf &= ~(REQ_PREFLUSH | REQ_FUA);
2291
		if (!nr_sectors) {
2292
			status = BLK_STS_OK;
2293 2294
			goto end_io;
		}
2295
	}
2296

2297 2298 2299 2300 2301 2302 2303 2304 2305 2306
	switch (bio_op(bio)) {
	case REQ_OP_DISCARD:
		if (!blk_queue_discard(q))
			goto not_supported;
		break;
	case REQ_OP_SECURE_ERASE:
		if (!blk_queue_secure_erase(q))
			goto not_supported;
		break;
	case REQ_OP_WRITE_SAME:
2307
		if (!q->limits.max_write_same_sectors)
2308
			goto not_supported;
2309
		break;
2310 2311
	case REQ_OP_ZONE_REPORT:
	case REQ_OP_ZONE_RESET:
2312
		if (!blk_queue_is_zoned(q))
2313
			goto not_supported;
2314
		break;
2315
	case REQ_OP_WRITE_ZEROES:
2316
		if (!q->limits.max_write_zeroes_sectors)
2317 2318
			goto not_supported;
		break;
2319 2320
	default:
		break;
2321
	}
2322

T
Tejun Heo 已提交
2323 2324 2325 2326 2327 2328 2329 2330
	/*
	 * Various block parts want %current->io_context and lazy ioc
	 * allocation ends up trading a lot of pain for a small amount of
	 * memory.  Just allocate it upfront.  This may fail and block
	 * layer knows how to live with it.
	 */
	create_io_context(GFP_ATOMIC, q->node);

2331 2332
	if (!blkcg_bio_issue_check(q, bio))
		return false;
2333

N
NeilBrown 已提交
2334 2335 2336 2337 2338 2339 2340
	if (!bio_flagged(bio, BIO_TRACE_COMPLETION)) {
		trace_block_bio_queue(q, bio);
		/* Now that enqueuing has been traced, we need to trace
		 * completion as well.
		 */
		bio_set_flag(bio, BIO_TRACE_COMPLETION);
	}
2341
	return true;
2342

2343
not_supported:
2344
	status = BLK_STS_NOTSUPP;
2345
end_io:
2346
	bio->bi_status = status;
2347
	bio_endio(bio);
2348
	return false;
L
Linus Torvalds 已提交
2349 2350
}

2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373
/**
 * generic_make_request - hand a buffer to its device driver for I/O
 * @bio:  The bio describing the location in memory and on the device.
 *
 * generic_make_request() is used to make I/O requests of block
 * devices. It is passed a &struct bio, which describes the I/O that needs
 * to be done.
 *
 * generic_make_request() does not return any status.  The
 * success/failure status of the request, along with notification of
 * completion, is delivered asynchronously through the bio->bi_end_io
 * function described (one day) else where.
 *
 * The caller of generic_make_request must make sure that bi_io_vec
 * are set to describe the memory buffer, and that bi_dev and bi_sector are
 * set to describe the device address, and the
 * bi_end_io and optionally bi_private are set to describe how
 * completion notification should be signaled.
 *
 * generic_make_request and the drivers it calls may use bi_next if this
 * bio happens to be merged with someone else, and may resubmit the bio to
 * a lower device by calling into generic_make_request recursively, which
 * means the bio should NOT be touched after the call to ->make_request_fn.
2374
 */
2375
blk_qc_t generic_make_request(struct bio *bio)
2376
{
2377 2378 2379 2380 2381 2382 2383 2384
	/*
	 * bio_list_on_stack[0] contains bios submitted by the current
	 * make_request_fn.
	 * bio_list_on_stack[1] contains bios that were submitted before
	 * the current make_request_fn, but that haven't been processed
	 * yet.
	 */
	struct bio_list bio_list_on_stack[2];
2385 2386
	blk_mq_req_flags_t flags = 0;
	struct request_queue *q = bio->bi_disk->queue;
2387
	blk_qc_t ret = BLK_QC_T_NONE;
2388

2389 2390
	if (bio->bi_opf & REQ_NOWAIT)
		flags = BLK_MQ_REQ_NOWAIT;
2391 2392 2393
	if (bio_flagged(bio, BIO_QUEUE_ENTERED))
		blk_queue_enter_live(q);
	else if (blk_queue_enter(q, flags) < 0) {
2394 2395 2396 2397 2398 2399 2400
		if (!blk_queue_dying(q) && (bio->bi_opf & REQ_NOWAIT))
			bio_wouldblock_error(bio);
		else
			bio_io_error(bio);
		return ret;
	}

2401
	if (!generic_make_request_checks(bio))
2402
		goto out;
2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413

	/*
	 * We only want one ->make_request_fn to be active at a time, else
	 * stack usage with stacked devices could be a problem.  So use
	 * current->bio_list to keep a list of requests submited by a
	 * make_request_fn function.  current->bio_list is also used as a
	 * flag to say if generic_make_request is currently active in this
	 * task or not.  If it is NULL, then no make_request is active.  If
	 * it is non-NULL, then a make_request is active, and new requests
	 * should be added at the tail
	 */
2414
	if (current->bio_list) {
2415
		bio_list_add(&current->bio_list[0], bio);
2416
		goto out;
2417
	}
2418

2419 2420 2421 2422 2423
	/* following loop may be a bit non-obvious, and so deserves some
	 * explanation.
	 * Before entering the loop, bio->bi_next is NULL (as all callers
	 * ensure that) so we have a list with a single bio.
	 * We pretend that we have just taken it off a longer list, so
2424 2425
	 * we assign bio_list to a pointer to the bio_list_on_stack,
	 * thus initialising the bio_list of new bios to be
2426
	 * added.  ->make_request() may indeed add some more bios
2427 2428 2429
	 * through a recursive call to generic_make_request.  If it
	 * did, we find a non-NULL value in bio_list and re-enter the loop
	 * from the top.  In this case we really did just take the bio
2430
	 * of the top of the list (no pretending) and so remove it from
2431
	 * bio_list, and call into ->make_request() again.
2432 2433
	 */
	BUG_ON(bio->bi_next);
2434 2435
	bio_list_init(&bio_list_on_stack[0]);
	current->bio_list = bio_list_on_stack;
2436
	do {
2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450
		bool enter_succeeded = true;

		if (unlikely(q != bio->bi_disk->queue)) {
			if (q)
				blk_queue_exit(q);
			q = bio->bi_disk->queue;
			flags = 0;
			if (bio->bi_opf & REQ_NOWAIT)
				flags = BLK_MQ_REQ_NOWAIT;
			if (blk_queue_enter(q, flags) < 0) {
				enter_succeeded = false;
				q = NULL;
			}
		}
2451

2452
		if (enter_succeeded) {
2453 2454 2455
			struct bio_list lower, same;

			/* Create a fresh bio_list for all subordinate requests */
2456 2457
			bio_list_on_stack[1] = bio_list_on_stack[0];
			bio_list_init(&bio_list_on_stack[0]);
2458
			ret = q->make_request_fn(q, bio);
2459

2460 2461 2462 2463 2464
			/* sort new bios into those for a lower level
			 * and those for the same level
			 */
			bio_list_init(&lower);
			bio_list_init(&same);
2465
			while ((bio = bio_list_pop(&bio_list_on_stack[0])) != NULL)
2466
				if (q == bio->bi_disk->queue)
2467 2468 2469 2470
					bio_list_add(&same, bio);
				else
					bio_list_add(&lower, bio);
			/* now assemble so we handle the lowest level first */
2471 2472 2473
			bio_list_merge(&bio_list_on_stack[0], &lower);
			bio_list_merge(&bio_list_on_stack[0], &same);
			bio_list_merge(&bio_list_on_stack[0], &bio_list_on_stack[1]);
2474
		} else {
2475 2476 2477 2478 2479
			if (unlikely(!blk_queue_dying(q) &&
					(bio->bi_opf & REQ_NOWAIT)))
				bio_wouldblock_error(bio);
			else
				bio_io_error(bio);
2480
		}
2481
		bio = bio_list_pop(&bio_list_on_stack[0]);
2482
	} while (bio);
2483
	current->bio_list = NULL; /* deactivate */
2484 2485

out:
2486 2487
	if (q)
		blk_queue_exit(q);
2488
	return ret;
2489
}
L
Linus Torvalds 已提交
2490 2491
EXPORT_SYMBOL(generic_make_request);

2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510
/**
 * direct_make_request - hand a buffer directly to its device driver for I/O
 * @bio:  The bio describing the location in memory and on the device.
 *
 * This function behaves like generic_make_request(), but does not protect
 * against recursion.  Must only be used if the called driver is known
 * to not call generic_make_request (or direct_make_request) again from
 * its make_request function.  (Calling direct_make_request again from
 * a workqueue is perfectly fine as that doesn't recurse).
 */
blk_qc_t direct_make_request(struct bio *bio)
{
	struct request_queue *q = bio->bi_disk->queue;
	bool nowait = bio->bi_opf & REQ_NOWAIT;
	blk_qc_t ret;

	if (!generic_make_request_checks(bio))
		return BLK_QC_T_NONE;

2511
	if (unlikely(blk_queue_enter(q, nowait ? BLK_MQ_REQ_NOWAIT : 0))) {
2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525
		if (nowait && !blk_queue_dying(q))
			bio->bi_status = BLK_STS_AGAIN;
		else
			bio->bi_status = BLK_STS_IOERR;
		bio_endio(bio);
		return BLK_QC_T_NONE;
	}

	ret = q->make_request_fn(q, bio);
	blk_queue_exit(q);
	return ret;
}
EXPORT_SYMBOL_GPL(direct_make_request);

L
Linus Torvalds 已提交
2526
/**
2527
 * submit_bio - submit a bio to the block device layer for I/O
L
Linus Torvalds 已提交
2528 2529 2530 2531
 * @bio: The &struct bio which describes the I/O
 *
 * submit_bio() is very similar in purpose to generic_make_request(), and
 * uses that function to do most of the work. Both are fairly rough
2532
 * interfaces; @bio must be presetup and ready for I/O.
L
Linus Torvalds 已提交
2533 2534
 *
 */
2535
blk_qc_t submit_bio(struct bio *bio)
L
Linus Torvalds 已提交
2536
{
2537 2538 2539 2540
	/*
	 * If it's a regular read/write or a barrier with data attached,
	 * go through the normal accounting stuff before submission.
	 */
2541
	if (bio_has_data(bio)) {
2542 2543
		unsigned int count;

2544
		if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
2545
			count = queue_logical_block_size(bio->bi_disk->queue) >> 9;
2546 2547 2548
		else
			count = bio_sectors(bio);

2549
		if (op_is_write(bio_op(bio))) {
2550 2551
			count_vm_events(PGPGOUT, count);
		} else {
2552
			task_io_account_read(bio->bi_iter.bi_size);
2553 2554 2555 2556 2557
			count_vm_events(PGPGIN, count);
		}

		if (unlikely(block_dump)) {
			char b[BDEVNAME_SIZE];
2558
			printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n",
2559
			current->comm, task_pid_nr(current),
2560
				op_is_write(bio_op(bio)) ? "WRITE" : "READ",
2561
				(unsigned long long)bio->bi_iter.bi_sector,
2562
				bio_devname(bio, b), count);
2563
		}
L
Linus Torvalds 已提交
2564 2565
	}

2566
	return generic_make_request(bio);
L
Linus Torvalds 已提交
2567 2568 2569
}
EXPORT_SYMBOL(submit_bio);

2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580
bool blk_poll(struct request_queue *q, blk_qc_t cookie)
{
	if (!q->poll_fn || !blk_qc_t_valid(cookie))
		return false;

	if (current->plug)
		blk_flush_plug_list(current->plug, false);
	return q->poll_fn(q, cookie);
}
EXPORT_SYMBOL_GPL(blk_poll);

2581
/**
2582 2583
 * blk_cloned_rq_check_limits - Helper function to check a cloned request
 *                              for new the queue limits
2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594
 * @q:  the queue
 * @rq: the request being checked
 *
 * Description:
 *    @rq may have been made based on weaker limitations of upper-level queues
 *    in request stacking drivers, and it may violate the limitation of @q.
 *    Since the block layer and the underlying device driver trust @rq
 *    after it is inserted to @q, it should be checked against @q before
 *    the insertion using this generic function.
 *
 *    Request stacking drivers like request-based dm may change the queue
2595 2596
 *    limits when retrying requests on other queues. Those requests need
 *    to be checked against the new queue limits again during dispatch.
2597
 */
2598 2599
static int blk_cloned_rq_check_limits(struct request_queue *q,
				      struct request *rq)
2600
{
2601
	if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, req_op(rq))) {
2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612
		printk(KERN_ERR "%s: over max size limit.\n", __func__);
		return -EIO;
	}

	/*
	 * queue's settings related to segment counting like q->bounce_pfn
	 * may differ from that of other stacking queues.
	 * Recalculate it to check the request correctly on this queue's
	 * limitation.
	 */
	blk_recalc_rq_segments(rq);
2613
	if (rq->nr_phys_segments > queue_max_segments(q)) {
2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625
		printk(KERN_ERR "%s: over max segments limit.\n", __func__);
		return -EIO;
	}

	return 0;
}

/**
 * blk_insert_cloned_request - Helper for stacking drivers to submit a request
 * @q:  the queue to submit the request
 * @rq: the request being queued
 */
2626
blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *rq)
2627 2628
{
	unsigned long flags;
2629
	int where = ELEVATOR_INSERT_BACK;
2630

2631
	if (blk_cloned_rq_check_limits(q, rq))
2632
		return BLK_STS_IOERR;
2633

2634 2635
	if (rq->rq_disk &&
	    should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
2636
		return BLK_STS_IOERR;
2637

2638 2639 2640
	if (q->mq_ops) {
		if (blk_queue_io_stat(q))
			blk_account_io_start(rq, true);
2641 2642 2643 2644 2645
		/*
		 * Since we have a scheduler attached on the top device,
		 * bypass a potential scheduler on the bottom device for
		 * insert.
		 */
2646
		return blk_mq_request_issue_directly(rq);
2647 2648
	}

2649
	spin_lock_irqsave(q->queue_lock, flags);
B
Bart Van Assche 已提交
2650
	if (unlikely(blk_queue_dying(q))) {
2651
		spin_unlock_irqrestore(q->queue_lock, flags);
2652
		return BLK_STS_IOERR;
2653
	}
2654 2655 2656 2657 2658 2659 2660

	/*
	 * Submitting request must be dequeued before calling this function
	 * because it will be linked to another request_queue
	 */
	BUG_ON(blk_queued_rq(rq));

2661
	if (op_is_flush(rq->cmd_flags))
2662 2663 2664
		where = ELEVATOR_INSERT_FLUSH;

	add_acct_request(q, rq, where);
J
Jeff Moyer 已提交
2665 2666
	if (where == ELEVATOR_INSERT_FLUSH)
		__blk_run_queue(q);
2667 2668
	spin_unlock_irqrestore(q->queue_lock, flags);

2669
	return BLK_STS_OK;
2670 2671 2672
}
EXPORT_SYMBOL_GPL(blk_insert_cloned_request);

2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691
/**
 * blk_rq_err_bytes - determine number of bytes till the next failure boundary
 * @rq: request to examine
 *
 * Description:
 *     A request could be merge of IOs which require different failure
 *     handling.  This function determines the number of bytes which
 *     can be failed from the beginning of the request without
 *     crossing into area which need to be retried further.
 *
 * Return:
 *     The number of bytes to fail.
 */
unsigned int blk_rq_err_bytes(const struct request *rq)
{
	unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
	unsigned int bytes = 0;
	struct bio *bio;

2692
	if (!(rq->rq_flags & RQF_MIXED_MERGE))
2693 2694 2695 2696 2697 2698 2699 2700 2701 2702
		return blk_rq_bytes(rq);

	/*
	 * Currently the only 'mixing' which can happen is between
	 * different fastfail types.  We can safely fail portions
	 * which have all the failfast bits that the first one has -
	 * the ones which are at least as eager to fail as the first
	 * one.
	 */
	for (bio = rq->bio; bio; bio = bio->bi_next) {
J
Jens Axboe 已提交
2703
		if ((bio->bi_opf & ff) != ff)
2704
			break;
2705
		bytes += bio->bi_iter.bi_size;
2706 2707 2708 2709 2710 2711 2712 2713
	}

	/* this could lead to infinite loop */
	BUG_ON(blk_rq_bytes(rq) && !bytes);
	return bytes;
}
EXPORT_SYMBOL_GPL(blk_rq_err_bytes);

2714
void blk_account_io_completion(struct request *req, unsigned int bytes)
2715
{
2716
	if (blk_do_io_stat(req)) {
2717
		const int sgrp = op_stat_group(req_op(req));
2718 2719 2720 2721
		struct hd_struct *part;
		int cpu;

		cpu = part_stat_lock();
2722
		part = req->part;
2723
		part_stat_add(cpu, part, sectors[sgrp], bytes >> 9);
2724 2725 2726 2727
		part_stat_unlock();
	}
}

2728
void blk_account_io_done(struct request *req, u64 now)
2729 2730
{
	/*
2731 2732 2733
	 * Account IO completion.  flush_rq isn't accounted as a
	 * normal IO on queueing nor completion.  Accounting the
	 * containing request is enough.
2734
	 */
2735
	if (blk_do_io_stat(req) && !(req->rq_flags & RQF_FLUSH_SEQ)) {
2736
		const int sgrp = op_stat_group(req_op(req));
2737 2738 2739 2740
		struct hd_struct *part;
		int cpu;

		cpu = part_stat_lock();
2741
		part = req->part;
2742

2743
		part_stat_inc(cpu, part, ios[sgrp]);
2744
		part_stat_add(cpu, part, nsecs[sgrp], now - req->start_time_ns);
2745
		part_round_stats(req->q, cpu, part);
2746
		part_dec_in_flight(req->q, part, rq_data_dir(req));
2747

2748
		hd_struct_put(part);
2749 2750 2751 2752
		part_stat_unlock();
	}
}

2753
#ifdef CONFIG_PM
L
Lin Ming 已提交
2754 2755 2756 2757
/*
 * Don't process normal requests when queue is suspended
 * or in the process of suspending/resuming
 */
2758
static bool blk_pm_allow_request(struct request *rq)
L
Lin Ming 已提交
2759
{
2760 2761 2762 2763 2764 2765
	switch (rq->q->rpm_status) {
	case RPM_RESUMING:
	case RPM_SUSPENDING:
		return rq->rq_flags & RQF_PM;
	case RPM_SUSPENDED:
		return false;
2766 2767
	default:
		return true;
2768
	}
L
Lin Ming 已提交
2769 2770
}
#else
2771
static bool blk_pm_allow_request(struct request *rq)
L
Lin Ming 已提交
2772
{
2773
	return true;
L
Lin Ming 已提交
2774 2775 2776
}
#endif

2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804
void blk_account_io_start(struct request *rq, bool new_io)
{
	struct hd_struct *part;
	int rw = rq_data_dir(rq);
	int cpu;

	if (!blk_do_io_stat(rq))
		return;

	cpu = part_stat_lock();

	if (!new_io) {
		part = rq->part;
		part_stat_inc(cpu, part, merges[rw]);
	} else {
		part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
		if (!hd_struct_try_get(part)) {
			/*
			 * The partition is already being removed,
			 * the request will be accounted on the disk only
			 *
			 * We take a reference on disk->part0 although that
			 * partition will never be deleted, so we can treat
			 * it as any other partition.
			 */
			part = &rq->rq_disk->part0;
			hd_struct_get(part);
		}
2805 2806
		part_round_stats(rq->q, cpu, part);
		part_inc_in_flight(rq->q, part, rw);
2807 2808 2809 2810 2811 2812
		rq->part = part;
	}

	part_stat_unlock();
}

2813 2814 2815 2816 2817 2818 2819 2820
static struct request *elv_next_request(struct request_queue *q)
{
	struct request *rq;
	struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);

	WARN_ON_ONCE(q->mq_ops);

	while (1) {
2821 2822 2823 2824 2825 2826
		list_for_each_entry(rq, &q->queue_head, queuelist) {
			if (blk_pm_allow_request(rq))
				return rq;

			if (rq->rq_flags & RQF_SOFTBARRIER)
				break;
2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854
		}

		/*
		 * Flush request is running and flush request isn't queueable
		 * in the drive, we can hold the queue till flush request is
		 * finished. Even we don't do this, driver can't dispatch next
		 * requests and will requeue them. And this can improve
		 * throughput too. For example, we have request flush1, write1,
		 * flush 2. flush1 is dispatched, then queue is hold, write1
		 * isn't inserted to queue. After flush1 is finished, flush2
		 * will be dispatched. Since disk cache is already clean,
		 * flush2 will be finished very soon, so looks like flush2 is
		 * folded to flush1.
		 * Since the queue is hold, a flag is set to indicate the queue
		 * should be restarted later. Please see flush_end_io() for
		 * details.
		 */
		if (fq->flush_pending_idx != fq->flush_running_idx &&
				!queue_flush_queueable(q)) {
			fq->flush_queue_delayed = 1;
			return NULL;
		}
		if (unlikely(blk_queue_bypass(q)) ||
		    !q->elevator->type->ops.sq.elevator_dispatch_fn(q, 0))
			return NULL;
	}
}

2855
/**
2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868
 * blk_peek_request - peek at the top of a request queue
 * @q: request queue to peek at
 *
 * Description:
 *     Return the request at the top of @q.  The returned request
 *     should be started using blk_start_request() before LLD starts
 *     processing it.
 *
 * Return:
 *     Pointer to the request at the top of @q if available.  Null
 *     otherwise.
 */
struct request *blk_peek_request(struct request_queue *q)
2869 2870 2871 2872
{
	struct request *rq;
	int ret;

2873
	lockdep_assert_held(q->queue_lock);
2874
	WARN_ON_ONCE(q->mq_ops);
2875

2876
	while ((rq = elv_next_request(q)) != NULL) {
2877
		if (!(rq->rq_flags & RQF_STARTED)) {
2878 2879 2880 2881 2882
			/*
			 * This is the first time the device driver
			 * sees this request (possibly after
			 * requeueing).  Notify IO scheduler.
			 */
2883
			if (rq->rq_flags & RQF_SORTED)
2884 2885 2886 2887 2888 2889 2890
				elv_activate_rq(q, rq);

			/*
			 * just mark as started even if we don't start
			 * it, a request that has been delayed should
			 * not be passed by new incoming requests
			 */
2891
			rq->rq_flags |= RQF_STARTED;
2892 2893 2894 2895 2896 2897 2898 2899
			trace_block_rq_issue(q, rq);
		}

		if (!q->boundary_rq || q->boundary_rq == rq) {
			q->end_sector = rq_end_sector(rq);
			q->boundary_rq = NULL;
		}

2900
		if (rq->rq_flags & RQF_DONTPREP)
2901 2902
			break;

2903
		if (q->dma_drain_size && blk_rq_bytes(rq)) {
2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922
			/*
			 * make sure space for the drain appears we
			 * know we can do this because max_hw_segments
			 * has been adjusted to be one fewer than the
			 * device can handle
			 */
			rq->nr_phys_segments++;
		}

		if (!q->prep_rq_fn)
			break;

		ret = q->prep_rq_fn(q, rq);
		if (ret == BLKPREP_OK) {
			break;
		} else if (ret == BLKPREP_DEFER) {
			/*
			 * the request may have been (partially) prepped.
			 * we need to keep this request in the front to
2923
			 * avoid resource deadlock.  RQF_STARTED will
2924 2925
			 * prevent other fs requests from passing this one.
			 */
2926
			if (q->dma_drain_size && blk_rq_bytes(rq) &&
2927
			    !(rq->rq_flags & RQF_DONTPREP)) {
2928 2929 2930 2931 2932 2933 2934 2935 2936
				/*
				 * remove the space for the drain we added
				 * so that we don't add it again
				 */
				--rq->nr_phys_segments;
			}

			rq = NULL;
			break;
2937
		} else if (ret == BLKPREP_KILL || ret == BLKPREP_INVALID) {
2938
			rq->rq_flags |= RQF_QUIET;
2939 2940 2941 2942 2943
			/*
			 * Mark this request as started so we don't trigger
			 * any debug logic in the end I/O path.
			 */
			blk_start_request(rq);
2944 2945
			__blk_end_request_all(rq, ret == BLKPREP_INVALID ?
					BLK_STS_TARGET : BLK_STS_IOERR);
2946 2947 2948 2949 2950 2951 2952 2953
		} else {
			printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
			break;
		}
	}

	return rq;
}
2954
EXPORT_SYMBOL(blk_peek_request);
2955

2956
static void blk_dequeue_request(struct request *rq)
2957
{
2958 2959
	struct request_queue *q = rq->q;

2960 2961 2962 2963 2964 2965 2966 2967 2968 2969
	BUG_ON(list_empty(&rq->queuelist));
	BUG_ON(ELV_ON_HASH(rq));

	list_del_init(&rq->queuelist);

	/*
	 * the time frame between a request being removed from the lists
	 * and to it is freed is accounted as io that is in progress at
	 * the driver side.
	 */
2970
	if (blk_account_rq(rq))
2971
		q->in_flight[rq_is_sync(rq)]++;
2972 2973
}

2974 2975 2976 2977 2978 2979 2980 2981 2982 2983
/**
 * blk_start_request - start request processing on the driver
 * @req: request to dequeue
 *
 * Description:
 *     Dequeue @req and start timeout timer on it.  This hands off the
 *     request to the driver.
 */
void blk_start_request(struct request *req)
{
2984
	lockdep_assert_held(req->q->queue_lock);
2985
	WARN_ON_ONCE(req->q->mq_ops);
2986

2987 2988
	blk_dequeue_request(req);

2989
	if (test_bit(QUEUE_FLAG_STATS, &req->q->queue_flags)) {
2990 2991 2992 2993
		req->io_start_time_ns = ktime_get_ns();
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
		req->throtl_size = blk_rq_sectors(req);
#endif
2994
		req->rq_flags |= RQF_STATS;
2995
		rq_qos_issue(req->q, req);
2996 2997
	}

2998
	BUG_ON(blk_rq_is_complete(req));
2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018
	blk_add_timer(req);
}
EXPORT_SYMBOL(blk_start_request);

/**
 * blk_fetch_request - fetch a request from a request queue
 * @q: request queue to fetch a request from
 *
 * Description:
 *     Return the request at the top of @q.  The request is started on
 *     return and LLD can start processing it immediately.
 *
 * Return:
 *     Pointer to the request at the top of @q if available.  Null
 *     otherwise.
 */
struct request *blk_fetch_request(struct request_queue *q)
{
	struct request *rq;

3019
	lockdep_assert_held(q->queue_lock);
3020
	WARN_ON_ONCE(q->mq_ops);
3021

3022 3023 3024 3025 3026 3027 3028
	rq = blk_peek_request(q);
	if (rq)
		blk_start_request(rq);
	return rq;
}
EXPORT_SYMBOL(blk_fetch_request);

3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049
/*
 * Steal bios from a request and add them to a bio list.
 * The request must not have been partially completed before.
 */
void blk_steal_bios(struct bio_list *list, struct request *rq)
{
	if (rq->bio) {
		if (list->tail)
			list->tail->bi_next = rq->bio;
		else
			list->head = rq->bio;
		list->tail = rq->biotail;

		rq->bio = NULL;
		rq->biotail = NULL;
	}

	rq->__data_len = 0;
}
EXPORT_SYMBOL_GPL(blk_steal_bios);

3050
/**
3051
 * blk_update_request - Special helper function for request stacking drivers
3052
 * @req:      the request being processed
3053
 * @error:    block status code
3054
 * @nr_bytes: number of bytes to complete @req
3055 3056
 *
 * Description:
3057 3058 3059
 *     Ends I/O on a number of bytes attached to @req, but doesn't complete
 *     the request structure even if @req doesn't have leftover.
 *     If @req has leftover, sets it up for the next range of segments.
3060 3061 3062 3063 3064 3065 3066
 *
 *     This special helper function is only for request stacking drivers
 *     (e.g. request-based dm) so that they can handle partial completion.
 *     Actual device drivers should use blk_end_request instead.
 *
 *     Passing the result of blk_rq_bytes() as @nr_bytes guarantees
 *     %false return from this function.
3067
 *
3068 3069 3070 3071
 * Note:
 *	The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in both
 *	blk_rq_bytes() and in blk_update_request().
 *
3072
 * Return:
3073 3074
 *     %false - this request doesn't have any more data
 *     %true  - this request has more data
3075
 **/
3076 3077
bool blk_update_request(struct request *req, blk_status_t error,
		unsigned int nr_bytes)
L
Linus Torvalds 已提交
3078
{
3079
	int total_bytes;
L
Linus Torvalds 已提交
3080

3081
	trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);
3082

3083 3084 3085
	if (!req->bio)
		return false;

3086 3087 3088
	if (unlikely(error && !blk_rq_is_passthrough(req) &&
		     !(req->rq_flags & RQF_QUIET)))
		print_req_error(req, error);
L
Linus Torvalds 已提交
3089

3090
	blk_account_io_completion(req, nr_bytes);
3091

3092 3093 3094
	total_bytes = 0;
	while (req->bio) {
		struct bio *bio = req->bio;
3095
		unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
L
Linus Torvalds 已提交
3096

3097
		if (bio_bytes == bio->bi_iter.bi_size)
L
Linus Torvalds 已提交
3098 3099
			req->bio = bio->bi_next;

N
NeilBrown 已提交
3100 3101
		/* Completion has already been traced */
		bio_clear_flag(bio, BIO_TRACE_COMPLETION);
3102
		req_bio_endio(req, bio, bio_bytes, error);
L
Linus Torvalds 已提交
3103

3104 3105
		total_bytes += bio_bytes;
		nr_bytes -= bio_bytes;
L
Linus Torvalds 已提交
3106

3107 3108
		if (!nr_bytes)
			break;
L
Linus Torvalds 已提交
3109 3110 3111 3112 3113
	}

	/*
	 * completely done
	 */
3114 3115 3116 3117 3118 3119
	if (!req->bio) {
		/*
		 * Reset counters so that the request stacking driver
		 * can find how many bytes remain in the request
		 * later.
		 */
3120
		req->__data_len = 0;
3121 3122
		return false;
	}
L
Linus Torvalds 已提交
3123

3124
	req->__data_len -= total_bytes;
3125 3126

	/* update sector only for requests with clear definition of sector */
3127
	if (!blk_rq_is_passthrough(req))
3128
		req->__sector += total_bytes >> 9;
3129

3130
	/* mixed attributes always follow the first bio */
3131
	if (req->rq_flags & RQF_MIXED_MERGE) {
3132
		req->cmd_flags &= ~REQ_FAILFAST_MASK;
J
Jens Axboe 已提交
3133
		req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
3134 3135
	}

3136 3137 3138 3139 3140 3141 3142 3143 3144
	if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
		/*
		 * If total number of sectors is less than the first segment
		 * size, something has gone terribly wrong.
		 */
		if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
			blk_dump_rq_flags(req, "request botched");
			req->__data_len = blk_rq_cur_bytes(req);
		}
3145

3146 3147 3148
		/* recalculate the number of segments */
		blk_recalc_rq_segments(req);
	}
3149

3150
	return true;
L
Linus Torvalds 已提交
3151
}
3152
EXPORT_SYMBOL_GPL(blk_update_request);
L
Linus Torvalds 已提交
3153

3154
static bool blk_update_bidi_request(struct request *rq, blk_status_t error,
3155 3156
				    unsigned int nr_bytes,
				    unsigned int bidi_bytes)
3157
{
3158 3159
	if (blk_update_request(rq, error, nr_bytes))
		return true;
3160

3161 3162 3163 3164
	/* Bidi request must be completed as a whole */
	if (unlikely(blk_bidi_rq(rq)) &&
	    blk_update_request(rq->next_rq, error, bidi_bytes))
		return true;
3165

3166 3167
	if (blk_queue_add_random(rq->q))
		add_disk_randomness(rq->rq_disk);
3168 3169

	return false;
L
Linus Torvalds 已提交
3170 3171
}

3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185
/**
 * blk_unprep_request - unprepare a request
 * @req:	the request
 *
 * This function makes a request ready for complete resubmission (or
 * completion).  It happens only after all error handling is complete,
 * so represents the appropriate moment to deallocate any resources
 * that were allocated to the request in the prep_rq_fn.  The queue
 * lock is held when calling this.
 */
void blk_unprep_request(struct request *req)
{
	struct request_queue *q = req->q;

3186
	req->rq_flags &= ~RQF_DONTPREP;
3187 3188 3189 3190 3191
	if (q->unprep_rq_fn)
		q->unprep_rq_fn(q, req);
}
EXPORT_SYMBOL_GPL(blk_unprep_request);

3192
void blk_finish_request(struct request *req, blk_status_t error)
L
Linus Torvalds 已提交
3193
{
3194
	struct request_queue *q = req->q;
3195
	u64 now = ktime_get_ns();
3196

3197
	lockdep_assert_held(req->q->queue_lock);
3198
	WARN_ON_ONCE(q->mq_ops);
3199

3200
	if (req->rq_flags & RQF_STATS)
3201
		blk_stat_add(req, now);
3202

3203
	if (req->rq_flags & RQF_QUEUED)
3204
		blk_queue_end_tag(q, req);
3205

3206
	BUG_ON(blk_queued_rq(req));
L
Linus Torvalds 已提交
3207

3208
	if (unlikely(laptop_mode) && !blk_rq_is_passthrough(req))
3209
		laptop_io_completion(req->q->backing_dev_info);
L
Linus Torvalds 已提交
3210

3211 3212
	blk_delete_timer(req);

3213
	if (req->rq_flags & RQF_DONTPREP)
3214 3215
		blk_unprep_request(req);

3216
	blk_account_io_done(req, now);
3217

J
Jens Axboe 已提交
3218
	if (req->end_io) {
3219
		rq_qos_done(q, req);
3220
		req->end_io(req, error);
J
Jens Axboe 已提交
3221
	} else {
3222 3223 3224
		if (blk_bidi_rq(req))
			__blk_put_request(req->next_rq->q, req->next_rq);

3225
		__blk_put_request(q, req);
3226
	}
L
Linus Torvalds 已提交
3227
}
3228
EXPORT_SYMBOL(blk_finish_request);
L
Linus Torvalds 已提交
3229

3230
/**
3231 3232
 * blk_end_bidi_request - Complete a bidi request
 * @rq:         the request to complete
3233
 * @error:      block status code
3234 3235
 * @nr_bytes:   number of bytes to complete @rq
 * @bidi_bytes: number of bytes to complete @rq->next_rq
3236 3237
 *
 * Description:
3238
 *     Ends I/O on a number of bytes attached to @rq and @rq->next_rq.
3239 3240 3241
 *     Drivers that supports bidi can safely call this member for any
 *     type of request, bidi or uni.  In the later case @bidi_bytes is
 *     just ignored.
3242 3243
 *
 * Return:
3244 3245
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
3246
 **/
3247
static bool blk_end_bidi_request(struct request *rq, blk_status_t error,
K
Kiyoshi Ueda 已提交
3248 3249
				 unsigned int nr_bytes, unsigned int bidi_bytes)
{
3250
	struct request_queue *q = rq->q;
3251
	unsigned long flags;
K
Kiyoshi Ueda 已提交
3252

3253 3254
	WARN_ON_ONCE(q->mq_ops);

3255 3256
	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
		return true;
K
Kiyoshi Ueda 已提交
3257

3258
	spin_lock_irqsave(q->queue_lock, flags);
3259
	blk_finish_request(rq, error);
3260 3261
	spin_unlock_irqrestore(q->queue_lock, flags);

3262
	return false;
K
Kiyoshi Ueda 已提交
3263 3264
}

3265
/**
3266 3267
 * __blk_end_bidi_request - Complete a bidi request with queue lock held
 * @rq:         the request to complete
3268
 * @error:      block status code
3269 3270
 * @nr_bytes:   number of bytes to complete @rq
 * @bidi_bytes: number of bytes to complete @rq->next_rq
3271 3272
 *
 * Description:
3273 3274
 *     Identical to blk_end_bidi_request() except that queue lock is
 *     assumed to be locked on entry and remains so on return.
3275 3276
 *
 * Return:
3277 3278
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
3279
 **/
3280
static bool __blk_end_bidi_request(struct request *rq, blk_status_t error,
3281
				   unsigned int nr_bytes, unsigned int bidi_bytes)
3282
{
3283
	lockdep_assert_held(rq->q->queue_lock);
3284
	WARN_ON_ONCE(rq->q->mq_ops);
3285

3286 3287
	if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
		return true;
3288

3289
	blk_finish_request(rq, error);
3290

3291
	return false;
3292
}
3293 3294 3295 3296

/**
 * blk_end_request - Helper function for drivers to complete the request.
 * @rq:       the request being processed
3297
 * @error:    block status code
3298 3299 3300 3301 3302 3303 3304
 * @nr_bytes: number of bytes to complete
 *
 * Description:
 *     Ends I/O on a number of bytes attached to @rq.
 *     If @rq has leftover, sets it up for the next range of segments.
 *
 * Return:
3305 3306
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
3307
 **/
3308 3309
bool blk_end_request(struct request *rq, blk_status_t error,
		unsigned int nr_bytes)
3310
{
3311
	WARN_ON_ONCE(rq->q->mq_ops);
3312
	return blk_end_bidi_request(rq, error, nr_bytes, 0);
3313
}
3314
EXPORT_SYMBOL(blk_end_request);
3315 3316

/**
3317 3318
 * blk_end_request_all - Helper function for drives to finish the request.
 * @rq: the request to finish
3319
 * @error: block status code
3320 3321
 *
 * Description:
3322 3323
 *     Completely finish @rq.
 */
3324
void blk_end_request_all(struct request *rq, blk_status_t error)
3325
{
3326 3327
	bool pending;
	unsigned int bidi_bytes = 0;
3328

3329 3330
	if (unlikely(blk_bidi_rq(rq)))
		bidi_bytes = blk_rq_bytes(rq->next_rq);
3331

3332 3333 3334
	pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
	BUG_ON(pending);
}
3335
EXPORT_SYMBOL(blk_end_request_all);
3336

3337
/**
3338 3339
 * __blk_end_request - Helper function for drivers to complete the request.
 * @rq:       the request being processed
3340
 * @error:    block status code
3341
 * @nr_bytes: number of bytes to complete
3342 3343
 *
 * Description:
3344
 *     Must be called with queue lock held unlike blk_end_request().
3345 3346
 *
 * Return:
3347 3348
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
3349
 **/
3350 3351
bool __blk_end_request(struct request *rq, blk_status_t error,
		unsigned int nr_bytes)
3352
{
3353
	lockdep_assert_held(rq->q->queue_lock);
3354
	WARN_ON_ONCE(rq->q->mq_ops);
3355

3356
	return __blk_end_bidi_request(rq, error, nr_bytes, 0);
3357
}
3358
EXPORT_SYMBOL(__blk_end_request);
3359

K
Kiyoshi Ueda 已提交
3360
/**
3361 3362
 * __blk_end_request_all - Helper function for drives to finish the request.
 * @rq: the request to finish
3363
 * @error:    block status code
K
Kiyoshi Ueda 已提交
3364 3365
 *
 * Description:
3366
 *     Completely finish @rq.  Must be called with queue lock held.
K
Kiyoshi Ueda 已提交
3367
 */
3368
void __blk_end_request_all(struct request *rq, blk_status_t error)
K
Kiyoshi Ueda 已提交
3369
{
3370 3371 3372
	bool pending;
	unsigned int bidi_bytes = 0;

3373
	lockdep_assert_held(rq->q->queue_lock);
3374
	WARN_ON_ONCE(rq->q->mq_ops);
3375

3376 3377 3378 3379 3380
	if (unlikely(blk_bidi_rq(rq)))
		bidi_bytes = blk_rq_bytes(rq->next_rq);

	pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
	BUG_ON(pending);
K
Kiyoshi Ueda 已提交
3381
}
3382
EXPORT_SYMBOL(__blk_end_request_all);
K
Kiyoshi Ueda 已提交
3383

3384
/**
3385 3386
 * __blk_end_request_cur - Helper function to finish the current request chunk.
 * @rq: the request to finish the current chunk for
3387
 * @error:    block status code
3388 3389
 *
 * Description:
3390 3391
 *     Complete the current consecutively mapped chunk from @rq.  Must
 *     be called with queue lock held.
3392 3393
 *
 * Return:
3394 3395 3396
 *     %false - we are done with this request
 *     %true  - still buffers pending for this request
 */
3397
bool __blk_end_request_cur(struct request *rq, blk_status_t error)
3398
{
3399
	return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
3400
}
3401
EXPORT_SYMBOL(__blk_end_request_cur);
3402

J
Jens Axboe 已提交
3403 3404
void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
		     struct bio *bio)
L
Linus Torvalds 已提交
3405
{
3406
	if (bio_has_data(bio))
D
David Woodhouse 已提交
3407
		rq->nr_phys_segments = bio_phys_segments(q, bio);
3408 3409
	else if (bio_op(bio) == REQ_OP_DISCARD)
		rq->nr_phys_segments = 1;
3410

3411
	rq->__data_len = bio->bi_iter.bi_size;
L
Linus Torvalds 已提交
3412 3413
	rq->bio = rq->biotail = bio;

3414 3415
	if (bio->bi_disk)
		rq->rq_disk = bio->bi_disk;
N
NeilBrown 已提交
3416
}
L
Linus Torvalds 已提交
3417

3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428
#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
/**
 * rq_flush_dcache_pages - Helper function to flush all pages in a request
 * @rq: the request to be flushed
 *
 * Description:
 *     Flush all pages in @rq.
 */
void rq_flush_dcache_pages(struct request *rq)
{
	struct req_iterator iter;
3429
	struct bio_vec bvec;
3430 3431

	rq_for_each_segment(bvec, rq, iter)
3432
		flush_dcache_page(bvec.bv_page);
3433 3434 3435 3436
}
EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
#endif

3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464
/**
 * blk_lld_busy - Check if underlying low-level drivers of a device are busy
 * @q : the queue of the device being checked
 *
 * Description:
 *    Check if underlying low-level drivers of a device are busy.
 *    If the drivers want to export their busy state, they must set own
 *    exporting function using blk_queue_lld_busy() first.
 *
 *    Basically, this function is used only by request stacking drivers
 *    to stop dispatching requests to underlying devices when underlying
 *    devices are busy.  This behavior helps more I/O merging on the queue
 *    of the request stacking driver and prevents I/O throughput regression
 *    on burst I/O load.
 *
 * Return:
 *    0 - Not busy (The request stacking driver should dispatch request)
 *    1 - Busy (The request stacking driver should stop dispatching request)
 */
int blk_lld_busy(struct request_queue *q)
{
	if (q->lld_busy_fn)
		return q->lld_busy_fn(q);

	return 0;
}
EXPORT_SYMBOL_GPL(blk_lld_busy);

3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488
/**
 * blk_rq_unprep_clone - Helper function to free all bios in a cloned request
 * @rq: the clone request to be cleaned up
 *
 * Description:
 *     Free all bios in @rq for a cloned request.
 */
void blk_rq_unprep_clone(struct request *rq)
{
	struct bio *bio;

	while ((bio = rq->bio) != NULL) {
		rq->bio = bio->bi_next;

		bio_put(bio);
	}
}
EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);

/*
 * Copy attributes of the original request to the clone request.
 * The actual data parts (e.g. ->cmd, ->sense) are not copied.
 */
static void __blk_rq_prep_clone(struct request *dst, struct request *src)
3489 3490 3491 3492
{
	dst->cpu = src->cpu;
	dst->__sector = blk_rq_pos(src);
	dst->__data_len = blk_rq_bytes(src);
3493 3494 3495 3496
	if (src->rq_flags & RQF_SPECIAL_PAYLOAD) {
		dst->rq_flags |= RQF_SPECIAL_PAYLOAD;
		dst->special_vec = src->special_vec;
	}
3497 3498 3499
	dst->nr_phys_segments = src->nr_phys_segments;
	dst->ioprio = src->ioprio;
	dst->extra_len = src->extra_len;
3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528
}

/**
 * blk_rq_prep_clone - Helper function to setup clone request
 * @rq: the request to be setup
 * @rq_src: original request to be cloned
 * @bs: bio_set that bios for clone are allocated from
 * @gfp_mask: memory allocation mask for bio
 * @bio_ctr: setup function to be called for each clone bio.
 *           Returns %0 for success, non %0 for failure.
 * @data: private data to be passed to @bio_ctr
 *
 * Description:
 *     Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq.
 *     The actual data parts of @rq_src (e.g. ->cmd, ->sense)
 *     are not copied, and copying such parts is the caller's responsibility.
 *     Also, pages which the original bios are pointing to are not copied
 *     and the cloned bios just point same pages.
 *     So cloned bios must be completed before original bios, which means
 *     the caller must complete @rq before @rq_src.
 */
int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
		      struct bio_set *bs, gfp_t gfp_mask,
		      int (*bio_ctr)(struct bio *, struct bio *, void *),
		      void *data)
{
	struct bio *bio, *bio_src;

	if (!bs)
3529
		bs = &fs_bio_set;
3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555

	__rq_for_each_bio(bio_src, rq_src) {
		bio = bio_clone_fast(bio_src, gfp_mask, bs);
		if (!bio)
			goto free_and_out;

		if (bio_ctr && bio_ctr(bio, bio_src, data))
			goto free_and_out;

		if (rq->bio) {
			rq->biotail->bi_next = bio;
			rq->biotail = bio;
		} else
			rq->bio = rq->biotail = bio;
	}

	__blk_rq_prep_clone(rq, rq_src);

	return 0;

free_and_out:
	if (bio)
		bio_put(bio);
	blk_rq_unprep_clone(rq);

	return -ENOMEM;
3556 3557 3558
}
EXPORT_SYMBOL_GPL(blk_rq_prep_clone);

3559
int kblockd_schedule_work(struct work_struct *work)
L
Linus Torvalds 已提交
3560 3561 3562 3563 3564
{
	return queue_work(kblockd_workqueue, work);
}
EXPORT_SYMBOL(kblockd_schedule_work);

3565 3566 3567 3568 3569 3570
int kblockd_schedule_work_on(int cpu, struct work_struct *work)
{
	return queue_work_on(cpu, kblockd_workqueue, work);
}
EXPORT_SYMBOL(kblockd_schedule_work_on);

3571 3572 3573 3574 3575 3576 3577
int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork,
				unsigned long delay)
{
	return mod_delayed_work_on(cpu, kblockd_workqueue, dwork, delay);
}
EXPORT_SYMBOL(kblockd_mod_delayed_work_on);

S
Suresh Jayaraman 已提交
3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591
/**
 * blk_start_plug - initialize blk_plug and track it inside the task_struct
 * @plug:	The &struct blk_plug that needs to be initialized
 *
 * Description:
 *   Tracking blk_plug inside the task_struct will help with auto-flushing the
 *   pending I/O should the task end up blocking between blk_start_plug() and
 *   blk_finish_plug(). This is important from a performance perspective, but
 *   also ensures that we don't deadlock. For instance, if the task is blocking
 *   for a memory allocation, memory reclaim could end up wanting to free a
 *   page belonging to that request that is currently residing in our private
 *   plug. By flushing the pending I/O when the process goes to sleep, we avoid
 *   this kind of deadlock.
 */
3592 3593 3594 3595
void blk_start_plug(struct blk_plug *plug)
{
	struct task_struct *tsk = current;

S
Shaohua Li 已提交
3596 3597 3598 3599 3600 3601
	/*
	 * If this is a nested plug, don't actually assign it.
	 */
	if (tsk->plug)
		return;

3602
	INIT_LIST_HEAD(&plug->list);
3603
	INIT_LIST_HEAD(&plug->mq_list);
3604
	INIT_LIST_HEAD(&plug->cb_list);
3605
	/*
S
Shaohua Li 已提交
3606 3607
	 * Store ordering should not be needed here, since a potential
	 * preempt will imply a full memory barrier
3608
	 */
S
Shaohua Li 已提交
3609
	tsk->plug = plug;
3610 3611 3612 3613 3614 3615 3616 3617
}
EXPORT_SYMBOL(blk_start_plug);

static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
{
	struct request *rqa = container_of(a, struct request, queuelist);
	struct request *rqb = container_of(b, struct request, queuelist);

3618 3619
	return !(rqa->q < rqb->q ||
		(rqa->q == rqb->q && blk_rq_pos(rqa) < blk_rq_pos(rqb)));
3620 3621
}

3622 3623 3624 3625 3626 3627
/*
 * If 'from_schedule' is true, then postpone the dispatch of requests
 * until a safe kblockd context. We due this to avoid accidental big
 * additional stack usage in driver dispatch, in places where the originally
 * plugger did not intend it.
 */
3628
static void queue_unplugged(struct request_queue *q, unsigned int depth,
3629
			    bool from_schedule)
3630
	__releases(q->queue_lock)
3631
{
3632 3633
	lockdep_assert_held(q->queue_lock);

3634
	trace_block_unplug(q, depth, !from_schedule);
3635

3636
	if (from_schedule)
3637
		blk_run_queue_async(q);
3638
	else
3639
		__blk_run_queue(q);
3640
	spin_unlock_irq(q->queue_lock);
3641 3642
}

3643
static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
3644 3645 3646
{
	LIST_HEAD(callbacks);

S
Shaohua Li 已提交
3647 3648
	while (!list_empty(&plug->cb_list)) {
		list_splice_init(&plug->cb_list, &callbacks);
3649

S
Shaohua Li 已提交
3650 3651
		while (!list_empty(&callbacks)) {
			struct blk_plug_cb *cb = list_first_entry(&callbacks,
3652 3653
							  struct blk_plug_cb,
							  list);
S
Shaohua Li 已提交
3654
			list_del(&cb->list);
3655
			cb->callback(cb, from_schedule);
S
Shaohua Li 已提交
3656
		}
3657 3658 3659
	}
}

3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684
struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
				      int size)
{
	struct blk_plug *plug = current->plug;
	struct blk_plug_cb *cb;

	if (!plug)
		return NULL;

	list_for_each_entry(cb, &plug->cb_list, list)
		if (cb->callback == unplug && cb->data == data)
			return cb;

	/* Not currently on the callback list */
	BUG_ON(size < sizeof(*cb));
	cb = kzalloc(size, GFP_ATOMIC);
	if (cb) {
		cb->data = data;
		cb->callback = unplug;
		list_add(&cb->list, &plug->cb_list);
	}
	return cb;
}
EXPORT_SYMBOL(blk_check_plugged);

3685
void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
3686 3687 3688
{
	struct request_queue *q;
	struct request *rq;
3689
	LIST_HEAD(list);
3690
	unsigned int depth;
3691

3692
	flush_plug_callbacks(plug, from_schedule);
3693 3694 3695 3696

	if (!list_empty(&plug->mq_list))
		blk_mq_flush_plug_list(plug, from_schedule);

3697 3698 3699
	if (list_empty(&plug->list))
		return;

3700 3701
	list_splice_init(&plug->list, &list);

3702
	list_sort(NULL, &list, plug_rq_cmp);
3703 3704

	q = NULL;
3705
	depth = 0;
3706

3707 3708
	while (!list_empty(&list)) {
		rq = list_entry_rq(list.next);
3709 3710 3711
		list_del_init(&rq->queuelist);
		BUG_ON(!rq->q);
		if (rq->q != q) {
3712 3713 3714 3715
			/*
			 * This drops the queue lock
			 */
			if (q)
3716
				queue_unplugged(q, depth, from_schedule);
3717
			q = rq->q;
3718
			depth = 0;
3719
			spin_lock_irq(q->queue_lock);
3720
		}
3721 3722 3723 3724

		/*
		 * Short-circuit if @q is dead
		 */
B
Bart Van Assche 已提交
3725
		if (unlikely(blk_queue_dying(q))) {
3726
			__blk_end_request_all(rq, BLK_STS_IOERR);
3727 3728 3729
			continue;
		}

3730 3731 3732
		/*
		 * rq is already accounted, so use raw insert
		 */
3733
		if (op_is_flush(rq->cmd_flags))
3734 3735 3736
			__elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH);
		else
			__elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE);
3737 3738

		depth++;
3739 3740
	}

3741 3742 3743 3744
	/*
	 * This drops the queue lock
	 */
	if (q)
3745
		queue_unplugged(q, depth, from_schedule);
3746 3747 3748 3749
}

void blk_finish_plug(struct blk_plug *plug)
{
S
Shaohua Li 已提交
3750 3751
	if (plug != current->plug)
		return;
3752
	blk_flush_plug_list(plug, false);
3753

S
Shaohua Li 已提交
3754
	current->plug = NULL;
3755
}
3756
EXPORT_SYMBOL(blk_finish_plug);
3757

3758
#ifdef CONFIG_PM
L
Lin Ming 已提交
3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781
/**
 * blk_pm_runtime_init - Block layer runtime PM initialization routine
 * @q: the queue of the device
 * @dev: the device the queue belongs to
 *
 * Description:
 *    Initialize runtime-PM-related fields for @q and start auto suspend for
 *    @dev. Drivers that want to take advantage of request-based runtime PM
 *    should call this function after @dev has been initialized, and its
 *    request queue @q has been allocated, and runtime PM for it can not happen
 *    yet(either due to disabled/forbidden or its usage_count > 0). In most
 *    cases, driver should call this function before any I/O has taken place.
 *
 *    This function takes care of setting up using auto suspend for the device,
 *    the autosuspend delay is set to -1 to make runtime suspend impossible
 *    until an updated value is either set by user or by driver. Drivers do
 *    not need to touch other autosuspend settings.
 *
 *    The block layer runtime PM is request based, so only works for drivers
 *    that use request as their IO unit instead of those directly use bio's.
 */
void blk_pm_runtime_init(struct request_queue *q, struct device *dev)
{
3782 3783 3784
	/* Don't enable runtime PM for blk-mq until it is ready */
	if (q->mq_ops) {
		pm_runtime_disable(dev);
3785
		return;
3786
	}
3787

L
Lin Ming 已提交
3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819
	q->dev = dev;
	q->rpm_status = RPM_ACTIVE;
	pm_runtime_set_autosuspend_delay(q->dev, -1);
	pm_runtime_use_autosuspend(q->dev);
}
EXPORT_SYMBOL(blk_pm_runtime_init);

/**
 * blk_pre_runtime_suspend - Pre runtime suspend check
 * @q: the queue of the device
 *
 * Description:
 *    This function will check if runtime suspend is allowed for the device
 *    by examining if there are any requests pending in the queue. If there
 *    are requests pending, the device can not be runtime suspended; otherwise,
 *    the queue's status will be updated to SUSPENDING and the driver can
 *    proceed to suspend the device.
 *
 *    For the not allowed case, we mark last busy for the device so that
 *    runtime PM core will try to autosuspend it some time later.
 *
 *    This function should be called near the start of the device's
 *    runtime_suspend callback.
 *
 * Return:
 *    0		- OK to runtime suspend the device
 *    -EBUSY	- Device should not be runtime suspended
 */
int blk_pre_runtime_suspend(struct request_queue *q)
{
	int ret = 0;

3820 3821 3822
	if (!q->dev)
		return ret;

L
Lin Ming 已提交
3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849
	spin_lock_irq(q->queue_lock);
	if (q->nr_pending) {
		ret = -EBUSY;
		pm_runtime_mark_last_busy(q->dev);
	} else {
		q->rpm_status = RPM_SUSPENDING;
	}
	spin_unlock_irq(q->queue_lock);
	return ret;
}
EXPORT_SYMBOL(blk_pre_runtime_suspend);

/**
 * blk_post_runtime_suspend - Post runtime suspend processing
 * @q: the queue of the device
 * @err: return value of the device's runtime_suspend function
 *
 * Description:
 *    Update the queue's runtime status according to the return value of the
 *    device's runtime suspend function and mark last busy for the device so
 *    that PM core will try to auto suspend the device at a later time.
 *
 *    This function should be called near the end of the device's
 *    runtime_suspend callback.
 */
void blk_post_runtime_suspend(struct request_queue *q, int err)
{
3850 3851 3852
	if (!q->dev)
		return;

L
Lin Ming 已提交
3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876
	spin_lock_irq(q->queue_lock);
	if (!err) {
		q->rpm_status = RPM_SUSPENDED;
	} else {
		q->rpm_status = RPM_ACTIVE;
		pm_runtime_mark_last_busy(q->dev);
	}
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL(blk_post_runtime_suspend);

/**
 * blk_pre_runtime_resume - Pre runtime resume processing
 * @q: the queue of the device
 *
 * Description:
 *    Update the queue's runtime status to RESUMING in preparation for the
 *    runtime resume of the device.
 *
 *    This function should be called near the start of the device's
 *    runtime_resume callback.
 */
void blk_pre_runtime_resume(struct request_queue *q)
{
3877 3878 3879
	if (!q->dev)
		return;

L
Lin Ming 已提交
3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901
	spin_lock_irq(q->queue_lock);
	q->rpm_status = RPM_RESUMING;
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL(blk_pre_runtime_resume);

/**
 * blk_post_runtime_resume - Post runtime resume processing
 * @q: the queue of the device
 * @err: return value of the device's runtime_resume function
 *
 * Description:
 *    Update the queue's runtime status according to the return value of the
 *    device's runtime_resume function. If it is successfully resumed, process
 *    the requests that are queued into the device's queue when it is resuming
 *    and then mark last busy and initiate autosuspend for it.
 *
 *    This function should be called near the end of the device's
 *    runtime_resume callback.
 */
void blk_post_runtime_resume(struct request_queue *q, int err)
{
3902 3903 3904
	if (!q->dev)
		return;

L
Lin Ming 已提交
3905 3906 3907 3908 3909
	spin_lock_irq(q->queue_lock);
	if (!err) {
		q->rpm_status = RPM_ACTIVE;
		__blk_run_queue(q);
		pm_runtime_mark_last_busy(q->dev);
3910
		pm_request_autosuspend(q->dev);
L
Lin Ming 已提交
3911 3912 3913 3914 3915 3916
	} else {
		q->rpm_status = RPM_SUSPENDED;
	}
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL(blk_post_runtime_resume);
3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940

/**
 * blk_set_runtime_active - Force runtime status of the queue to be active
 * @q: the queue of the device
 *
 * If the device is left runtime suspended during system suspend the resume
 * hook typically resumes the device and corrects runtime status
 * accordingly. However, that does not affect the queue runtime PM status
 * which is still "suspended". This prevents processing requests from the
 * queue.
 *
 * This function can be used in driver's resume hook to correct queue
 * runtime PM status and re-enable peeking requests from the queue. It
 * should be called before first request is added to the queue.
 */
void blk_set_runtime_active(struct request_queue *q)
{
	spin_lock_irq(q->queue_lock);
	q->rpm_status = RPM_ACTIVE;
	pm_runtime_mark_last_busy(q->dev);
	pm_request_autosuspend(q->dev);
	spin_unlock_irq(q->queue_lock);
}
EXPORT_SYMBOL(blk_set_runtime_active);
L
Lin Ming 已提交
3941 3942
#endif

L
Linus Torvalds 已提交
3943 3944
int __init blk_dev_init(void)
{
3945 3946
	BUILD_BUG_ON(REQ_OP_LAST >= (1 << REQ_OP_BITS));
	BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
3947
			FIELD_SIZEOF(struct request, cmd_flags));
3948 3949
	BUILD_BUG_ON(REQ_OP_BITS + REQ_FLAG_BITS > 8 *
			FIELD_SIZEOF(struct bio, bi_opf));
3950

3951 3952
	/* used for unplugging and affects IO latency/throughput - HIGHPRI */
	kblockd_workqueue = alloc_workqueue("kblockd",
3953
					    WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
L
Linus Torvalds 已提交
3954 3955 3956 3957
	if (!kblockd_workqueue)
		panic("Failed to create kblockd\n");

	request_cachep = kmem_cache_create("blkdev_requests",
3958
			sizeof(struct request), 0, SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
3959

3960
	blk_requestq_cachep = kmem_cache_create("request_queue",
3961
			sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
L
Linus Torvalds 已提交
3962

3963 3964 3965 3966
#ifdef CONFIG_DEBUG_FS
	blk_debugfs_root = debugfs_create_dir("block", NULL);
#endif

3967
	return 0;
L
Linus Torvalds 已提交
3968
}