blk-mq.c 67.4 KB
Newer Older
1 2 3 4 5 6
/*
 * Block multiqueue core code
 *
 * Copyright (C) 2013-2014 Jens Axboe
 * Copyright (C) 2013-2014 Christoph Hellwig
 */
7 8 9 10 11
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
12
#include <linux/kmemleak.h>
13 14 15 16 17 18 19 20 21 22 23
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>
#include <linux/llist.h>
#include <linux/list_sort.h>
#include <linux/cpu.h>
#include <linux/cache.h>
#include <linux/sched/sysctl.h>
#include <linux/delay.h>
24
#include <linux/crash_dump.h>
25
#include <linux/prefetch.h>
26 27 28 29 30 31 32

#include <trace/events/block.h>

#include <linux/blk-mq.h>
#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-tag.h"
33
#include "blk-stat.h"
J
Jens Axboe 已提交
34
#include "blk-wbt.h"
35
#include "blk-mq-sched.h"
36 37 38 39 40 41 42

static DEFINE_MUTEX(all_q_mutex);
static LIST_HEAD(all_q_list);

/*
 * Check if any of the ctx's have pending work in this hardware queue
 */
43
bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
44
{
45 46 47
	return sbitmap_any_bit_set(&hctx->ctx_map) ||
			!list_empty_careful(&hctx->dispatch) ||
			blk_mq_sched_has_work(hctx);
48 49
}

50 51 52 53 54 55
/*
 * Mark this ctx as having pending work in this hardware queue
 */
static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
				     struct blk_mq_ctx *ctx)
{
56 57
	if (!sbitmap_test_bit(&hctx->ctx_map, ctx->index_hw))
		sbitmap_set_bit(&hctx->ctx_map, ctx->index_hw);
58 59 60 61 62
}

static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
				      struct blk_mq_ctx *ctx)
{
63
	sbitmap_clear_bit(&hctx->ctx_map, ctx->index_hw);
64 65
}

66
void blk_mq_freeze_queue_start(struct request_queue *q)
67
{
68
	int freeze_depth;
69

70 71
	freeze_depth = atomic_inc_return(&q->mq_freeze_depth);
	if (freeze_depth == 1) {
72
		percpu_ref_kill(&q->q_usage_counter);
73
		blk_mq_run_hw_queues(q, false);
74
	}
75
}
76
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_start);
77 78 79

static void blk_mq_freeze_queue_wait(struct request_queue *q)
{
80
	wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
81 82
}

83 84 85 86
/*
 * Guarantee no request is in use, so we can change any data structure of
 * the queue afterward.
 */
87
void blk_freeze_queue(struct request_queue *q)
88
{
89 90 91 92 93 94 95
	/*
	 * In the !blk_mq case we are only calling this to kill the
	 * q_usage_counter, otherwise this increases the freeze depth
	 * and waits for it to return to zero.  For this reason there is
	 * no blk_unfreeze_queue(), and blk_freeze_queue() is not
	 * exported to drivers as the only user for unfreeze is blk_mq.
	 */
96 97 98
	blk_mq_freeze_queue_start(q);
	blk_mq_freeze_queue_wait(q);
}
99 100 101 102 103 104 105 106 107

void blk_mq_freeze_queue(struct request_queue *q)
{
	/*
	 * ...just an alias to keep freeze and unfreeze actions balanced
	 * in the blk_mq_* namespace
	 */
	blk_freeze_queue(q);
}
108
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
109

110
void blk_mq_unfreeze_queue(struct request_queue *q)
111
{
112
	int freeze_depth;
113

114 115 116
	freeze_depth = atomic_dec_return(&q->mq_freeze_depth);
	WARN_ON_ONCE(freeze_depth < 0);
	if (!freeze_depth) {
117
		percpu_ref_reinit(&q->q_usage_counter);
118
		wake_up_all(&q->mq_freeze_wq);
119
	}
120
}
121
EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
122

123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
/**
 * blk_mq_quiesce_queue() - wait until all ongoing queue_rq calls have finished
 * @q: request queue.
 *
 * Note: this function does not prevent that the struct request end_io()
 * callback function is invoked. Additionally, it is not prevented that
 * new queue_rq() calls occur unless the queue has been stopped first.
 */
void blk_mq_quiesce_queue(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;
	bool rcu = false;

	blk_mq_stop_hw_queues(q);

	queue_for_each_hw_ctx(q, hctx, i) {
		if (hctx->flags & BLK_MQ_F_BLOCKING)
			synchronize_srcu(&hctx->queue_rq_srcu);
		else
			rcu = true;
	}
	if (rcu)
		synchronize_rcu();
}
EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);

150 151 152 153 154 155 156 157
void blk_mq_wake_waiters(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i)
		if (blk_mq_hw_queue_mapped(hctx))
			blk_mq_tag_wakeup_all(hctx->tags, true);
158 159 160 161 162 163 164

	/*
	 * If we are called because the queue has now been marked as
	 * dying, we need to ensure that processes currently waiting on
	 * the queue are notified as well.
	 */
	wake_up_all(&q->mq_freeze_wq);
165 166
}

167 168 169 170 171 172
bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
{
	return blk_mq_has_free_tags(hctx->tags);
}
EXPORT_SYMBOL(blk_mq_can_queue);

173 174
void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
			struct request *rq, unsigned int op)
175
{
176 177 178
	INIT_LIST_HEAD(&rq->queuelist);
	/* csd/requeue_work/fifo_time is initialized before use */
	rq->q = q;
179
	rq->mq_ctx = ctx;
180
	rq->cmd_flags = op;
181 182
	if (blk_queue_io_stat(q))
		rq->rq_flags |= RQF_IO_STAT;
183 184 185 186 187 188
	/* do not touch atomic flags, it needs atomic ops against the timer */
	rq->cpu = -1;
	INIT_HLIST_NODE(&rq->hash);
	RB_CLEAR_NODE(&rq->rb_node);
	rq->rq_disk = NULL;
	rq->part = NULL;
189
	rq->start_time = jiffies;
190 191
#ifdef CONFIG_BLK_CGROUP
	rq->rl = NULL;
192
	set_start_time_ns(rq);
193 194 195 196 197 198 199 200 201 202
	rq->io_start_time_ns = 0;
#endif
	rq->nr_phys_segments = 0;
#if defined(CONFIG_BLK_DEV_INTEGRITY)
	rq->nr_integrity_segments = 0;
#endif
	rq->special = NULL;
	/* tag was already set */
	rq->errors = 0;

203 204
	rq->cmd = rq->__cmd;

205 206 207 208 209 210
	rq->extra_len = 0;
	rq->sense_len = 0;
	rq->resid_len = 0;
	rq->sense = NULL;

	INIT_LIST_HEAD(&rq->timeout_list);
211 212
	rq->timeout = 0;

213 214 215 216
	rq->end_io = NULL;
	rq->end_io_data = NULL;
	rq->next_rq = NULL;

217
	ctx->rq_dispatched[op_is_sync(op)]++;
218
}
219
EXPORT_SYMBOL_GPL(blk_mq_rq_ctx_init);
220

221 222
struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data,
				       unsigned int op)
223 224 225 226
{
	struct request *rq;
	unsigned int tag;

227
	tag = blk_mq_get_tag(data);
228
	if (tag != BLK_MQ_TAG_FAIL) {
229 230 231
		struct blk_mq_tags *tags = blk_mq_tags_from_data(data);

		rq = tags->static_rqs[tag];
232

233 234 235 236
		if (data->flags & BLK_MQ_REQ_INTERNAL) {
			rq->tag = -1;
			rq->internal_tag = tag;
		} else {
237 238 239 240
			if (blk_mq_tag_busy(data->hctx)) {
				rq->rq_flags = RQF_MQ_INFLIGHT;
				atomic_inc(&data->hctx->nr_active);
			}
241 242 243 244
			rq->tag = tag;
			rq->internal_tag = -1;
		}

245
		blk_mq_rq_ctx_init(data->q, data->ctx, rq, op);
246 247 248 249 250
		return rq;
	}

	return NULL;
}
251
EXPORT_SYMBOL_GPL(__blk_mq_alloc_request);
252

253 254
struct request *blk_mq_alloc_request(struct request_queue *q, int rw,
		unsigned int flags)
255
{
256
	struct blk_mq_alloc_data alloc_data = { .flags = flags };
257
	struct request *rq;
258
	int ret;
259

260
	ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT);
261 262
	if (ret)
		return ERR_PTR(ret);
263

264
	rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data);
265

266 267 268 269
	blk_mq_put_ctx(alloc_data.ctx);
	blk_queue_exit(q);

	if (!rq)
270
		return ERR_PTR(-EWOULDBLOCK);
271 272 273 274

	rq->__data_len = 0;
	rq->__sector = (sector_t) -1;
	rq->bio = rq->biotail = NULL;
275 276
	return rq;
}
277
EXPORT_SYMBOL(blk_mq_alloc_request);
278

M
Ming Lin 已提交
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw,
		unsigned int flags, unsigned int hctx_idx)
{
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
	struct request *rq;
	struct blk_mq_alloc_data alloc_data;
	int ret;

	/*
	 * If the tag allocator sleeps we could get an allocation for a
	 * different hardware context.  No need to complicate the low level
	 * allocator for this for the rare use case of a command tied to
	 * a specific queue.
	 */
	if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)))
		return ERR_PTR(-EINVAL);

	if (hctx_idx >= q->nr_hw_queues)
		return ERR_PTR(-EIO);

	ret = blk_queue_enter(q, true);
	if (ret)
		return ERR_PTR(ret);

304 305 306 307
	/*
	 * Check if the hardware context is actually mapped to anything.
	 * If not tell the caller that it should skip this queue.
	 */
M
Ming Lin 已提交
308
	hctx = q->queue_hw_ctx[hctx_idx];
309 310 311 312
	if (!blk_mq_hw_queue_mapped(hctx)) {
		ret = -EXDEV;
		goto out_queue_exit;
	}
M
Ming Lin 已提交
313 314 315
	ctx = __blk_mq_get_ctx(q, cpumask_first(hctx->cpumask));

	blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx);
316
	rq = __blk_mq_alloc_request(&alloc_data, rw);
M
Ming Lin 已提交
317
	if (!rq) {
318 319
		ret = -EWOULDBLOCK;
		goto out_queue_exit;
M
Ming Lin 已提交
320 321 322
	}

	return rq;
323 324 325 326

out_queue_exit:
	blk_queue_exit(q);
	return ERR_PTR(ret);
M
Ming Lin 已提交
327 328 329
}
EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);

330 331
void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
			     struct request *rq)
332
{
333
	const int sched_tag = rq->internal_tag;
334 335
	struct request_queue *q = rq->q;

336
	if (rq->rq_flags & RQF_MQ_INFLIGHT)
337
		atomic_dec(&hctx->nr_active);
J
Jens Axboe 已提交
338 339

	wbt_done(q->rq_wb, &rq->issue_stat);
340
	rq->rq_flags = 0;
341

342
	clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
343
	clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
344 345 346 347
	if (rq->tag != -1)
		blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
	if (sched_tag != -1)
		blk_mq_sched_completed_request(hctx, rq);
348
	blk_mq_sched_restart_queues(hctx);
349
	blk_queue_exit(q);
350 351
}

352
static void blk_mq_finish_hctx_request(struct blk_mq_hw_ctx *hctx,
353
				     struct request *rq)
354 355 356 357
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;

	ctx->rq_completed[rq_is_sync(rq)]++;
358 359 360 361 362 363
	__blk_mq_finish_request(hctx, ctx, rq);
}

void blk_mq_finish_request(struct request *rq)
{
	blk_mq_finish_hctx_request(blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), rq);
364 365 366 367
}

void blk_mq_free_request(struct request *rq)
{
368
	blk_mq_sched_put_request(rq);
369
}
J
Jens Axboe 已提交
370
EXPORT_SYMBOL_GPL(blk_mq_free_request);
371

372
inline void __blk_mq_end_request(struct request *rq, int error)
373
{
M
Ming Lei 已提交
374 375
	blk_account_io_done(rq);

C
Christoph Hellwig 已提交
376
	if (rq->end_io) {
J
Jens Axboe 已提交
377
		wbt_done(rq->q->rq_wb, &rq->issue_stat);
378
		rq->end_io(rq, error);
C
Christoph Hellwig 已提交
379 380 381
	} else {
		if (unlikely(blk_bidi_rq(rq)))
			blk_mq_free_request(rq->next_rq);
382
		blk_mq_free_request(rq);
C
Christoph Hellwig 已提交
383
	}
384
}
385
EXPORT_SYMBOL(__blk_mq_end_request);
386

387
void blk_mq_end_request(struct request *rq, int error)
388 389 390
{
	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
		BUG();
391
	__blk_mq_end_request(rq, error);
392
}
393
EXPORT_SYMBOL(blk_mq_end_request);
394

395
static void __blk_mq_complete_request_remote(void *data)
396
{
397
	struct request *rq = data;
398

399
	rq->q->softirq_done_fn(rq);
400 401
}

402
static void blk_mq_ipi_complete_request(struct request *rq)
403 404
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;
C
Christoph Hellwig 已提交
405
	bool shared = false;
406 407
	int cpu;

C
Christoph Hellwig 已提交
408
	if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
409 410 411
		rq->q->softirq_done_fn(rq);
		return;
	}
412 413

	cpu = get_cpu();
C
Christoph Hellwig 已提交
414 415 416 417
	if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
		shared = cpus_share_cache(cpu, ctx->cpu);

	if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
418
		rq->csd.func = __blk_mq_complete_request_remote;
419 420
		rq->csd.info = rq;
		rq->csd.flags = 0;
421
		smp_call_function_single_async(ctx->cpu, &rq->csd);
422
	} else {
423
		rq->q->softirq_done_fn(rq);
424
	}
425 426
	put_cpu();
}
427

428 429 430 431 432 433 434 435 436 437 438 439 440 441 442
static void blk_mq_stat_add(struct request *rq)
{
	if (rq->rq_flags & RQF_STATS) {
		/*
		 * We could rq->mq_ctx here, but there's less of a risk
		 * of races if we have the completion event add the stats
		 * to the local software queue.
		 */
		struct blk_mq_ctx *ctx;

		ctx = __blk_mq_get_ctx(rq->q, raw_smp_processor_id());
		blk_stat_add(&ctx->stat[rq_data_dir(rq)], rq);
	}
}

443
static void __blk_mq_complete_request(struct request *rq)
444 445 446
{
	struct request_queue *q = rq->q;

447 448
	blk_mq_stat_add(rq);

449
	if (!q->softirq_done_fn)
450
		blk_mq_end_request(rq, rq->errors);
451 452 453 454
	else
		blk_mq_ipi_complete_request(rq);
}

455 456 457 458 459 460 461 462
/**
 * blk_mq_complete_request - end I/O on a request
 * @rq:		the request being processed
 *
 * Description:
 *	Ends all I/O on a request. It does not handle partial completions.
 *	The actual completion happens out-of-order, through a IPI handler.
 **/
463
void blk_mq_complete_request(struct request *rq, int error)
464
{
465 466 467
	struct request_queue *q = rq->q;

	if (unlikely(blk_should_fake_timeout(q)))
468
		return;
469 470
	if (!blk_mark_rq_complete(rq)) {
		rq->errors = error;
471
		__blk_mq_complete_request(rq);
472
	}
473 474
}
EXPORT_SYMBOL(blk_mq_complete_request);
475

476 477 478 479 480 481
int blk_mq_request_started(struct request *rq)
{
	return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
}
EXPORT_SYMBOL_GPL(blk_mq_request_started);

482
void blk_mq_start_request(struct request *rq)
483 484 485
{
	struct request_queue *q = rq->q;

486 487
	blk_mq_sched_started_request(rq);

488 489
	trace_block_rq_issue(q, rq);

C
Christoph Hellwig 已提交
490
	rq->resid_len = blk_rq_bytes(rq);
C
Christoph Hellwig 已提交
491 492
	if (unlikely(blk_bidi_rq(rq)))
		rq->next_rq->resid_len = blk_rq_bytes(rq->next_rq);
C
Christoph Hellwig 已提交
493

494 495 496
	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
		blk_stat_set_issue_time(&rq->issue_stat);
		rq->rq_flags |= RQF_STATS;
J
Jens Axboe 已提交
497
		wbt_issue(q->rq_wb, &rq->issue_stat);
498 499
	}

500
	blk_add_timer(rq);
501

502 503 504 505 506 507
	/*
	 * Ensure that ->deadline is visible before set the started
	 * flag and clear the completed flag.
	 */
	smp_mb__before_atomic();

508 509 510 511 512 513
	/*
	 * Mark us as started and clear complete. Complete might have been
	 * set if requeue raced with timeout, which then marked it as
	 * complete. So be sure to clear complete again when we start
	 * the request, otherwise we'll ignore the completion event.
	 */
514 515 516 517
	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
		set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
	if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
		clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
518 519 520 521 522 523 524 525 526

	if (q->dma_drain_size && blk_rq_bytes(rq)) {
		/*
		 * Make sure space for the drain appears.  We know we can do
		 * this because max_hw_segments has been adjusted to be one
		 * fewer than the device can handle.
		 */
		rq->nr_phys_segments++;
	}
527
}
528
EXPORT_SYMBOL(blk_mq_start_request);
529

530
static void __blk_mq_requeue_request(struct request *rq)
531 532 533 534
{
	struct request_queue *q = rq->q;

	trace_block_rq_requeue(q, rq);
J
Jens Axboe 已提交
535
	wbt_requeue(q->rq_wb, &rq->issue_stat);
536
	blk_mq_sched_requeue_request(rq);
537

538 539 540 541
	if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
		if (q->dma_drain_size && blk_rq_bytes(rq))
			rq->nr_phys_segments--;
	}
542 543
}

544
void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
545 546 547 548
{
	__blk_mq_requeue_request(rq);

	BUG_ON(blk_queued_rq(rq));
549
	blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
550 551 552
}
EXPORT_SYMBOL(blk_mq_requeue_request);

553 554 555
static void blk_mq_requeue_work(struct work_struct *work)
{
	struct request_queue *q =
556
		container_of(work, struct request_queue, requeue_work.work);
557 558 559 560 561 562 563 564 565
	LIST_HEAD(rq_list);
	struct request *rq, *next;
	unsigned long flags;

	spin_lock_irqsave(&q->requeue_lock, flags);
	list_splice_init(&q->requeue_list, &rq_list);
	spin_unlock_irqrestore(&q->requeue_lock, flags);

	list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
566
		if (!(rq->rq_flags & RQF_SOFTBARRIER))
567 568
			continue;

569
		rq->rq_flags &= ~RQF_SOFTBARRIER;
570
		list_del_init(&rq->queuelist);
571
		blk_mq_sched_insert_request(rq, true, false, false);
572 573 574 575 576
	}

	while (!list_empty(&rq_list)) {
		rq = list_entry(rq_list.next, struct request, queuelist);
		list_del_init(&rq->queuelist);
577
		blk_mq_sched_insert_request(rq, false, false, false);
578 579
	}

580
	blk_mq_run_hw_queues(q, false);
581 582
}

583 584
void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
				bool kick_requeue_list)
585 586 587 588 589 590 591 592
{
	struct request_queue *q = rq->q;
	unsigned long flags;

	/*
	 * We abuse this flag that is otherwise used by the I/O scheduler to
	 * request head insertation from the workqueue.
	 */
593
	BUG_ON(rq->rq_flags & RQF_SOFTBARRIER);
594 595 596

	spin_lock_irqsave(&q->requeue_lock, flags);
	if (at_head) {
597
		rq->rq_flags |= RQF_SOFTBARRIER;
598 599 600 601 602
		list_add(&rq->queuelist, &q->requeue_list);
	} else {
		list_add_tail(&rq->queuelist, &q->requeue_list);
	}
	spin_unlock_irqrestore(&q->requeue_lock, flags);
603 604 605

	if (kick_requeue_list)
		blk_mq_kick_requeue_list(q);
606 607 608 609 610
}
EXPORT_SYMBOL(blk_mq_add_to_requeue_list);

void blk_mq_kick_requeue_list(struct request_queue *q)
{
611
	kblockd_schedule_delayed_work(&q->requeue_work, 0);
612 613 614
}
EXPORT_SYMBOL(blk_mq_kick_requeue_list);

615 616 617 618 619 620 621 622
void blk_mq_delay_kick_requeue_list(struct request_queue *q,
				    unsigned long msecs)
{
	kblockd_schedule_delayed_work(&q->requeue_work,
				      msecs_to_jiffies(msecs));
}
EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);

623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642
void blk_mq_abort_requeue_list(struct request_queue *q)
{
	unsigned long flags;
	LIST_HEAD(rq_list);

	spin_lock_irqsave(&q->requeue_lock, flags);
	list_splice_init(&q->requeue_list, &rq_list);
	spin_unlock_irqrestore(&q->requeue_lock, flags);

	while (!list_empty(&rq_list)) {
		struct request *rq;

		rq = list_first_entry(&rq_list, struct request, queuelist);
		list_del_init(&rq->queuelist);
		rq->errors = -EIO;
		blk_mq_end_request(rq, rq->errors);
	}
}
EXPORT_SYMBOL(blk_mq_abort_requeue_list);

643 644
struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
{
645 646
	if (tag < tags->nr_tags) {
		prefetch(tags->rqs[tag]);
647
		return tags->rqs[tag];
648
	}
649 650

	return NULL;
651 652 653
}
EXPORT_SYMBOL(blk_mq_tag_to_rq);

654
struct blk_mq_timeout_data {
655 656
	unsigned long next;
	unsigned int next_set;
657 658
};

659
void blk_mq_rq_timed_out(struct request *req, bool reserved)
660
{
J
Jens Axboe 已提交
661
	const struct blk_mq_ops *ops = req->q->mq_ops;
662
	enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
663 664 665 666 667 668 669 670 671 672

	/*
	 * We know that complete is set at this point. If STARTED isn't set
	 * anymore, then the request isn't active and the "timeout" should
	 * just be ignored. This can happen due to the bitflag ordering.
	 * Timeout first checks if STARTED is set, and if it is, assumes
	 * the request is active. But if we race with completion, then
	 * we both flags will get cleared. So check here again, and ignore
	 * a timeout event with a request that isn't active.
	 */
673 674
	if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
		return;
675

676
	if (ops->timeout)
677
		ret = ops->timeout(req, reserved);
678 679 680 681 682 683 684 685 686 687 688 689 690 691 692

	switch (ret) {
	case BLK_EH_HANDLED:
		__blk_mq_complete_request(req);
		break;
	case BLK_EH_RESET_TIMER:
		blk_add_timer(req);
		blk_clear_rq_complete(req);
		break;
	case BLK_EH_NOT_HANDLED:
		break;
	default:
		printk(KERN_ERR "block: bad eh return: %d\n", ret);
		break;
	}
693
}
694

695 696 697 698
static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
		struct request *rq, void *priv, bool reserved)
{
	struct blk_mq_timeout_data *data = priv;
699

700 701 702 703 704
	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
		/*
		 * If a request wasn't started before the queue was
		 * marked dying, kill it here or it'll go unnoticed.
		 */
705 706 707 708
		if (unlikely(blk_queue_dying(rq->q))) {
			rq->errors = -EIO;
			blk_mq_end_request(rq, rq->errors);
		}
709
		return;
710
	}
711

712 713
	if (time_after_eq(jiffies, rq->deadline)) {
		if (!blk_mark_rq_complete(rq))
714
			blk_mq_rq_timed_out(rq, reserved);
715 716 717 718
	} else if (!data->next_set || time_after(data->next, rq->deadline)) {
		data->next = rq->deadline;
		data->next_set = 1;
	}
719 720
}

721
static void blk_mq_timeout_work(struct work_struct *work)
722
{
723 724
	struct request_queue *q =
		container_of(work, struct request_queue, timeout_work);
725 726 727 728 729
	struct blk_mq_timeout_data data = {
		.next		= 0,
		.next_set	= 0,
	};
	int i;
730

731 732 733 734 735 736 737 738 739 740 741 742 743 744
	/* A deadlock might occur if a request is stuck requiring a
	 * timeout at the same time a queue freeze is waiting
	 * completion, since the timeout code would not be able to
	 * acquire the queue reference here.
	 *
	 * That's why we don't use blk_queue_enter here; instead, we use
	 * percpu_ref_tryget directly, because we need to be able to
	 * obtain a reference even in the short window between the queue
	 * starting to freeze, by dropping the first reference in
	 * blk_mq_freeze_queue_start, and the moment the last request is
	 * consumed, marked by the instant q_usage_counter reaches
	 * zero.
	 */
	if (!percpu_ref_tryget(&q->q_usage_counter))
745 746
		return;

747
	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data);
748

749 750 751
	if (data.next_set) {
		data.next = blk_rq_timeout(round_jiffies_up(data.next));
		mod_timer(&q->timeout, data.next);
752
	} else {
753 754
		struct blk_mq_hw_ctx *hctx;

755 756 757 758 759
		queue_for_each_hw_ctx(q, hctx, i) {
			/* the hctx may be unmapped, so check it here */
			if (blk_mq_hw_queue_mapped(hctx))
				blk_mq_tag_idle(hctx);
		}
760
	}
761
	blk_queue_exit(q);
762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784
}

/*
 * Reverse check our software queue for entries that we could potentially
 * merge with. Currently includes a hand-wavy stop count of 8, to not spend
 * too much time checking for merges.
 */
static bool blk_mq_attempt_merge(struct request_queue *q,
				 struct blk_mq_ctx *ctx, struct bio *bio)
{
	struct request *rq;
	int checked = 8;

	list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
		int el_ret;

		if (!checked--)
			break;

		if (!blk_rq_merge_ok(rq, bio))
			continue;

		el_ret = blk_try_merge(rq, bio);
785 786 787 788 789 790
		if (el_ret == ELEVATOR_NO_MERGE)
			continue;

		if (!blk_mq_sched_allow_merge(q, rq, bio))
			break;

791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808
		if (el_ret == ELEVATOR_BACK_MERGE) {
			if (bio_attempt_back_merge(q, rq, bio)) {
				ctx->rq_merged++;
				return true;
			}
			break;
		} else if (el_ret == ELEVATOR_FRONT_MERGE) {
			if (bio_attempt_front_merge(q, rq, bio)) {
				ctx->rq_merged++;
				return true;
			}
			break;
		}
	}

	return false;
}

809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826
struct flush_busy_ctx_data {
	struct blk_mq_hw_ctx *hctx;
	struct list_head *list;
};

static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
{
	struct flush_busy_ctx_data *flush_data = data;
	struct blk_mq_hw_ctx *hctx = flush_data->hctx;
	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];

	sbitmap_clear_bit(sb, bitnr);
	spin_lock(&ctx->lock);
	list_splice_tail_init(&ctx->rq_list, flush_data->list);
	spin_unlock(&ctx->lock);
	return true;
}

827 828 829 830
/*
 * Process software queues that have been marked busy, splicing them
 * to the for-dispatch
 */
831
void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
832
{
833 834 835 836
	struct flush_busy_ctx_data data = {
		.hctx = hctx,
		.list = list,
	};
837

838
	sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
839
}
840
EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
841

842 843 844 845
static inline unsigned int queued_to_index(unsigned int queued)
{
	if (!queued)
		return 0;
846

847
	return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
848 849
}

850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871
static bool blk_mq_get_driver_tag(struct request *rq,
				  struct blk_mq_hw_ctx **hctx, bool wait)
{
	struct blk_mq_alloc_data data = {
		.q = rq->q,
		.ctx = rq->mq_ctx,
		.hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu),
		.flags = wait ? 0 : BLK_MQ_REQ_NOWAIT,
	};

	if (blk_mq_hctx_stopped(data.hctx))
		return false;

	if (rq->tag != -1) {
done:
		if (hctx)
			*hctx = data.hctx;
		return true;
	}

	rq->tag = blk_mq_get_tag(&data);
	if (rq->tag >= 0) {
872 873 874 875
		if (blk_mq_tag_busy(data.hctx)) {
			rq->rq_flags |= RQF_MQ_INFLIGHT;
			atomic_inc(&data.hctx->nr_active);
		}
876 877 878 879 880 881 882
		data.hctx->tags->rqs[rq->tag] = rq;
		goto done;
	}

	return false;
}

883 884 885 886 887 888 889 890 891 892 893 894 895 896 897
static void blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,
				  struct request *rq)
{
	if (rq->tag == -1 || rq->internal_tag == -1)
		return;

	blk_mq_put_tag(hctx, hctx->tags, rq->mq_ctx, rq->tag);
	rq->tag = -1;

	if (rq->rq_flags & RQF_MQ_INFLIGHT) {
		rq->rq_flags &= ~RQF_MQ_INFLIGHT;
		atomic_dec(&hctx->nr_active);
	}
}

898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921
/*
 * If we fail getting a driver tag because all the driver tags are already
 * assigned and on the dispatch list, BUT the first entry does not have a
 * tag, then we could deadlock. For that case, move entries with assigned
 * driver tags to the front, leaving the set of tagged requests in the
 * same order, and the untagged set in the same order.
 */
static bool reorder_tags_to_front(struct list_head *list)
{
	struct request *rq, *tmp, *first = NULL;

	list_for_each_entry_safe_reverse(rq, tmp, list, queuelist) {
		if (rq == first)
			break;
		if (rq->tag != -1) {
			list_move(&rq->queuelist, list);
			if (!first)
				first = rq;
		}
	}

	return first != NULL;
}

922
bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list)
923 924 925
{
	struct request_queue *q = hctx->queue;
	struct request *rq;
926 927
	LIST_HEAD(driver_list);
	struct list_head *dptr;
928
	int queued, ret = BLK_MQ_RQ_QUEUE_OK;
929

930 931 932 933 934 935
	/*
	 * Start off with dptr being NULL, so we start the first request
	 * immediately, even if we have more pending.
	 */
	dptr = NULL;

936 937 938
	/*
	 * Now process all the entries, sending them to the driver.
	 */
939
	queued = 0;
940
	while (!list_empty(list)) {
941
		struct blk_mq_queue_data bd;
942

943
		rq = list_first_entry(list, struct request, queuelist);
944 945 946
		if (!blk_mq_get_driver_tag(rq, &hctx, false)) {
			if (!queued && reorder_tags_to_front(list))
				continue;
947 948 949 950 951 952 953

			/*
			 * We failed getting a driver tag. Mark the queue(s)
			 * as needing a restart. Retry getting a tag again,
			 * in case the needed IO completed right before we
			 * marked the queue as needing a restart.
			 */
954
			blk_mq_sched_mark_restart(hctx);
955 956
			if (!blk_mq_get_driver_tag(rq, &hctx, false))
				break;
957
		}
958 959
		list_del_init(&rq->queuelist);

960 961
		bd.rq = rq;
		bd.list = dptr;
962
		bd.last = list_empty(list);
963 964

		ret = q->mq_ops->queue_rq(hctx, &bd);
965 966 967
		switch (ret) {
		case BLK_MQ_RQ_QUEUE_OK:
			queued++;
968
			break;
969
		case BLK_MQ_RQ_QUEUE_BUSY:
970
			blk_mq_put_driver_tag(hctx, rq);
971
			list_add(&rq->queuelist, list);
972
			__blk_mq_requeue_request(rq);
973 974 975 976
			break;
		default:
			pr_err("blk-mq: bad return on queue: %d\n", ret);
		case BLK_MQ_RQ_QUEUE_ERROR:
977
			rq->errors = -EIO;
978
			blk_mq_end_request(rq, rq->errors);
979 980 981 982 983
			break;
		}

		if (ret == BLK_MQ_RQ_QUEUE_BUSY)
			break;
984 985 986 987 988

		/*
		 * We've done the first request. If we have more than 1
		 * left in the list, set dptr to defer issue.
		 */
989
		if (!dptr && list->next != list->prev)
990
			dptr = &driver_list;
991 992
	}

993
	hctx->dispatched[queued_to_index(queued)]++;
994 995 996 997 998

	/*
	 * Any items that need requeuing? Stuff them into hctx->dispatch,
	 * that is where we will continue on next queue run.
	 */
999
	if (!list_empty(list)) {
1000
		spin_lock(&hctx->lock);
1001
		list_splice_init(list, &hctx->dispatch);
1002
		spin_unlock(&hctx->lock);
1003

1004 1005 1006 1007 1008 1009 1010 1011
		/*
		 * the queue is expected stopped with BLK_MQ_RQ_QUEUE_BUSY, but
		 * it's possible the queue is stopped and restarted again
		 * before this. Queue restart will dispatch requests. And since
		 * requests in rq_list aren't added into hctx->dispatch yet,
		 * the requests in rq_list might get lost.
		 *
		 * blk_mq_run_hw_queue() already checks the STOPPED bit
1012 1013 1014 1015 1016 1017
		 *
		 * If RESTART is set, then let completion restart the queue
		 * instead of potentially looping here.
		 */
		if (!blk_mq_sched_needs_restart(hctx))
			blk_mq_run_hw_queue(hctx, true);
1018
	}
1019 1020 1021 1022

	return ret != BLK_MQ_RQ_QUEUE_BUSY;
}

1023 1024 1025 1026 1027 1028 1029 1030 1031
static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	int srcu_idx;

	WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) &&
		cpu_online(hctx->next_cpu));

	if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
		rcu_read_lock();
1032
		blk_mq_sched_dispatch_requests(hctx);
1033 1034 1035
		rcu_read_unlock();
	} else {
		srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu);
1036
		blk_mq_sched_dispatch_requests(hctx);
1037 1038 1039 1040
		srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx);
	}
}

1041 1042 1043 1044 1045 1046 1047 1048
/*
 * It'd be great if the workqueue API had a way to pass
 * in a mask and had some smarts for more clever placement.
 * For now we just round-robin here, switching for every
 * BLK_MQ_CPU_WORK_BATCH queued items.
 */
static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
{
1049 1050
	if (hctx->queue->nr_hw_queues == 1)
		return WORK_CPU_UNBOUND;
1051 1052

	if (--hctx->next_cpu_batch <= 0) {
1053
		int next_cpu;
1054 1055 1056 1057 1058 1059 1060 1061 1062

		next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
		if (next_cpu >= nr_cpu_ids)
			next_cpu = cpumask_first(hctx->cpumask);

		hctx->next_cpu = next_cpu;
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}

1063
	return hctx->next_cpu;
1064 1065
}

1066 1067
void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
1068 1069
	if (unlikely(blk_mq_hctx_stopped(hctx) ||
		     !blk_mq_hw_queue_mapped(hctx)))
1070 1071
		return;

1072
	if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
1073 1074
		int cpu = get_cpu();
		if (cpumask_test_cpu(cpu, hctx->cpumask)) {
1075
			__blk_mq_run_hw_queue(hctx);
1076
			put_cpu();
1077 1078
			return;
		}
1079

1080
		put_cpu();
1081
	}
1082

1083
	kblockd_schedule_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work);
1084 1085
}

1086
void blk_mq_run_hw_queues(struct request_queue *q, bool async)
1087 1088 1089 1090 1091
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i) {
1092
		if (!blk_mq_hctx_has_pending(hctx) ||
1093
		    blk_mq_hctx_stopped(hctx))
1094 1095
			continue;

1096
		blk_mq_run_hw_queue(hctx, async);
1097 1098
	}
}
1099
EXPORT_SYMBOL(blk_mq_run_hw_queues);
1100

1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120
/**
 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped
 * @q: request queue.
 *
 * The caller is responsible for serializing this function against
 * blk_mq_{start,stop}_hw_queue().
 */
bool blk_mq_queue_stopped(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		if (blk_mq_hctx_stopped(hctx))
			return true;

	return false;
}
EXPORT_SYMBOL(blk_mq_queue_stopped);

1121 1122
void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
{
1123
	cancel_work(&hctx->run_work);
1124
	cancel_delayed_work(&hctx->delay_work);
1125 1126 1127 1128
	set_bit(BLK_MQ_S_STOPPED, &hctx->state);
}
EXPORT_SYMBOL(blk_mq_stop_hw_queue);

1129 1130 1131 1132 1133 1134 1135 1136 1137 1138
void blk_mq_stop_hw_queues(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_stop_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_stop_hw_queues);

1139 1140 1141
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1142

1143
	blk_mq_run_hw_queue(hctx, false);
1144 1145 1146
}
EXPORT_SYMBOL(blk_mq_start_hw_queue);

1147 1148 1149 1150 1151 1152 1153 1154 1155 1156
void blk_mq_start_hw_queues(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_start_hw_queues);

1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
	if (!blk_mq_hctx_stopped(hctx))
		return;

	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
	blk_mq_run_hw_queue(hctx, async);
}
EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);

1167
void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
1168 1169 1170 1171
{
	struct blk_mq_hw_ctx *hctx;
	int i;

1172 1173
	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_stopped_hw_queue(hctx, async);
1174 1175 1176
}
EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);

1177
static void blk_mq_run_work_fn(struct work_struct *work)
1178 1179 1180
{
	struct blk_mq_hw_ctx *hctx;

1181
	hctx = container_of(work, struct blk_mq_hw_ctx, run_work);
1182

1183 1184 1185
	__blk_mq_run_hw_queue(hctx);
}

1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
static void blk_mq_delay_work_fn(struct work_struct *work)
{
	struct blk_mq_hw_ctx *hctx;

	hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);

	if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
		__blk_mq_run_hw_queue(hctx);
}

void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
{
1198 1199
	if (unlikely(!blk_mq_hw_queue_mapped(hctx)))
		return;
1200

1201
	blk_mq_stop_hw_queue(hctx);
1202 1203
	kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
			&hctx->delay_work, msecs_to_jiffies(msecs));
1204 1205 1206
}
EXPORT_SYMBOL(blk_mq_delay_queue);

1207 1208 1209
static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,
					    struct request *rq,
					    bool at_head)
1210
{
J
Jens Axboe 已提交
1211 1212
	struct blk_mq_ctx *ctx = rq->mq_ctx;

1213 1214
	trace_block_rq_insert(hctx->queue, rq);

1215 1216 1217 1218
	if (at_head)
		list_add(&rq->queuelist, &ctx->rq_list);
	else
		list_add_tail(&rq->queuelist, &ctx->rq_list);
1219
}
1220

1221 1222
void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
			     bool at_head)
1223 1224 1225
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;

J
Jens Axboe 已提交
1226
	__blk_mq_insert_req_list(hctx, rq, at_head);
1227 1228 1229
	blk_mq_hctx_mark_pending(hctx, ctx);
}

1230 1231
void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
			    struct list_head *list)
1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242

{
	/*
	 * preemption doesn't flush plug list, so it's possible ctx->cpu is
	 * offline now
	 */
	spin_lock(&ctx->lock);
	while (!list_empty(list)) {
		struct request *rq;

		rq = list_first_entry(list, struct request, queuelist);
J
Jens Axboe 已提交
1243
		BUG_ON(rq->mq_ctx != ctx);
1244
		list_del_init(&rq->queuelist);
J
Jens Axboe 已提交
1245
		__blk_mq_insert_req_list(hctx, rq, false);
1246
	}
1247
	blk_mq_hctx_mark_pending(hctx, ctx);
1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283
	spin_unlock(&ctx->lock);
}

static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
{
	struct request *rqa = container_of(a, struct request, queuelist);
	struct request *rqb = container_of(b, struct request, queuelist);

	return !(rqa->mq_ctx < rqb->mq_ctx ||
		 (rqa->mq_ctx == rqb->mq_ctx &&
		  blk_rq_pos(rqa) < blk_rq_pos(rqb)));
}

void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
{
	struct blk_mq_ctx *this_ctx;
	struct request_queue *this_q;
	struct request *rq;
	LIST_HEAD(list);
	LIST_HEAD(ctx_list);
	unsigned int depth;

	list_splice_init(&plug->mq_list, &list);

	list_sort(NULL, &list, plug_ctx_cmp);

	this_q = NULL;
	this_ctx = NULL;
	depth = 0;

	while (!list_empty(&list)) {
		rq = list_entry_rq(list.next);
		list_del_init(&rq->queuelist);
		BUG_ON(!rq->q);
		if (rq->mq_ctx != this_ctx) {
			if (this_ctx) {
1284 1285 1286 1287
				trace_block_unplug(this_q, depth, from_schedule);
				blk_mq_sched_insert_requests(this_q, this_ctx,
								&ctx_list,
								from_schedule);
1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303
			}

			this_ctx = rq->mq_ctx;
			this_q = rq->q;
			depth = 0;
		}

		depth++;
		list_add_tail(&rq->queuelist, &ctx_list);
	}

	/*
	 * If 'this_ctx' is set, we know we have entries to complete
	 * on 'ctx_list'. Do those.
	 */
	if (this_ctx) {
1304 1305 1306
		trace_block_unplug(this_q, depth, from_schedule);
		blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list,
						from_schedule);
1307 1308 1309 1310 1311 1312
	}
}

static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
{
	init_request_from_bio(rq, bio);
1313

1314
	blk_account_io_start(rq, true);
1315 1316
}

1317 1318 1319 1320 1321 1322
static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx)
{
	return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
		!blk_queue_nomerges(hctx->queue);
}

1323 1324 1325
static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
					 struct blk_mq_ctx *ctx,
					 struct request *rq, struct bio *bio)
1326
{
1327
	if (!hctx_allow_merges(hctx) || !bio_mergeable(bio)) {
1328 1329 1330 1331 1332 1333 1334
		blk_mq_bio_to_request(rq, bio);
		spin_lock(&ctx->lock);
insert_rq:
		__blk_mq_insert_request(hctx, rq, false);
		spin_unlock(&ctx->lock);
		return false;
	} else {
1335 1336
		struct request_queue *q = hctx->queue;

1337 1338 1339 1340 1341
		spin_lock(&ctx->lock);
		if (!blk_mq_attempt_merge(q, ctx, bio)) {
			blk_mq_bio_to_request(rq, bio);
			goto insert_rq;
		}
1342

1343
		spin_unlock(&ctx->lock);
1344
		__blk_mq_finish_request(hctx, ctx, rq);
1345
		return true;
1346
	}
1347
}
1348

1349 1350
static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq)
{
1351 1352 1353 1354
	if (rq->tag != -1)
		return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false);

	return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true);
1355 1356
}

1357
static void blk_mq_try_issue_directly(struct request *rq, blk_qc_t *cookie)
1358 1359 1360 1361 1362 1363 1364
{
	struct request_queue *q = rq->q;
	struct blk_mq_queue_data bd = {
		.rq = rq,
		.list = NULL,
		.last = 1
	};
1365 1366 1367
	struct blk_mq_hw_ctx *hctx;
	blk_qc_t new_cookie;
	int ret;
1368

1369
	if (q->elevator)
1370 1371
		goto insert;

1372 1373 1374 1375 1376
	if (!blk_mq_get_driver_tag(rq, &hctx, false))
		goto insert;

	new_cookie = request_to_qc_t(hctx, rq);

1377 1378 1379 1380 1381 1382
	/*
	 * For OK queue, we are done. For error, kill it. Any other
	 * error (busy), just add it to our list as we previously
	 * would have done
	 */
	ret = q->mq_ops->queue_rq(hctx, &bd);
1383 1384
	if (ret == BLK_MQ_RQ_QUEUE_OK) {
		*cookie = new_cookie;
1385
		return;
1386
	}
1387

1388 1389 1390 1391 1392 1393
	__blk_mq_requeue_request(rq);

	if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
		*cookie = BLK_QC_T_NONE;
		rq->errors = -EIO;
		blk_mq_end_request(rq, rq->errors);
1394
		return;
1395
	}
1396

1397
insert:
1398
	blk_mq_sched_insert_request(rq, false, true, true);
1399 1400
}

1401 1402 1403 1404 1405
/*
 * Multiple hardware queue variant. This will not use per-process plugs,
 * but will attempt to bypass the hctx queueing if we can go straight to
 * hardware for SYNC IO.
 */
1406
static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
1407
{
1408
	const int is_sync = op_is_sync(bio->bi_opf);
J
Jens Axboe 已提交
1409
	const int is_flush_fua = bio->bi_opf & (REQ_PREFLUSH | REQ_FUA);
1410
	struct blk_mq_alloc_data data = { .flags = 0 };
1411
	struct request *rq;
1412
	unsigned int request_count = 0, srcu_idx;
1413
	struct blk_plug *plug;
1414
	struct request *same_queue_rq = NULL;
1415
	blk_qc_t cookie;
J
Jens Axboe 已提交
1416
	unsigned int wb_acct;
1417 1418 1419 1420

	blk_queue_bounce(q, &bio);

	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1421
		bio_io_error(bio);
1422
		return BLK_QC_T_NONE;
1423 1424
	}

1425 1426
	blk_queue_split(q, &bio, q->bio_split);

1427 1428 1429
	if (!is_flush_fua && !blk_queue_nomerges(q) &&
	    blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq))
		return BLK_QC_T_NONE;
1430

1431 1432 1433
	if (blk_mq_sched_bio_merge(q, bio))
		return BLK_QC_T_NONE;

J
Jens Axboe 已提交
1434 1435
	wb_acct = wbt_wait(q->rq_wb, bio, NULL);

1436 1437 1438
	trace_block_getrq(q, bio, bio->bi_opf);

	rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
J
Jens Axboe 已提交
1439 1440
	if (unlikely(!rq)) {
		__wbt_done(q->rq_wb, wb_acct);
1441
		return BLK_QC_T_NONE;
J
Jens Axboe 已提交
1442 1443 1444
	}

	wbt_track(&rq->issue_stat, wb_acct);
1445

1446
	cookie = request_to_qc_t(data.hctx, rq);
1447 1448 1449

	if (unlikely(is_flush_fua)) {
		blk_mq_bio_to_request(rq, bio);
1450
		blk_mq_get_driver_tag(rq, NULL, true);
1451 1452 1453 1454
		blk_insert_flush(rq);
		goto run_queue;
	}

1455
	plug = current->plug;
1456 1457 1458 1459 1460
	/*
	 * If the driver supports defer issued based on 'last', then
	 * queue it up like normal since we can potentially save some
	 * CPU this way.
	 */
1461 1462 1463
	if (((plug && !blk_queue_nomerges(q)) || is_sync) &&
	    !(data.hctx->flags & BLK_MQ_F_DEFER_ISSUE)) {
		struct request *old_rq = NULL;
1464 1465 1466 1467

		blk_mq_bio_to_request(rq, bio);

		/*
1468
		 * We do limited plugging. If the bio can be merged, do that.
1469 1470
		 * Otherwise the existing request in the plug list will be
		 * issued. So the plug list will have one request at most
1471
		 */
1472
		if (plug) {
1473 1474
			/*
			 * The plug list might get flushed before this. If that
1475 1476 1477
			 * happens, same_queue_rq is invalid and plug list is
			 * empty
			 */
1478 1479
			if (same_queue_rq && !list_empty(&plug->mq_list)) {
				old_rq = same_queue_rq;
1480
				list_del_init(&old_rq->queuelist);
1481
			}
1482 1483 1484 1485 1486
			list_add_tail(&rq->queuelist, &plug->mq_list);
		} else /* is_sync */
			old_rq = rq;
		blk_mq_put_ctx(data.ctx);
		if (!old_rq)
1487
			goto done;
1488 1489 1490

		if (!(data.hctx->flags & BLK_MQ_F_BLOCKING)) {
			rcu_read_lock();
1491
			blk_mq_try_issue_directly(old_rq, &cookie);
1492 1493 1494
			rcu_read_unlock();
		} else {
			srcu_idx = srcu_read_lock(&data.hctx->queue_rq_srcu);
1495
			blk_mq_try_issue_directly(old_rq, &cookie);
1496 1497
			srcu_read_unlock(&data.hctx->queue_rq_srcu, srcu_idx);
		}
1498
		goto done;
1499 1500
	}

1501 1502 1503
	if (q->elevator) {
		blk_mq_put_ctx(data.ctx);
		blk_mq_bio_to_request(rq, bio);
1504 1505
		blk_mq_sched_insert_request(rq, false, true,
						!is_sync || is_flush_fua);
1506 1507
		goto done;
	}
1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518
	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
		/*
		 * For a SYNC request, send it to the hardware immediately. For
		 * an ASYNC request, just ensure that we run it later on. The
		 * latter allows for merging opportunities and more efficient
		 * dispatching.
		 */
run_queue:
		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
	}
	blk_mq_put_ctx(data.ctx);
1519 1520
done:
	return cookie;
1521 1522 1523 1524 1525 1526
}

/*
 * Single hardware queue variant. This will attempt to use any per-process
 * plug for merging and IO deferral.
 */
1527
static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio)
1528
{
1529
	const int is_sync = op_is_sync(bio->bi_opf);
J
Jens Axboe 已提交
1530
	const int is_flush_fua = bio->bi_opf & (REQ_PREFLUSH | REQ_FUA);
1531 1532
	struct blk_plug *plug;
	unsigned int request_count = 0;
1533
	struct blk_mq_alloc_data data = { .flags = 0 };
1534
	struct request *rq;
1535
	blk_qc_t cookie;
J
Jens Axboe 已提交
1536
	unsigned int wb_acct;
1537 1538 1539 1540

	blk_queue_bounce(q, &bio);

	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1541
		bio_io_error(bio);
1542
		return BLK_QC_T_NONE;
1543 1544
	}

1545 1546
	blk_queue_split(q, &bio, q->bio_split);

1547 1548 1549 1550 1551
	if (!is_flush_fua && !blk_queue_nomerges(q)) {
		if (blk_attempt_plug_merge(q, bio, &request_count, NULL))
			return BLK_QC_T_NONE;
	} else
		request_count = blk_plug_queued_count(q);
1552

1553 1554 1555
	if (blk_mq_sched_bio_merge(q, bio))
		return BLK_QC_T_NONE;

J
Jens Axboe 已提交
1556 1557
	wb_acct = wbt_wait(q->rq_wb, bio, NULL);

1558 1559 1560
	trace_block_getrq(q, bio, bio->bi_opf);

	rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
J
Jens Axboe 已提交
1561 1562
	if (unlikely(!rq)) {
		__wbt_done(q->rq_wb, wb_acct);
1563
		return BLK_QC_T_NONE;
J
Jens Axboe 已提交
1564 1565 1566
	}

	wbt_track(&rq->issue_stat, wb_acct);
1567

1568
	cookie = request_to_qc_t(data.hctx, rq);
1569 1570 1571

	if (unlikely(is_flush_fua)) {
		blk_mq_bio_to_request(rq, bio);
1572
		blk_mq_get_driver_tag(rq, NULL, true);
1573 1574 1575 1576 1577 1578 1579 1580 1581
		blk_insert_flush(rq);
		goto run_queue;
	}

	/*
	 * A task plug currently exists. Since this is completely lockless,
	 * utilize that to temporarily store requests until the task is
	 * either done or scheduled away.
	 */
1582 1583
	plug = current->plug;
	if (plug) {
1584 1585
		struct request *last = NULL;

1586
		blk_mq_bio_to_request(rq, bio);
1587 1588 1589 1590 1591 1592 1593

		/*
		 * @request_count may become stale because of schedule
		 * out, so check the list again.
		 */
		if (list_empty(&plug->mq_list))
			request_count = 0;
M
Ming Lei 已提交
1594
		if (!request_count)
1595
			trace_block_plug(q);
1596 1597
		else
			last = list_entry_rq(plug->mq_list.prev);
1598 1599 1600

		blk_mq_put_ctx(data.ctx);

1601 1602
		if (request_count >= BLK_MAX_REQUEST_COUNT || (last &&
		    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
1603 1604
			blk_flush_plug_list(plug, false);
			trace_block_plug(q);
1605
		}
1606

1607
		list_add_tail(&rq->queuelist, &plug->mq_list);
1608
		return cookie;
1609 1610
	}

1611 1612 1613
	if (q->elevator) {
		blk_mq_put_ctx(data.ctx);
		blk_mq_bio_to_request(rq, bio);
1614 1615
		blk_mq_sched_insert_request(rq, false, true,
						!is_sync || is_flush_fua);
1616 1617
		goto done;
	}
1618 1619 1620 1621 1622 1623 1624 1625 1626
	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
		/*
		 * For a SYNC request, send it to the hardware immediately. For
		 * an ASYNC request, just ensure that we run it later on. The
		 * latter allows for merging opportunities and more efficient
		 * dispatching.
		 */
run_queue:
		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1627 1628
	}

1629
	blk_mq_put_ctx(data.ctx);
1630
done:
1631
	return cookie;
1632 1633
}

1634 1635
void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx)
1636
{
1637
	struct page *page;
1638

1639
	if (tags->rqs && set->ops->exit_request) {
1640
		int i;
1641

1642
		for (i = 0; i < tags->nr_tags; i++) {
J
Jens Axboe 已提交
1643 1644 1645
			struct request *rq = tags->static_rqs[i];

			if (!rq)
1646
				continue;
J
Jens Axboe 已提交
1647
			set->ops->exit_request(set->driver_data, rq,
1648
						hctx_idx, i);
J
Jens Axboe 已提交
1649
			tags->static_rqs[i] = NULL;
1650
		}
1651 1652
	}

1653 1654
	while (!list_empty(&tags->page_list)) {
		page = list_first_entry(&tags->page_list, struct page, lru);
1655
		list_del_init(&page->lru);
1656 1657 1658 1659 1660
		/*
		 * Remove kmemleak object previously allocated in
		 * blk_mq_init_rq_map().
		 */
		kmemleak_free(page_address(page));
1661 1662
		__free_pages(page, page->private);
	}
1663
}
1664

1665 1666
void blk_mq_free_rq_map(struct blk_mq_tags *tags)
{
1667
	kfree(tags->rqs);
1668
	tags->rqs = NULL;
J
Jens Axboe 已提交
1669 1670
	kfree(tags->static_rqs);
	tags->static_rqs = NULL;
1671

1672
	blk_mq_free_tags(tags);
1673 1674
}

1675 1676 1677 1678
struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
					unsigned int hctx_idx,
					unsigned int nr_tags,
					unsigned int reserved_tags)
1679
{
1680
	struct blk_mq_tags *tags;
1681

1682
	tags = blk_mq_init_tags(nr_tags, reserved_tags,
S
Shaohua Li 已提交
1683 1684
				set->numa_node,
				BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
1685 1686
	if (!tags)
		return NULL;
1687

1688
	tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *),
1689
				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
1690
				 set->numa_node);
1691 1692 1693 1694
	if (!tags->rqs) {
		blk_mq_free_tags(tags);
		return NULL;
	}
1695

J
Jens Axboe 已提交
1696 1697 1698 1699 1700 1701 1702 1703 1704
	tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *),
				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
				 set->numa_node);
	if (!tags->static_rqs) {
		kfree(tags->rqs);
		blk_mq_free_tags(tags);
		return NULL;
	}

1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720
	return tags;
}

static size_t order_to_size(unsigned int order)
{
	return (size_t)PAGE_SIZE << order;
}

int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx, unsigned int depth)
{
	unsigned int i, j, entries_per_page, max_order = 4;
	size_t rq_size, left;

	INIT_LIST_HEAD(&tags->page_list);

1721 1722 1723 1724
	/*
	 * rq_size is the size of the request plus driver payload, rounded
	 * to the cacheline size
	 */
1725
	rq_size = round_up(sizeof(struct request) + set->cmd_size,
1726
				cache_line_size());
1727
	left = rq_size * depth;
1728

1729
	for (i = 0; i < depth; ) {
1730 1731 1732 1733 1734
		int this_order = max_order;
		struct page *page;
		int to_do;
		void *p;

1735
		while (this_order && left < order_to_size(this_order - 1))
1736 1737 1738
			this_order--;

		do {
1739
			page = alloc_pages_node(set->numa_node,
1740
				GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
1741
				this_order);
1742 1743 1744 1745 1746 1747 1748 1749 1750
			if (page)
				break;
			if (!this_order--)
				break;
			if (order_to_size(this_order) < rq_size)
				break;
		} while (1);

		if (!page)
1751
			goto fail;
1752 1753

		page->private = this_order;
1754
		list_add_tail(&page->lru, &tags->page_list);
1755 1756

		p = page_address(page);
1757 1758 1759 1760
		/*
		 * Allow kmemleak to scan these pages as they contain pointers
		 * to additional allocations like via ops->init_request().
		 */
1761
		kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
1762
		entries_per_page = order_to_size(this_order) / rq_size;
1763
		to_do = min(entries_per_page, depth - i);
1764 1765
		left -= to_do * rq_size;
		for (j = 0; j < to_do; j++) {
J
Jens Axboe 已提交
1766 1767 1768
			struct request *rq = p;

			tags->static_rqs[i] = rq;
1769 1770
			if (set->ops->init_request) {
				if (set->ops->init_request(set->driver_data,
J
Jens Axboe 已提交
1771
						rq, hctx_idx, i,
1772
						set->numa_node)) {
J
Jens Axboe 已提交
1773
					tags->static_rqs[i] = NULL;
1774
					goto fail;
1775
				}
1776 1777
			}

1778 1779 1780 1781
			p += rq_size;
			i++;
		}
	}
1782
	return 0;
1783

1784
fail:
1785 1786
	blk_mq_free_rqs(set, tags, hctx_idx);
	return -ENOMEM;
1787 1788
}

J
Jens Axboe 已提交
1789 1790 1791 1792 1793
/*
 * 'cpu' is going away. splice any existing rq_list entries from this
 * software queue to the hw queue dispatch list, and ensure that it
 * gets run.
 */
1794
static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
1795
{
1796
	struct blk_mq_hw_ctx *hctx;
1797 1798 1799
	struct blk_mq_ctx *ctx;
	LIST_HEAD(tmp);

1800
	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
J
Jens Axboe 已提交
1801
	ctx = __blk_mq_get_ctx(hctx->queue, cpu);
1802 1803 1804 1805 1806 1807 1808 1809 1810

	spin_lock(&ctx->lock);
	if (!list_empty(&ctx->rq_list)) {
		list_splice_init(&ctx->rq_list, &tmp);
		blk_mq_hctx_clear_pending(hctx, ctx);
	}
	spin_unlock(&ctx->lock);

	if (list_empty(&tmp))
1811
		return 0;
1812

J
Jens Axboe 已提交
1813 1814 1815
	spin_lock(&hctx->lock);
	list_splice_tail_init(&tmp, &hctx->dispatch);
	spin_unlock(&hctx->lock);
1816 1817

	blk_mq_run_hw_queue(hctx, true);
1818
	return 0;
1819 1820
}

1821
static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
1822
{
1823 1824
	cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
					    &hctx->cpuhp_dead);
1825 1826
}

1827
/* hctx->ctxs will be freed in queue's release handler */
1828 1829 1830 1831
static void blk_mq_exit_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
1832 1833
	unsigned flush_start_tag = set->queue_depth;

1834 1835
	blk_mq_tag_idle(hctx);

1836 1837 1838 1839 1840
	if (set->ops->exit_request)
		set->ops->exit_request(set->driver_data,
				       hctx->fq->flush_rq, hctx_idx,
				       flush_start_tag + hctx_idx);

1841 1842 1843
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);

1844 1845 1846
	if (hctx->flags & BLK_MQ_F_BLOCKING)
		cleanup_srcu_struct(&hctx->queue_rq_srcu);

1847
	blk_mq_remove_cpuhp(hctx);
1848
	blk_free_flush_queue(hctx->fq);
1849
	sbitmap_free(&hctx->ctx_map);
1850 1851
}

M
Ming Lei 已提交
1852 1853 1854 1855 1856 1857 1858 1859 1860
static void blk_mq_exit_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set, int nr_queue)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if (i == nr_queue)
			break;
1861
		blk_mq_exit_hctx(q, set, hctx, i);
M
Ming Lei 已提交
1862 1863 1864 1865 1866 1867 1868 1869 1870
	}
}

static void blk_mq_free_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

1871
	queue_for_each_hw_ctx(q, hctx, i)
M
Ming Lei 已提交
1872 1873 1874
		free_cpumask_var(hctx->cpumask);
}

1875 1876 1877
static int blk_mq_init_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
1878
{
1879
	int node;
1880
	unsigned flush_start_tag = set->queue_depth;
1881 1882 1883 1884 1885

	node = hctx->numa_node;
	if (node == NUMA_NO_NODE)
		node = hctx->numa_node = set->numa_node;

1886
	INIT_WORK(&hctx->run_work, blk_mq_run_work_fn);
1887 1888 1889 1890 1891
	INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
	spin_lock_init(&hctx->lock);
	INIT_LIST_HEAD(&hctx->dispatch);
	hctx->queue = q;
	hctx->queue_num = hctx_idx;
1892
	hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED;
1893

1894
	cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
1895 1896

	hctx->tags = set->tags[hctx_idx];
1897 1898

	/*
1899 1900
	 * Allocate space for all possible cpus to avoid allocation at
	 * runtime
1901
	 */
1902 1903 1904 1905
	hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
					GFP_KERNEL, node);
	if (!hctx->ctxs)
		goto unregister_cpu_notifier;
1906

1907 1908
	if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL,
			      node))
1909
		goto free_ctxs;
1910

1911
	hctx->nr_ctx = 0;
1912

1913 1914 1915
	if (set->ops->init_hctx &&
	    set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
		goto free_bitmap;
1916

1917 1918 1919
	hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size);
	if (!hctx->fq)
		goto exit_hctx;
1920

1921 1922 1923 1924 1925
	if (set->ops->init_request &&
	    set->ops->init_request(set->driver_data,
				   hctx->fq->flush_rq, hctx_idx,
				   flush_start_tag + hctx_idx, node))
		goto free_fq;
1926

1927 1928 1929
	if (hctx->flags & BLK_MQ_F_BLOCKING)
		init_srcu_struct(&hctx->queue_rq_srcu);

1930
	return 0;
1931

1932 1933 1934 1935 1936
 free_fq:
	kfree(hctx->fq);
 exit_hctx:
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);
1937
 free_bitmap:
1938
	sbitmap_free(&hctx->ctx_map);
1939 1940 1941
 free_ctxs:
	kfree(hctx->ctxs);
 unregister_cpu_notifier:
1942
	blk_mq_remove_cpuhp(hctx);
1943 1944
	return -1;
}
1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959

static void blk_mq_init_cpu_queues(struct request_queue *q,
				   unsigned int nr_hw_queues)
{
	unsigned int i;

	for_each_possible_cpu(i) {
		struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
		struct blk_mq_hw_ctx *hctx;

		memset(__ctx, 0, sizeof(*__ctx));
		__ctx->cpu = i;
		spin_lock_init(&__ctx->lock);
		INIT_LIST_HEAD(&__ctx->rq_list);
		__ctx->queue = q;
1960 1961
		blk_stat_init(&__ctx->stat[BLK_STAT_READ]);
		blk_stat_init(&__ctx->stat[BLK_STAT_WRITE]);
1962 1963 1964 1965 1966

		/* If the cpu isn't online, the cpu is mapped to first hctx */
		if (!cpu_online(i))
			continue;

C
Christoph Hellwig 已提交
1967
		hctx = blk_mq_map_queue(q, i);
1968

1969 1970 1971 1972 1973
		/*
		 * Set local node, IFF we have more than one hw queue. If
		 * not, we remain on the home node of the device
		 */
		if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
1974
			hctx->numa_node = local_memory_node(cpu_to_node(i));
1975 1976 1977
	}
}

1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999
static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx)
{
	int ret = 0;

	set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
					set->queue_depth, set->reserved_tags);
	if (!set->tags[hctx_idx])
		return false;

	ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
				set->queue_depth);
	if (!ret)
		return true;

	blk_mq_free_rq_map(set->tags[hctx_idx]);
	set->tags[hctx_idx] = NULL;
	return false;
}

static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
					 unsigned int hctx_idx)
{
2000 2001 2002 2003 2004
	if (set->tags[hctx_idx]) {
		blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
		blk_mq_free_rq_map(set->tags[hctx_idx]);
		set->tags[hctx_idx] = NULL;
	}
2005 2006
}

2007 2008
static void blk_mq_map_swqueue(struct request_queue *q,
			       const struct cpumask *online_mask)
2009
{
2010
	unsigned int i, hctx_idx;
2011 2012
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
M
Ming Lei 已提交
2013
	struct blk_mq_tag_set *set = q->tag_set;
2014

2015 2016 2017 2018 2019
	/*
	 * Avoid others reading imcomplete hctx->cpumask through sysfs
	 */
	mutex_lock(&q->sysfs_lock);

2020
	queue_for_each_hw_ctx(q, hctx, i) {
2021
		cpumask_clear(hctx->cpumask);
2022 2023 2024 2025 2026 2027
		hctx->nr_ctx = 0;
	}

	/*
	 * Map software to hardware queues
	 */
2028
	for_each_possible_cpu(i) {
2029
		/* If the cpu isn't online, the cpu is mapped to first hctx */
2030
		if (!cpumask_test_cpu(i, online_mask))
2031 2032
			continue;

2033 2034
		hctx_idx = q->mq_map[i];
		/* unmapped hw queue can be remapped after CPU topo changed */
2035 2036
		if (!set->tags[hctx_idx] &&
		    !__blk_mq_alloc_rq_map(set, hctx_idx)) {
2037 2038 2039 2040 2041 2042
			/*
			 * If tags initialization fail for some hctx,
			 * that hctx won't be brought online.  In this
			 * case, remap the current ctx to hctx[0] which
			 * is guaranteed to always have tags allocated
			 */
2043
			q->mq_map[i] = 0;
2044 2045
		}

2046
		ctx = per_cpu_ptr(q->queue_ctx, i);
C
Christoph Hellwig 已提交
2047
		hctx = blk_mq_map_queue(q, i);
K
Keith Busch 已提交
2048

2049
		cpumask_set_cpu(i, hctx->cpumask);
2050 2051 2052
		ctx->index_hw = hctx->nr_ctx;
		hctx->ctxs[hctx->nr_ctx++] = ctx;
	}
2053

2054 2055
	mutex_unlock(&q->sysfs_lock);

2056
	queue_for_each_hw_ctx(q, hctx, i) {
2057
		/*
2058 2059
		 * If no software queues are mapped to this hardware queue,
		 * disable it and free the request entries.
2060 2061
		 */
		if (!hctx->nr_ctx) {
2062 2063 2064 2065
			/* Never unmap queue 0.  We need it as a
			 * fallback in case of a new remap fails
			 * allocation
			 */
2066 2067 2068
			if (i && set->tags[i])
				blk_mq_free_map_and_requests(set, i);

M
Ming Lei 已提交
2069
			hctx->tags = NULL;
2070 2071 2072
			continue;
		}

M
Ming Lei 已提交
2073 2074 2075
		hctx->tags = set->tags[i];
		WARN_ON(!hctx->tags);

2076 2077 2078 2079 2080
		/*
		 * Set the map size to the number of mapped software queues.
		 * This is more accurate and more efficient than looping
		 * over all possibly mapped software queues.
		 */
2081
		sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
2082

2083 2084 2085
		/*
		 * Initialize batch roundrobin counts
		 */
2086 2087 2088
		hctx->next_cpu = cpumask_first(hctx->cpumask);
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}
2089 2090
}

2091
static void queue_set_hctx_shared(struct request_queue *q, bool shared)
2092 2093 2094 2095
{
	struct blk_mq_hw_ctx *hctx;
	int i;

2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106
	queue_for_each_hw_ctx(q, hctx, i) {
		if (shared)
			hctx->flags |= BLK_MQ_F_TAG_SHARED;
		else
			hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
	}
}

static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, bool shared)
{
	struct request_queue *q;
2107 2108 2109

	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_freeze_queue(q);
2110
		queue_set_hctx_shared(q, shared);
2111 2112 2113 2114 2115 2116 2117 2118 2119 2120
		blk_mq_unfreeze_queue(q);
	}
}

static void blk_mq_del_queue_tag_set(struct request_queue *q)
{
	struct blk_mq_tag_set *set = q->tag_set;

	mutex_lock(&set->tag_list_lock);
	list_del_init(&q->tag_set_list);
2121 2122 2123 2124 2125 2126
	if (list_is_singular(&set->tag_list)) {
		/* just transitioned to unshared */
		set->flags &= ~BLK_MQ_F_TAG_SHARED;
		/* update existing queue */
		blk_mq_update_tag_set_depth(set, false);
	}
2127 2128 2129 2130 2131 2132 2133 2134 2135
	mutex_unlock(&set->tag_list_lock);
}

static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
				     struct request_queue *q)
{
	q->tag_set = set;

	mutex_lock(&set->tag_list_lock);
2136 2137 2138 2139 2140 2141 2142 2143 2144

	/* Check to see if we're transitioning to shared (from 1 to 2 queues). */
	if (!list_empty(&set->tag_list) && !(set->flags & BLK_MQ_F_TAG_SHARED)) {
		set->flags |= BLK_MQ_F_TAG_SHARED;
		/* update existing queue */
		blk_mq_update_tag_set_depth(set, true);
	}
	if (set->flags & BLK_MQ_F_TAG_SHARED)
		queue_set_hctx_shared(q, true);
2145
	list_add_tail(&q->tag_set_list, &set->tag_list);
2146

2147 2148 2149
	mutex_unlock(&set->tag_list_lock);
}

2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160
/*
 * It is the actual release handler for mq, but we do it from
 * request queue's release handler for avoiding use-after-free
 * and headache because q->mq_kobj shouldn't have been introduced,
 * but we can't group ctx/kctx kobj without it.
 */
void blk_mq_release(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

2161 2162
	blk_mq_sched_teardown(q);

2163
	/* hctx kobj stays in hctx */
2164 2165 2166 2167
	queue_for_each_hw_ctx(q, hctx, i) {
		if (!hctx)
			continue;
		kfree(hctx->ctxs);
2168
		kfree(hctx);
2169
	}
2170

2171 2172
	q->mq_map = NULL;

2173 2174 2175 2176 2177 2178
	kfree(q->queue_hw_ctx);

	/* ctx kobj stays in queue_ctx */
	free_percpu(q->queue_ctx);
}

2179
struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194
{
	struct request_queue *uninit_q, *q;

	uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
	if (!uninit_q)
		return ERR_PTR(-ENOMEM);

	q = blk_mq_init_allocated_queue(set, uninit_q);
	if (IS_ERR(q))
		blk_cleanup_queue(uninit_q);

	return q;
}
EXPORT_SYMBOL(blk_mq_init_queue);

K
Keith Busch 已提交
2195 2196
static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
						struct request_queue *q)
2197
{
K
Keith Busch 已提交
2198 2199
	int i, j;
	struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
2200

K
Keith Busch 已提交
2201
	blk_mq_sysfs_unregister(q);
2202
	for (i = 0; i < set->nr_hw_queues; i++) {
K
Keith Busch 已提交
2203
		int node;
2204

K
Keith Busch 已提交
2205 2206 2207 2208
		if (hctxs[i])
			continue;

		node = blk_mq_hw_queue_to_node(q->mq_map, i);
2209 2210
		hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
					GFP_KERNEL, node);
2211
		if (!hctxs[i])
K
Keith Busch 已提交
2212
			break;
2213

2214
		if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL,
K
Keith Busch 已提交
2215 2216 2217 2218 2219
						node)) {
			kfree(hctxs[i]);
			hctxs[i] = NULL;
			break;
		}
2220

2221
		atomic_set(&hctxs[i]->nr_active, 0);
2222
		hctxs[i]->numa_node = node;
2223
		hctxs[i]->queue_num = i;
K
Keith Busch 已提交
2224 2225 2226 2227 2228 2229 2230 2231

		if (blk_mq_init_hctx(q, set, hctxs[i], i)) {
			free_cpumask_var(hctxs[i]->cpumask);
			kfree(hctxs[i]);
			hctxs[i] = NULL;
			break;
		}
		blk_mq_hctx_kobj_init(hctxs[i]);
2232
	}
K
Keith Busch 已提交
2233 2234 2235 2236
	for (j = i; j < q->nr_hw_queues; j++) {
		struct blk_mq_hw_ctx *hctx = hctxs[j];

		if (hctx) {
2237 2238
			if (hctx->tags)
				blk_mq_free_map_and_requests(set, j);
K
Keith Busch 已提交
2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254
			blk_mq_exit_hctx(q, set, hctx, j);
			free_cpumask_var(hctx->cpumask);
			kobject_put(&hctx->kobj);
			kfree(hctx->ctxs);
			kfree(hctx);
			hctxs[j] = NULL;

		}
	}
	q->nr_hw_queues = i;
	blk_mq_sysfs_register(q);
}

struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
						  struct request_queue *q)
{
M
Ming Lei 已提交
2255 2256 2257
	/* mark the queue as mq asap */
	q->mq_ops = set->ops;

K
Keith Busch 已提交
2258 2259
	q->queue_ctx = alloc_percpu(struct blk_mq_ctx);
	if (!q->queue_ctx)
M
Ming Lin 已提交
2260
		goto err_exit;
K
Keith Busch 已提交
2261 2262 2263 2264 2265 2266

	q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)),
						GFP_KERNEL, set->numa_node);
	if (!q->queue_hw_ctx)
		goto err_percpu;

2267
	q->mq_map = set->mq_map;
K
Keith Busch 已提交
2268 2269 2270 2271

	blk_mq_realloc_hw_ctxs(set, q);
	if (!q->nr_hw_queues)
		goto err_hctxs;
2272

2273
	INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
2274
	blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
2275 2276 2277

	q->nr_queues = nr_cpu_ids;

2278
	q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
2279

2280 2281 2282
	if (!(set->flags & BLK_MQ_F_SG_MERGE))
		q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;

2283 2284
	q->sg_reserved_size = INT_MAX;

2285
	INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
2286 2287 2288
	INIT_LIST_HEAD(&q->requeue_list);
	spin_lock_init(&q->requeue_lock);

2289 2290 2291 2292 2293
	if (q->nr_hw_queues > 1)
		blk_queue_make_request(q, blk_mq_make_request);
	else
		blk_queue_make_request(q, blk_sq_make_request);

2294 2295 2296 2297 2298
	/*
	 * Do this after blk_queue_make_request() overrides it...
	 */
	q->nr_requests = set->queue_depth;

2299 2300 2301 2302 2303
	/*
	 * Default to classic polling
	 */
	q->poll_nsec = -1;

2304 2305
	if (set->ops->complete)
		blk_queue_softirq_done(q, set->ops->complete);
2306

2307
	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
2308

2309
	get_online_cpus();
2310 2311
	mutex_lock(&all_q_mutex);

2312
	list_add_tail(&q->all_q_node, &all_q_list);
2313
	blk_mq_add_queue_tag_set(set, q);
2314
	blk_mq_map_swqueue(q, cpu_online_mask);
2315

2316
	mutex_unlock(&all_q_mutex);
2317
	put_online_cpus();
2318

2319 2320 2321 2322 2323 2324 2325 2326
	if (!(set->flags & BLK_MQ_F_NO_SCHED)) {
		int ret;

		ret = blk_mq_sched_init(q);
		if (ret)
			return ERR_PTR(ret);
	}

2327
	return q;
2328

2329
err_hctxs:
K
Keith Busch 已提交
2330
	kfree(q->queue_hw_ctx);
2331
err_percpu:
K
Keith Busch 已提交
2332
	free_percpu(q->queue_ctx);
M
Ming Lin 已提交
2333 2334
err_exit:
	q->mq_ops = NULL;
2335 2336
	return ERR_PTR(-ENOMEM);
}
2337
EXPORT_SYMBOL(blk_mq_init_allocated_queue);
2338 2339 2340

void blk_mq_free_queue(struct request_queue *q)
{
M
Ming Lei 已提交
2341
	struct blk_mq_tag_set	*set = q->tag_set;
2342

2343 2344 2345 2346
	mutex_lock(&all_q_mutex);
	list_del_init(&q->all_q_node);
	mutex_unlock(&all_q_mutex);

J
Jens Axboe 已提交
2347 2348
	wbt_exit(q);

2349 2350
	blk_mq_del_queue_tag_set(q);

M
Ming Lei 已提交
2351 2352
	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
	blk_mq_free_hw_queues(q, set);
2353 2354 2355
}

/* Basically redo blk_mq_init_queue with queue frozen */
2356 2357
static void blk_mq_queue_reinit(struct request_queue *q,
				const struct cpumask *online_mask)
2358
{
2359
	WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth));
2360

2361 2362
	blk_mq_sysfs_unregister(q);

2363 2364 2365 2366 2367 2368
	/*
	 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
	 * we should change hctx numa_node according to new topology (this
	 * involves free and re-allocate memory, worthy doing?)
	 */

2369
	blk_mq_map_swqueue(q, online_mask);
2370

2371
	blk_mq_sysfs_register(q);
2372 2373
}

2374 2375 2376 2377 2378 2379 2380 2381
/*
 * New online cpumask which is going to be set in this hotplug event.
 * Declare this cpumasks as global as cpu-hotplug operation is invoked
 * one-by-one and dynamically allocating this could result in a failure.
 */
static struct cpumask cpuhp_online_new;

static void blk_mq_queue_reinit_work(void)
2382 2383 2384 2385
{
	struct request_queue *q;

	mutex_lock(&all_q_mutex);
2386 2387 2388 2389 2390 2391 2392 2393 2394
	/*
	 * We need to freeze and reinit all existing queues.  Freezing
	 * involves synchronous wait for an RCU grace period and doing it
	 * one by one may take a long time.  Start freezing all queues in
	 * one swoop and then wait for the completions so that freezing can
	 * take place in parallel.
	 */
	list_for_each_entry(q, &all_q_list, all_q_node)
		blk_mq_freeze_queue_start(q);
2395
	list_for_each_entry(q, &all_q_list, all_q_node)
2396 2397
		blk_mq_freeze_queue_wait(q);

2398
	list_for_each_entry(q, &all_q_list, all_q_node)
2399
		blk_mq_queue_reinit(q, &cpuhp_online_new);
2400 2401 2402 2403

	list_for_each_entry(q, &all_q_list, all_q_node)
		blk_mq_unfreeze_queue(q);

2404
	mutex_unlock(&all_q_mutex);
2405 2406 2407 2408
}

static int blk_mq_queue_reinit_dead(unsigned int cpu)
{
2409
	cpumask_copy(&cpuhp_online_new, cpu_online_mask);
2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424
	blk_mq_queue_reinit_work();
	return 0;
}

/*
 * Before hotadded cpu starts handling requests, new mappings must be
 * established.  Otherwise, these requests in hw queue might never be
 * dispatched.
 *
 * For example, there is a single hw queue (hctx) and two CPU queues (ctx0
 * for CPU0, and ctx1 for CPU1).
 *
 * Now CPU1 is just onlined and a request is inserted into ctx1->rq_list
 * and set bit0 in pending bitmap as ctx1->index_hw is still zero.
 *
2425 2426 2427 2428
 * And then while running hw queue, blk_mq_flush_busy_ctxs() finds bit0 is set
 * in pending bitmap and tries to retrieve requests in hctx->ctxs[0]->rq_list.
 * But htx->ctxs[0] is a pointer to ctx0, so the request in ctx1->rq_list is
 * ignored.
2429 2430 2431 2432 2433 2434 2435
 */
static int blk_mq_queue_reinit_prepare(unsigned int cpu)
{
	cpumask_copy(&cpuhp_online_new, cpu_online_mask);
	cpumask_set_cpu(cpu, &cpuhp_online_new);
	blk_mq_queue_reinit_work();
	return 0;
2436 2437
}

2438 2439 2440 2441
static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	int i;

2442 2443
	for (i = 0; i < set->nr_hw_queues; i++)
		if (!__blk_mq_alloc_rq_map(set, i))
2444 2445 2446 2447 2448 2449
			goto out_unwind;

	return 0;

out_unwind:
	while (--i >= 0)
2450
		blk_mq_free_rq_map(set->tags[i]);
2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489

	return -ENOMEM;
}

/*
 * Allocate the request maps associated with this tag_set. Note that this
 * may reduce the depth asked for, if memory is tight. set->queue_depth
 * will be updated to reflect the allocated depth.
 */
static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	unsigned int depth;
	int err;

	depth = set->queue_depth;
	do {
		err = __blk_mq_alloc_rq_maps(set);
		if (!err)
			break;

		set->queue_depth >>= 1;
		if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
			err = -ENOMEM;
			break;
		}
	} while (set->queue_depth);

	if (!set->queue_depth || err) {
		pr_err("blk-mq: failed to allocate request map\n");
		return -ENOMEM;
	}

	if (depth != set->queue_depth)
		pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
						depth, set->queue_depth);

	return 0;
}

2490 2491 2492 2493 2494 2495
/*
 * Alloc a tag set to be associated with one or more request queues.
 * May fail with EINVAL for various error conditions. May adjust the
 * requested depth down, if if it too large. In that case, the set
 * value will be stored in set->queue_depth.
 */
2496 2497
int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
{
2498 2499
	int ret;

B
Bart Van Assche 已提交
2500 2501
	BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);

2502 2503
	if (!set->nr_hw_queues)
		return -EINVAL;
2504
	if (!set->queue_depth)
2505 2506 2507 2508
		return -EINVAL;
	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
		return -EINVAL;

C
Christoph Hellwig 已提交
2509
	if (!set->ops->queue_rq)
2510 2511
		return -EINVAL;

2512 2513 2514 2515 2516
	if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
		pr_info("blk-mq: reduced tag depth to %u\n",
			BLK_MQ_MAX_DEPTH);
		set->queue_depth = BLK_MQ_MAX_DEPTH;
	}
2517

2518 2519 2520 2521 2522 2523 2524 2525 2526
	/*
	 * If a crashdump is active, then we are potentially in a very
	 * memory constrained environment. Limit us to 1 queue and
	 * 64 tags to prevent using too much memory.
	 */
	if (is_kdump_kernel()) {
		set->nr_hw_queues = 1;
		set->queue_depth = min(64U, set->queue_depth);
	}
K
Keith Busch 已提交
2527 2528 2529 2530 2531
	/*
	 * There is no use for more h/w queues than cpus.
	 */
	if (set->nr_hw_queues > nr_cpu_ids)
		set->nr_hw_queues = nr_cpu_ids;
2532

K
Keith Busch 已提交
2533
	set->tags = kzalloc_node(nr_cpu_ids * sizeof(struct blk_mq_tags *),
2534 2535
				 GFP_KERNEL, set->numa_node);
	if (!set->tags)
2536
		return -ENOMEM;
2537

2538 2539 2540
	ret = -ENOMEM;
	set->mq_map = kzalloc_node(sizeof(*set->mq_map) * nr_cpu_ids,
			GFP_KERNEL, set->numa_node);
2541 2542 2543
	if (!set->mq_map)
		goto out_free_tags;

2544 2545 2546 2547 2548 2549 2550 2551 2552
	if (set->ops->map_queues)
		ret = set->ops->map_queues(set);
	else
		ret = blk_mq_map_queues(set);
	if (ret)
		goto out_free_mq_map;

	ret = blk_mq_alloc_rq_maps(set);
	if (ret)
2553
		goto out_free_mq_map;
2554

2555 2556 2557
	mutex_init(&set->tag_list_lock);
	INIT_LIST_HEAD(&set->tag_list);

2558
	return 0;
2559 2560 2561 2562 2563

out_free_mq_map:
	kfree(set->mq_map);
	set->mq_map = NULL;
out_free_tags:
2564 2565
	kfree(set->tags);
	set->tags = NULL;
2566
	return ret;
2567 2568 2569 2570 2571 2572 2573
}
EXPORT_SYMBOL(blk_mq_alloc_tag_set);

void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
{
	int i;

2574 2575
	for (i = 0; i < nr_cpu_ids; i++)
		blk_mq_free_map_and_requests(set, i);
2576

2577 2578 2579
	kfree(set->mq_map);
	set->mq_map = NULL;

M
Ming Lei 已提交
2580
	kfree(set->tags);
2581
	set->tags = NULL;
2582 2583 2584
}
EXPORT_SYMBOL(blk_mq_free_tag_set);

2585 2586 2587 2588 2589 2590
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
{
	struct blk_mq_tag_set *set = q->tag_set;
	struct blk_mq_hw_ctx *hctx;
	int i, ret;

2591
	if (!set)
2592 2593
		return -EINVAL;

2594 2595 2596
	blk_mq_freeze_queue(q);
	blk_mq_quiesce_queue(q);

2597 2598
	ret = 0;
	queue_for_each_hw_ctx(q, hctx, i) {
2599 2600
		if (!hctx->tags)
			continue;
2601 2602 2603 2604
		/*
		 * If we're using an MQ scheduler, just update the scheduler
		 * queue depth. This is similar to what the old code would do.
		 */
2605 2606 2607 2608 2609 2610 2611 2612
		if (!hctx->sched_tags) {
			ret = blk_mq_tag_update_depth(hctx, &hctx->tags,
							min(nr, set->queue_depth),
							false);
		} else {
			ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
							nr, true);
		}
2613 2614 2615 2616 2617 2618 2619
		if (ret)
			break;
	}

	if (!ret)
		q->nr_requests = nr;

2620 2621 2622
	blk_mq_unfreeze_queue(q);
	blk_mq_start_stopped_hw_queues(q, true);

2623 2624 2625
	return ret;
}

K
Keith Busch 已提交
2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654
void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
{
	struct request_queue *q;

	if (nr_hw_queues > nr_cpu_ids)
		nr_hw_queues = nr_cpu_ids;
	if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues)
		return;

	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_freeze_queue(q);

	set->nr_hw_queues = nr_hw_queues;
	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_realloc_hw_ctxs(set, q);

		if (q->nr_hw_queues > 1)
			blk_queue_make_request(q, blk_mq_make_request);
		else
			blk_queue_make_request(q, blk_sq_make_request);

		blk_mq_queue_reinit(q, cpu_online_mask);
	}

	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_unfreeze_queue(q);
}
EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);

2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691
static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
				       struct blk_mq_hw_ctx *hctx,
				       struct request *rq)
{
	struct blk_rq_stat stat[2];
	unsigned long ret = 0;

	/*
	 * If stats collection isn't on, don't sleep but turn it on for
	 * future users
	 */
	if (!blk_stat_enable(q))
		return 0;

	/*
	 * We don't have to do this once per IO, should optimize this
	 * to just use the current window of stats until it changes
	 */
	memset(&stat, 0, sizeof(stat));
	blk_hctx_stat_get(hctx, stat);

	/*
	 * As an optimistic guess, use half of the mean service time
	 * for this type of request. We can (and should) make this smarter.
	 * For instance, if the completion latencies are tight, we can
	 * get closer than just half the mean. This is especially
	 * important on devices where the completion latencies are longer
	 * than ~10 usec.
	 */
	if (req_op(rq) == REQ_OP_READ && stat[BLK_STAT_READ].nr_samples)
		ret = (stat[BLK_STAT_READ].mean + 1) / 2;
	else if (req_op(rq) == REQ_OP_WRITE && stat[BLK_STAT_WRITE].nr_samples)
		ret = (stat[BLK_STAT_WRITE].mean + 1) / 2;

	return ret;
}

2692
static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
2693
				     struct blk_mq_hw_ctx *hctx,
2694 2695 2696 2697
				     struct request *rq)
{
	struct hrtimer_sleeper hs;
	enum hrtimer_mode mode;
2698
	unsigned int nsecs;
2699 2700
	ktime_t kt;

2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718
	if (test_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags))
		return false;

	/*
	 * poll_nsec can be:
	 *
	 * -1:	don't ever hybrid sleep
	 *  0:	use half of prev avg
	 * >0:	use this specific value
	 */
	if (q->poll_nsec == -1)
		return false;
	else if (q->poll_nsec > 0)
		nsecs = q->poll_nsec;
	else
		nsecs = blk_mq_poll_nsecs(q, hctx, rq);

	if (!nsecs)
2719 2720 2721 2722 2723 2724 2725 2726
		return false;

	set_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);

	/*
	 * This will be replaced with the stats tracking code, using
	 * 'avg_completion_time / 2' as the pre-sleep target.
	 */
T
Thomas Gleixner 已提交
2727
	kt = nsecs;
2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749

	mode = HRTIMER_MODE_REL;
	hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode);
	hrtimer_set_expires(&hs.timer, kt);

	hrtimer_init_sleeper(&hs, current);
	do {
		if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
			break;
		set_current_state(TASK_UNINTERRUPTIBLE);
		hrtimer_start_expires(&hs.timer, mode);
		if (hs.task)
			io_schedule();
		hrtimer_cancel(&hs.timer);
		mode = HRTIMER_MODE_ABS;
	} while (hs.task && !signal_pending(current));

	__set_current_state(TASK_RUNNING);
	destroy_hrtimer_on_stack(&hs.timer);
	return true;
}

J
Jens Axboe 已提交
2750 2751 2752 2753 2754
static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq)
{
	struct request_queue *q = hctx->queue;
	long state;

2755 2756 2757 2758 2759 2760 2761
	/*
	 * If we sleep, have the caller restart the poll loop to reset
	 * the state. Like for the other success return cases, the
	 * caller is responsible for checking if the IO completed. If
	 * the IO isn't complete, we'll get called again and will go
	 * straight to the busy poll loop.
	 */
2762
	if (blk_mq_poll_hybrid_sleep(q, hctx, rq))
2763 2764
		return true;

J
Jens Axboe 已提交
2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807
	hctx->poll_considered++;

	state = current->state;
	while (!need_resched()) {
		int ret;

		hctx->poll_invoked++;

		ret = q->mq_ops->poll(hctx, rq->tag);
		if (ret > 0) {
			hctx->poll_success++;
			set_current_state(TASK_RUNNING);
			return true;
		}

		if (signal_pending_state(state, current))
			set_current_state(TASK_RUNNING);

		if (current->state == TASK_RUNNING)
			return true;
		if (ret < 0)
			break;
		cpu_relax();
	}

	return false;
}

bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie)
{
	struct blk_mq_hw_ctx *hctx;
	struct blk_plug *plug;
	struct request *rq;

	if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) ||
	    !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
		return false;

	plug = current->plug;
	if (plug)
		blk_flush_plug_list(plug, false);

	hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
2808 2809 2810 2811
	if (!blk_qc_t_is_internal(cookie))
		rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
	else
		rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
J
Jens Axboe 已提交
2812 2813 2814 2815 2816

	return __blk_mq_poll(hctx, rq);
}
EXPORT_SYMBOL_GPL(blk_mq_poll);

2817 2818 2819 2820 2821 2822 2823 2824 2825 2826
void blk_mq_disable_hotplug(void)
{
	mutex_lock(&all_q_mutex);
}

void blk_mq_enable_hotplug(void)
{
	mutex_unlock(&all_q_mutex);
}

2827 2828
static int __init blk_mq_init(void)
{
2829 2830
	blk_mq_debugfs_init();

2831 2832
	cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
				blk_mq_hctx_notify_dead);
2833

2834 2835 2836
	cpuhp_setup_state_nocalls(CPUHP_BLK_MQ_PREPARE, "block/mq:prepare",
				  blk_mq_queue_reinit_prepare,
				  blk_mq_queue_reinit_dead);
2837 2838 2839
	return 0;
}
subsys_initcall(blk_mq_init);