blk-mq.c 66.4 KB
Newer Older
1 2 3 4 5 6
/*
 * Block multiqueue core code
 *
 * Copyright (C) 2013-2014 Jens Axboe
 * Copyright (C) 2013-2014 Christoph Hellwig
 */
7 8 9 10 11
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
12
#include <linux/kmemleak.h>
13 14 15 16 17 18 19 20 21 22 23
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>
#include <linux/llist.h>
#include <linux/list_sort.h>
#include <linux/cpu.h>
#include <linux/cache.h>
#include <linux/sched/sysctl.h>
#include <linux/delay.h>
24
#include <linux/crash_dump.h>
25
#include <linux/prefetch.h>
26 27 28 29 30 31 32

#include <trace/events/block.h>

#include <linux/blk-mq.h>
#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-tag.h"
33
#include "blk-stat.h"
J
Jens Axboe 已提交
34
#include "blk-wbt.h"
35
#include "blk-mq-sched.h"
36 37 38 39 40 41 42 43 44

static DEFINE_MUTEX(all_q_mutex);
static LIST_HEAD(all_q_list);

/*
 * Check if any of the ctx's have pending work in this hardware queue
 */
static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
{
45 46 47
	return sbitmap_any_bit_set(&hctx->ctx_map) ||
			!list_empty_careful(&hctx->dispatch) ||
			blk_mq_sched_has_work(hctx);
48 49
}

50 51 52 53 54 55
/*
 * Mark this ctx as having pending work in this hardware queue
 */
static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
				     struct blk_mq_ctx *ctx)
{
56 57
	if (!sbitmap_test_bit(&hctx->ctx_map, ctx->index_hw))
		sbitmap_set_bit(&hctx->ctx_map, ctx->index_hw);
58 59 60 61 62
}

static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
				      struct blk_mq_ctx *ctx)
{
63
	sbitmap_clear_bit(&hctx->ctx_map, ctx->index_hw);
64 65
}

66
void blk_mq_freeze_queue_start(struct request_queue *q)
67
{
68
	int freeze_depth;
69

70 71
	freeze_depth = atomic_inc_return(&q->mq_freeze_depth);
	if (freeze_depth == 1) {
72
		percpu_ref_kill(&q->q_usage_counter);
73
		blk_mq_run_hw_queues(q, false);
74
	}
75
}
76
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_start);
77 78 79

static void blk_mq_freeze_queue_wait(struct request_queue *q)
{
80
	wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
81 82
}

83 84 85 86
/*
 * Guarantee no request is in use, so we can change any data structure of
 * the queue afterward.
 */
87
void blk_freeze_queue(struct request_queue *q)
88
{
89 90 91 92 93 94 95
	/*
	 * In the !blk_mq case we are only calling this to kill the
	 * q_usage_counter, otherwise this increases the freeze depth
	 * and waits for it to return to zero.  For this reason there is
	 * no blk_unfreeze_queue(), and blk_freeze_queue() is not
	 * exported to drivers as the only user for unfreeze is blk_mq.
	 */
96 97 98
	blk_mq_freeze_queue_start(q);
	blk_mq_freeze_queue_wait(q);
}
99 100 101 102 103 104 105 106 107

void blk_mq_freeze_queue(struct request_queue *q)
{
	/*
	 * ...just an alias to keep freeze and unfreeze actions balanced
	 * in the blk_mq_* namespace
	 */
	blk_freeze_queue(q);
}
108
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
109

110
void blk_mq_unfreeze_queue(struct request_queue *q)
111
{
112
	int freeze_depth;
113

114 115 116
	freeze_depth = atomic_dec_return(&q->mq_freeze_depth);
	WARN_ON_ONCE(freeze_depth < 0);
	if (!freeze_depth) {
117
		percpu_ref_reinit(&q->q_usage_counter);
118
		wake_up_all(&q->mq_freeze_wq);
119
	}
120
}
121
EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
122

123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
/**
 * blk_mq_quiesce_queue() - wait until all ongoing queue_rq calls have finished
 * @q: request queue.
 *
 * Note: this function does not prevent that the struct request end_io()
 * callback function is invoked. Additionally, it is not prevented that
 * new queue_rq() calls occur unless the queue has been stopped first.
 */
void blk_mq_quiesce_queue(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;
	bool rcu = false;

	blk_mq_stop_hw_queues(q);

	queue_for_each_hw_ctx(q, hctx, i) {
		if (hctx->flags & BLK_MQ_F_BLOCKING)
			synchronize_srcu(&hctx->queue_rq_srcu);
		else
			rcu = true;
	}
	if (rcu)
		synchronize_rcu();
}
EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);

150 151 152 153 154 155 156 157
void blk_mq_wake_waiters(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i)
		if (blk_mq_hw_queue_mapped(hctx))
			blk_mq_tag_wakeup_all(hctx->tags, true);
158 159 160 161 162 163 164

	/*
	 * If we are called because the queue has now been marked as
	 * dying, we need to ensure that processes currently waiting on
	 * the queue are notified as well.
	 */
	wake_up_all(&q->mq_freeze_wq);
165 166
}

167 168 169 170 171 172
bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
{
	return blk_mq_has_free_tags(hctx->tags);
}
EXPORT_SYMBOL(blk_mq_can_queue);

173 174
void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
			struct request *rq, unsigned int op)
175
{
176 177 178
	INIT_LIST_HEAD(&rq->queuelist);
	/* csd/requeue_work/fifo_time is initialized before use */
	rq->q = q;
179
	rq->mq_ctx = ctx;
180
	rq->cmd_flags = op;
181 182
	if (blk_queue_io_stat(q))
		rq->rq_flags |= RQF_IO_STAT;
183 184 185 186 187 188
	/* do not touch atomic flags, it needs atomic ops against the timer */
	rq->cpu = -1;
	INIT_HLIST_NODE(&rq->hash);
	RB_CLEAR_NODE(&rq->rb_node);
	rq->rq_disk = NULL;
	rq->part = NULL;
189
	rq->start_time = jiffies;
190 191
#ifdef CONFIG_BLK_CGROUP
	rq->rl = NULL;
192
	set_start_time_ns(rq);
193 194 195 196 197 198 199 200 201 202
	rq->io_start_time_ns = 0;
#endif
	rq->nr_phys_segments = 0;
#if defined(CONFIG_BLK_DEV_INTEGRITY)
	rq->nr_integrity_segments = 0;
#endif
	rq->special = NULL;
	/* tag was already set */
	rq->errors = 0;

203 204
	rq->cmd = rq->__cmd;

205 206 207 208 209 210
	rq->extra_len = 0;
	rq->sense_len = 0;
	rq->resid_len = 0;
	rq->sense = NULL;

	INIT_LIST_HEAD(&rq->timeout_list);
211 212
	rq->timeout = 0;

213 214 215 216
	rq->end_io = NULL;
	rq->end_io_data = NULL;
	rq->next_rq = NULL;

217
	ctx->rq_dispatched[op_is_sync(op)]++;
218
}
219
EXPORT_SYMBOL_GPL(blk_mq_rq_ctx_init);
220

221 222
struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data,
				       unsigned int op)
223 224 225 226
{
	struct request *rq;
	unsigned int tag;

227
	tag = blk_mq_get_tag(data);
228
	if (tag != BLK_MQ_TAG_FAIL) {
229 230 231
		struct blk_mq_tags *tags = blk_mq_tags_from_data(data);

		rq = tags->static_rqs[tag];
232

233
		if (blk_mq_tag_busy(data->hctx)) {
234
			rq->rq_flags = RQF_MQ_INFLIGHT;
235
			atomic_inc(&data->hctx->nr_active);
236 237
		}

238 239 240 241 242 243 244 245
		if (data->flags & BLK_MQ_REQ_INTERNAL) {
			rq->tag = -1;
			rq->internal_tag = tag;
		} else {
			rq->tag = tag;
			rq->internal_tag = -1;
		}

246
		blk_mq_rq_ctx_init(data->q, data->ctx, rq, op);
247 248 249 250 251
		return rq;
	}

	return NULL;
}
252
EXPORT_SYMBOL_GPL(__blk_mq_alloc_request);
253

254 255
struct request *blk_mq_alloc_request(struct request_queue *q, int rw,
		unsigned int flags)
256
{
257
	struct blk_mq_alloc_data alloc_data;
258
	struct request *rq;
259
	int ret;
260

261
	ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT);
262 263
	if (ret)
		return ERR_PTR(ret);
264

265
	rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data);
266

267 268 269 270
	blk_mq_put_ctx(alloc_data.ctx);
	blk_queue_exit(q);

	if (!rq)
271
		return ERR_PTR(-EWOULDBLOCK);
272 273 274 275

	rq->__data_len = 0;
	rq->__sector = (sector_t) -1;
	rq->bio = rq->biotail = NULL;
276 277
	return rq;
}
278
EXPORT_SYMBOL(blk_mq_alloc_request);
279

M
Ming Lin 已提交
280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw,
		unsigned int flags, unsigned int hctx_idx)
{
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
	struct request *rq;
	struct blk_mq_alloc_data alloc_data;
	int ret;

	/*
	 * If the tag allocator sleeps we could get an allocation for a
	 * different hardware context.  No need to complicate the low level
	 * allocator for this for the rare use case of a command tied to
	 * a specific queue.
	 */
	if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)))
		return ERR_PTR(-EINVAL);

	if (hctx_idx >= q->nr_hw_queues)
		return ERR_PTR(-EIO);

	ret = blk_queue_enter(q, true);
	if (ret)
		return ERR_PTR(ret);

305 306 307 308
	/*
	 * Check if the hardware context is actually mapped to anything.
	 * If not tell the caller that it should skip this queue.
	 */
M
Ming Lin 已提交
309
	hctx = q->queue_hw_ctx[hctx_idx];
310 311 312 313
	if (!blk_mq_hw_queue_mapped(hctx)) {
		ret = -EXDEV;
		goto out_queue_exit;
	}
M
Ming Lin 已提交
314 315 316
	ctx = __blk_mq_get_ctx(q, cpumask_first(hctx->cpumask));

	blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx);
317
	rq = __blk_mq_alloc_request(&alloc_data, rw);
M
Ming Lin 已提交
318
	if (!rq) {
319 320
		ret = -EWOULDBLOCK;
		goto out_queue_exit;
M
Ming Lin 已提交
321 322 323
	}

	return rq;
324 325 326 327

out_queue_exit:
	blk_queue_exit(q);
	return ERR_PTR(ret);
M
Ming Lin 已提交
328 329 330
}
EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);

331 332
void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
			     struct request *rq)
333
{
334
	const int sched_tag = rq->internal_tag;
335 336
	struct request_queue *q = rq->q;

337
	if (rq->rq_flags & RQF_MQ_INFLIGHT)
338
		atomic_dec(&hctx->nr_active);
J
Jens Axboe 已提交
339 340

	wbt_done(q->rq_wb, &rq->issue_stat);
341
	rq->rq_flags = 0;
342

343
	clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
344
	clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
345 346 347 348
	if (rq->tag != -1)
		blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
	if (sched_tag != -1)
		blk_mq_sched_completed_request(hctx, rq);
349
	blk_queue_exit(q);
350 351
}

352
static void blk_mq_finish_hctx_request(struct blk_mq_hw_ctx *hctx,
353
				     struct request *rq)
354 355 356 357
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;

	ctx->rq_completed[rq_is_sync(rq)]++;
358 359 360 361 362 363
	__blk_mq_finish_request(hctx, ctx, rq);
}

void blk_mq_finish_request(struct request *rq)
{
	blk_mq_finish_hctx_request(blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), rq);
364 365 366 367
}

void blk_mq_free_request(struct request *rq)
{
368
	blk_mq_sched_put_request(rq);
369
}
J
Jens Axboe 已提交
370
EXPORT_SYMBOL_GPL(blk_mq_free_request);
371

372
inline void __blk_mq_end_request(struct request *rq, int error)
373
{
M
Ming Lei 已提交
374 375
	blk_account_io_done(rq);

C
Christoph Hellwig 已提交
376
	if (rq->end_io) {
J
Jens Axboe 已提交
377
		wbt_done(rq->q->rq_wb, &rq->issue_stat);
378
		rq->end_io(rq, error);
C
Christoph Hellwig 已提交
379 380 381
	} else {
		if (unlikely(blk_bidi_rq(rq)))
			blk_mq_free_request(rq->next_rq);
382
		blk_mq_free_request(rq);
C
Christoph Hellwig 已提交
383
	}
384
}
385
EXPORT_SYMBOL(__blk_mq_end_request);
386

387
void blk_mq_end_request(struct request *rq, int error)
388 389 390
{
	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
		BUG();
391
	__blk_mq_end_request(rq, error);
392
}
393
EXPORT_SYMBOL(blk_mq_end_request);
394

395
static void __blk_mq_complete_request_remote(void *data)
396
{
397
	struct request *rq = data;
398

399
	rq->q->softirq_done_fn(rq);
400 401
}

402
static void blk_mq_ipi_complete_request(struct request *rq)
403 404
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;
C
Christoph Hellwig 已提交
405
	bool shared = false;
406 407
	int cpu;

C
Christoph Hellwig 已提交
408
	if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
409 410 411
		rq->q->softirq_done_fn(rq);
		return;
	}
412 413

	cpu = get_cpu();
C
Christoph Hellwig 已提交
414 415 416 417
	if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
		shared = cpus_share_cache(cpu, ctx->cpu);

	if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
418
		rq->csd.func = __blk_mq_complete_request_remote;
419 420
		rq->csd.info = rq;
		rq->csd.flags = 0;
421
		smp_call_function_single_async(ctx->cpu, &rq->csd);
422
	} else {
423
		rq->q->softirq_done_fn(rq);
424
	}
425 426
	put_cpu();
}
427

428 429 430 431 432 433 434 435 436 437 438 439 440 441 442
static void blk_mq_stat_add(struct request *rq)
{
	if (rq->rq_flags & RQF_STATS) {
		/*
		 * We could rq->mq_ctx here, but there's less of a risk
		 * of races if we have the completion event add the stats
		 * to the local software queue.
		 */
		struct blk_mq_ctx *ctx;

		ctx = __blk_mq_get_ctx(rq->q, raw_smp_processor_id());
		blk_stat_add(&ctx->stat[rq_data_dir(rq)], rq);
	}
}

443
static void __blk_mq_complete_request(struct request *rq)
444 445 446
{
	struct request_queue *q = rq->q;

447 448
	blk_mq_stat_add(rq);

449
	if (!q->softirq_done_fn)
450
		blk_mq_end_request(rq, rq->errors);
451 452 453 454
	else
		blk_mq_ipi_complete_request(rq);
}

455 456 457 458 459 460 461 462
/**
 * blk_mq_complete_request - end I/O on a request
 * @rq:		the request being processed
 *
 * Description:
 *	Ends all I/O on a request. It does not handle partial completions.
 *	The actual completion happens out-of-order, through a IPI handler.
 **/
463
void blk_mq_complete_request(struct request *rq, int error)
464
{
465 466 467
	struct request_queue *q = rq->q;

	if (unlikely(blk_should_fake_timeout(q)))
468
		return;
469 470
	if (!blk_mark_rq_complete(rq)) {
		rq->errors = error;
471
		__blk_mq_complete_request(rq);
472
	}
473 474
}
EXPORT_SYMBOL(blk_mq_complete_request);
475

476 477 478 479 480 481
int blk_mq_request_started(struct request *rq)
{
	return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
}
EXPORT_SYMBOL_GPL(blk_mq_request_started);

482
void blk_mq_start_request(struct request *rq)
483 484 485
{
	struct request_queue *q = rq->q;

486 487
	blk_mq_sched_started_request(rq);

488 489
	trace_block_rq_issue(q, rq);

C
Christoph Hellwig 已提交
490
	rq->resid_len = blk_rq_bytes(rq);
C
Christoph Hellwig 已提交
491 492
	if (unlikely(blk_bidi_rq(rq)))
		rq->next_rq->resid_len = blk_rq_bytes(rq->next_rq);
C
Christoph Hellwig 已提交
493

494 495 496
	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
		blk_stat_set_issue_time(&rq->issue_stat);
		rq->rq_flags |= RQF_STATS;
J
Jens Axboe 已提交
497
		wbt_issue(q->rq_wb, &rq->issue_stat);
498 499
	}

500
	blk_add_timer(rq);
501

502 503 504 505 506 507
	/*
	 * Ensure that ->deadline is visible before set the started
	 * flag and clear the completed flag.
	 */
	smp_mb__before_atomic();

508 509 510 511 512 513
	/*
	 * Mark us as started and clear complete. Complete might have been
	 * set if requeue raced with timeout, which then marked it as
	 * complete. So be sure to clear complete again when we start
	 * the request, otherwise we'll ignore the completion event.
	 */
514 515 516 517
	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
		set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
	if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
		clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
518 519 520 521 522 523 524 525 526

	if (q->dma_drain_size && blk_rq_bytes(rq)) {
		/*
		 * Make sure space for the drain appears.  We know we can do
		 * this because max_hw_segments has been adjusted to be one
		 * fewer than the device can handle.
		 */
		rq->nr_phys_segments++;
	}
527
}
528
EXPORT_SYMBOL(blk_mq_start_request);
529

530
static void __blk_mq_requeue_request(struct request *rq)
531 532 533 534
{
	struct request_queue *q = rq->q;

	trace_block_rq_requeue(q, rq);
J
Jens Axboe 已提交
535
	wbt_requeue(q->rq_wb, &rq->issue_stat);
536
	blk_mq_sched_requeue_request(rq);
537

538 539 540 541
	if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
		if (q->dma_drain_size && blk_rq_bytes(rq))
			rq->nr_phys_segments--;
	}
542 543
}

544
void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
545 546 547 548
{
	__blk_mq_requeue_request(rq);

	BUG_ON(blk_queued_rq(rq));
549
	blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
550 551 552
}
EXPORT_SYMBOL(blk_mq_requeue_request);

553 554 555
static void blk_mq_requeue_work(struct work_struct *work)
{
	struct request_queue *q =
556
		container_of(work, struct request_queue, requeue_work.work);
557 558 559 560 561 562 563 564 565
	LIST_HEAD(rq_list);
	struct request *rq, *next;
	unsigned long flags;

	spin_lock_irqsave(&q->requeue_lock, flags);
	list_splice_init(&q->requeue_list, &rq_list);
	spin_unlock_irqrestore(&q->requeue_lock, flags);

	list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
566
		if (!(rq->rq_flags & RQF_SOFTBARRIER))
567 568
			continue;

569
		rq->rq_flags &= ~RQF_SOFTBARRIER;
570
		list_del_init(&rq->queuelist);
571
		blk_mq_sched_insert_request(rq, true, false, false);
572 573 574 575 576
	}

	while (!list_empty(&rq_list)) {
		rq = list_entry(rq_list.next, struct request, queuelist);
		list_del_init(&rq->queuelist);
577
		blk_mq_sched_insert_request(rq, false, false, false);
578 579
	}

580
	blk_mq_run_hw_queues(q, false);
581 582
}

583 584
void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
				bool kick_requeue_list)
585 586 587 588 589 590 591 592
{
	struct request_queue *q = rq->q;
	unsigned long flags;

	/*
	 * We abuse this flag that is otherwise used by the I/O scheduler to
	 * request head insertation from the workqueue.
	 */
593
	BUG_ON(rq->rq_flags & RQF_SOFTBARRIER);
594 595 596

	spin_lock_irqsave(&q->requeue_lock, flags);
	if (at_head) {
597
		rq->rq_flags |= RQF_SOFTBARRIER;
598 599 600 601 602
		list_add(&rq->queuelist, &q->requeue_list);
	} else {
		list_add_tail(&rq->queuelist, &q->requeue_list);
	}
	spin_unlock_irqrestore(&q->requeue_lock, flags);
603 604 605

	if (kick_requeue_list)
		blk_mq_kick_requeue_list(q);
606 607 608 609 610
}
EXPORT_SYMBOL(blk_mq_add_to_requeue_list);

void blk_mq_kick_requeue_list(struct request_queue *q)
{
611
	kblockd_schedule_delayed_work(&q->requeue_work, 0);
612 613 614
}
EXPORT_SYMBOL(blk_mq_kick_requeue_list);

615 616 617 618 619 620 621 622
void blk_mq_delay_kick_requeue_list(struct request_queue *q,
				    unsigned long msecs)
{
	kblockd_schedule_delayed_work(&q->requeue_work,
				      msecs_to_jiffies(msecs));
}
EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);

623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642
void blk_mq_abort_requeue_list(struct request_queue *q)
{
	unsigned long flags;
	LIST_HEAD(rq_list);

	spin_lock_irqsave(&q->requeue_lock, flags);
	list_splice_init(&q->requeue_list, &rq_list);
	spin_unlock_irqrestore(&q->requeue_lock, flags);

	while (!list_empty(&rq_list)) {
		struct request *rq;

		rq = list_first_entry(&rq_list, struct request, queuelist);
		list_del_init(&rq->queuelist);
		rq->errors = -EIO;
		blk_mq_end_request(rq, rq->errors);
	}
}
EXPORT_SYMBOL(blk_mq_abort_requeue_list);

643 644
struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
{
645 646
	if (tag < tags->nr_tags) {
		prefetch(tags->rqs[tag]);
647
		return tags->rqs[tag];
648
	}
649 650

	return NULL;
651 652 653
}
EXPORT_SYMBOL(blk_mq_tag_to_rq);

654
struct blk_mq_timeout_data {
655 656
	unsigned long next;
	unsigned int next_set;
657 658
};

659
void blk_mq_rq_timed_out(struct request *req, bool reserved)
660
{
J
Jens Axboe 已提交
661
	const struct blk_mq_ops *ops = req->q->mq_ops;
662
	enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
663 664 665 666 667 668 669 670 671 672

	/*
	 * We know that complete is set at this point. If STARTED isn't set
	 * anymore, then the request isn't active and the "timeout" should
	 * just be ignored. This can happen due to the bitflag ordering.
	 * Timeout first checks if STARTED is set, and if it is, assumes
	 * the request is active. But if we race with completion, then
	 * we both flags will get cleared. So check here again, and ignore
	 * a timeout event with a request that isn't active.
	 */
673 674
	if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
		return;
675

676
	if (ops->timeout)
677
		ret = ops->timeout(req, reserved);
678 679 680 681 682 683 684 685 686 687 688 689 690 691 692

	switch (ret) {
	case BLK_EH_HANDLED:
		__blk_mq_complete_request(req);
		break;
	case BLK_EH_RESET_TIMER:
		blk_add_timer(req);
		blk_clear_rq_complete(req);
		break;
	case BLK_EH_NOT_HANDLED:
		break;
	default:
		printk(KERN_ERR "block: bad eh return: %d\n", ret);
		break;
	}
693
}
694

695 696 697 698
static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
		struct request *rq, void *priv, bool reserved)
{
	struct blk_mq_timeout_data *data = priv;
699

700 701 702 703 704
	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
		/*
		 * If a request wasn't started before the queue was
		 * marked dying, kill it here or it'll go unnoticed.
		 */
705 706 707 708
		if (unlikely(blk_queue_dying(rq->q))) {
			rq->errors = -EIO;
			blk_mq_end_request(rq, rq->errors);
		}
709
		return;
710
	}
711

712 713
	if (time_after_eq(jiffies, rq->deadline)) {
		if (!blk_mark_rq_complete(rq))
714
			blk_mq_rq_timed_out(rq, reserved);
715 716 717 718
	} else if (!data->next_set || time_after(data->next, rq->deadline)) {
		data->next = rq->deadline;
		data->next_set = 1;
	}
719 720
}

721
static void blk_mq_timeout_work(struct work_struct *work)
722
{
723 724
	struct request_queue *q =
		container_of(work, struct request_queue, timeout_work);
725 726 727 728 729
	struct blk_mq_timeout_data data = {
		.next		= 0,
		.next_set	= 0,
	};
	int i;
730

731 732 733 734 735 736 737 738 739 740 741 742 743 744
	/* A deadlock might occur if a request is stuck requiring a
	 * timeout at the same time a queue freeze is waiting
	 * completion, since the timeout code would not be able to
	 * acquire the queue reference here.
	 *
	 * That's why we don't use blk_queue_enter here; instead, we use
	 * percpu_ref_tryget directly, because we need to be able to
	 * obtain a reference even in the short window between the queue
	 * starting to freeze, by dropping the first reference in
	 * blk_mq_freeze_queue_start, and the moment the last request is
	 * consumed, marked by the instant q_usage_counter reaches
	 * zero.
	 */
	if (!percpu_ref_tryget(&q->q_usage_counter))
745 746
		return;

747
	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data);
748

749 750 751
	if (data.next_set) {
		data.next = blk_rq_timeout(round_jiffies_up(data.next));
		mod_timer(&q->timeout, data.next);
752
	} else {
753 754
		struct blk_mq_hw_ctx *hctx;

755 756 757 758 759
		queue_for_each_hw_ctx(q, hctx, i) {
			/* the hctx may be unmapped, so check it here */
			if (blk_mq_hw_queue_mapped(hctx))
				blk_mq_tag_idle(hctx);
		}
760
	}
761
	blk_queue_exit(q);
762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784
}

/*
 * Reverse check our software queue for entries that we could potentially
 * merge with. Currently includes a hand-wavy stop count of 8, to not spend
 * too much time checking for merges.
 */
static bool blk_mq_attempt_merge(struct request_queue *q,
				 struct blk_mq_ctx *ctx, struct bio *bio)
{
	struct request *rq;
	int checked = 8;

	list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
		int el_ret;

		if (!checked--)
			break;

		if (!blk_rq_merge_ok(rq, bio))
			continue;

		el_ret = blk_try_merge(rq, bio);
785 786 787 788 789 790
		if (el_ret == ELEVATOR_NO_MERGE)
			continue;

		if (!blk_mq_sched_allow_merge(q, rq, bio))
			break;

791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808
		if (el_ret == ELEVATOR_BACK_MERGE) {
			if (bio_attempt_back_merge(q, rq, bio)) {
				ctx->rq_merged++;
				return true;
			}
			break;
		} else if (el_ret == ELEVATOR_FRONT_MERGE) {
			if (bio_attempt_front_merge(q, rq, bio)) {
				ctx->rq_merged++;
				return true;
			}
			break;
		}
	}

	return false;
}

809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826
struct flush_busy_ctx_data {
	struct blk_mq_hw_ctx *hctx;
	struct list_head *list;
};

static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
{
	struct flush_busy_ctx_data *flush_data = data;
	struct blk_mq_hw_ctx *hctx = flush_data->hctx;
	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];

	sbitmap_clear_bit(sb, bitnr);
	spin_lock(&ctx->lock);
	list_splice_tail_init(&ctx->rq_list, flush_data->list);
	spin_unlock(&ctx->lock);
	return true;
}

827 828 829 830
/*
 * Process software queues that have been marked busy, splicing them
 * to the for-dispatch
 */
831
void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
832
{
833 834 835 836
	struct flush_busy_ctx_data data = {
		.hctx = hctx,
		.list = list,
	};
837

838
	sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
839
}
840
EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
841

842 843 844 845
static inline unsigned int queued_to_index(unsigned int queued)
{
	if (!queued)
		return 0;
846

847
	return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
848 849
}

850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902
static bool blk_mq_get_driver_tag(struct request *rq,
				  struct blk_mq_hw_ctx **hctx, bool wait)
{
	struct blk_mq_alloc_data data = {
		.q = rq->q,
		.ctx = rq->mq_ctx,
		.hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu),
		.flags = wait ? 0 : BLK_MQ_REQ_NOWAIT,
	};

	if (blk_mq_hctx_stopped(data.hctx))
		return false;

	if (rq->tag != -1) {
done:
		if (hctx)
			*hctx = data.hctx;
		return true;
	}

	rq->tag = blk_mq_get_tag(&data);
	if (rq->tag >= 0) {
		data.hctx->tags->rqs[rq->tag] = rq;
		goto done;
	}

	return false;
}

/*
 * If we fail getting a driver tag because all the driver tags are already
 * assigned and on the dispatch list, BUT the first entry does not have a
 * tag, then we could deadlock. For that case, move entries with assigned
 * driver tags to the front, leaving the set of tagged requests in the
 * same order, and the untagged set in the same order.
 */
static bool reorder_tags_to_front(struct list_head *list)
{
	struct request *rq, *tmp, *first = NULL;

	list_for_each_entry_safe_reverse(rq, tmp, list, queuelist) {
		if (rq == first)
			break;
		if (rq->tag != -1) {
			list_move(&rq->queuelist, list);
			if (!first)
				first = rq;
		}
	}

	return first != NULL;
}

903
bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list)
904 905 906
{
	struct request_queue *q = hctx->queue;
	struct request *rq;
907 908
	LIST_HEAD(driver_list);
	struct list_head *dptr;
909
	int queued, ret = BLK_MQ_RQ_QUEUE_OK;
910

911 912 913 914 915 916
	/*
	 * Start off with dptr being NULL, so we start the first request
	 * immediately, even if we have more pending.
	 */
	dptr = NULL;

917 918 919
	/*
	 * Now process all the entries, sending them to the driver.
	 */
920
	queued = 0;
921
	while (!list_empty(list)) {
922
		struct blk_mq_queue_data bd;
923

924
		rq = list_first_entry(list, struct request, queuelist);
925 926 927 928 929 930
		if (!blk_mq_get_driver_tag(rq, &hctx, false)) {
			if (!queued && reorder_tags_to_front(list))
				continue;
			blk_mq_sched_mark_restart(hctx);
			break;
		}
931 932
		list_del_init(&rq->queuelist);

933 934
		bd.rq = rq;
		bd.list = dptr;
935
		bd.last = list_empty(list);
936 937

		ret = q->mq_ops->queue_rq(hctx, &bd);
938 939 940
		switch (ret) {
		case BLK_MQ_RQ_QUEUE_OK:
			queued++;
941
			break;
942
		case BLK_MQ_RQ_QUEUE_BUSY:
943
			list_add(&rq->queuelist, list);
944
			__blk_mq_requeue_request(rq);
945 946 947 948
			break;
		default:
			pr_err("blk-mq: bad return on queue: %d\n", ret);
		case BLK_MQ_RQ_QUEUE_ERROR:
949
			rq->errors = -EIO;
950
			blk_mq_end_request(rq, rq->errors);
951 952 953 954 955
			break;
		}

		if (ret == BLK_MQ_RQ_QUEUE_BUSY)
			break;
956 957 958 959 960

		/*
		 * We've done the first request. If we have more than 1
		 * left in the list, set dptr to defer issue.
		 */
961
		if (!dptr && list->next != list->prev)
962
			dptr = &driver_list;
963 964
	}

965
	hctx->dispatched[queued_to_index(queued)]++;
966 967 968 969 970

	/*
	 * Any items that need requeuing? Stuff them into hctx->dispatch,
	 * that is where we will continue on next queue run.
	 */
971
	if (!list_empty(list)) {
972
		spin_lock(&hctx->lock);
973
		list_splice(list, &hctx->dispatch);
974
		spin_unlock(&hctx->lock);
975

976 977 978 979 980 981 982 983
		/*
		 * the queue is expected stopped with BLK_MQ_RQ_QUEUE_BUSY, but
		 * it's possible the queue is stopped and restarted again
		 * before this. Queue restart will dispatch requests. And since
		 * requests in rq_list aren't added into hctx->dispatch yet,
		 * the requests in rq_list might get lost.
		 *
		 * blk_mq_run_hw_queue() already checks the STOPPED bit
984 985 986 987 988 989
		 *
		 * If RESTART is set, then let completion restart the queue
		 * instead of potentially looping here.
		 */
		if (!blk_mq_sched_needs_restart(hctx))
			blk_mq_run_hw_queue(hctx, true);
990
	}
991 992 993 994

	return ret != BLK_MQ_RQ_QUEUE_BUSY;
}

995 996 997 998 999 1000 1001 1002 1003
static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	int srcu_idx;

	WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) &&
		cpu_online(hctx->next_cpu));

	if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
		rcu_read_lock();
1004
		blk_mq_sched_dispatch_requests(hctx);
1005 1006 1007
		rcu_read_unlock();
	} else {
		srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu);
1008
		blk_mq_sched_dispatch_requests(hctx);
1009 1010 1011 1012
		srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx);
	}
}

1013 1014 1015 1016 1017 1018 1019 1020
/*
 * It'd be great if the workqueue API had a way to pass
 * in a mask and had some smarts for more clever placement.
 * For now we just round-robin here, switching for every
 * BLK_MQ_CPU_WORK_BATCH queued items.
 */
static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
{
1021 1022
	if (hctx->queue->nr_hw_queues == 1)
		return WORK_CPU_UNBOUND;
1023 1024

	if (--hctx->next_cpu_batch <= 0) {
1025
		int next_cpu;
1026 1027 1028 1029 1030 1031 1032 1033 1034

		next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
		if (next_cpu >= nr_cpu_ids)
			next_cpu = cpumask_first(hctx->cpumask);

		hctx->next_cpu = next_cpu;
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}

1035
	return hctx->next_cpu;
1036 1037
}

1038 1039
void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
1040 1041
	if (unlikely(blk_mq_hctx_stopped(hctx) ||
		     !blk_mq_hw_queue_mapped(hctx)))
1042 1043
		return;

1044
	if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
1045 1046
		int cpu = get_cpu();
		if (cpumask_test_cpu(cpu, hctx->cpumask)) {
1047
			__blk_mq_run_hw_queue(hctx);
1048
			put_cpu();
1049 1050
			return;
		}
1051

1052
		put_cpu();
1053
	}
1054

1055
	kblockd_schedule_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work);
1056 1057
}

1058
void blk_mq_run_hw_queues(struct request_queue *q, bool async)
1059 1060 1061 1062 1063
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i) {
1064
		if (!blk_mq_hctx_has_pending(hctx) ||
1065
		    blk_mq_hctx_stopped(hctx))
1066 1067
			continue;

1068
		blk_mq_run_hw_queue(hctx, async);
1069 1070
	}
}
1071
EXPORT_SYMBOL(blk_mq_run_hw_queues);
1072

1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
/**
 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped
 * @q: request queue.
 *
 * The caller is responsible for serializing this function against
 * blk_mq_{start,stop}_hw_queue().
 */
bool blk_mq_queue_stopped(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		if (blk_mq_hctx_stopped(hctx))
			return true;

	return false;
}
EXPORT_SYMBOL(blk_mq_queue_stopped);

1093 1094
void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
{
1095
	cancel_work(&hctx->run_work);
1096
	cancel_delayed_work(&hctx->delay_work);
1097 1098 1099 1100
	set_bit(BLK_MQ_S_STOPPED, &hctx->state);
}
EXPORT_SYMBOL(blk_mq_stop_hw_queue);

1101 1102 1103 1104 1105 1106 1107 1108 1109 1110
void blk_mq_stop_hw_queues(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_stop_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_stop_hw_queues);

1111 1112 1113
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1114

1115
	blk_mq_run_hw_queue(hctx, false);
1116 1117 1118
}
EXPORT_SYMBOL(blk_mq_start_hw_queue);

1119 1120 1121 1122 1123 1124 1125 1126 1127 1128
void blk_mq_start_hw_queues(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_start_hw_queues);

1129 1130 1131 1132 1133 1134 1135 1136 1137 1138
void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
	if (!blk_mq_hctx_stopped(hctx))
		return;

	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
	blk_mq_run_hw_queue(hctx, async);
}
EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);

1139
void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
1140 1141 1142 1143
{
	struct blk_mq_hw_ctx *hctx;
	int i;

1144 1145
	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_stopped_hw_queue(hctx, async);
1146 1147 1148
}
EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);

1149
static void blk_mq_run_work_fn(struct work_struct *work)
1150 1151 1152
{
	struct blk_mq_hw_ctx *hctx;

1153
	hctx = container_of(work, struct blk_mq_hw_ctx, run_work);
1154

1155 1156 1157
	__blk_mq_run_hw_queue(hctx);
}

1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169
static void blk_mq_delay_work_fn(struct work_struct *work)
{
	struct blk_mq_hw_ctx *hctx;

	hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);

	if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
		__blk_mq_run_hw_queue(hctx);
}

void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
{
1170 1171
	if (unlikely(!blk_mq_hw_queue_mapped(hctx)))
		return;
1172

1173
	blk_mq_stop_hw_queue(hctx);
1174 1175
	kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
			&hctx->delay_work, msecs_to_jiffies(msecs));
1176 1177 1178
}
EXPORT_SYMBOL(blk_mq_delay_queue);

1179 1180 1181
static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,
					    struct request *rq,
					    bool at_head)
1182
{
J
Jens Axboe 已提交
1183 1184
	struct blk_mq_ctx *ctx = rq->mq_ctx;

1185 1186
	trace_block_rq_insert(hctx->queue, rq);

1187 1188 1189 1190
	if (at_head)
		list_add(&rq->queuelist, &ctx->rq_list);
	else
		list_add_tail(&rq->queuelist, &ctx->rq_list);
1191
}
1192

1193 1194
void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
			     bool at_head)
1195 1196 1197
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;

J
Jens Axboe 已提交
1198
	__blk_mq_insert_req_list(hctx, rq, at_head);
1199 1200 1201
	blk_mq_hctx_mark_pending(hctx, ctx);
}

1202 1203
void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
			    struct list_head *list)
1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214

{
	/*
	 * preemption doesn't flush plug list, so it's possible ctx->cpu is
	 * offline now
	 */
	spin_lock(&ctx->lock);
	while (!list_empty(list)) {
		struct request *rq;

		rq = list_first_entry(list, struct request, queuelist);
J
Jens Axboe 已提交
1215
		BUG_ON(rq->mq_ctx != ctx);
1216
		list_del_init(&rq->queuelist);
J
Jens Axboe 已提交
1217
		__blk_mq_insert_req_list(hctx, rq, false);
1218
	}
1219
	blk_mq_hctx_mark_pending(hctx, ctx);
1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255
	spin_unlock(&ctx->lock);
}

static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
{
	struct request *rqa = container_of(a, struct request, queuelist);
	struct request *rqb = container_of(b, struct request, queuelist);

	return !(rqa->mq_ctx < rqb->mq_ctx ||
		 (rqa->mq_ctx == rqb->mq_ctx &&
		  blk_rq_pos(rqa) < blk_rq_pos(rqb)));
}

void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
{
	struct blk_mq_ctx *this_ctx;
	struct request_queue *this_q;
	struct request *rq;
	LIST_HEAD(list);
	LIST_HEAD(ctx_list);
	unsigned int depth;

	list_splice_init(&plug->mq_list, &list);

	list_sort(NULL, &list, plug_ctx_cmp);

	this_q = NULL;
	this_ctx = NULL;
	depth = 0;

	while (!list_empty(&list)) {
		rq = list_entry_rq(list.next);
		list_del_init(&rq->queuelist);
		BUG_ON(!rq->q);
		if (rq->mq_ctx != this_ctx) {
			if (this_ctx) {
1256 1257 1258 1259
				trace_block_unplug(this_q, depth, from_schedule);
				blk_mq_sched_insert_requests(this_q, this_ctx,
								&ctx_list,
								from_schedule);
1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275
			}

			this_ctx = rq->mq_ctx;
			this_q = rq->q;
			depth = 0;
		}

		depth++;
		list_add_tail(&rq->queuelist, &ctx_list);
	}

	/*
	 * If 'this_ctx' is set, we know we have entries to complete
	 * on 'ctx_list'. Do those.
	 */
	if (this_ctx) {
1276 1277 1278
		trace_block_unplug(this_q, depth, from_schedule);
		blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list,
						from_schedule);
1279 1280 1281 1282 1283 1284
	}
}

static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
{
	init_request_from_bio(rq, bio);
1285

1286
	blk_account_io_start(rq, true);
1287 1288
}

1289 1290 1291 1292 1293 1294
static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx)
{
	return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
		!blk_queue_nomerges(hctx->queue);
}

1295 1296 1297
static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
					 struct blk_mq_ctx *ctx,
					 struct request *rq, struct bio *bio)
1298
{
1299
	if (!hctx_allow_merges(hctx) || !bio_mergeable(bio)) {
1300 1301 1302 1303 1304 1305 1306
		blk_mq_bio_to_request(rq, bio);
		spin_lock(&ctx->lock);
insert_rq:
		__blk_mq_insert_request(hctx, rq, false);
		spin_unlock(&ctx->lock);
		return false;
	} else {
1307 1308
		struct request_queue *q = hctx->queue;

1309 1310 1311 1312 1313
		spin_lock(&ctx->lock);
		if (!blk_mq_attempt_merge(q, ctx, bio)) {
			blk_mq_bio_to_request(rq, bio);
			goto insert_rq;
		}
1314

1315
		spin_unlock(&ctx->lock);
1316
		__blk_mq_finish_request(hctx, ctx, rq);
1317
		return true;
1318
	}
1319
}
1320

1321 1322
static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq)
{
1323 1324 1325 1326
	if (rq->tag != -1)
		return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false);

	return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true);
1327 1328
}

1329
static void blk_mq_try_issue_directly(struct request *rq, blk_qc_t *cookie)
1330 1331 1332 1333 1334 1335 1336
{
	struct request_queue *q = rq->q;
	struct blk_mq_queue_data bd = {
		.rq = rq,
		.list = NULL,
		.last = 1
	};
1337 1338 1339
	struct blk_mq_hw_ctx *hctx;
	blk_qc_t new_cookie;
	int ret;
1340

1341
	if (q->elevator)
1342 1343
		goto insert;

1344 1345 1346 1347 1348
	if (!blk_mq_get_driver_tag(rq, &hctx, false))
		goto insert;

	new_cookie = request_to_qc_t(hctx, rq);

1349 1350 1351 1352 1353 1354
	/*
	 * For OK queue, we are done. For error, kill it. Any other
	 * error (busy), just add it to our list as we previously
	 * would have done
	 */
	ret = q->mq_ops->queue_rq(hctx, &bd);
1355 1356
	if (ret == BLK_MQ_RQ_QUEUE_OK) {
		*cookie = new_cookie;
1357
		return;
1358
	}
1359

1360 1361 1362 1363 1364 1365
	__blk_mq_requeue_request(rq);

	if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
		*cookie = BLK_QC_T_NONE;
		rq->errors = -EIO;
		blk_mq_end_request(rq, rq->errors);
1366
		return;
1367
	}
1368

1369
insert:
1370
	blk_mq_sched_insert_request(rq, false, true, true);
1371 1372
}

1373 1374 1375 1376 1377
/*
 * Multiple hardware queue variant. This will not use per-process plugs,
 * but will attempt to bypass the hctx queueing if we can go straight to
 * hardware for SYNC IO.
 */
1378
static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
1379
{
1380
	const int is_sync = op_is_sync(bio->bi_opf);
J
Jens Axboe 已提交
1381
	const int is_flush_fua = bio->bi_opf & (REQ_PREFLUSH | REQ_FUA);
1382
	struct blk_mq_alloc_data data;
1383
	struct request *rq;
1384
	unsigned int request_count = 0, srcu_idx;
1385
	struct blk_plug *plug;
1386
	struct request *same_queue_rq = NULL;
1387
	blk_qc_t cookie;
J
Jens Axboe 已提交
1388
	unsigned int wb_acct;
1389 1390 1391 1392

	blk_queue_bounce(q, &bio);

	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1393
		bio_io_error(bio);
1394
		return BLK_QC_T_NONE;
1395 1396
	}

1397 1398
	blk_queue_split(q, &bio, q->bio_split);

1399 1400 1401
	if (!is_flush_fua && !blk_queue_nomerges(q) &&
	    blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq))
		return BLK_QC_T_NONE;
1402

1403 1404 1405
	if (blk_mq_sched_bio_merge(q, bio))
		return BLK_QC_T_NONE;

J
Jens Axboe 已提交
1406 1407
	wb_acct = wbt_wait(q->rq_wb, bio, NULL);

1408 1409 1410
	trace_block_getrq(q, bio, bio->bi_opf);

	rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
J
Jens Axboe 已提交
1411 1412
	if (unlikely(!rq)) {
		__wbt_done(q->rq_wb, wb_acct);
1413
		return BLK_QC_T_NONE;
J
Jens Axboe 已提交
1414 1415 1416
	}

	wbt_track(&rq->issue_stat, wb_acct);
1417

1418
	cookie = request_to_qc_t(data.hctx, rq);
1419 1420 1421

	if (unlikely(is_flush_fua)) {
		blk_mq_bio_to_request(rq, bio);
1422
		blk_mq_get_driver_tag(rq, NULL, true);
1423 1424 1425 1426
		blk_insert_flush(rq);
		goto run_queue;
	}

1427
	plug = current->plug;
1428 1429 1430 1431 1432
	/*
	 * If the driver supports defer issued based on 'last', then
	 * queue it up like normal since we can potentially save some
	 * CPU this way.
	 */
1433 1434 1435
	if (((plug && !blk_queue_nomerges(q)) || is_sync) &&
	    !(data.hctx->flags & BLK_MQ_F_DEFER_ISSUE)) {
		struct request *old_rq = NULL;
1436 1437 1438 1439

		blk_mq_bio_to_request(rq, bio);

		/*
1440
		 * We do limited plugging. If the bio can be merged, do that.
1441 1442
		 * Otherwise the existing request in the plug list will be
		 * issued. So the plug list will have one request at most
1443
		 */
1444
		if (plug) {
1445 1446
			/*
			 * The plug list might get flushed before this. If that
1447 1448 1449
			 * happens, same_queue_rq is invalid and plug list is
			 * empty
			 */
1450 1451
			if (same_queue_rq && !list_empty(&plug->mq_list)) {
				old_rq = same_queue_rq;
1452
				list_del_init(&old_rq->queuelist);
1453
			}
1454 1455 1456 1457 1458
			list_add_tail(&rq->queuelist, &plug->mq_list);
		} else /* is_sync */
			old_rq = rq;
		blk_mq_put_ctx(data.ctx);
		if (!old_rq)
1459
			goto done;
1460 1461 1462

		if (!(data.hctx->flags & BLK_MQ_F_BLOCKING)) {
			rcu_read_lock();
1463
			blk_mq_try_issue_directly(old_rq, &cookie);
1464 1465 1466
			rcu_read_unlock();
		} else {
			srcu_idx = srcu_read_lock(&data.hctx->queue_rq_srcu);
1467
			blk_mq_try_issue_directly(old_rq, &cookie);
1468 1469
			srcu_read_unlock(&data.hctx->queue_rq_srcu, srcu_idx);
		}
1470
		goto done;
1471 1472
	}

1473 1474 1475 1476 1477 1478
	if (q->elevator) {
		blk_mq_put_ctx(data.ctx);
		blk_mq_bio_to_request(rq, bio);
		blk_mq_sched_insert_request(rq, false, true, true);
		goto done;
	}
1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489
	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
		/*
		 * For a SYNC request, send it to the hardware immediately. For
		 * an ASYNC request, just ensure that we run it later on. The
		 * latter allows for merging opportunities and more efficient
		 * dispatching.
		 */
run_queue:
		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
	}
	blk_mq_put_ctx(data.ctx);
1490 1491
done:
	return cookie;
1492 1493 1494 1495 1496 1497
}

/*
 * Single hardware queue variant. This will attempt to use any per-process
 * plug for merging and IO deferral.
 */
1498
static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio)
1499
{
1500
	const int is_sync = op_is_sync(bio->bi_opf);
J
Jens Axboe 已提交
1501
	const int is_flush_fua = bio->bi_opf & (REQ_PREFLUSH | REQ_FUA);
1502 1503
	struct blk_plug *plug;
	unsigned int request_count = 0;
1504
	struct blk_mq_alloc_data data;
1505
	struct request *rq;
1506
	blk_qc_t cookie;
J
Jens Axboe 已提交
1507
	unsigned int wb_acct;
1508 1509 1510 1511

	blk_queue_bounce(q, &bio);

	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1512
		bio_io_error(bio);
1513
		return BLK_QC_T_NONE;
1514 1515
	}

1516 1517
	blk_queue_split(q, &bio, q->bio_split);

1518 1519 1520 1521 1522
	if (!is_flush_fua && !blk_queue_nomerges(q)) {
		if (blk_attempt_plug_merge(q, bio, &request_count, NULL))
			return BLK_QC_T_NONE;
	} else
		request_count = blk_plug_queued_count(q);
1523

1524 1525 1526
	if (blk_mq_sched_bio_merge(q, bio))
		return BLK_QC_T_NONE;

J
Jens Axboe 已提交
1527 1528
	wb_acct = wbt_wait(q->rq_wb, bio, NULL);

1529 1530 1531
	trace_block_getrq(q, bio, bio->bi_opf);

	rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
J
Jens Axboe 已提交
1532 1533
	if (unlikely(!rq)) {
		__wbt_done(q->rq_wb, wb_acct);
1534
		return BLK_QC_T_NONE;
J
Jens Axboe 已提交
1535 1536 1537
	}

	wbt_track(&rq->issue_stat, wb_acct);
1538

1539
	cookie = request_to_qc_t(data.hctx, rq);
1540 1541 1542

	if (unlikely(is_flush_fua)) {
		blk_mq_bio_to_request(rq, bio);
1543
		blk_mq_get_driver_tag(rq, NULL, true);
1544 1545 1546 1547 1548 1549 1550 1551 1552
		blk_insert_flush(rq);
		goto run_queue;
	}

	/*
	 * A task plug currently exists. Since this is completely lockless,
	 * utilize that to temporarily store requests until the task is
	 * either done or scheduled away.
	 */
1553 1554
	plug = current->plug;
	if (plug) {
1555 1556
		struct request *last = NULL;

1557
		blk_mq_bio_to_request(rq, bio);
1558 1559 1560 1561 1562 1563 1564

		/*
		 * @request_count may become stale because of schedule
		 * out, so check the list again.
		 */
		if (list_empty(&plug->mq_list))
			request_count = 0;
M
Ming Lei 已提交
1565
		if (!request_count)
1566
			trace_block_plug(q);
1567 1568
		else
			last = list_entry_rq(plug->mq_list.prev);
1569 1570 1571

		blk_mq_put_ctx(data.ctx);

1572 1573
		if (request_count >= BLK_MAX_REQUEST_COUNT || (last &&
		    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
1574 1575
			blk_flush_plug_list(plug, false);
			trace_block_plug(q);
1576
		}
1577

1578
		list_add_tail(&rq->queuelist, &plug->mq_list);
1579
		return cookie;
1580 1581
	}

1582 1583 1584 1585 1586 1587
	if (q->elevator) {
		blk_mq_put_ctx(data.ctx);
		blk_mq_bio_to_request(rq, bio);
		blk_mq_sched_insert_request(rq, false, true, true);
		goto done;
	}
1588 1589 1590 1591 1592 1593 1594 1595 1596
	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
		/*
		 * For a SYNC request, send it to the hardware immediately. For
		 * an ASYNC request, just ensure that we run it later on. The
		 * latter allows for merging opportunities and more efficient
		 * dispatching.
		 */
run_queue:
		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1597 1598
	}

1599
	blk_mq_put_ctx(data.ctx);
1600
done:
1601
	return cookie;
1602 1603
}

1604 1605
void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx)
1606
{
1607
	struct page *page;
1608

1609
	if (tags->rqs && set->ops->exit_request) {
1610
		int i;
1611

1612
		for (i = 0; i < tags->nr_tags; i++) {
J
Jens Axboe 已提交
1613 1614 1615
			struct request *rq = tags->static_rqs[i];

			if (!rq)
1616
				continue;
J
Jens Axboe 已提交
1617
			set->ops->exit_request(set->driver_data, rq,
1618
						hctx_idx, i);
J
Jens Axboe 已提交
1619
			tags->static_rqs[i] = NULL;
1620
		}
1621 1622
	}

1623 1624
	while (!list_empty(&tags->page_list)) {
		page = list_first_entry(&tags->page_list, struct page, lru);
1625
		list_del_init(&page->lru);
1626 1627 1628 1629 1630
		/*
		 * Remove kmemleak object previously allocated in
		 * blk_mq_init_rq_map().
		 */
		kmemleak_free(page_address(page));
1631 1632
		__free_pages(page, page->private);
	}
1633
}
1634

1635 1636
void blk_mq_free_rq_map(struct blk_mq_tags *tags)
{
1637
	kfree(tags->rqs);
1638
	tags->rqs = NULL;
J
Jens Axboe 已提交
1639 1640
	kfree(tags->static_rqs);
	tags->static_rqs = NULL;
1641

1642
	blk_mq_free_tags(tags);
1643 1644
}

1645 1646 1647 1648
struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
					unsigned int hctx_idx,
					unsigned int nr_tags,
					unsigned int reserved_tags)
1649
{
1650
	struct blk_mq_tags *tags;
1651

1652
	tags = blk_mq_init_tags(nr_tags, reserved_tags,
S
Shaohua Li 已提交
1653 1654
				set->numa_node,
				BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
1655 1656
	if (!tags)
		return NULL;
1657

1658
	tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *),
1659
				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
1660
				 set->numa_node);
1661 1662 1663 1664
	if (!tags->rqs) {
		blk_mq_free_tags(tags);
		return NULL;
	}
1665

J
Jens Axboe 已提交
1666 1667 1668 1669 1670 1671 1672 1673 1674
	tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *),
				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
				 set->numa_node);
	if (!tags->static_rqs) {
		kfree(tags->rqs);
		blk_mq_free_tags(tags);
		return NULL;
	}

1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690
	return tags;
}

static size_t order_to_size(unsigned int order)
{
	return (size_t)PAGE_SIZE << order;
}

int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx, unsigned int depth)
{
	unsigned int i, j, entries_per_page, max_order = 4;
	size_t rq_size, left;

	INIT_LIST_HEAD(&tags->page_list);

1691 1692 1693 1694
	/*
	 * rq_size is the size of the request plus driver payload, rounded
	 * to the cacheline size
	 */
1695
	rq_size = round_up(sizeof(struct request) + set->cmd_size,
1696
				cache_line_size());
1697
	left = rq_size * depth;
1698

1699
	for (i = 0; i < depth; ) {
1700 1701 1702 1703 1704
		int this_order = max_order;
		struct page *page;
		int to_do;
		void *p;

1705
		while (this_order && left < order_to_size(this_order - 1))
1706 1707 1708
			this_order--;

		do {
1709
			page = alloc_pages_node(set->numa_node,
1710
				GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
1711
				this_order);
1712 1713 1714 1715 1716 1717 1718 1719 1720
			if (page)
				break;
			if (!this_order--)
				break;
			if (order_to_size(this_order) < rq_size)
				break;
		} while (1);

		if (!page)
1721
			goto fail;
1722 1723

		page->private = this_order;
1724
		list_add_tail(&page->lru, &tags->page_list);
1725 1726

		p = page_address(page);
1727 1728 1729 1730
		/*
		 * Allow kmemleak to scan these pages as they contain pointers
		 * to additional allocations like via ops->init_request().
		 */
1731
		kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
1732
		entries_per_page = order_to_size(this_order) / rq_size;
1733
		to_do = min(entries_per_page, depth - i);
1734 1735
		left -= to_do * rq_size;
		for (j = 0; j < to_do; j++) {
J
Jens Axboe 已提交
1736 1737 1738
			struct request *rq = p;

			tags->static_rqs[i] = rq;
1739 1740
			if (set->ops->init_request) {
				if (set->ops->init_request(set->driver_data,
J
Jens Axboe 已提交
1741
						rq, hctx_idx, i,
1742
						set->numa_node)) {
J
Jens Axboe 已提交
1743
					tags->static_rqs[i] = NULL;
1744
					goto fail;
1745
				}
1746 1747
			}

1748 1749 1750 1751
			p += rq_size;
			i++;
		}
	}
1752
	return 0;
1753

1754
fail:
1755 1756
	blk_mq_free_rqs(set, tags, hctx_idx);
	return -ENOMEM;
1757 1758
}

J
Jens Axboe 已提交
1759 1760 1761 1762 1763
/*
 * 'cpu' is going away. splice any existing rq_list entries from this
 * software queue to the hw queue dispatch list, and ensure that it
 * gets run.
 */
1764
static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
1765
{
1766
	struct blk_mq_hw_ctx *hctx;
1767 1768 1769
	struct blk_mq_ctx *ctx;
	LIST_HEAD(tmp);

1770
	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
J
Jens Axboe 已提交
1771
	ctx = __blk_mq_get_ctx(hctx->queue, cpu);
1772 1773 1774 1775 1776 1777 1778 1779 1780

	spin_lock(&ctx->lock);
	if (!list_empty(&ctx->rq_list)) {
		list_splice_init(&ctx->rq_list, &tmp);
		blk_mq_hctx_clear_pending(hctx, ctx);
	}
	spin_unlock(&ctx->lock);

	if (list_empty(&tmp))
1781
		return 0;
1782

J
Jens Axboe 已提交
1783 1784 1785
	spin_lock(&hctx->lock);
	list_splice_tail_init(&tmp, &hctx->dispatch);
	spin_unlock(&hctx->lock);
1786 1787

	blk_mq_run_hw_queue(hctx, true);
1788
	return 0;
1789 1790
}

1791
static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
1792
{
1793 1794
	cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
					    &hctx->cpuhp_dead);
1795 1796
}

1797
/* hctx->ctxs will be freed in queue's release handler */
1798 1799 1800 1801
static void blk_mq_exit_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
1802 1803
	unsigned flush_start_tag = set->queue_depth;

1804 1805
	blk_mq_tag_idle(hctx);

1806 1807 1808 1809 1810
	if (set->ops->exit_request)
		set->ops->exit_request(set->driver_data,
				       hctx->fq->flush_rq, hctx_idx,
				       flush_start_tag + hctx_idx);

1811 1812 1813
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);

1814 1815 1816
	if (hctx->flags & BLK_MQ_F_BLOCKING)
		cleanup_srcu_struct(&hctx->queue_rq_srcu);

1817
	blk_mq_remove_cpuhp(hctx);
1818
	blk_free_flush_queue(hctx->fq);
1819
	sbitmap_free(&hctx->ctx_map);
1820 1821
}

M
Ming Lei 已提交
1822 1823 1824 1825 1826 1827 1828 1829 1830
static void blk_mq_exit_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set, int nr_queue)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if (i == nr_queue)
			break;
1831
		blk_mq_exit_hctx(q, set, hctx, i);
M
Ming Lei 已提交
1832 1833 1834 1835 1836 1837 1838 1839 1840
	}
}

static void blk_mq_free_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

1841
	queue_for_each_hw_ctx(q, hctx, i)
M
Ming Lei 已提交
1842 1843 1844
		free_cpumask_var(hctx->cpumask);
}

1845 1846 1847
static int blk_mq_init_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
1848
{
1849
	int node;
1850
	unsigned flush_start_tag = set->queue_depth;
1851 1852 1853 1854 1855

	node = hctx->numa_node;
	if (node == NUMA_NO_NODE)
		node = hctx->numa_node = set->numa_node;

1856
	INIT_WORK(&hctx->run_work, blk_mq_run_work_fn);
1857 1858 1859 1860 1861
	INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
	spin_lock_init(&hctx->lock);
	INIT_LIST_HEAD(&hctx->dispatch);
	hctx->queue = q;
	hctx->queue_num = hctx_idx;
1862
	hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED;
1863

1864
	cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
1865 1866

	hctx->tags = set->tags[hctx_idx];
1867 1868

	/*
1869 1870
	 * Allocate space for all possible cpus to avoid allocation at
	 * runtime
1871
	 */
1872 1873 1874 1875
	hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
					GFP_KERNEL, node);
	if (!hctx->ctxs)
		goto unregister_cpu_notifier;
1876

1877 1878
	if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL,
			      node))
1879
		goto free_ctxs;
1880

1881
	hctx->nr_ctx = 0;
1882

1883 1884 1885
	if (set->ops->init_hctx &&
	    set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
		goto free_bitmap;
1886

1887 1888 1889
	hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size);
	if (!hctx->fq)
		goto exit_hctx;
1890

1891 1892 1893 1894 1895
	if (set->ops->init_request &&
	    set->ops->init_request(set->driver_data,
				   hctx->fq->flush_rq, hctx_idx,
				   flush_start_tag + hctx_idx, node))
		goto free_fq;
1896

1897 1898 1899
	if (hctx->flags & BLK_MQ_F_BLOCKING)
		init_srcu_struct(&hctx->queue_rq_srcu);

1900
	return 0;
1901

1902 1903 1904 1905 1906
 free_fq:
	kfree(hctx->fq);
 exit_hctx:
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);
1907
 free_bitmap:
1908
	sbitmap_free(&hctx->ctx_map);
1909 1910 1911
 free_ctxs:
	kfree(hctx->ctxs);
 unregister_cpu_notifier:
1912
	blk_mq_remove_cpuhp(hctx);
1913 1914
	return -1;
}
1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929

static void blk_mq_init_cpu_queues(struct request_queue *q,
				   unsigned int nr_hw_queues)
{
	unsigned int i;

	for_each_possible_cpu(i) {
		struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
		struct blk_mq_hw_ctx *hctx;

		memset(__ctx, 0, sizeof(*__ctx));
		__ctx->cpu = i;
		spin_lock_init(&__ctx->lock);
		INIT_LIST_HEAD(&__ctx->rq_list);
		__ctx->queue = q;
1930 1931
		blk_stat_init(&__ctx->stat[BLK_STAT_READ]);
		blk_stat_init(&__ctx->stat[BLK_STAT_WRITE]);
1932 1933 1934 1935 1936

		/* If the cpu isn't online, the cpu is mapped to first hctx */
		if (!cpu_online(i))
			continue;

C
Christoph Hellwig 已提交
1937
		hctx = blk_mq_map_queue(q, i);
1938

1939 1940 1941 1942 1943
		/*
		 * Set local node, IFF we have more than one hw queue. If
		 * not, we remain on the home node of the device
		 */
		if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
1944
			hctx->numa_node = local_memory_node(cpu_to_node(i));
1945 1946 1947
	}
}

1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969
static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx)
{
	int ret = 0;

	set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
					set->queue_depth, set->reserved_tags);
	if (!set->tags[hctx_idx])
		return false;

	ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
				set->queue_depth);
	if (!ret)
		return true;

	blk_mq_free_rq_map(set->tags[hctx_idx]);
	set->tags[hctx_idx] = NULL;
	return false;
}

static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
					 unsigned int hctx_idx)
{
1970 1971 1972 1973 1974
	if (set->tags[hctx_idx]) {
		blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
		blk_mq_free_rq_map(set->tags[hctx_idx]);
		set->tags[hctx_idx] = NULL;
	}
1975 1976
}

1977 1978
static void blk_mq_map_swqueue(struct request_queue *q,
			       const struct cpumask *online_mask)
1979
{
1980
	unsigned int i, hctx_idx;
1981 1982
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
M
Ming Lei 已提交
1983
	struct blk_mq_tag_set *set = q->tag_set;
1984

1985 1986 1987 1988 1989
	/*
	 * Avoid others reading imcomplete hctx->cpumask through sysfs
	 */
	mutex_lock(&q->sysfs_lock);

1990
	queue_for_each_hw_ctx(q, hctx, i) {
1991
		cpumask_clear(hctx->cpumask);
1992 1993 1994 1995 1996 1997
		hctx->nr_ctx = 0;
	}

	/*
	 * Map software to hardware queues
	 */
1998
	for_each_possible_cpu(i) {
1999
		/* If the cpu isn't online, the cpu is mapped to first hctx */
2000
		if (!cpumask_test_cpu(i, online_mask))
2001 2002
			continue;

2003 2004
		hctx_idx = q->mq_map[i];
		/* unmapped hw queue can be remapped after CPU topo changed */
2005 2006
		if (!set->tags[hctx_idx] &&
		    !__blk_mq_alloc_rq_map(set, hctx_idx)) {
2007 2008 2009 2010 2011 2012
			/*
			 * If tags initialization fail for some hctx,
			 * that hctx won't be brought online.  In this
			 * case, remap the current ctx to hctx[0] which
			 * is guaranteed to always have tags allocated
			 */
2013
			q->mq_map[i] = 0;
2014 2015
		}

2016
		ctx = per_cpu_ptr(q->queue_ctx, i);
C
Christoph Hellwig 已提交
2017
		hctx = blk_mq_map_queue(q, i);
K
Keith Busch 已提交
2018

2019
		cpumask_set_cpu(i, hctx->cpumask);
2020 2021 2022
		ctx->index_hw = hctx->nr_ctx;
		hctx->ctxs[hctx->nr_ctx++] = ctx;
	}
2023

2024 2025
	mutex_unlock(&q->sysfs_lock);

2026
	queue_for_each_hw_ctx(q, hctx, i) {
2027
		/*
2028 2029
		 * If no software queues are mapped to this hardware queue,
		 * disable it and free the request entries.
2030 2031
		 */
		if (!hctx->nr_ctx) {
2032 2033 2034 2035
			/* Never unmap queue 0.  We need it as a
			 * fallback in case of a new remap fails
			 * allocation
			 */
2036 2037 2038
			if (i && set->tags[i])
				blk_mq_free_map_and_requests(set, i);

M
Ming Lei 已提交
2039
			hctx->tags = NULL;
2040 2041 2042
			continue;
		}

M
Ming Lei 已提交
2043 2044 2045
		hctx->tags = set->tags[i];
		WARN_ON(!hctx->tags);

2046 2047 2048 2049 2050
		/*
		 * Set the map size to the number of mapped software queues.
		 * This is more accurate and more efficient than looping
		 * over all possibly mapped software queues.
		 */
2051
		sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
2052

2053 2054 2055
		/*
		 * Initialize batch roundrobin counts
		 */
2056 2057 2058
		hctx->next_cpu = cpumask_first(hctx->cpumask);
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}
2059 2060
}

2061
static void queue_set_hctx_shared(struct request_queue *q, bool shared)
2062 2063 2064 2065
{
	struct blk_mq_hw_ctx *hctx;
	int i;

2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076
	queue_for_each_hw_ctx(q, hctx, i) {
		if (shared)
			hctx->flags |= BLK_MQ_F_TAG_SHARED;
		else
			hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
	}
}

static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, bool shared)
{
	struct request_queue *q;
2077 2078 2079

	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_freeze_queue(q);
2080
		queue_set_hctx_shared(q, shared);
2081 2082 2083 2084 2085 2086 2087 2088 2089 2090
		blk_mq_unfreeze_queue(q);
	}
}

static void blk_mq_del_queue_tag_set(struct request_queue *q)
{
	struct blk_mq_tag_set *set = q->tag_set;

	mutex_lock(&set->tag_list_lock);
	list_del_init(&q->tag_set_list);
2091 2092 2093 2094 2095 2096
	if (list_is_singular(&set->tag_list)) {
		/* just transitioned to unshared */
		set->flags &= ~BLK_MQ_F_TAG_SHARED;
		/* update existing queue */
		blk_mq_update_tag_set_depth(set, false);
	}
2097 2098 2099 2100 2101 2102 2103 2104 2105
	mutex_unlock(&set->tag_list_lock);
}

static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
				     struct request_queue *q)
{
	q->tag_set = set;

	mutex_lock(&set->tag_list_lock);
2106 2107 2108 2109 2110 2111 2112 2113 2114

	/* Check to see if we're transitioning to shared (from 1 to 2 queues). */
	if (!list_empty(&set->tag_list) && !(set->flags & BLK_MQ_F_TAG_SHARED)) {
		set->flags |= BLK_MQ_F_TAG_SHARED;
		/* update existing queue */
		blk_mq_update_tag_set_depth(set, true);
	}
	if (set->flags & BLK_MQ_F_TAG_SHARED)
		queue_set_hctx_shared(q, true);
2115
	list_add_tail(&q->tag_set_list, &set->tag_list);
2116

2117 2118 2119
	mutex_unlock(&set->tag_list_lock);
}

2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130
/*
 * It is the actual release handler for mq, but we do it from
 * request queue's release handler for avoiding use-after-free
 * and headache because q->mq_kobj shouldn't have been introduced,
 * but we can't group ctx/kctx kobj without it.
 */
void blk_mq_release(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

2131 2132
	blk_mq_sched_teardown(q);

2133
	/* hctx kobj stays in hctx */
2134 2135 2136 2137
	queue_for_each_hw_ctx(q, hctx, i) {
		if (!hctx)
			continue;
		kfree(hctx->ctxs);
2138
		kfree(hctx);
2139
	}
2140

2141 2142
	q->mq_map = NULL;

2143 2144 2145 2146 2147 2148
	kfree(q->queue_hw_ctx);

	/* ctx kobj stays in queue_ctx */
	free_percpu(q->queue_ctx);
}

2149
struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164
{
	struct request_queue *uninit_q, *q;

	uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
	if (!uninit_q)
		return ERR_PTR(-ENOMEM);

	q = blk_mq_init_allocated_queue(set, uninit_q);
	if (IS_ERR(q))
		blk_cleanup_queue(uninit_q);

	return q;
}
EXPORT_SYMBOL(blk_mq_init_queue);

K
Keith Busch 已提交
2165 2166
static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
						struct request_queue *q)
2167
{
K
Keith Busch 已提交
2168 2169
	int i, j;
	struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
2170

K
Keith Busch 已提交
2171
	blk_mq_sysfs_unregister(q);
2172
	for (i = 0; i < set->nr_hw_queues; i++) {
K
Keith Busch 已提交
2173
		int node;
2174

K
Keith Busch 已提交
2175 2176 2177 2178
		if (hctxs[i])
			continue;

		node = blk_mq_hw_queue_to_node(q->mq_map, i);
2179 2180
		hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
					GFP_KERNEL, node);
2181
		if (!hctxs[i])
K
Keith Busch 已提交
2182
			break;
2183

2184
		if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL,
K
Keith Busch 已提交
2185 2186 2187 2188 2189
						node)) {
			kfree(hctxs[i]);
			hctxs[i] = NULL;
			break;
		}
2190

2191
		atomic_set(&hctxs[i]->nr_active, 0);
2192
		hctxs[i]->numa_node = node;
2193
		hctxs[i]->queue_num = i;
K
Keith Busch 已提交
2194 2195 2196 2197 2198 2199 2200 2201

		if (blk_mq_init_hctx(q, set, hctxs[i], i)) {
			free_cpumask_var(hctxs[i]->cpumask);
			kfree(hctxs[i]);
			hctxs[i] = NULL;
			break;
		}
		blk_mq_hctx_kobj_init(hctxs[i]);
2202
	}
K
Keith Busch 已提交
2203 2204 2205 2206
	for (j = i; j < q->nr_hw_queues; j++) {
		struct blk_mq_hw_ctx *hctx = hctxs[j];

		if (hctx) {
2207 2208
			if (hctx->tags)
				blk_mq_free_map_and_requests(set, j);
K
Keith Busch 已提交
2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224
			blk_mq_exit_hctx(q, set, hctx, j);
			free_cpumask_var(hctx->cpumask);
			kobject_put(&hctx->kobj);
			kfree(hctx->ctxs);
			kfree(hctx);
			hctxs[j] = NULL;

		}
	}
	q->nr_hw_queues = i;
	blk_mq_sysfs_register(q);
}

struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
						  struct request_queue *q)
{
M
Ming Lei 已提交
2225 2226 2227
	/* mark the queue as mq asap */
	q->mq_ops = set->ops;

K
Keith Busch 已提交
2228 2229
	q->queue_ctx = alloc_percpu(struct blk_mq_ctx);
	if (!q->queue_ctx)
M
Ming Lin 已提交
2230
		goto err_exit;
K
Keith Busch 已提交
2231 2232 2233 2234 2235 2236

	q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)),
						GFP_KERNEL, set->numa_node);
	if (!q->queue_hw_ctx)
		goto err_percpu;

2237
	q->mq_map = set->mq_map;
K
Keith Busch 已提交
2238 2239 2240 2241

	blk_mq_realloc_hw_ctxs(set, q);
	if (!q->nr_hw_queues)
		goto err_hctxs;
2242

2243
	INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
2244
	blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
2245 2246 2247

	q->nr_queues = nr_cpu_ids;

2248
	q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
2249

2250 2251 2252
	if (!(set->flags & BLK_MQ_F_SG_MERGE))
		q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;

2253 2254
	q->sg_reserved_size = INT_MAX;

2255
	INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
2256 2257 2258
	INIT_LIST_HEAD(&q->requeue_list);
	spin_lock_init(&q->requeue_lock);

2259 2260 2261 2262 2263
	if (q->nr_hw_queues > 1)
		blk_queue_make_request(q, blk_mq_make_request);
	else
		blk_queue_make_request(q, blk_sq_make_request);

2264 2265 2266 2267 2268
	/*
	 * Do this after blk_queue_make_request() overrides it...
	 */
	q->nr_requests = set->queue_depth;

2269 2270 2271 2272 2273
	/*
	 * Default to classic polling
	 */
	q->poll_nsec = -1;

2274 2275
	if (set->ops->complete)
		blk_queue_softirq_done(q, set->ops->complete);
2276

2277
	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
2278

2279
	get_online_cpus();
2280 2281
	mutex_lock(&all_q_mutex);

2282
	list_add_tail(&q->all_q_node, &all_q_list);
2283
	blk_mq_add_queue_tag_set(set, q);
2284
	blk_mq_map_swqueue(q, cpu_online_mask);
2285

2286
	mutex_unlock(&all_q_mutex);
2287
	put_online_cpus();
2288

2289 2290 2291 2292 2293 2294 2295 2296
	if (!(set->flags & BLK_MQ_F_NO_SCHED)) {
		int ret;

		ret = blk_mq_sched_init(q);
		if (ret)
			return ERR_PTR(ret);
	}

2297
	return q;
2298

2299
err_hctxs:
K
Keith Busch 已提交
2300
	kfree(q->queue_hw_ctx);
2301
err_percpu:
K
Keith Busch 已提交
2302
	free_percpu(q->queue_ctx);
M
Ming Lin 已提交
2303 2304
err_exit:
	q->mq_ops = NULL;
2305 2306
	return ERR_PTR(-ENOMEM);
}
2307
EXPORT_SYMBOL(blk_mq_init_allocated_queue);
2308 2309 2310

void blk_mq_free_queue(struct request_queue *q)
{
M
Ming Lei 已提交
2311
	struct blk_mq_tag_set	*set = q->tag_set;
2312

2313 2314 2315 2316
	mutex_lock(&all_q_mutex);
	list_del_init(&q->all_q_node);
	mutex_unlock(&all_q_mutex);

J
Jens Axboe 已提交
2317 2318
	wbt_exit(q);

2319 2320
	blk_mq_del_queue_tag_set(q);

M
Ming Lei 已提交
2321 2322
	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
	blk_mq_free_hw_queues(q, set);
2323 2324 2325
}

/* Basically redo blk_mq_init_queue with queue frozen */
2326 2327
static void blk_mq_queue_reinit(struct request_queue *q,
				const struct cpumask *online_mask)
2328
{
2329
	WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth));
2330

2331 2332
	blk_mq_sysfs_unregister(q);

2333 2334 2335 2336 2337 2338
	/*
	 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
	 * we should change hctx numa_node according to new topology (this
	 * involves free and re-allocate memory, worthy doing?)
	 */

2339
	blk_mq_map_swqueue(q, online_mask);
2340

2341
	blk_mq_sysfs_register(q);
2342 2343
}

2344 2345 2346 2347 2348 2349 2350 2351
/*
 * New online cpumask which is going to be set in this hotplug event.
 * Declare this cpumasks as global as cpu-hotplug operation is invoked
 * one-by-one and dynamically allocating this could result in a failure.
 */
static struct cpumask cpuhp_online_new;

static void blk_mq_queue_reinit_work(void)
2352 2353 2354 2355
{
	struct request_queue *q;

	mutex_lock(&all_q_mutex);
2356 2357 2358 2359 2360 2361 2362 2363 2364
	/*
	 * We need to freeze and reinit all existing queues.  Freezing
	 * involves synchronous wait for an RCU grace period and doing it
	 * one by one may take a long time.  Start freezing all queues in
	 * one swoop and then wait for the completions so that freezing can
	 * take place in parallel.
	 */
	list_for_each_entry(q, &all_q_list, all_q_node)
		blk_mq_freeze_queue_start(q);
2365
	list_for_each_entry(q, &all_q_list, all_q_node)
2366 2367
		blk_mq_freeze_queue_wait(q);

2368
	list_for_each_entry(q, &all_q_list, all_q_node)
2369
		blk_mq_queue_reinit(q, &cpuhp_online_new);
2370 2371 2372 2373

	list_for_each_entry(q, &all_q_list, all_q_node)
		blk_mq_unfreeze_queue(q);

2374
	mutex_unlock(&all_q_mutex);
2375 2376 2377 2378
}

static int blk_mq_queue_reinit_dead(unsigned int cpu)
{
2379
	cpumask_copy(&cpuhp_online_new, cpu_online_mask);
2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394
	blk_mq_queue_reinit_work();
	return 0;
}

/*
 * Before hotadded cpu starts handling requests, new mappings must be
 * established.  Otherwise, these requests in hw queue might never be
 * dispatched.
 *
 * For example, there is a single hw queue (hctx) and two CPU queues (ctx0
 * for CPU0, and ctx1 for CPU1).
 *
 * Now CPU1 is just onlined and a request is inserted into ctx1->rq_list
 * and set bit0 in pending bitmap as ctx1->index_hw is still zero.
 *
2395 2396 2397 2398
 * And then while running hw queue, blk_mq_flush_busy_ctxs() finds bit0 is set
 * in pending bitmap and tries to retrieve requests in hctx->ctxs[0]->rq_list.
 * But htx->ctxs[0] is a pointer to ctx0, so the request in ctx1->rq_list is
 * ignored.
2399 2400 2401 2402 2403 2404 2405
 */
static int blk_mq_queue_reinit_prepare(unsigned int cpu)
{
	cpumask_copy(&cpuhp_online_new, cpu_online_mask);
	cpumask_set_cpu(cpu, &cpuhp_online_new);
	blk_mq_queue_reinit_work();
	return 0;
2406 2407
}

2408 2409 2410 2411
static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	int i;

2412 2413
	for (i = 0; i < set->nr_hw_queues; i++)
		if (!__blk_mq_alloc_rq_map(set, i))
2414 2415 2416 2417 2418 2419
			goto out_unwind;

	return 0;

out_unwind:
	while (--i >= 0)
2420
		blk_mq_free_rq_map(set->tags[i]);
2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459

	return -ENOMEM;
}

/*
 * Allocate the request maps associated with this tag_set. Note that this
 * may reduce the depth asked for, if memory is tight. set->queue_depth
 * will be updated to reflect the allocated depth.
 */
static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	unsigned int depth;
	int err;

	depth = set->queue_depth;
	do {
		err = __blk_mq_alloc_rq_maps(set);
		if (!err)
			break;

		set->queue_depth >>= 1;
		if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
			err = -ENOMEM;
			break;
		}
	} while (set->queue_depth);

	if (!set->queue_depth || err) {
		pr_err("blk-mq: failed to allocate request map\n");
		return -ENOMEM;
	}

	if (depth != set->queue_depth)
		pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
						depth, set->queue_depth);

	return 0;
}

2460 2461 2462 2463 2464 2465
/*
 * Alloc a tag set to be associated with one or more request queues.
 * May fail with EINVAL for various error conditions. May adjust the
 * requested depth down, if if it too large. In that case, the set
 * value will be stored in set->queue_depth.
 */
2466 2467
int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
{
2468 2469
	int ret;

B
Bart Van Assche 已提交
2470 2471
	BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);

2472 2473
	if (!set->nr_hw_queues)
		return -EINVAL;
2474
	if (!set->queue_depth)
2475 2476 2477 2478
		return -EINVAL;
	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
		return -EINVAL;

C
Christoph Hellwig 已提交
2479
	if (!set->ops->queue_rq)
2480 2481
		return -EINVAL;

2482 2483 2484 2485 2486
	if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
		pr_info("blk-mq: reduced tag depth to %u\n",
			BLK_MQ_MAX_DEPTH);
		set->queue_depth = BLK_MQ_MAX_DEPTH;
	}
2487

2488 2489 2490 2491 2492 2493 2494 2495 2496
	/*
	 * If a crashdump is active, then we are potentially in a very
	 * memory constrained environment. Limit us to 1 queue and
	 * 64 tags to prevent using too much memory.
	 */
	if (is_kdump_kernel()) {
		set->nr_hw_queues = 1;
		set->queue_depth = min(64U, set->queue_depth);
	}
K
Keith Busch 已提交
2497 2498 2499 2500 2501
	/*
	 * There is no use for more h/w queues than cpus.
	 */
	if (set->nr_hw_queues > nr_cpu_ids)
		set->nr_hw_queues = nr_cpu_ids;
2502

K
Keith Busch 已提交
2503
	set->tags = kzalloc_node(nr_cpu_ids * sizeof(struct blk_mq_tags *),
2504 2505
				 GFP_KERNEL, set->numa_node);
	if (!set->tags)
2506
		return -ENOMEM;
2507

2508 2509 2510
	ret = -ENOMEM;
	set->mq_map = kzalloc_node(sizeof(*set->mq_map) * nr_cpu_ids,
			GFP_KERNEL, set->numa_node);
2511 2512 2513
	if (!set->mq_map)
		goto out_free_tags;

2514 2515 2516 2517 2518 2519 2520 2521 2522
	if (set->ops->map_queues)
		ret = set->ops->map_queues(set);
	else
		ret = blk_mq_map_queues(set);
	if (ret)
		goto out_free_mq_map;

	ret = blk_mq_alloc_rq_maps(set);
	if (ret)
2523
		goto out_free_mq_map;
2524

2525 2526 2527
	mutex_init(&set->tag_list_lock);
	INIT_LIST_HEAD(&set->tag_list);

2528
	return 0;
2529 2530 2531 2532 2533

out_free_mq_map:
	kfree(set->mq_map);
	set->mq_map = NULL;
out_free_tags:
2534 2535
	kfree(set->tags);
	set->tags = NULL;
2536
	return ret;
2537 2538 2539 2540 2541 2542 2543
}
EXPORT_SYMBOL(blk_mq_alloc_tag_set);

void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
{
	int i;

2544 2545
	for (i = 0; i < nr_cpu_ids; i++)
		blk_mq_free_map_and_requests(set, i);
2546

2547 2548 2549
	kfree(set->mq_map);
	set->mq_map = NULL;

M
Ming Lei 已提交
2550
	kfree(set->tags);
2551
	set->tags = NULL;
2552 2553 2554
}
EXPORT_SYMBOL(blk_mq_free_tag_set);

2555 2556 2557 2558 2559 2560
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
{
	struct blk_mq_tag_set *set = q->tag_set;
	struct blk_mq_hw_ctx *hctx;
	int i, ret;

2561
	if (!set)
2562 2563 2564 2565
		return -EINVAL;

	ret = 0;
	queue_for_each_hw_ctx(q, hctx, i) {
2566 2567
		if (!hctx->tags)
			continue;
2568 2569 2570 2571 2572 2573 2574 2575 2576
		/*
		 * If we're using an MQ scheduler, just update the scheduler
		 * queue depth. This is similar to what the old code would do.
		 */
		if (!hctx->sched_tags)
			ret = blk_mq_tag_update_depth(hctx->tags,
							min(nr, set->queue_depth));
		else
			ret = blk_mq_tag_update_depth(hctx->sched_tags, nr);
2577 2578 2579 2580 2581 2582 2583 2584 2585 2586
		if (ret)
			break;
	}

	if (!ret)
		q->nr_requests = nr;

	return ret;
}

K
Keith Busch 已提交
2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615
void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
{
	struct request_queue *q;

	if (nr_hw_queues > nr_cpu_ids)
		nr_hw_queues = nr_cpu_ids;
	if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues)
		return;

	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_freeze_queue(q);

	set->nr_hw_queues = nr_hw_queues;
	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_realloc_hw_ctxs(set, q);

		if (q->nr_hw_queues > 1)
			blk_queue_make_request(q, blk_mq_make_request);
		else
			blk_queue_make_request(q, blk_sq_make_request);

		blk_mq_queue_reinit(q, cpu_online_mask);
	}

	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_unfreeze_queue(q);
}
EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);

2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652
static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
				       struct blk_mq_hw_ctx *hctx,
				       struct request *rq)
{
	struct blk_rq_stat stat[2];
	unsigned long ret = 0;

	/*
	 * If stats collection isn't on, don't sleep but turn it on for
	 * future users
	 */
	if (!blk_stat_enable(q))
		return 0;

	/*
	 * We don't have to do this once per IO, should optimize this
	 * to just use the current window of stats until it changes
	 */
	memset(&stat, 0, sizeof(stat));
	blk_hctx_stat_get(hctx, stat);

	/*
	 * As an optimistic guess, use half of the mean service time
	 * for this type of request. We can (and should) make this smarter.
	 * For instance, if the completion latencies are tight, we can
	 * get closer than just half the mean. This is especially
	 * important on devices where the completion latencies are longer
	 * than ~10 usec.
	 */
	if (req_op(rq) == REQ_OP_READ && stat[BLK_STAT_READ].nr_samples)
		ret = (stat[BLK_STAT_READ].mean + 1) / 2;
	else if (req_op(rq) == REQ_OP_WRITE && stat[BLK_STAT_WRITE].nr_samples)
		ret = (stat[BLK_STAT_WRITE].mean + 1) / 2;

	return ret;
}

2653
static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
2654
				     struct blk_mq_hw_ctx *hctx,
2655 2656 2657 2658
				     struct request *rq)
{
	struct hrtimer_sleeper hs;
	enum hrtimer_mode mode;
2659
	unsigned int nsecs;
2660 2661
	ktime_t kt;

2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679
	if (test_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags))
		return false;

	/*
	 * poll_nsec can be:
	 *
	 * -1:	don't ever hybrid sleep
	 *  0:	use half of prev avg
	 * >0:	use this specific value
	 */
	if (q->poll_nsec == -1)
		return false;
	else if (q->poll_nsec > 0)
		nsecs = q->poll_nsec;
	else
		nsecs = blk_mq_poll_nsecs(q, hctx, rq);

	if (!nsecs)
2680 2681 2682 2683 2684 2685 2686 2687
		return false;

	set_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);

	/*
	 * This will be replaced with the stats tracking code, using
	 * 'avg_completion_time / 2' as the pre-sleep target.
	 */
T
Thomas Gleixner 已提交
2688
	kt = nsecs;
2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710

	mode = HRTIMER_MODE_REL;
	hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode);
	hrtimer_set_expires(&hs.timer, kt);

	hrtimer_init_sleeper(&hs, current);
	do {
		if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
			break;
		set_current_state(TASK_UNINTERRUPTIBLE);
		hrtimer_start_expires(&hs.timer, mode);
		if (hs.task)
			io_schedule();
		hrtimer_cancel(&hs.timer);
		mode = HRTIMER_MODE_ABS;
	} while (hs.task && !signal_pending(current));

	__set_current_state(TASK_RUNNING);
	destroy_hrtimer_on_stack(&hs.timer);
	return true;
}

J
Jens Axboe 已提交
2711 2712 2713 2714 2715
static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq)
{
	struct request_queue *q = hctx->queue;
	long state;

2716 2717 2718 2719 2720 2721 2722
	/*
	 * If we sleep, have the caller restart the poll loop to reset
	 * the state. Like for the other success return cases, the
	 * caller is responsible for checking if the IO completed. If
	 * the IO isn't complete, we'll get called again and will go
	 * straight to the busy poll loop.
	 */
2723
	if (blk_mq_poll_hybrid_sleep(q, hctx, rq))
2724 2725
		return true;

J
Jens Axboe 已提交
2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768
	hctx->poll_considered++;

	state = current->state;
	while (!need_resched()) {
		int ret;

		hctx->poll_invoked++;

		ret = q->mq_ops->poll(hctx, rq->tag);
		if (ret > 0) {
			hctx->poll_success++;
			set_current_state(TASK_RUNNING);
			return true;
		}

		if (signal_pending_state(state, current))
			set_current_state(TASK_RUNNING);

		if (current->state == TASK_RUNNING)
			return true;
		if (ret < 0)
			break;
		cpu_relax();
	}

	return false;
}

bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie)
{
	struct blk_mq_hw_ctx *hctx;
	struct blk_plug *plug;
	struct request *rq;

	if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) ||
	    !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
		return false;

	plug = current->plug;
	if (plug)
		blk_flush_plug_list(plug, false);

	hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
2769 2770 2771 2772
	if (!blk_qc_t_is_internal(cookie))
		rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
	else
		rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
J
Jens Axboe 已提交
2773 2774 2775 2776 2777

	return __blk_mq_poll(hctx, rq);
}
EXPORT_SYMBOL_GPL(blk_mq_poll);

2778 2779 2780 2781 2782 2783 2784 2785 2786 2787
void blk_mq_disable_hotplug(void)
{
	mutex_lock(&all_q_mutex);
}

void blk_mq_enable_hotplug(void)
{
	mutex_unlock(&all_q_mutex);
}

2788 2789
static int __init blk_mq_init(void)
{
2790 2791
	cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
				blk_mq_hctx_notify_dead);
2792

2793 2794 2795
	cpuhp_setup_state_nocalls(CPUHP_BLK_MQ_PREPARE, "block/mq:prepare",
				  blk_mq_queue_reinit_prepare,
				  blk_mq_queue_reinit_dead);
2796 2797 2798
	return 0;
}
subsys_initcall(blk_mq_init);