blk-mq.c 66.3 KB
Newer Older
1 2 3 4 5 6
/*
 * Block multiqueue core code
 *
 * Copyright (C) 2013-2014 Jens Axboe
 * Copyright (C) 2013-2014 Christoph Hellwig
 */
7 8 9 10 11
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
12
#include <linux/kmemleak.h>
13 14 15 16 17 18 19 20 21 22 23
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>
#include <linux/llist.h>
#include <linux/list_sort.h>
#include <linux/cpu.h>
#include <linux/cache.h>
#include <linux/sched/sysctl.h>
#include <linux/delay.h>
24
#include <linux/crash_dump.h>
25
#include <linux/prefetch.h>
26 27 28 29 30 31 32

#include <trace/events/block.h>

#include <linux/blk-mq.h>
#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-tag.h"
33
#include "blk-stat.h"
J
Jens Axboe 已提交
34
#include "blk-wbt.h"
35
#include "blk-mq-sched.h"
36 37 38 39 40 41 42 43 44

static DEFINE_MUTEX(all_q_mutex);
static LIST_HEAD(all_q_list);

/*
 * Check if any of the ctx's have pending work in this hardware queue
 */
static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
{
45 46 47
	return sbitmap_any_bit_set(&hctx->ctx_map) ||
			!list_empty_careful(&hctx->dispatch) ||
			blk_mq_sched_has_work(hctx);
48 49
}

50 51 52 53 54 55
/*
 * Mark this ctx as having pending work in this hardware queue
 */
static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
				     struct blk_mq_ctx *ctx)
{
56 57
	if (!sbitmap_test_bit(&hctx->ctx_map, ctx->index_hw))
		sbitmap_set_bit(&hctx->ctx_map, ctx->index_hw);
58 59 60 61 62
}

static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
				      struct blk_mq_ctx *ctx)
{
63
	sbitmap_clear_bit(&hctx->ctx_map, ctx->index_hw);
64 65
}

66
void blk_mq_freeze_queue_start(struct request_queue *q)
67
{
68
	int freeze_depth;
69

70 71
	freeze_depth = atomic_inc_return(&q->mq_freeze_depth);
	if (freeze_depth == 1) {
72
		percpu_ref_kill(&q->q_usage_counter);
73
		blk_mq_run_hw_queues(q, false);
74
	}
75
}
76
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_start);
77 78 79

static void blk_mq_freeze_queue_wait(struct request_queue *q)
{
80
	wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
81 82
}

83 84 85 86
/*
 * Guarantee no request is in use, so we can change any data structure of
 * the queue afterward.
 */
87
void blk_freeze_queue(struct request_queue *q)
88
{
89 90 91 92 93 94 95
	/*
	 * In the !blk_mq case we are only calling this to kill the
	 * q_usage_counter, otherwise this increases the freeze depth
	 * and waits for it to return to zero.  For this reason there is
	 * no blk_unfreeze_queue(), and blk_freeze_queue() is not
	 * exported to drivers as the only user for unfreeze is blk_mq.
	 */
96 97 98
	blk_mq_freeze_queue_start(q);
	blk_mq_freeze_queue_wait(q);
}
99 100 101 102 103 104 105 106 107

void blk_mq_freeze_queue(struct request_queue *q)
{
	/*
	 * ...just an alias to keep freeze and unfreeze actions balanced
	 * in the blk_mq_* namespace
	 */
	blk_freeze_queue(q);
}
108
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
109

110
void blk_mq_unfreeze_queue(struct request_queue *q)
111
{
112
	int freeze_depth;
113

114 115 116
	freeze_depth = atomic_dec_return(&q->mq_freeze_depth);
	WARN_ON_ONCE(freeze_depth < 0);
	if (!freeze_depth) {
117
		percpu_ref_reinit(&q->q_usage_counter);
118
		wake_up_all(&q->mq_freeze_wq);
119
	}
120
}
121
EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
122

123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
/**
 * blk_mq_quiesce_queue() - wait until all ongoing queue_rq calls have finished
 * @q: request queue.
 *
 * Note: this function does not prevent that the struct request end_io()
 * callback function is invoked. Additionally, it is not prevented that
 * new queue_rq() calls occur unless the queue has been stopped first.
 */
void blk_mq_quiesce_queue(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;
	bool rcu = false;

	blk_mq_stop_hw_queues(q);

	queue_for_each_hw_ctx(q, hctx, i) {
		if (hctx->flags & BLK_MQ_F_BLOCKING)
			synchronize_srcu(&hctx->queue_rq_srcu);
		else
			rcu = true;
	}
	if (rcu)
		synchronize_rcu();
}
EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);

150 151 152 153 154 155 156 157
void blk_mq_wake_waiters(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i)
		if (blk_mq_hw_queue_mapped(hctx))
			blk_mq_tag_wakeup_all(hctx->tags, true);
158 159 160 161 162 163 164

	/*
	 * If we are called because the queue has now been marked as
	 * dying, we need to ensure that processes currently waiting on
	 * the queue are notified as well.
	 */
	wake_up_all(&q->mq_freeze_wq);
165 166
}

167 168 169 170 171 172
bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
{
	return blk_mq_has_free_tags(hctx->tags);
}
EXPORT_SYMBOL(blk_mq_can_queue);

173 174
void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
			struct request *rq, unsigned int op)
175
{
176 177 178
	INIT_LIST_HEAD(&rq->queuelist);
	/* csd/requeue_work/fifo_time is initialized before use */
	rq->q = q;
179
	rq->mq_ctx = ctx;
180
	rq->cmd_flags = op;
181 182
	if (blk_queue_io_stat(q))
		rq->rq_flags |= RQF_IO_STAT;
183 184 185 186 187 188
	/* do not touch atomic flags, it needs atomic ops against the timer */
	rq->cpu = -1;
	INIT_HLIST_NODE(&rq->hash);
	RB_CLEAR_NODE(&rq->rb_node);
	rq->rq_disk = NULL;
	rq->part = NULL;
189
	rq->start_time = jiffies;
190 191
#ifdef CONFIG_BLK_CGROUP
	rq->rl = NULL;
192
	set_start_time_ns(rq);
193 194 195 196 197 198 199 200 201 202
	rq->io_start_time_ns = 0;
#endif
	rq->nr_phys_segments = 0;
#if defined(CONFIG_BLK_DEV_INTEGRITY)
	rq->nr_integrity_segments = 0;
#endif
	rq->special = NULL;
	/* tag was already set */
	rq->errors = 0;

203 204
	rq->cmd = rq->__cmd;

205 206 207 208 209 210
	rq->extra_len = 0;
	rq->sense_len = 0;
	rq->resid_len = 0;
	rq->sense = NULL;

	INIT_LIST_HEAD(&rq->timeout_list);
211 212
	rq->timeout = 0;

213 214 215 216
	rq->end_io = NULL;
	rq->end_io_data = NULL;
	rq->next_rq = NULL;

217
	ctx->rq_dispatched[op_is_sync(op)]++;
218
}
219
EXPORT_SYMBOL_GPL(blk_mq_rq_ctx_init);
220

221 222
struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data,
				       unsigned int op)
223 224 225 226
{
	struct request *rq;
	unsigned int tag;

227
	tag = blk_mq_get_tag(data);
228
	if (tag != BLK_MQ_TAG_FAIL) {
229 230 231
		struct blk_mq_tags *tags = blk_mq_tags_from_data(data);

		rq = tags->static_rqs[tag];
232

233
		if (blk_mq_tag_busy(data->hctx)) {
234
			rq->rq_flags = RQF_MQ_INFLIGHT;
235
			atomic_inc(&data->hctx->nr_active);
236 237
		}

238 239 240 241 242 243 244 245
		if (data->flags & BLK_MQ_REQ_INTERNAL) {
			rq->tag = -1;
			rq->internal_tag = tag;
		} else {
			rq->tag = tag;
			rq->internal_tag = -1;
		}

246
		blk_mq_rq_ctx_init(data->q, data->ctx, rq, op);
247 248 249 250 251
		return rq;
	}

	return NULL;
}
252
EXPORT_SYMBOL_GPL(__blk_mq_alloc_request);
253

254 255
struct request *blk_mq_alloc_request(struct request_queue *q, int rw,
		unsigned int flags)
256
{
257
	struct blk_mq_alloc_data alloc_data;
258
	struct request *rq;
259
	int ret;
260

261
	ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT);
262 263
	if (ret)
		return ERR_PTR(ret);
264

265
	rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data);
266

267 268 269 270
	blk_mq_put_ctx(alloc_data.ctx);
	blk_queue_exit(q);

	if (!rq)
271
		return ERR_PTR(-EWOULDBLOCK);
272 273 274 275

	rq->__data_len = 0;
	rq->__sector = (sector_t) -1;
	rq->bio = rq->biotail = NULL;
276 277
	return rq;
}
278
EXPORT_SYMBOL(blk_mq_alloc_request);
279

M
Ming Lin 已提交
280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304
struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw,
		unsigned int flags, unsigned int hctx_idx)
{
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
	struct request *rq;
	struct blk_mq_alloc_data alloc_data;
	int ret;

	/*
	 * If the tag allocator sleeps we could get an allocation for a
	 * different hardware context.  No need to complicate the low level
	 * allocator for this for the rare use case of a command tied to
	 * a specific queue.
	 */
	if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)))
		return ERR_PTR(-EINVAL);

	if (hctx_idx >= q->nr_hw_queues)
		return ERR_PTR(-EIO);

	ret = blk_queue_enter(q, true);
	if (ret)
		return ERR_PTR(ret);

305 306 307 308
	/*
	 * Check if the hardware context is actually mapped to anything.
	 * If not tell the caller that it should skip this queue.
	 */
M
Ming Lin 已提交
309
	hctx = q->queue_hw_ctx[hctx_idx];
310 311 312 313
	if (!blk_mq_hw_queue_mapped(hctx)) {
		ret = -EXDEV;
		goto out_queue_exit;
	}
M
Ming Lin 已提交
314 315 316
	ctx = __blk_mq_get_ctx(q, cpumask_first(hctx->cpumask));

	blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx);
317
	rq = __blk_mq_alloc_request(&alloc_data, rw);
M
Ming Lin 已提交
318
	if (!rq) {
319 320
		ret = -EWOULDBLOCK;
		goto out_queue_exit;
M
Ming Lin 已提交
321 322 323
	}

	return rq;
324 325 326 327

out_queue_exit:
	blk_queue_exit(q);
	return ERR_PTR(ret);
M
Ming Lin 已提交
328 329 330
}
EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);

331 332
void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
			     struct request *rq)
333
{
334
	const int sched_tag = rq->internal_tag;
335 336
	struct request_queue *q = rq->q;

337
	if (rq->rq_flags & RQF_MQ_INFLIGHT)
338
		atomic_dec(&hctx->nr_active);
J
Jens Axboe 已提交
339 340

	wbt_done(q->rq_wb, &rq->issue_stat);
341
	rq->rq_flags = 0;
342

343
	clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
344
	clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
345 346 347 348
	if (rq->tag != -1)
		blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
	if (sched_tag != -1)
		blk_mq_sched_completed_request(hctx, rq);
349
	blk_queue_exit(q);
350 351
}

352
static void blk_mq_finish_hctx_request(struct blk_mq_hw_ctx *hctx,
353
				     struct request *rq)
354 355 356 357
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;

	ctx->rq_completed[rq_is_sync(rq)]++;
358 359 360 361 362 363
	__blk_mq_finish_request(hctx, ctx, rq);
}

void blk_mq_finish_request(struct request *rq)
{
	blk_mq_finish_hctx_request(blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), rq);
364 365 366 367
}

void blk_mq_free_request(struct request *rq)
{
368
	blk_mq_sched_put_request(rq);
369
}
J
Jens Axboe 已提交
370
EXPORT_SYMBOL_GPL(blk_mq_free_request);
371

372
inline void __blk_mq_end_request(struct request *rq, int error)
373
{
M
Ming Lei 已提交
374 375
	blk_account_io_done(rq);

C
Christoph Hellwig 已提交
376
	if (rq->end_io) {
J
Jens Axboe 已提交
377
		wbt_done(rq->q->rq_wb, &rq->issue_stat);
378
		rq->end_io(rq, error);
C
Christoph Hellwig 已提交
379 380 381
	} else {
		if (unlikely(blk_bidi_rq(rq)))
			blk_mq_free_request(rq->next_rq);
382
		blk_mq_free_request(rq);
C
Christoph Hellwig 已提交
383
	}
384
}
385
EXPORT_SYMBOL(__blk_mq_end_request);
386

387
void blk_mq_end_request(struct request *rq, int error)
388 389 390
{
	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
		BUG();
391
	__blk_mq_end_request(rq, error);
392
}
393
EXPORT_SYMBOL(blk_mq_end_request);
394

395
static void __blk_mq_complete_request_remote(void *data)
396
{
397
	struct request *rq = data;
398

399
	rq->q->softirq_done_fn(rq);
400 401
}

402
static void blk_mq_ipi_complete_request(struct request *rq)
403 404
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;
C
Christoph Hellwig 已提交
405
	bool shared = false;
406 407
	int cpu;

C
Christoph Hellwig 已提交
408
	if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
409 410 411
		rq->q->softirq_done_fn(rq);
		return;
	}
412 413

	cpu = get_cpu();
C
Christoph Hellwig 已提交
414 415 416 417
	if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
		shared = cpus_share_cache(cpu, ctx->cpu);

	if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
418
		rq->csd.func = __blk_mq_complete_request_remote;
419 420
		rq->csd.info = rq;
		rq->csd.flags = 0;
421
		smp_call_function_single_async(ctx->cpu, &rq->csd);
422
	} else {
423
		rq->q->softirq_done_fn(rq);
424
	}
425 426
	put_cpu();
}
427

428 429 430 431 432 433 434 435 436 437 438 439 440 441 442
static void blk_mq_stat_add(struct request *rq)
{
	if (rq->rq_flags & RQF_STATS) {
		/*
		 * We could rq->mq_ctx here, but there's less of a risk
		 * of races if we have the completion event add the stats
		 * to the local software queue.
		 */
		struct blk_mq_ctx *ctx;

		ctx = __blk_mq_get_ctx(rq->q, raw_smp_processor_id());
		blk_stat_add(&ctx->stat[rq_data_dir(rq)], rq);
	}
}

443
static void __blk_mq_complete_request(struct request *rq)
444 445 446
{
	struct request_queue *q = rq->q;

447 448
	blk_mq_stat_add(rq);

449
	if (!q->softirq_done_fn)
450
		blk_mq_end_request(rq, rq->errors);
451 452 453 454
	else
		blk_mq_ipi_complete_request(rq);
}

455 456 457 458 459 460 461 462
/**
 * blk_mq_complete_request - end I/O on a request
 * @rq:		the request being processed
 *
 * Description:
 *	Ends all I/O on a request. It does not handle partial completions.
 *	The actual completion happens out-of-order, through a IPI handler.
 **/
463
void blk_mq_complete_request(struct request *rq, int error)
464
{
465 466 467
	struct request_queue *q = rq->q;

	if (unlikely(blk_should_fake_timeout(q)))
468
		return;
469 470
	if (!blk_mark_rq_complete(rq)) {
		rq->errors = error;
471
		__blk_mq_complete_request(rq);
472
	}
473 474
}
EXPORT_SYMBOL(blk_mq_complete_request);
475

476 477 478 479 480 481
int blk_mq_request_started(struct request *rq)
{
	return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
}
EXPORT_SYMBOL_GPL(blk_mq_request_started);

482
void blk_mq_start_request(struct request *rq)
483 484 485
{
	struct request_queue *q = rq->q;

486 487
	blk_mq_sched_started_request(rq);

488 489
	trace_block_rq_issue(q, rq);

C
Christoph Hellwig 已提交
490
	rq->resid_len = blk_rq_bytes(rq);
C
Christoph Hellwig 已提交
491 492
	if (unlikely(blk_bidi_rq(rq)))
		rq->next_rq->resid_len = blk_rq_bytes(rq->next_rq);
C
Christoph Hellwig 已提交
493

494 495 496
	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
		blk_stat_set_issue_time(&rq->issue_stat);
		rq->rq_flags |= RQF_STATS;
J
Jens Axboe 已提交
497
		wbt_issue(q->rq_wb, &rq->issue_stat);
498 499
	}

500
	blk_add_timer(rq);
501

502 503 504 505 506 507
	/*
	 * Ensure that ->deadline is visible before set the started
	 * flag and clear the completed flag.
	 */
	smp_mb__before_atomic();

508 509 510 511 512 513
	/*
	 * Mark us as started and clear complete. Complete might have been
	 * set if requeue raced with timeout, which then marked it as
	 * complete. So be sure to clear complete again when we start
	 * the request, otherwise we'll ignore the completion event.
	 */
514 515 516 517
	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
		set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
	if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
		clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
518 519 520 521 522 523 524 525 526

	if (q->dma_drain_size && blk_rq_bytes(rq)) {
		/*
		 * Make sure space for the drain appears.  We know we can do
		 * this because max_hw_segments has been adjusted to be one
		 * fewer than the device can handle.
		 */
		rq->nr_phys_segments++;
	}
527
}
528
EXPORT_SYMBOL(blk_mq_start_request);
529

530
static void __blk_mq_requeue_request(struct request *rq)
531 532 533 534
{
	struct request_queue *q = rq->q;

	trace_block_rq_requeue(q, rq);
J
Jens Axboe 已提交
535
	wbt_requeue(q->rq_wb, &rq->issue_stat);
536
	blk_mq_sched_requeue_request(rq);
537

538 539 540 541
	if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
		if (q->dma_drain_size && blk_rq_bytes(rq))
			rq->nr_phys_segments--;
	}
542 543
}

544
void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
545 546 547 548
{
	__blk_mq_requeue_request(rq);

	BUG_ON(blk_queued_rq(rq));
549
	blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
550 551 552
}
EXPORT_SYMBOL(blk_mq_requeue_request);

553 554 555
static void blk_mq_requeue_work(struct work_struct *work)
{
	struct request_queue *q =
556
		container_of(work, struct request_queue, requeue_work.work);
557 558 559 560 561 562 563 564 565
	LIST_HEAD(rq_list);
	struct request *rq, *next;
	unsigned long flags;

	spin_lock_irqsave(&q->requeue_lock, flags);
	list_splice_init(&q->requeue_list, &rq_list);
	spin_unlock_irqrestore(&q->requeue_lock, flags);

	list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
566
		if (!(rq->rq_flags & RQF_SOFTBARRIER))
567 568
			continue;

569
		rq->rq_flags &= ~RQF_SOFTBARRIER;
570
		list_del_init(&rq->queuelist);
571
		blk_mq_sched_insert_request(rq, true, false, false);
572 573 574 575 576
	}

	while (!list_empty(&rq_list)) {
		rq = list_entry(rq_list.next, struct request, queuelist);
		list_del_init(&rq->queuelist);
577
		blk_mq_sched_insert_request(rq, false, false, false);
578 579
	}

580
	blk_mq_run_hw_queues(q, false);
581 582
}

583 584
void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
				bool kick_requeue_list)
585 586 587 588 589 590 591 592
{
	struct request_queue *q = rq->q;
	unsigned long flags;

	/*
	 * We abuse this flag that is otherwise used by the I/O scheduler to
	 * request head insertation from the workqueue.
	 */
593
	BUG_ON(rq->rq_flags & RQF_SOFTBARRIER);
594 595 596

	spin_lock_irqsave(&q->requeue_lock, flags);
	if (at_head) {
597
		rq->rq_flags |= RQF_SOFTBARRIER;
598 599 600 601 602
		list_add(&rq->queuelist, &q->requeue_list);
	} else {
		list_add_tail(&rq->queuelist, &q->requeue_list);
	}
	spin_unlock_irqrestore(&q->requeue_lock, flags);
603 604 605

	if (kick_requeue_list)
		blk_mq_kick_requeue_list(q);
606 607 608 609 610
}
EXPORT_SYMBOL(blk_mq_add_to_requeue_list);

void blk_mq_kick_requeue_list(struct request_queue *q)
{
611
	kblockd_schedule_delayed_work(&q->requeue_work, 0);
612 613 614
}
EXPORT_SYMBOL(blk_mq_kick_requeue_list);

615 616 617 618 619 620 621 622
void blk_mq_delay_kick_requeue_list(struct request_queue *q,
				    unsigned long msecs)
{
	kblockd_schedule_delayed_work(&q->requeue_work,
				      msecs_to_jiffies(msecs));
}
EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);

623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642
void blk_mq_abort_requeue_list(struct request_queue *q)
{
	unsigned long flags;
	LIST_HEAD(rq_list);

	spin_lock_irqsave(&q->requeue_lock, flags);
	list_splice_init(&q->requeue_list, &rq_list);
	spin_unlock_irqrestore(&q->requeue_lock, flags);

	while (!list_empty(&rq_list)) {
		struct request *rq;

		rq = list_first_entry(&rq_list, struct request, queuelist);
		list_del_init(&rq->queuelist);
		rq->errors = -EIO;
		blk_mq_end_request(rq, rq->errors);
	}
}
EXPORT_SYMBOL(blk_mq_abort_requeue_list);

643 644
struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
{
645 646
	if (tag < tags->nr_tags) {
		prefetch(tags->rqs[tag]);
647
		return tags->rqs[tag];
648
	}
649 650

	return NULL;
651 652 653
}
EXPORT_SYMBOL(blk_mq_tag_to_rq);

654
struct blk_mq_timeout_data {
655 656
	unsigned long next;
	unsigned int next_set;
657 658
};

659
void blk_mq_rq_timed_out(struct request *req, bool reserved)
660
{
J
Jens Axboe 已提交
661
	const struct blk_mq_ops *ops = req->q->mq_ops;
662
	enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
663 664 665 666 667 668 669 670 671 672

	/*
	 * We know that complete is set at this point. If STARTED isn't set
	 * anymore, then the request isn't active and the "timeout" should
	 * just be ignored. This can happen due to the bitflag ordering.
	 * Timeout first checks if STARTED is set, and if it is, assumes
	 * the request is active. But if we race with completion, then
	 * we both flags will get cleared. So check here again, and ignore
	 * a timeout event with a request that isn't active.
	 */
673 674
	if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
		return;
675

676
	if (ops->timeout)
677
		ret = ops->timeout(req, reserved);
678 679 680 681 682 683 684 685 686 687 688 689 690 691 692

	switch (ret) {
	case BLK_EH_HANDLED:
		__blk_mq_complete_request(req);
		break;
	case BLK_EH_RESET_TIMER:
		blk_add_timer(req);
		blk_clear_rq_complete(req);
		break;
	case BLK_EH_NOT_HANDLED:
		break;
	default:
		printk(KERN_ERR "block: bad eh return: %d\n", ret);
		break;
	}
693
}
694

695 696 697 698
static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
		struct request *rq, void *priv, bool reserved)
{
	struct blk_mq_timeout_data *data = priv;
699

700 701 702 703 704
	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
		/*
		 * If a request wasn't started before the queue was
		 * marked dying, kill it here or it'll go unnoticed.
		 */
705 706 707 708
		if (unlikely(blk_queue_dying(rq->q))) {
			rq->errors = -EIO;
			blk_mq_end_request(rq, rq->errors);
		}
709
		return;
710
	}
711

712 713
	if (time_after_eq(jiffies, rq->deadline)) {
		if (!blk_mark_rq_complete(rq))
714
			blk_mq_rq_timed_out(rq, reserved);
715 716 717 718
	} else if (!data->next_set || time_after(data->next, rq->deadline)) {
		data->next = rq->deadline;
		data->next_set = 1;
	}
719 720
}

721
static void blk_mq_timeout_work(struct work_struct *work)
722
{
723 724
	struct request_queue *q =
		container_of(work, struct request_queue, timeout_work);
725 726 727 728 729
	struct blk_mq_timeout_data data = {
		.next		= 0,
		.next_set	= 0,
	};
	int i;
730

731 732 733 734 735 736 737 738 739 740 741 742 743 744
	/* A deadlock might occur if a request is stuck requiring a
	 * timeout at the same time a queue freeze is waiting
	 * completion, since the timeout code would not be able to
	 * acquire the queue reference here.
	 *
	 * That's why we don't use blk_queue_enter here; instead, we use
	 * percpu_ref_tryget directly, because we need to be able to
	 * obtain a reference even in the short window between the queue
	 * starting to freeze, by dropping the first reference in
	 * blk_mq_freeze_queue_start, and the moment the last request is
	 * consumed, marked by the instant q_usage_counter reaches
	 * zero.
	 */
	if (!percpu_ref_tryget(&q->q_usage_counter))
745 746
		return;

747
	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data);
748

749 750 751
	if (data.next_set) {
		data.next = blk_rq_timeout(round_jiffies_up(data.next));
		mod_timer(&q->timeout, data.next);
752
	} else {
753 754
		struct blk_mq_hw_ctx *hctx;

755 756 757 758 759
		queue_for_each_hw_ctx(q, hctx, i) {
			/* the hctx may be unmapped, so check it here */
			if (blk_mq_hw_queue_mapped(hctx))
				blk_mq_tag_idle(hctx);
		}
760
	}
761
	blk_queue_exit(q);
762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784
}

/*
 * Reverse check our software queue for entries that we could potentially
 * merge with. Currently includes a hand-wavy stop count of 8, to not spend
 * too much time checking for merges.
 */
static bool blk_mq_attempt_merge(struct request_queue *q,
				 struct blk_mq_ctx *ctx, struct bio *bio)
{
	struct request *rq;
	int checked = 8;

	list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
		int el_ret;

		if (!checked--)
			break;

		if (!blk_rq_merge_ok(rq, bio))
			continue;

		el_ret = blk_try_merge(rq, bio);
785 786 787 788 789 790
		if (el_ret == ELEVATOR_NO_MERGE)
			continue;

		if (!blk_mq_sched_allow_merge(q, rq, bio))
			break;

791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808
		if (el_ret == ELEVATOR_BACK_MERGE) {
			if (bio_attempt_back_merge(q, rq, bio)) {
				ctx->rq_merged++;
				return true;
			}
			break;
		} else if (el_ret == ELEVATOR_FRONT_MERGE) {
			if (bio_attempt_front_merge(q, rq, bio)) {
				ctx->rq_merged++;
				return true;
			}
			break;
		}
	}

	return false;
}

809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826
struct flush_busy_ctx_data {
	struct blk_mq_hw_ctx *hctx;
	struct list_head *list;
};

static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
{
	struct flush_busy_ctx_data *flush_data = data;
	struct blk_mq_hw_ctx *hctx = flush_data->hctx;
	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];

	sbitmap_clear_bit(sb, bitnr);
	spin_lock(&ctx->lock);
	list_splice_tail_init(&ctx->rq_list, flush_data->list);
	spin_unlock(&ctx->lock);
	return true;
}

827 828 829 830
/*
 * Process software queues that have been marked busy, splicing them
 * to the for-dispatch
 */
831
void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
832
{
833 834 835 836
	struct flush_busy_ctx_data data = {
		.hctx = hctx,
		.list = list,
	};
837

838
	sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
839
}
840
EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
841

842 843 844 845
static inline unsigned int queued_to_index(unsigned int queued)
{
	if (!queued)
		return 0;
846

847
	return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
848 849
}

850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902
static bool blk_mq_get_driver_tag(struct request *rq,
				  struct blk_mq_hw_ctx **hctx, bool wait)
{
	struct blk_mq_alloc_data data = {
		.q = rq->q,
		.ctx = rq->mq_ctx,
		.hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu),
		.flags = wait ? 0 : BLK_MQ_REQ_NOWAIT,
	};

	if (blk_mq_hctx_stopped(data.hctx))
		return false;

	if (rq->tag != -1) {
done:
		if (hctx)
			*hctx = data.hctx;
		return true;
	}

	rq->tag = blk_mq_get_tag(&data);
	if (rq->tag >= 0) {
		data.hctx->tags->rqs[rq->tag] = rq;
		goto done;
	}

	return false;
}

/*
 * If we fail getting a driver tag because all the driver tags are already
 * assigned and on the dispatch list, BUT the first entry does not have a
 * tag, then we could deadlock. For that case, move entries with assigned
 * driver tags to the front, leaving the set of tagged requests in the
 * same order, and the untagged set in the same order.
 */
static bool reorder_tags_to_front(struct list_head *list)
{
	struct request *rq, *tmp, *first = NULL;

	list_for_each_entry_safe_reverse(rq, tmp, list, queuelist) {
		if (rq == first)
			break;
		if (rq->tag != -1) {
			list_move(&rq->queuelist, list);
			if (!first)
				first = rq;
		}
	}

	return first != NULL;
}

903
bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list)
904 905 906
{
	struct request_queue *q = hctx->queue;
	struct request *rq;
907 908
	LIST_HEAD(driver_list);
	struct list_head *dptr;
909
	int queued, ret = BLK_MQ_RQ_QUEUE_OK;
910

911 912 913 914 915 916
	/*
	 * Start off with dptr being NULL, so we start the first request
	 * immediately, even if we have more pending.
	 */
	dptr = NULL;

917 918 919
	/*
	 * Now process all the entries, sending them to the driver.
	 */
920
	queued = 0;
921
	while (!list_empty(list)) {
922
		struct blk_mq_queue_data bd;
923

924
		rq = list_first_entry(list, struct request, queuelist);
925 926 927 928 929 930
		if (!blk_mq_get_driver_tag(rq, &hctx, false)) {
			if (!queued && reorder_tags_to_front(list))
				continue;
			blk_mq_sched_mark_restart(hctx);
			break;
		}
931 932
		list_del_init(&rq->queuelist);

933 934
		bd.rq = rq;
		bd.list = dptr;
935
		bd.last = list_empty(list);
936 937

		ret = q->mq_ops->queue_rq(hctx, &bd);
938 939 940
		switch (ret) {
		case BLK_MQ_RQ_QUEUE_OK:
			queued++;
941
			break;
942
		case BLK_MQ_RQ_QUEUE_BUSY:
943
			list_add(&rq->queuelist, list);
944
			__blk_mq_requeue_request(rq);
945 946 947 948
			break;
		default:
			pr_err("blk-mq: bad return on queue: %d\n", ret);
		case BLK_MQ_RQ_QUEUE_ERROR:
949
			rq->errors = -EIO;
950
			blk_mq_end_request(rq, rq->errors);
951 952 953 954 955
			break;
		}

		if (ret == BLK_MQ_RQ_QUEUE_BUSY)
			break;
956 957 958 959 960

		/*
		 * We've done the first request. If we have more than 1
		 * left in the list, set dptr to defer issue.
		 */
961
		if (!dptr && list->next != list->prev)
962
			dptr = &driver_list;
963 964
	}

965
	hctx->dispatched[queued_to_index(queued)]++;
966 967 968 969 970

	/*
	 * Any items that need requeuing? Stuff them into hctx->dispatch,
	 * that is where we will continue on next queue run.
	 */
971
	if (!list_empty(list)) {
972
		spin_lock(&hctx->lock);
973
		list_splice(list, &hctx->dispatch);
974
		spin_unlock(&hctx->lock);
975

976 977 978 979 980 981 982 983
		/*
		 * the queue is expected stopped with BLK_MQ_RQ_QUEUE_BUSY, but
		 * it's possible the queue is stopped and restarted again
		 * before this. Queue restart will dispatch requests. And since
		 * requests in rq_list aren't added into hctx->dispatch yet,
		 * the requests in rq_list might get lost.
		 *
		 * blk_mq_run_hw_queue() already checks the STOPPED bit
984 985 986 987 988 989
		 *
		 * If RESTART is set, then let completion restart the queue
		 * instead of potentially looping here.
		 */
		if (!blk_mq_sched_needs_restart(hctx))
			blk_mq_run_hw_queue(hctx, true);
990
	}
991 992 993 994

	return ret != BLK_MQ_RQ_QUEUE_BUSY;
}

995 996 997 998 999 1000 1001 1002 1003
static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	int srcu_idx;

	WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) &&
		cpu_online(hctx->next_cpu));

	if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
		rcu_read_lock();
1004
		blk_mq_sched_dispatch_requests(hctx);
1005 1006 1007
		rcu_read_unlock();
	} else {
		srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu);
1008
		blk_mq_sched_dispatch_requests(hctx);
1009 1010 1011 1012
		srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx);
	}
}

1013 1014 1015 1016 1017 1018 1019 1020
/*
 * It'd be great if the workqueue API had a way to pass
 * in a mask and had some smarts for more clever placement.
 * For now we just round-robin here, switching for every
 * BLK_MQ_CPU_WORK_BATCH queued items.
 */
static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
{
1021 1022
	if (hctx->queue->nr_hw_queues == 1)
		return WORK_CPU_UNBOUND;
1023 1024

	if (--hctx->next_cpu_batch <= 0) {
1025
		int next_cpu;
1026 1027 1028 1029 1030 1031 1032 1033 1034

		next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
		if (next_cpu >= nr_cpu_ids)
			next_cpu = cpumask_first(hctx->cpumask);

		hctx->next_cpu = next_cpu;
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}

1035
	return hctx->next_cpu;
1036 1037
}

1038 1039
void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
1040 1041
	if (unlikely(blk_mq_hctx_stopped(hctx) ||
		     !blk_mq_hw_queue_mapped(hctx)))
1042 1043
		return;

1044
	if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
1045 1046
		int cpu = get_cpu();
		if (cpumask_test_cpu(cpu, hctx->cpumask)) {
1047
			__blk_mq_run_hw_queue(hctx);
1048
			put_cpu();
1049 1050
			return;
		}
1051

1052
		put_cpu();
1053
	}
1054

1055
	kblockd_schedule_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work);
1056 1057
}

1058
void blk_mq_run_hw_queues(struct request_queue *q, bool async)
1059 1060 1061 1062 1063
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i) {
1064
		if (!blk_mq_hctx_has_pending(hctx) ||
1065
		    blk_mq_hctx_stopped(hctx))
1066 1067
			continue;

1068
		blk_mq_run_hw_queue(hctx, async);
1069 1070
	}
}
1071
EXPORT_SYMBOL(blk_mq_run_hw_queues);
1072

1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
/**
 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped
 * @q: request queue.
 *
 * The caller is responsible for serializing this function against
 * blk_mq_{start,stop}_hw_queue().
 */
bool blk_mq_queue_stopped(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		if (blk_mq_hctx_stopped(hctx))
			return true;

	return false;
}
EXPORT_SYMBOL(blk_mq_queue_stopped);

1093 1094
void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
{
1095
	cancel_work(&hctx->run_work);
1096
	cancel_delayed_work(&hctx->delay_work);
1097 1098 1099 1100
	set_bit(BLK_MQ_S_STOPPED, &hctx->state);
}
EXPORT_SYMBOL(blk_mq_stop_hw_queue);

1101 1102 1103 1104 1105 1106 1107 1108 1109 1110
void blk_mq_stop_hw_queues(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_stop_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_stop_hw_queues);

1111 1112 1113
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1114

1115
	blk_mq_run_hw_queue(hctx, false);
1116 1117 1118
}
EXPORT_SYMBOL(blk_mq_start_hw_queue);

1119 1120 1121 1122 1123 1124 1125 1126 1127 1128
void blk_mq_start_hw_queues(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_start_hw_queues);

1129 1130 1131 1132 1133 1134 1135 1136 1137 1138
void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
	if (!blk_mq_hctx_stopped(hctx))
		return;

	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
	blk_mq_run_hw_queue(hctx, async);
}
EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);

1139
void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
1140 1141 1142 1143
{
	struct blk_mq_hw_ctx *hctx;
	int i;

1144 1145
	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_stopped_hw_queue(hctx, async);
1146 1147 1148
}
EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);

1149
static void blk_mq_run_work_fn(struct work_struct *work)
1150 1151 1152
{
	struct blk_mq_hw_ctx *hctx;

1153
	hctx = container_of(work, struct blk_mq_hw_ctx, run_work);
1154

1155 1156 1157
	__blk_mq_run_hw_queue(hctx);
}

1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169
static void blk_mq_delay_work_fn(struct work_struct *work)
{
	struct blk_mq_hw_ctx *hctx;

	hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);

	if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
		__blk_mq_run_hw_queue(hctx);
}

void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
{
1170 1171
	if (unlikely(!blk_mq_hw_queue_mapped(hctx)))
		return;
1172

1173 1174
	kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
			&hctx->delay_work, msecs_to_jiffies(msecs));
1175 1176 1177
}
EXPORT_SYMBOL(blk_mq_delay_queue);

1178 1179 1180
static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,
					    struct request *rq,
					    bool at_head)
1181
{
J
Jens Axboe 已提交
1182 1183
	struct blk_mq_ctx *ctx = rq->mq_ctx;

1184 1185
	trace_block_rq_insert(hctx->queue, rq);

1186 1187 1188 1189
	if (at_head)
		list_add(&rq->queuelist, &ctx->rq_list);
	else
		list_add_tail(&rq->queuelist, &ctx->rq_list);
1190
}
1191

1192 1193
void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
			     bool at_head)
1194 1195 1196
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;

J
Jens Axboe 已提交
1197
	__blk_mq_insert_req_list(hctx, rq, at_head);
1198 1199 1200
	blk_mq_hctx_mark_pending(hctx, ctx);
}

1201 1202
void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
			    struct list_head *list)
1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213

{
	/*
	 * preemption doesn't flush plug list, so it's possible ctx->cpu is
	 * offline now
	 */
	spin_lock(&ctx->lock);
	while (!list_empty(list)) {
		struct request *rq;

		rq = list_first_entry(list, struct request, queuelist);
J
Jens Axboe 已提交
1214
		BUG_ON(rq->mq_ctx != ctx);
1215
		list_del_init(&rq->queuelist);
J
Jens Axboe 已提交
1216
		__blk_mq_insert_req_list(hctx, rq, false);
1217
	}
1218
	blk_mq_hctx_mark_pending(hctx, ctx);
1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254
	spin_unlock(&ctx->lock);
}

static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
{
	struct request *rqa = container_of(a, struct request, queuelist);
	struct request *rqb = container_of(b, struct request, queuelist);

	return !(rqa->mq_ctx < rqb->mq_ctx ||
		 (rqa->mq_ctx == rqb->mq_ctx &&
		  blk_rq_pos(rqa) < blk_rq_pos(rqb)));
}

void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
{
	struct blk_mq_ctx *this_ctx;
	struct request_queue *this_q;
	struct request *rq;
	LIST_HEAD(list);
	LIST_HEAD(ctx_list);
	unsigned int depth;

	list_splice_init(&plug->mq_list, &list);

	list_sort(NULL, &list, plug_ctx_cmp);

	this_q = NULL;
	this_ctx = NULL;
	depth = 0;

	while (!list_empty(&list)) {
		rq = list_entry_rq(list.next);
		list_del_init(&rq->queuelist);
		BUG_ON(!rq->q);
		if (rq->mq_ctx != this_ctx) {
			if (this_ctx) {
1255 1256 1257 1258
				trace_block_unplug(this_q, depth, from_schedule);
				blk_mq_sched_insert_requests(this_q, this_ctx,
								&ctx_list,
								from_schedule);
1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
			}

			this_ctx = rq->mq_ctx;
			this_q = rq->q;
			depth = 0;
		}

		depth++;
		list_add_tail(&rq->queuelist, &ctx_list);
	}

	/*
	 * If 'this_ctx' is set, we know we have entries to complete
	 * on 'ctx_list'. Do those.
	 */
	if (this_ctx) {
1275 1276 1277
		trace_block_unplug(this_q, depth, from_schedule);
		blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list,
						from_schedule);
1278 1279 1280 1281 1282 1283
	}
}

static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
{
	init_request_from_bio(rq, bio);
1284

1285
	blk_account_io_start(rq, true);
1286 1287
}

1288 1289 1290 1291 1292 1293
static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx)
{
	return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
		!blk_queue_nomerges(hctx->queue);
}

1294 1295 1296
static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
					 struct blk_mq_ctx *ctx,
					 struct request *rq, struct bio *bio)
1297
{
1298
	if (!hctx_allow_merges(hctx) || !bio_mergeable(bio)) {
1299 1300 1301 1302 1303 1304 1305
		blk_mq_bio_to_request(rq, bio);
		spin_lock(&ctx->lock);
insert_rq:
		__blk_mq_insert_request(hctx, rq, false);
		spin_unlock(&ctx->lock);
		return false;
	} else {
1306 1307
		struct request_queue *q = hctx->queue;

1308 1309 1310 1311 1312
		spin_lock(&ctx->lock);
		if (!blk_mq_attempt_merge(q, ctx, bio)) {
			blk_mq_bio_to_request(rq, bio);
			goto insert_rq;
		}
1313

1314
		spin_unlock(&ctx->lock);
1315
		__blk_mq_finish_request(hctx, ctx, rq);
1316
		return true;
1317
	}
1318
}
1319

1320 1321
static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq)
{
1322 1323 1324 1325
	if (rq->tag != -1)
		return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false);

	return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true);
1326 1327
}

1328
static void blk_mq_try_issue_directly(struct request *rq, blk_qc_t *cookie)
1329 1330 1331 1332 1333 1334 1335
{
	struct request_queue *q = rq->q;
	struct blk_mq_queue_data bd = {
		.rq = rq,
		.list = NULL,
		.last = 1
	};
1336 1337 1338
	struct blk_mq_hw_ctx *hctx;
	blk_qc_t new_cookie;
	int ret;
1339

1340
	if (q->elevator)
1341 1342
		goto insert;

1343 1344 1345 1346 1347
	if (!blk_mq_get_driver_tag(rq, &hctx, false))
		goto insert;

	new_cookie = request_to_qc_t(hctx, rq);

1348 1349 1350 1351 1352 1353
	/*
	 * For OK queue, we are done. For error, kill it. Any other
	 * error (busy), just add it to our list as we previously
	 * would have done
	 */
	ret = q->mq_ops->queue_rq(hctx, &bd);
1354 1355
	if (ret == BLK_MQ_RQ_QUEUE_OK) {
		*cookie = new_cookie;
1356
		return;
1357
	}
1358

1359 1360 1361 1362 1363 1364
	__blk_mq_requeue_request(rq);

	if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
		*cookie = BLK_QC_T_NONE;
		rq->errors = -EIO;
		blk_mq_end_request(rq, rq->errors);
1365
		return;
1366
	}
1367

1368
insert:
1369
	blk_mq_sched_insert_request(rq, false, true, true);
1370 1371
}

1372 1373 1374 1375 1376
/*
 * Multiple hardware queue variant. This will not use per-process plugs,
 * but will attempt to bypass the hctx queueing if we can go straight to
 * hardware for SYNC IO.
 */
1377
static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
1378
{
1379
	const int is_sync = op_is_sync(bio->bi_opf);
J
Jens Axboe 已提交
1380
	const int is_flush_fua = bio->bi_opf & (REQ_PREFLUSH | REQ_FUA);
1381
	struct blk_mq_alloc_data data;
1382
	struct request *rq;
1383
	unsigned int request_count = 0, srcu_idx;
1384
	struct blk_plug *plug;
1385
	struct request *same_queue_rq = NULL;
1386
	blk_qc_t cookie;
J
Jens Axboe 已提交
1387
	unsigned int wb_acct;
1388 1389 1390 1391

	blk_queue_bounce(q, &bio);

	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1392
		bio_io_error(bio);
1393
		return BLK_QC_T_NONE;
1394 1395
	}

1396 1397
	blk_queue_split(q, &bio, q->bio_split);

1398 1399 1400
	if (!is_flush_fua && !blk_queue_nomerges(q) &&
	    blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq))
		return BLK_QC_T_NONE;
1401

1402 1403 1404
	if (blk_mq_sched_bio_merge(q, bio))
		return BLK_QC_T_NONE;

J
Jens Axboe 已提交
1405 1406
	wb_acct = wbt_wait(q->rq_wb, bio, NULL);

1407 1408 1409
	trace_block_getrq(q, bio, bio->bi_opf);

	rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
J
Jens Axboe 已提交
1410 1411
	if (unlikely(!rq)) {
		__wbt_done(q->rq_wb, wb_acct);
1412
		return BLK_QC_T_NONE;
J
Jens Axboe 已提交
1413 1414 1415
	}

	wbt_track(&rq->issue_stat, wb_acct);
1416

1417
	cookie = request_to_qc_t(data.hctx, rq);
1418 1419 1420

	if (unlikely(is_flush_fua)) {
		blk_mq_bio_to_request(rq, bio);
1421
		blk_mq_get_driver_tag(rq, NULL, true);
1422 1423 1424 1425
		blk_insert_flush(rq);
		goto run_queue;
	}

1426
	plug = current->plug;
1427 1428 1429 1430 1431
	/*
	 * If the driver supports defer issued based on 'last', then
	 * queue it up like normal since we can potentially save some
	 * CPU this way.
	 */
1432 1433 1434
	if (((plug && !blk_queue_nomerges(q)) || is_sync) &&
	    !(data.hctx->flags & BLK_MQ_F_DEFER_ISSUE)) {
		struct request *old_rq = NULL;
1435 1436 1437 1438

		blk_mq_bio_to_request(rq, bio);

		/*
1439
		 * We do limited plugging. If the bio can be merged, do that.
1440 1441
		 * Otherwise the existing request in the plug list will be
		 * issued. So the plug list will have one request at most
1442
		 */
1443
		if (plug) {
1444 1445
			/*
			 * The plug list might get flushed before this. If that
1446 1447 1448
			 * happens, same_queue_rq is invalid and plug list is
			 * empty
			 */
1449 1450
			if (same_queue_rq && !list_empty(&plug->mq_list)) {
				old_rq = same_queue_rq;
1451
				list_del_init(&old_rq->queuelist);
1452
			}
1453 1454 1455 1456 1457
			list_add_tail(&rq->queuelist, &plug->mq_list);
		} else /* is_sync */
			old_rq = rq;
		blk_mq_put_ctx(data.ctx);
		if (!old_rq)
1458
			goto done;
1459 1460 1461

		if (!(data.hctx->flags & BLK_MQ_F_BLOCKING)) {
			rcu_read_lock();
1462
			blk_mq_try_issue_directly(old_rq, &cookie);
1463 1464 1465
			rcu_read_unlock();
		} else {
			srcu_idx = srcu_read_lock(&data.hctx->queue_rq_srcu);
1466
			blk_mq_try_issue_directly(old_rq, &cookie);
1467 1468
			srcu_read_unlock(&data.hctx->queue_rq_srcu, srcu_idx);
		}
1469
		goto done;
1470 1471
	}

1472 1473 1474 1475 1476 1477
	if (q->elevator) {
		blk_mq_put_ctx(data.ctx);
		blk_mq_bio_to_request(rq, bio);
		blk_mq_sched_insert_request(rq, false, true, true);
		goto done;
	}
1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488
	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
		/*
		 * For a SYNC request, send it to the hardware immediately. For
		 * an ASYNC request, just ensure that we run it later on. The
		 * latter allows for merging opportunities and more efficient
		 * dispatching.
		 */
run_queue:
		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
	}
	blk_mq_put_ctx(data.ctx);
1489 1490
done:
	return cookie;
1491 1492 1493 1494 1495 1496
}

/*
 * Single hardware queue variant. This will attempt to use any per-process
 * plug for merging and IO deferral.
 */
1497
static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio)
1498
{
1499
	const int is_sync = op_is_sync(bio->bi_opf);
J
Jens Axboe 已提交
1500
	const int is_flush_fua = bio->bi_opf & (REQ_PREFLUSH | REQ_FUA);
1501 1502
	struct blk_plug *plug;
	unsigned int request_count = 0;
1503
	struct blk_mq_alloc_data data;
1504
	struct request *rq;
1505
	blk_qc_t cookie;
J
Jens Axboe 已提交
1506
	unsigned int wb_acct;
1507 1508 1509 1510

	blk_queue_bounce(q, &bio);

	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1511
		bio_io_error(bio);
1512
		return BLK_QC_T_NONE;
1513 1514
	}

1515 1516
	blk_queue_split(q, &bio, q->bio_split);

1517 1518 1519 1520 1521
	if (!is_flush_fua && !blk_queue_nomerges(q)) {
		if (blk_attempt_plug_merge(q, bio, &request_count, NULL))
			return BLK_QC_T_NONE;
	} else
		request_count = blk_plug_queued_count(q);
1522

1523 1524 1525
	if (blk_mq_sched_bio_merge(q, bio))
		return BLK_QC_T_NONE;

J
Jens Axboe 已提交
1526 1527
	wb_acct = wbt_wait(q->rq_wb, bio, NULL);

1528 1529 1530
	trace_block_getrq(q, bio, bio->bi_opf);

	rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
J
Jens Axboe 已提交
1531 1532
	if (unlikely(!rq)) {
		__wbt_done(q->rq_wb, wb_acct);
1533
		return BLK_QC_T_NONE;
J
Jens Axboe 已提交
1534 1535 1536
	}

	wbt_track(&rq->issue_stat, wb_acct);
1537

1538
	cookie = request_to_qc_t(data.hctx, rq);
1539 1540 1541

	if (unlikely(is_flush_fua)) {
		blk_mq_bio_to_request(rq, bio);
1542
		blk_mq_get_driver_tag(rq, NULL, true);
1543 1544 1545 1546 1547 1548 1549 1550 1551
		blk_insert_flush(rq);
		goto run_queue;
	}

	/*
	 * A task plug currently exists. Since this is completely lockless,
	 * utilize that to temporarily store requests until the task is
	 * either done or scheduled away.
	 */
1552 1553
	plug = current->plug;
	if (plug) {
1554 1555
		struct request *last = NULL;

1556
		blk_mq_bio_to_request(rq, bio);
1557 1558 1559 1560 1561 1562 1563

		/*
		 * @request_count may become stale because of schedule
		 * out, so check the list again.
		 */
		if (list_empty(&plug->mq_list))
			request_count = 0;
M
Ming Lei 已提交
1564
		if (!request_count)
1565
			trace_block_plug(q);
1566 1567
		else
			last = list_entry_rq(plug->mq_list.prev);
1568 1569 1570

		blk_mq_put_ctx(data.ctx);

1571 1572
		if (request_count >= BLK_MAX_REQUEST_COUNT || (last &&
		    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
1573 1574
			blk_flush_plug_list(plug, false);
			trace_block_plug(q);
1575
		}
1576

1577
		list_add_tail(&rq->queuelist, &plug->mq_list);
1578
		return cookie;
1579 1580
	}

1581 1582 1583 1584 1585 1586
	if (q->elevator) {
		blk_mq_put_ctx(data.ctx);
		blk_mq_bio_to_request(rq, bio);
		blk_mq_sched_insert_request(rq, false, true, true);
		goto done;
	}
1587 1588 1589 1590 1591 1592 1593 1594 1595
	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
		/*
		 * For a SYNC request, send it to the hardware immediately. For
		 * an ASYNC request, just ensure that we run it later on. The
		 * latter allows for merging opportunities and more efficient
		 * dispatching.
		 */
run_queue:
		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1596 1597
	}

1598
	blk_mq_put_ctx(data.ctx);
1599
done:
1600
	return cookie;
1601 1602
}

1603 1604
void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx)
1605
{
1606
	struct page *page;
1607

1608
	if (tags->rqs && set->ops->exit_request) {
1609
		int i;
1610

1611
		for (i = 0; i < tags->nr_tags; i++) {
J
Jens Axboe 已提交
1612 1613 1614
			struct request *rq = tags->static_rqs[i];

			if (!rq)
1615
				continue;
J
Jens Axboe 已提交
1616
			set->ops->exit_request(set->driver_data, rq,
1617
						hctx_idx, i);
J
Jens Axboe 已提交
1618
			tags->static_rqs[i] = NULL;
1619
		}
1620 1621
	}

1622 1623
	while (!list_empty(&tags->page_list)) {
		page = list_first_entry(&tags->page_list, struct page, lru);
1624
		list_del_init(&page->lru);
1625 1626 1627 1628 1629
		/*
		 * Remove kmemleak object previously allocated in
		 * blk_mq_init_rq_map().
		 */
		kmemleak_free(page_address(page));
1630 1631
		__free_pages(page, page->private);
	}
1632
}
1633

1634 1635
void blk_mq_free_rq_map(struct blk_mq_tags *tags)
{
1636
	kfree(tags->rqs);
1637
	tags->rqs = NULL;
J
Jens Axboe 已提交
1638 1639
	kfree(tags->static_rqs);
	tags->static_rqs = NULL;
1640

1641
	blk_mq_free_tags(tags);
1642 1643
}

1644 1645 1646 1647
struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
					unsigned int hctx_idx,
					unsigned int nr_tags,
					unsigned int reserved_tags)
1648
{
1649
	struct blk_mq_tags *tags;
1650

1651
	tags = blk_mq_init_tags(nr_tags, reserved_tags,
S
Shaohua Li 已提交
1652 1653
				set->numa_node,
				BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
1654 1655
	if (!tags)
		return NULL;
1656

1657
	tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *),
1658
				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
1659
				 set->numa_node);
1660 1661 1662 1663
	if (!tags->rqs) {
		blk_mq_free_tags(tags);
		return NULL;
	}
1664

J
Jens Axboe 已提交
1665 1666 1667 1668 1669 1670 1671 1672 1673
	tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *),
				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
				 set->numa_node);
	if (!tags->static_rqs) {
		kfree(tags->rqs);
		blk_mq_free_tags(tags);
		return NULL;
	}

1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689
	return tags;
}

static size_t order_to_size(unsigned int order)
{
	return (size_t)PAGE_SIZE << order;
}

int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx, unsigned int depth)
{
	unsigned int i, j, entries_per_page, max_order = 4;
	size_t rq_size, left;

	INIT_LIST_HEAD(&tags->page_list);

1690 1691 1692 1693
	/*
	 * rq_size is the size of the request plus driver payload, rounded
	 * to the cacheline size
	 */
1694
	rq_size = round_up(sizeof(struct request) + set->cmd_size,
1695
				cache_line_size());
1696
	left = rq_size * depth;
1697

1698
	for (i = 0; i < depth; ) {
1699 1700 1701 1702 1703
		int this_order = max_order;
		struct page *page;
		int to_do;
		void *p;

1704
		while (this_order && left < order_to_size(this_order - 1))
1705 1706 1707
			this_order--;

		do {
1708
			page = alloc_pages_node(set->numa_node,
1709
				GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
1710
				this_order);
1711 1712 1713 1714 1715 1716 1717 1718 1719
			if (page)
				break;
			if (!this_order--)
				break;
			if (order_to_size(this_order) < rq_size)
				break;
		} while (1);

		if (!page)
1720
			goto fail;
1721 1722

		page->private = this_order;
1723
		list_add_tail(&page->lru, &tags->page_list);
1724 1725

		p = page_address(page);
1726 1727 1728 1729
		/*
		 * Allow kmemleak to scan these pages as they contain pointers
		 * to additional allocations like via ops->init_request().
		 */
1730
		kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
1731
		entries_per_page = order_to_size(this_order) / rq_size;
1732
		to_do = min(entries_per_page, depth - i);
1733 1734
		left -= to_do * rq_size;
		for (j = 0; j < to_do; j++) {
J
Jens Axboe 已提交
1735 1736 1737
			struct request *rq = p;

			tags->static_rqs[i] = rq;
1738 1739
			if (set->ops->init_request) {
				if (set->ops->init_request(set->driver_data,
J
Jens Axboe 已提交
1740
						rq, hctx_idx, i,
1741
						set->numa_node)) {
J
Jens Axboe 已提交
1742
					tags->static_rqs[i] = NULL;
1743
					goto fail;
1744
				}
1745 1746
			}

1747 1748 1749 1750
			p += rq_size;
			i++;
		}
	}
1751
	return 0;
1752

1753
fail:
1754 1755
	blk_mq_free_rqs(set, tags, hctx_idx);
	return -ENOMEM;
1756 1757
}

J
Jens Axboe 已提交
1758 1759 1760 1761 1762
/*
 * 'cpu' is going away. splice any existing rq_list entries from this
 * software queue to the hw queue dispatch list, and ensure that it
 * gets run.
 */
1763
static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
1764
{
1765
	struct blk_mq_hw_ctx *hctx;
1766 1767 1768
	struct blk_mq_ctx *ctx;
	LIST_HEAD(tmp);

1769
	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
J
Jens Axboe 已提交
1770
	ctx = __blk_mq_get_ctx(hctx->queue, cpu);
1771 1772 1773 1774 1775 1776 1777 1778 1779

	spin_lock(&ctx->lock);
	if (!list_empty(&ctx->rq_list)) {
		list_splice_init(&ctx->rq_list, &tmp);
		blk_mq_hctx_clear_pending(hctx, ctx);
	}
	spin_unlock(&ctx->lock);

	if (list_empty(&tmp))
1780
		return 0;
1781

J
Jens Axboe 已提交
1782 1783 1784
	spin_lock(&hctx->lock);
	list_splice_tail_init(&tmp, &hctx->dispatch);
	spin_unlock(&hctx->lock);
1785 1786

	blk_mq_run_hw_queue(hctx, true);
1787
	return 0;
1788 1789
}

1790
static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
1791
{
1792 1793
	cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
					    &hctx->cpuhp_dead);
1794 1795
}

1796
/* hctx->ctxs will be freed in queue's release handler */
1797 1798 1799 1800
static void blk_mq_exit_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
1801 1802
	unsigned flush_start_tag = set->queue_depth;

1803 1804
	blk_mq_tag_idle(hctx);

1805 1806 1807 1808 1809
	if (set->ops->exit_request)
		set->ops->exit_request(set->driver_data,
				       hctx->fq->flush_rq, hctx_idx,
				       flush_start_tag + hctx_idx);

1810 1811 1812
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);

1813 1814 1815
	if (hctx->flags & BLK_MQ_F_BLOCKING)
		cleanup_srcu_struct(&hctx->queue_rq_srcu);

1816
	blk_mq_remove_cpuhp(hctx);
1817
	blk_free_flush_queue(hctx->fq);
1818
	sbitmap_free(&hctx->ctx_map);
1819 1820
}

M
Ming Lei 已提交
1821 1822 1823 1824 1825 1826 1827 1828 1829
static void blk_mq_exit_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set, int nr_queue)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if (i == nr_queue)
			break;
1830
		blk_mq_exit_hctx(q, set, hctx, i);
M
Ming Lei 已提交
1831 1832 1833 1834 1835 1836 1837 1838 1839
	}
}

static void blk_mq_free_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

1840
	queue_for_each_hw_ctx(q, hctx, i)
M
Ming Lei 已提交
1841 1842 1843
		free_cpumask_var(hctx->cpumask);
}

1844 1845 1846
static int blk_mq_init_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
1847
{
1848
	int node;
1849
	unsigned flush_start_tag = set->queue_depth;
1850 1851 1852 1853 1854

	node = hctx->numa_node;
	if (node == NUMA_NO_NODE)
		node = hctx->numa_node = set->numa_node;

1855
	INIT_WORK(&hctx->run_work, blk_mq_run_work_fn);
1856 1857 1858 1859 1860
	INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
	spin_lock_init(&hctx->lock);
	INIT_LIST_HEAD(&hctx->dispatch);
	hctx->queue = q;
	hctx->queue_num = hctx_idx;
1861
	hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED;
1862

1863
	cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
1864 1865

	hctx->tags = set->tags[hctx_idx];
1866 1867

	/*
1868 1869
	 * Allocate space for all possible cpus to avoid allocation at
	 * runtime
1870
	 */
1871 1872 1873 1874
	hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
					GFP_KERNEL, node);
	if (!hctx->ctxs)
		goto unregister_cpu_notifier;
1875

1876 1877
	if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL,
			      node))
1878
		goto free_ctxs;
1879

1880
	hctx->nr_ctx = 0;
1881

1882 1883 1884
	if (set->ops->init_hctx &&
	    set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
		goto free_bitmap;
1885

1886 1887 1888
	hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size);
	if (!hctx->fq)
		goto exit_hctx;
1889

1890 1891 1892 1893 1894
	if (set->ops->init_request &&
	    set->ops->init_request(set->driver_data,
				   hctx->fq->flush_rq, hctx_idx,
				   flush_start_tag + hctx_idx, node))
		goto free_fq;
1895

1896 1897 1898
	if (hctx->flags & BLK_MQ_F_BLOCKING)
		init_srcu_struct(&hctx->queue_rq_srcu);

1899
	return 0;
1900

1901 1902 1903 1904 1905
 free_fq:
	kfree(hctx->fq);
 exit_hctx:
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);
1906
 free_bitmap:
1907
	sbitmap_free(&hctx->ctx_map);
1908 1909 1910
 free_ctxs:
	kfree(hctx->ctxs);
 unregister_cpu_notifier:
1911
	blk_mq_remove_cpuhp(hctx);
1912 1913
	return -1;
}
1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928

static void blk_mq_init_cpu_queues(struct request_queue *q,
				   unsigned int nr_hw_queues)
{
	unsigned int i;

	for_each_possible_cpu(i) {
		struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
		struct blk_mq_hw_ctx *hctx;

		memset(__ctx, 0, sizeof(*__ctx));
		__ctx->cpu = i;
		spin_lock_init(&__ctx->lock);
		INIT_LIST_HEAD(&__ctx->rq_list);
		__ctx->queue = q;
1929 1930
		blk_stat_init(&__ctx->stat[BLK_STAT_READ]);
		blk_stat_init(&__ctx->stat[BLK_STAT_WRITE]);
1931 1932 1933 1934 1935

		/* If the cpu isn't online, the cpu is mapped to first hctx */
		if (!cpu_online(i))
			continue;

C
Christoph Hellwig 已提交
1936
		hctx = blk_mq_map_queue(q, i);
1937

1938 1939 1940 1941 1942
		/*
		 * Set local node, IFF we have more than one hw queue. If
		 * not, we remain on the home node of the device
		 */
		if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
1943
			hctx->numa_node = local_memory_node(cpu_to_node(i));
1944 1945 1946
	}
}

1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968
static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx)
{
	int ret = 0;

	set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
					set->queue_depth, set->reserved_tags);
	if (!set->tags[hctx_idx])
		return false;

	ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
				set->queue_depth);
	if (!ret)
		return true;

	blk_mq_free_rq_map(set->tags[hctx_idx]);
	set->tags[hctx_idx] = NULL;
	return false;
}

static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
					 unsigned int hctx_idx)
{
1969 1970 1971 1972 1973
	if (set->tags[hctx_idx]) {
		blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
		blk_mq_free_rq_map(set->tags[hctx_idx]);
		set->tags[hctx_idx] = NULL;
	}
1974 1975
}

1976 1977
static void blk_mq_map_swqueue(struct request_queue *q,
			       const struct cpumask *online_mask)
1978
{
1979
	unsigned int i, hctx_idx;
1980 1981
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
M
Ming Lei 已提交
1982
	struct blk_mq_tag_set *set = q->tag_set;
1983

1984 1985 1986 1987 1988
	/*
	 * Avoid others reading imcomplete hctx->cpumask through sysfs
	 */
	mutex_lock(&q->sysfs_lock);

1989
	queue_for_each_hw_ctx(q, hctx, i) {
1990
		cpumask_clear(hctx->cpumask);
1991 1992 1993 1994 1995 1996
		hctx->nr_ctx = 0;
	}

	/*
	 * Map software to hardware queues
	 */
1997
	for_each_possible_cpu(i) {
1998
		/* If the cpu isn't online, the cpu is mapped to first hctx */
1999
		if (!cpumask_test_cpu(i, online_mask))
2000 2001
			continue;

2002 2003
		hctx_idx = q->mq_map[i];
		/* unmapped hw queue can be remapped after CPU topo changed */
2004 2005
		if (!set->tags[hctx_idx] &&
		    !__blk_mq_alloc_rq_map(set, hctx_idx)) {
2006 2007 2008 2009 2010 2011
			/*
			 * If tags initialization fail for some hctx,
			 * that hctx won't be brought online.  In this
			 * case, remap the current ctx to hctx[0] which
			 * is guaranteed to always have tags allocated
			 */
2012
			q->mq_map[i] = 0;
2013 2014
		}

2015
		ctx = per_cpu_ptr(q->queue_ctx, i);
C
Christoph Hellwig 已提交
2016
		hctx = blk_mq_map_queue(q, i);
K
Keith Busch 已提交
2017

2018
		cpumask_set_cpu(i, hctx->cpumask);
2019 2020 2021
		ctx->index_hw = hctx->nr_ctx;
		hctx->ctxs[hctx->nr_ctx++] = ctx;
	}
2022

2023 2024
	mutex_unlock(&q->sysfs_lock);

2025
	queue_for_each_hw_ctx(q, hctx, i) {
2026
		/*
2027 2028
		 * If no software queues are mapped to this hardware queue,
		 * disable it and free the request entries.
2029 2030
		 */
		if (!hctx->nr_ctx) {
2031 2032 2033 2034
			/* Never unmap queue 0.  We need it as a
			 * fallback in case of a new remap fails
			 * allocation
			 */
2035 2036 2037
			if (i && set->tags[i])
				blk_mq_free_map_and_requests(set, i);

M
Ming Lei 已提交
2038
			hctx->tags = NULL;
2039 2040 2041
			continue;
		}

M
Ming Lei 已提交
2042 2043 2044
		hctx->tags = set->tags[i];
		WARN_ON(!hctx->tags);

2045 2046 2047 2048 2049
		/*
		 * Set the map size to the number of mapped software queues.
		 * This is more accurate and more efficient than looping
		 * over all possibly mapped software queues.
		 */
2050
		sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
2051

2052 2053 2054
		/*
		 * Initialize batch roundrobin counts
		 */
2055 2056 2057
		hctx->next_cpu = cpumask_first(hctx->cpumask);
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}
2058 2059
}

2060
static void queue_set_hctx_shared(struct request_queue *q, bool shared)
2061 2062 2063 2064
{
	struct blk_mq_hw_ctx *hctx;
	int i;

2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075
	queue_for_each_hw_ctx(q, hctx, i) {
		if (shared)
			hctx->flags |= BLK_MQ_F_TAG_SHARED;
		else
			hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
	}
}

static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, bool shared)
{
	struct request_queue *q;
2076 2077 2078

	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_freeze_queue(q);
2079
		queue_set_hctx_shared(q, shared);
2080 2081 2082 2083 2084 2085 2086 2087 2088 2089
		blk_mq_unfreeze_queue(q);
	}
}

static void blk_mq_del_queue_tag_set(struct request_queue *q)
{
	struct blk_mq_tag_set *set = q->tag_set;

	mutex_lock(&set->tag_list_lock);
	list_del_init(&q->tag_set_list);
2090 2091 2092 2093 2094 2095
	if (list_is_singular(&set->tag_list)) {
		/* just transitioned to unshared */
		set->flags &= ~BLK_MQ_F_TAG_SHARED;
		/* update existing queue */
		blk_mq_update_tag_set_depth(set, false);
	}
2096 2097 2098 2099 2100 2101 2102 2103 2104
	mutex_unlock(&set->tag_list_lock);
}

static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
				     struct request_queue *q)
{
	q->tag_set = set;

	mutex_lock(&set->tag_list_lock);
2105 2106 2107 2108 2109 2110 2111 2112 2113

	/* Check to see if we're transitioning to shared (from 1 to 2 queues). */
	if (!list_empty(&set->tag_list) && !(set->flags & BLK_MQ_F_TAG_SHARED)) {
		set->flags |= BLK_MQ_F_TAG_SHARED;
		/* update existing queue */
		blk_mq_update_tag_set_depth(set, true);
	}
	if (set->flags & BLK_MQ_F_TAG_SHARED)
		queue_set_hctx_shared(q, true);
2114
	list_add_tail(&q->tag_set_list, &set->tag_list);
2115

2116 2117 2118
	mutex_unlock(&set->tag_list_lock);
}

2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129
/*
 * It is the actual release handler for mq, but we do it from
 * request queue's release handler for avoiding use-after-free
 * and headache because q->mq_kobj shouldn't have been introduced,
 * but we can't group ctx/kctx kobj without it.
 */
void blk_mq_release(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

2130 2131
	blk_mq_sched_teardown(q);

2132
	/* hctx kobj stays in hctx */
2133 2134 2135 2136
	queue_for_each_hw_ctx(q, hctx, i) {
		if (!hctx)
			continue;
		kfree(hctx->ctxs);
2137
		kfree(hctx);
2138
	}
2139

2140 2141
	q->mq_map = NULL;

2142 2143 2144 2145 2146 2147
	kfree(q->queue_hw_ctx);

	/* ctx kobj stays in queue_ctx */
	free_percpu(q->queue_ctx);
}

2148
struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163
{
	struct request_queue *uninit_q, *q;

	uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
	if (!uninit_q)
		return ERR_PTR(-ENOMEM);

	q = blk_mq_init_allocated_queue(set, uninit_q);
	if (IS_ERR(q))
		blk_cleanup_queue(uninit_q);

	return q;
}
EXPORT_SYMBOL(blk_mq_init_queue);

K
Keith Busch 已提交
2164 2165
static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
						struct request_queue *q)
2166
{
K
Keith Busch 已提交
2167 2168
	int i, j;
	struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
2169

K
Keith Busch 已提交
2170
	blk_mq_sysfs_unregister(q);
2171
	for (i = 0; i < set->nr_hw_queues; i++) {
K
Keith Busch 已提交
2172
		int node;
2173

K
Keith Busch 已提交
2174 2175 2176 2177
		if (hctxs[i])
			continue;

		node = blk_mq_hw_queue_to_node(q->mq_map, i);
2178 2179
		hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
					GFP_KERNEL, node);
2180
		if (!hctxs[i])
K
Keith Busch 已提交
2181
			break;
2182

2183
		if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL,
K
Keith Busch 已提交
2184 2185 2186 2187 2188
						node)) {
			kfree(hctxs[i]);
			hctxs[i] = NULL;
			break;
		}
2189

2190
		atomic_set(&hctxs[i]->nr_active, 0);
2191
		hctxs[i]->numa_node = node;
2192
		hctxs[i]->queue_num = i;
K
Keith Busch 已提交
2193 2194 2195 2196 2197 2198 2199 2200

		if (blk_mq_init_hctx(q, set, hctxs[i], i)) {
			free_cpumask_var(hctxs[i]->cpumask);
			kfree(hctxs[i]);
			hctxs[i] = NULL;
			break;
		}
		blk_mq_hctx_kobj_init(hctxs[i]);
2201
	}
K
Keith Busch 已提交
2202 2203 2204 2205
	for (j = i; j < q->nr_hw_queues; j++) {
		struct blk_mq_hw_ctx *hctx = hctxs[j];

		if (hctx) {
2206 2207
			if (hctx->tags)
				blk_mq_free_map_and_requests(set, j);
K
Keith Busch 已提交
2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223
			blk_mq_exit_hctx(q, set, hctx, j);
			free_cpumask_var(hctx->cpumask);
			kobject_put(&hctx->kobj);
			kfree(hctx->ctxs);
			kfree(hctx);
			hctxs[j] = NULL;

		}
	}
	q->nr_hw_queues = i;
	blk_mq_sysfs_register(q);
}

struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
						  struct request_queue *q)
{
M
Ming Lei 已提交
2224 2225 2226
	/* mark the queue as mq asap */
	q->mq_ops = set->ops;

K
Keith Busch 已提交
2227 2228
	q->queue_ctx = alloc_percpu(struct blk_mq_ctx);
	if (!q->queue_ctx)
M
Ming Lin 已提交
2229
		goto err_exit;
K
Keith Busch 已提交
2230 2231 2232 2233 2234 2235

	q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)),
						GFP_KERNEL, set->numa_node);
	if (!q->queue_hw_ctx)
		goto err_percpu;

2236
	q->mq_map = set->mq_map;
K
Keith Busch 已提交
2237 2238 2239 2240

	blk_mq_realloc_hw_ctxs(set, q);
	if (!q->nr_hw_queues)
		goto err_hctxs;
2241

2242
	INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
2243
	blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
2244 2245 2246

	q->nr_queues = nr_cpu_ids;

2247
	q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
2248

2249 2250 2251
	if (!(set->flags & BLK_MQ_F_SG_MERGE))
		q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;

2252 2253
	q->sg_reserved_size = INT_MAX;

2254
	INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
2255 2256 2257
	INIT_LIST_HEAD(&q->requeue_list);
	spin_lock_init(&q->requeue_lock);

2258 2259 2260 2261 2262
	if (q->nr_hw_queues > 1)
		blk_queue_make_request(q, blk_mq_make_request);
	else
		blk_queue_make_request(q, blk_sq_make_request);

2263 2264 2265 2266 2267
	/*
	 * Do this after blk_queue_make_request() overrides it...
	 */
	q->nr_requests = set->queue_depth;

2268 2269 2270 2271 2272
	/*
	 * Default to classic polling
	 */
	q->poll_nsec = -1;

2273 2274
	if (set->ops->complete)
		blk_queue_softirq_done(q, set->ops->complete);
2275

2276
	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
2277

2278
	get_online_cpus();
2279 2280
	mutex_lock(&all_q_mutex);

2281
	list_add_tail(&q->all_q_node, &all_q_list);
2282
	blk_mq_add_queue_tag_set(set, q);
2283
	blk_mq_map_swqueue(q, cpu_online_mask);
2284

2285
	mutex_unlock(&all_q_mutex);
2286
	put_online_cpus();
2287

2288 2289 2290 2291 2292 2293 2294 2295
	if (!(set->flags & BLK_MQ_F_NO_SCHED)) {
		int ret;

		ret = blk_mq_sched_init(q);
		if (ret)
			return ERR_PTR(ret);
	}

2296
	return q;
2297

2298
err_hctxs:
K
Keith Busch 已提交
2299
	kfree(q->queue_hw_ctx);
2300
err_percpu:
K
Keith Busch 已提交
2301
	free_percpu(q->queue_ctx);
M
Ming Lin 已提交
2302 2303
err_exit:
	q->mq_ops = NULL;
2304 2305
	return ERR_PTR(-ENOMEM);
}
2306
EXPORT_SYMBOL(blk_mq_init_allocated_queue);
2307 2308 2309

void blk_mq_free_queue(struct request_queue *q)
{
M
Ming Lei 已提交
2310
	struct blk_mq_tag_set	*set = q->tag_set;
2311

2312 2313 2314 2315
	mutex_lock(&all_q_mutex);
	list_del_init(&q->all_q_node);
	mutex_unlock(&all_q_mutex);

J
Jens Axboe 已提交
2316 2317
	wbt_exit(q);

2318 2319
	blk_mq_del_queue_tag_set(q);

M
Ming Lei 已提交
2320 2321
	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
	blk_mq_free_hw_queues(q, set);
2322 2323 2324
}

/* Basically redo blk_mq_init_queue with queue frozen */
2325 2326
static void blk_mq_queue_reinit(struct request_queue *q,
				const struct cpumask *online_mask)
2327
{
2328
	WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth));
2329

2330 2331
	blk_mq_sysfs_unregister(q);

2332 2333 2334 2335 2336 2337
	/*
	 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
	 * we should change hctx numa_node according to new topology (this
	 * involves free and re-allocate memory, worthy doing?)
	 */

2338
	blk_mq_map_swqueue(q, online_mask);
2339

2340
	blk_mq_sysfs_register(q);
2341 2342
}

2343 2344 2345 2346 2347 2348 2349 2350
/*
 * New online cpumask which is going to be set in this hotplug event.
 * Declare this cpumasks as global as cpu-hotplug operation is invoked
 * one-by-one and dynamically allocating this could result in a failure.
 */
static struct cpumask cpuhp_online_new;

static void blk_mq_queue_reinit_work(void)
2351 2352 2353 2354
{
	struct request_queue *q;

	mutex_lock(&all_q_mutex);
2355 2356 2357 2358 2359 2360 2361 2362 2363
	/*
	 * We need to freeze and reinit all existing queues.  Freezing
	 * involves synchronous wait for an RCU grace period and doing it
	 * one by one may take a long time.  Start freezing all queues in
	 * one swoop and then wait for the completions so that freezing can
	 * take place in parallel.
	 */
	list_for_each_entry(q, &all_q_list, all_q_node)
		blk_mq_freeze_queue_start(q);
2364
	list_for_each_entry(q, &all_q_list, all_q_node)
2365 2366
		blk_mq_freeze_queue_wait(q);

2367
	list_for_each_entry(q, &all_q_list, all_q_node)
2368
		blk_mq_queue_reinit(q, &cpuhp_online_new);
2369 2370 2371 2372

	list_for_each_entry(q, &all_q_list, all_q_node)
		blk_mq_unfreeze_queue(q);

2373
	mutex_unlock(&all_q_mutex);
2374 2375 2376 2377
}

static int blk_mq_queue_reinit_dead(unsigned int cpu)
{
2378
	cpumask_copy(&cpuhp_online_new, cpu_online_mask);
2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393
	blk_mq_queue_reinit_work();
	return 0;
}

/*
 * Before hotadded cpu starts handling requests, new mappings must be
 * established.  Otherwise, these requests in hw queue might never be
 * dispatched.
 *
 * For example, there is a single hw queue (hctx) and two CPU queues (ctx0
 * for CPU0, and ctx1 for CPU1).
 *
 * Now CPU1 is just onlined and a request is inserted into ctx1->rq_list
 * and set bit0 in pending bitmap as ctx1->index_hw is still zero.
 *
2394 2395 2396 2397
 * And then while running hw queue, blk_mq_flush_busy_ctxs() finds bit0 is set
 * in pending bitmap and tries to retrieve requests in hctx->ctxs[0]->rq_list.
 * But htx->ctxs[0] is a pointer to ctx0, so the request in ctx1->rq_list is
 * ignored.
2398 2399 2400 2401 2402 2403 2404
 */
static int blk_mq_queue_reinit_prepare(unsigned int cpu)
{
	cpumask_copy(&cpuhp_online_new, cpu_online_mask);
	cpumask_set_cpu(cpu, &cpuhp_online_new);
	blk_mq_queue_reinit_work();
	return 0;
2405 2406
}

2407 2408 2409 2410
static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	int i;

2411 2412
	for (i = 0; i < set->nr_hw_queues; i++)
		if (!__blk_mq_alloc_rq_map(set, i))
2413 2414 2415 2416 2417 2418
			goto out_unwind;

	return 0;

out_unwind:
	while (--i >= 0)
2419
		blk_mq_free_rq_map(set->tags[i]);
2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458

	return -ENOMEM;
}

/*
 * Allocate the request maps associated with this tag_set. Note that this
 * may reduce the depth asked for, if memory is tight. set->queue_depth
 * will be updated to reflect the allocated depth.
 */
static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	unsigned int depth;
	int err;

	depth = set->queue_depth;
	do {
		err = __blk_mq_alloc_rq_maps(set);
		if (!err)
			break;

		set->queue_depth >>= 1;
		if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
			err = -ENOMEM;
			break;
		}
	} while (set->queue_depth);

	if (!set->queue_depth || err) {
		pr_err("blk-mq: failed to allocate request map\n");
		return -ENOMEM;
	}

	if (depth != set->queue_depth)
		pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
						depth, set->queue_depth);

	return 0;
}

2459 2460 2461 2462 2463 2464
/*
 * Alloc a tag set to be associated with one or more request queues.
 * May fail with EINVAL for various error conditions. May adjust the
 * requested depth down, if if it too large. In that case, the set
 * value will be stored in set->queue_depth.
 */
2465 2466
int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
{
2467 2468
	int ret;

B
Bart Van Assche 已提交
2469 2470
	BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);

2471 2472
	if (!set->nr_hw_queues)
		return -EINVAL;
2473
	if (!set->queue_depth)
2474 2475 2476 2477
		return -EINVAL;
	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
		return -EINVAL;

C
Christoph Hellwig 已提交
2478
	if (!set->ops->queue_rq)
2479 2480
		return -EINVAL;

2481 2482 2483 2484 2485
	if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
		pr_info("blk-mq: reduced tag depth to %u\n",
			BLK_MQ_MAX_DEPTH);
		set->queue_depth = BLK_MQ_MAX_DEPTH;
	}
2486

2487 2488 2489 2490 2491 2492 2493 2494 2495
	/*
	 * If a crashdump is active, then we are potentially in a very
	 * memory constrained environment. Limit us to 1 queue and
	 * 64 tags to prevent using too much memory.
	 */
	if (is_kdump_kernel()) {
		set->nr_hw_queues = 1;
		set->queue_depth = min(64U, set->queue_depth);
	}
K
Keith Busch 已提交
2496 2497 2498 2499 2500
	/*
	 * There is no use for more h/w queues than cpus.
	 */
	if (set->nr_hw_queues > nr_cpu_ids)
		set->nr_hw_queues = nr_cpu_ids;
2501

K
Keith Busch 已提交
2502
	set->tags = kzalloc_node(nr_cpu_ids * sizeof(struct blk_mq_tags *),
2503 2504
				 GFP_KERNEL, set->numa_node);
	if (!set->tags)
2505
		return -ENOMEM;
2506

2507 2508 2509
	ret = -ENOMEM;
	set->mq_map = kzalloc_node(sizeof(*set->mq_map) * nr_cpu_ids,
			GFP_KERNEL, set->numa_node);
2510 2511 2512
	if (!set->mq_map)
		goto out_free_tags;

2513 2514 2515 2516 2517 2518 2519 2520 2521
	if (set->ops->map_queues)
		ret = set->ops->map_queues(set);
	else
		ret = blk_mq_map_queues(set);
	if (ret)
		goto out_free_mq_map;

	ret = blk_mq_alloc_rq_maps(set);
	if (ret)
2522
		goto out_free_mq_map;
2523

2524 2525 2526
	mutex_init(&set->tag_list_lock);
	INIT_LIST_HEAD(&set->tag_list);

2527
	return 0;
2528 2529 2530 2531 2532

out_free_mq_map:
	kfree(set->mq_map);
	set->mq_map = NULL;
out_free_tags:
2533 2534
	kfree(set->tags);
	set->tags = NULL;
2535
	return ret;
2536 2537 2538 2539 2540 2541 2542
}
EXPORT_SYMBOL(blk_mq_alloc_tag_set);

void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
{
	int i;

2543 2544
	for (i = 0; i < nr_cpu_ids; i++)
		blk_mq_free_map_and_requests(set, i);
2545

2546 2547 2548
	kfree(set->mq_map);
	set->mq_map = NULL;

M
Ming Lei 已提交
2549
	kfree(set->tags);
2550
	set->tags = NULL;
2551 2552 2553
}
EXPORT_SYMBOL(blk_mq_free_tag_set);

2554 2555 2556 2557 2558 2559
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
{
	struct blk_mq_tag_set *set = q->tag_set;
	struct blk_mq_hw_ctx *hctx;
	int i, ret;

2560
	if (!set)
2561 2562 2563 2564
		return -EINVAL;

	ret = 0;
	queue_for_each_hw_ctx(q, hctx, i) {
2565 2566
		if (!hctx->tags)
			continue;
2567 2568 2569 2570 2571 2572 2573 2574 2575
		/*
		 * If we're using an MQ scheduler, just update the scheduler
		 * queue depth. This is similar to what the old code would do.
		 */
		if (!hctx->sched_tags)
			ret = blk_mq_tag_update_depth(hctx->tags,
							min(nr, set->queue_depth));
		else
			ret = blk_mq_tag_update_depth(hctx->sched_tags, nr);
2576 2577 2578 2579 2580 2581 2582 2583 2584 2585
		if (ret)
			break;
	}

	if (!ret)
		q->nr_requests = nr;

	return ret;
}

K
Keith Busch 已提交
2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614
void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
{
	struct request_queue *q;

	if (nr_hw_queues > nr_cpu_ids)
		nr_hw_queues = nr_cpu_ids;
	if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues)
		return;

	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_freeze_queue(q);

	set->nr_hw_queues = nr_hw_queues;
	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_realloc_hw_ctxs(set, q);

		if (q->nr_hw_queues > 1)
			blk_queue_make_request(q, blk_mq_make_request);
		else
			blk_queue_make_request(q, blk_sq_make_request);

		blk_mq_queue_reinit(q, cpu_online_mask);
	}

	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_unfreeze_queue(q);
}
EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);

2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651
static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
				       struct blk_mq_hw_ctx *hctx,
				       struct request *rq)
{
	struct blk_rq_stat stat[2];
	unsigned long ret = 0;

	/*
	 * If stats collection isn't on, don't sleep but turn it on for
	 * future users
	 */
	if (!blk_stat_enable(q))
		return 0;

	/*
	 * We don't have to do this once per IO, should optimize this
	 * to just use the current window of stats until it changes
	 */
	memset(&stat, 0, sizeof(stat));
	blk_hctx_stat_get(hctx, stat);

	/*
	 * As an optimistic guess, use half of the mean service time
	 * for this type of request. We can (and should) make this smarter.
	 * For instance, if the completion latencies are tight, we can
	 * get closer than just half the mean. This is especially
	 * important on devices where the completion latencies are longer
	 * than ~10 usec.
	 */
	if (req_op(rq) == REQ_OP_READ && stat[BLK_STAT_READ].nr_samples)
		ret = (stat[BLK_STAT_READ].mean + 1) / 2;
	else if (req_op(rq) == REQ_OP_WRITE && stat[BLK_STAT_WRITE].nr_samples)
		ret = (stat[BLK_STAT_WRITE].mean + 1) / 2;

	return ret;
}

2652
static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
2653
				     struct blk_mq_hw_ctx *hctx,
2654 2655 2656 2657
				     struct request *rq)
{
	struct hrtimer_sleeper hs;
	enum hrtimer_mode mode;
2658
	unsigned int nsecs;
2659 2660
	ktime_t kt;

2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678
	if (test_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags))
		return false;

	/*
	 * poll_nsec can be:
	 *
	 * -1:	don't ever hybrid sleep
	 *  0:	use half of prev avg
	 * >0:	use this specific value
	 */
	if (q->poll_nsec == -1)
		return false;
	else if (q->poll_nsec > 0)
		nsecs = q->poll_nsec;
	else
		nsecs = blk_mq_poll_nsecs(q, hctx, rq);

	if (!nsecs)
2679 2680 2681 2682 2683 2684 2685 2686
		return false;

	set_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);

	/*
	 * This will be replaced with the stats tracking code, using
	 * 'avg_completion_time / 2' as the pre-sleep target.
	 */
T
Thomas Gleixner 已提交
2687
	kt = nsecs;
2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709

	mode = HRTIMER_MODE_REL;
	hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode);
	hrtimer_set_expires(&hs.timer, kt);

	hrtimer_init_sleeper(&hs, current);
	do {
		if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
			break;
		set_current_state(TASK_UNINTERRUPTIBLE);
		hrtimer_start_expires(&hs.timer, mode);
		if (hs.task)
			io_schedule();
		hrtimer_cancel(&hs.timer);
		mode = HRTIMER_MODE_ABS;
	} while (hs.task && !signal_pending(current));

	__set_current_state(TASK_RUNNING);
	destroy_hrtimer_on_stack(&hs.timer);
	return true;
}

J
Jens Axboe 已提交
2710 2711 2712 2713 2714
static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq)
{
	struct request_queue *q = hctx->queue;
	long state;

2715 2716 2717 2718 2719 2720 2721
	/*
	 * If we sleep, have the caller restart the poll loop to reset
	 * the state. Like for the other success return cases, the
	 * caller is responsible for checking if the IO completed. If
	 * the IO isn't complete, we'll get called again and will go
	 * straight to the busy poll loop.
	 */
2722
	if (blk_mq_poll_hybrid_sleep(q, hctx, rq))
2723 2724
		return true;

J
Jens Axboe 已提交
2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767
	hctx->poll_considered++;

	state = current->state;
	while (!need_resched()) {
		int ret;

		hctx->poll_invoked++;

		ret = q->mq_ops->poll(hctx, rq->tag);
		if (ret > 0) {
			hctx->poll_success++;
			set_current_state(TASK_RUNNING);
			return true;
		}

		if (signal_pending_state(state, current))
			set_current_state(TASK_RUNNING);

		if (current->state == TASK_RUNNING)
			return true;
		if (ret < 0)
			break;
		cpu_relax();
	}

	return false;
}

bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie)
{
	struct blk_mq_hw_ctx *hctx;
	struct blk_plug *plug;
	struct request *rq;

	if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) ||
	    !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
		return false;

	plug = current->plug;
	if (plug)
		blk_flush_plug_list(plug, false);

	hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
2768 2769 2770 2771
	if (!blk_qc_t_is_internal(cookie))
		rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
	else
		rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
J
Jens Axboe 已提交
2772 2773 2774 2775 2776

	return __blk_mq_poll(hctx, rq);
}
EXPORT_SYMBOL_GPL(blk_mq_poll);

2777 2778 2779 2780 2781 2782 2783 2784 2785 2786
void blk_mq_disable_hotplug(void)
{
	mutex_lock(&all_q_mutex);
}

void blk_mq_enable_hotplug(void)
{
	mutex_unlock(&all_q_mutex);
}

2787 2788
static int __init blk_mq_init(void)
{
2789 2790
	cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
				blk_mq_hctx_notify_dead);
2791

2792 2793 2794
	cpuhp_setup_state_nocalls(CPUHP_BLK_MQ_PREPARE, "block/mq:prepare",
				  blk_mq_queue_reinit_prepare,
				  blk_mq_queue_reinit_dead);
2795 2796 2797
	return 0;
}
subsys_initcall(blk_mq_init);