blk-mq.c 48.7 KB
Newer Older
1 2 3 4 5 6
/*
 * Block multiqueue core code
 *
 * Copyright (C) 2013-2014 Jens Axboe
 * Copyright (C) 2013-2014 Christoph Hellwig
 */
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>
#include <linux/llist.h>
#include <linux/list_sort.h>
#include <linux/cpu.h>
#include <linux/cache.h>
#include <linux/sched/sysctl.h>
#include <linux/delay.h>
23
#include <linux/crash_dump.h>
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43

#include <trace/events/block.h>

#include <linux/blk-mq.h>
#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-tag.h"

static DEFINE_MUTEX(all_q_mutex);
static LIST_HEAD(all_q_list);

static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx);

/*
 * Check if any of the ctx's have pending work in this hardware queue
 */
static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
{
	unsigned int i;

44 45
	for (i = 0; i < hctx->ctx_map.map_size; i++)
		if (hctx->ctx_map.map[i].word)
46 47 48 49 50
			return true;

	return false;
}

51 52 53 54 55 56 57 58 59
static inline struct blk_align_bitmap *get_bm(struct blk_mq_hw_ctx *hctx,
					      struct blk_mq_ctx *ctx)
{
	return &hctx->ctx_map.map[ctx->index_hw / hctx->ctx_map.bits_per_word];
}

#define CTX_TO_BIT(hctx, ctx)	\
	((ctx)->index_hw & ((hctx)->ctx_map.bits_per_word - 1))

60 61 62 63 64 65
/*
 * Mark this ctx as having pending work in this hardware queue
 */
static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
				     struct blk_mq_ctx *ctx)
{
66 67 68 69 70 71 72 73 74 75 76 77
	struct blk_align_bitmap *bm = get_bm(hctx, ctx);

	if (!test_bit(CTX_TO_BIT(hctx, ctx), &bm->word))
		set_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
}

static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
				      struct blk_mq_ctx *ctx)
{
	struct blk_align_bitmap *bm = get_bm(hctx, ctx);

	clear_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
78 79 80 81
}

static int blk_mq_queue_enter(struct request_queue *q)
{
82 83
	while (true) {
		int ret;
84

85 86
		if (percpu_ref_tryget_live(&q->mq_usage_counter))
			return 0;
87

88 89 90 91 92 93 94
		ret = wait_event_interruptible(q->mq_freeze_wq,
				!q->mq_freeze_depth || blk_queue_dying(q));
		if (blk_queue_dying(q))
			return -ENODEV;
		if (ret)
			return ret;
	}
95 96 97 98
}

static void blk_mq_queue_exit(struct request_queue *q)
{
99 100 101 102 103 104 105 106 107
	percpu_ref_put(&q->mq_usage_counter);
}

static void blk_mq_usage_counter_release(struct percpu_ref *ref)
{
	struct request_queue *q =
		container_of(ref, struct request_queue, mq_usage_counter);

	wake_up_all(&q->mq_freeze_wq);
108 109
}

110 111 112 113 114
/*
 * Guarantee no request is in use, so we can change any data structure of
 * the queue afterward.
 */
void blk_mq_freeze_queue(struct request_queue *q)
115
{
116 117
	bool freeze;

118
	spin_lock_irq(q->queue_lock);
119
	freeze = !q->mq_freeze_depth++;
120 121
	spin_unlock_irq(q->queue_lock);

122
	if (freeze) {
123
		percpu_ref_kill(&q->mq_usage_counter);
124 125
		blk_mq_run_queues(q, false);
	}
126
	wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->mq_usage_counter));
127 128
}

129 130
static void blk_mq_unfreeze_queue(struct request_queue *q)
{
131
	bool wake;
132 133

	spin_lock_irq(q->queue_lock);
134 135
	wake = !--q->mq_freeze_depth;
	WARN_ON_ONCE(q->mq_freeze_depth < 0);
136
	spin_unlock_irq(q->queue_lock);
137 138
	if (wake) {
		percpu_ref_reinit(&q->mq_usage_counter);
139
		wake_up_all(&q->mq_freeze_wq);
140
	}
141 142 143 144 145 146 147 148
}

bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
{
	return blk_mq_has_free_tags(hctx->tags);
}
EXPORT_SYMBOL(blk_mq_can_queue);

149 150
static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
			       struct request *rq, unsigned int rw_flags)
151
{
152 153 154
	if (blk_queue_io_stat(q))
		rw_flags |= REQ_IO_STAT;

155 156 157
	INIT_LIST_HEAD(&rq->queuelist);
	/* csd/requeue_work/fifo_time is initialized before use */
	rq->q = q;
158
	rq->mq_ctx = ctx;
159
	rq->cmd_flags |= rw_flags;
160 161 162 163 164 165
	/* do not touch atomic flags, it needs atomic ops against the timer */
	rq->cpu = -1;
	INIT_HLIST_NODE(&rq->hash);
	RB_CLEAR_NODE(&rq->rb_node);
	rq->rq_disk = NULL;
	rq->part = NULL;
166
	rq->start_time = jiffies;
167 168
#ifdef CONFIG_BLK_CGROUP
	rq->rl = NULL;
169
	set_start_time_ns(rq);
170 171 172 173 174 175 176 177 178 179
	rq->io_start_time_ns = 0;
#endif
	rq->nr_phys_segments = 0;
#if defined(CONFIG_BLK_DEV_INTEGRITY)
	rq->nr_integrity_segments = 0;
#endif
	rq->special = NULL;
	/* tag was already set */
	rq->errors = 0;

180 181
	rq->cmd = rq->__cmd;

182 183 184 185 186 187
	rq->extra_len = 0;
	rq->sense_len = 0;
	rq->resid_len = 0;
	rq->sense = NULL;

	INIT_LIST_HEAD(&rq->timeout_list);
188 189
	rq->timeout = 0;

190 191 192 193
	rq->end_io = NULL;
	rq->end_io_data = NULL;
	rq->next_rq = NULL;

194 195 196
	ctx->rq_dispatched[rw_is_sync(rw_flags)]++;
}

197
static struct request *
198
__blk_mq_alloc_request(struct blk_mq_alloc_data *data, int rw)
199 200 201 202
{
	struct request *rq;
	unsigned int tag;

203
	tag = blk_mq_get_tag(data);
204
	if (tag != BLK_MQ_TAG_FAIL) {
205
		rq = data->hctx->tags->rqs[tag];
206

207
		if (blk_mq_tag_busy(data->hctx)) {
208
			rq->cmd_flags = REQ_MQ_INFLIGHT;
209
			atomic_inc(&data->hctx->nr_active);
210 211 212
		}

		rq->tag = tag;
213
		blk_mq_rq_ctx_init(data->q, data->ctx, rq, rw);
214 215 216 217 218 219
		return rq;
	}

	return NULL;
}

220 221
struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp,
		bool reserved)
222
{
223 224
	struct blk_mq_ctx *ctx;
	struct blk_mq_hw_ctx *hctx;
225
	struct request *rq;
226
	struct blk_mq_alloc_data alloc_data;
227
	int ret;
228

229 230 231
	ret = blk_mq_queue_enter(q);
	if (ret)
		return ERR_PTR(ret);
232

233 234
	ctx = blk_mq_get_ctx(q);
	hctx = q->mq_ops->map_queue(q, ctx->cpu);
235 236
	blk_mq_set_alloc_data(&alloc_data, q, gfp & ~__GFP_WAIT,
			reserved, ctx, hctx);
237

238
	rq = __blk_mq_alloc_request(&alloc_data, rw);
239 240 241 242 243 244
	if (!rq && (gfp & __GFP_WAIT)) {
		__blk_mq_run_hw_queue(hctx);
		blk_mq_put_ctx(ctx);

		ctx = blk_mq_get_ctx(q);
		hctx = q->mq_ops->map_queue(q, ctx->cpu);
245 246 247 248
		blk_mq_set_alloc_data(&alloc_data, q, gfp, reserved, ctx,
				hctx);
		rq =  __blk_mq_alloc_request(&alloc_data, rw);
		ctx = alloc_data.ctx;
249 250
	}
	blk_mq_put_ctx(ctx);
251 252
	if (!rq)
		return ERR_PTR(-EWOULDBLOCK);
253 254
	return rq;
}
255
EXPORT_SYMBOL(blk_mq_alloc_request);
256 257 258 259 260 261 262

static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx,
				  struct blk_mq_ctx *ctx, struct request *rq)
{
	const int tag = rq->tag;
	struct request_queue *q = rq->q;

263 264
	if (rq->cmd_flags & REQ_MQ_INFLIGHT)
		atomic_dec(&hctx->nr_active);
265
	rq->cmd_flags = 0;
266

267
	clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
268
	blk_mq_put_tag(hctx, tag, &ctx->last_tag);
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283
	blk_mq_queue_exit(q);
}

void blk_mq_free_request(struct request *rq)
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q = rq->q;

	ctx->rq_completed[rq_is_sync(rq)]++;

	hctx = q->mq_ops->map_queue(q, ctx->cpu);
	__blk_mq_free_request(hctx, ctx, rq);
}

284
inline void __blk_mq_end_request(struct request *rq, int error)
285
{
M
Ming Lei 已提交
286 287
	blk_account_io_done(rq);

C
Christoph Hellwig 已提交
288
	if (rq->end_io) {
289
		rq->end_io(rq, error);
C
Christoph Hellwig 已提交
290 291 292
	} else {
		if (unlikely(blk_bidi_rq(rq)))
			blk_mq_free_request(rq->next_rq);
293
		blk_mq_free_request(rq);
C
Christoph Hellwig 已提交
294
	}
295
}
296
EXPORT_SYMBOL(__blk_mq_end_request);
297

298
void blk_mq_end_request(struct request *rq, int error)
299 300 301
{
	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
		BUG();
302
	__blk_mq_end_request(rq, error);
303
}
304
EXPORT_SYMBOL(blk_mq_end_request);
305

306
static void __blk_mq_complete_request_remote(void *data)
307
{
308
	struct request *rq = data;
309

310
	rq->q->softirq_done_fn(rq);
311 312
}

313
static void blk_mq_ipi_complete_request(struct request *rq)
314 315
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;
C
Christoph Hellwig 已提交
316
	bool shared = false;
317 318
	int cpu;

C
Christoph Hellwig 已提交
319
	if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
320 321 322
		rq->q->softirq_done_fn(rq);
		return;
	}
323 324

	cpu = get_cpu();
C
Christoph Hellwig 已提交
325 326 327 328
	if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
		shared = cpus_share_cache(cpu, ctx->cpu);

	if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
329
		rq->csd.func = __blk_mq_complete_request_remote;
330 331
		rq->csd.info = rq;
		rq->csd.flags = 0;
332
		smp_call_function_single_async(ctx->cpu, &rq->csd);
333
	} else {
334
		rq->q->softirq_done_fn(rq);
335
	}
336 337
	put_cpu();
}
338

339 340 341 342 343
void __blk_mq_complete_request(struct request *rq)
{
	struct request_queue *q = rq->q;

	if (!q->softirq_done_fn)
344
		blk_mq_end_request(rq, rq->errors);
345 346 347 348
	else
		blk_mq_ipi_complete_request(rq);
}

349 350 351 352 353 354 355 356 357 358
/**
 * blk_mq_complete_request - end I/O on a request
 * @rq:		the request being processed
 *
 * Description:
 *	Ends all I/O on a request. It does not handle partial completions.
 *	The actual completion happens out-of-order, through a IPI handler.
 **/
void blk_mq_complete_request(struct request *rq)
{
359 360 361
	struct request_queue *q = rq->q;

	if (unlikely(blk_should_fake_timeout(q)))
362
		return;
363 364
	if (!blk_mark_rq_complete(rq))
		__blk_mq_complete_request(rq);
365 366
}
EXPORT_SYMBOL(blk_mq_complete_request);
367

368
void blk_mq_start_request(struct request *rq)
369 370 371 372 373
{
	struct request_queue *q = rq->q;

	trace_block_rq_issue(q, rq);

C
Christoph Hellwig 已提交
374
	rq->resid_len = blk_rq_bytes(rq);
C
Christoph Hellwig 已提交
375 376
	if (unlikely(blk_bidi_rq(rq)))
		rq->next_rq->resid_len = blk_rq_bytes(rq->next_rq);
C
Christoph Hellwig 已提交
377

378
	blk_add_timer(rq);
379

380 381 382 383 384 385
	/*
	 * Ensure that ->deadline is visible before set the started
	 * flag and clear the completed flag.
	 */
	smp_mb__before_atomic();

386 387 388 389 390 391
	/*
	 * Mark us as started and clear complete. Complete might have been
	 * set if requeue raced with timeout, which then marked it as
	 * complete. So be sure to clear complete again when we start
	 * the request, otherwise we'll ignore the completion event.
	 */
392 393 394 395
	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
		set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
	if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
		clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
396 397 398 399 400 401 402 403 404

	if (q->dma_drain_size && blk_rq_bytes(rq)) {
		/*
		 * Make sure space for the drain appears.  We know we can do
		 * this because max_hw_segments has been adjusted to be one
		 * fewer than the device can handle.
		 */
		rq->nr_phys_segments++;
	}
405
}
406
EXPORT_SYMBOL(blk_mq_start_request);
407

408
static void __blk_mq_requeue_request(struct request *rq)
409 410 411 412
{
	struct request_queue *q = rq->q;

	trace_block_rq_requeue(q, rq);
413

414 415 416 417
	if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
		if (q->dma_drain_size && blk_rq_bytes(rq))
			rq->nr_phys_segments--;
	}
418 419
}

420 421 422 423 424
void blk_mq_requeue_request(struct request *rq)
{
	__blk_mq_requeue_request(rq);

	BUG_ON(blk_queued_rq(rq));
425
	blk_mq_add_to_requeue_list(rq, true);
426 427 428
}
EXPORT_SYMBOL(blk_mq_requeue_request);

429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455
static void blk_mq_requeue_work(struct work_struct *work)
{
	struct request_queue *q =
		container_of(work, struct request_queue, requeue_work);
	LIST_HEAD(rq_list);
	struct request *rq, *next;
	unsigned long flags;

	spin_lock_irqsave(&q->requeue_lock, flags);
	list_splice_init(&q->requeue_list, &rq_list);
	spin_unlock_irqrestore(&q->requeue_lock, flags);

	list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
		if (!(rq->cmd_flags & REQ_SOFTBARRIER))
			continue;

		rq->cmd_flags &= ~REQ_SOFTBARRIER;
		list_del_init(&rq->queuelist);
		blk_mq_insert_request(rq, true, false, false);
	}

	while (!list_empty(&rq_list)) {
		rq = list_entry(rq_list.next, struct request, queuelist);
		list_del_init(&rq->queuelist);
		blk_mq_insert_request(rq, false, false, false);
	}

456 457 458 459 460
	/*
	 * Use the start variant of queue running here, so that running
	 * the requeue work will kick stopped queues.
	 */
	blk_mq_start_hw_queues(q);
461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490
}

void blk_mq_add_to_requeue_list(struct request *rq, bool at_head)
{
	struct request_queue *q = rq->q;
	unsigned long flags;

	/*
	 * We abuse this flag that is otherwise used by the I/O scheduler to
	 * request head insertation from the workqueue.
	 */
	BUG_ON(rq->cmd_flags & REQ_SOFTBARRIER);

	spin_lock_irqsave(&q->requeue_lock, flags);
	if (at_head) {
		rq->cmd_flags |= REQ_SOFTBARRIER;
		list_add(&rq->queuelist, &q->requeue_list);
	} else {
		list_add_tail(&rq->queuelist, &q->requeue_list);
	}
	spin_unlock_irqrestore(&q->requeue_lock, flags);
}
EXPORT_SYMBOL(blk_mq_add_to_requeue_list);

void blk_mq_kick_requeue_list(struct request_queue *q)
{
	kblockd_schedule_work(&q->requeue_work);
}
EXPORT_SYMBOL(blk_mq_kick_requeue_list);

491 492
static inline bool is_flush_request(struct request *rq,
		struct blk_flush_queue *fq, unsigned int tag)
493
{
494
	return ((rq->cmd_flags & REQ_FLUSH_SEQ) &&
495
			fq->flush_rq->tag == tag);
496 497 498 499 500
}

struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
{
	struct request *rq = tags->rqs[tag];
501 502
	/* mq_ctx of flush rq is always cloned from the corresponding req */
	struct blk_flush_queue *fq = blk_get_flush_queue(rq->q, rq->mq_ctx);
503

504
	if (!is_flush_request(rq, fq, tag))
505
		return rq;
506

507
	return fq->flush_rq;
508 509 510
}
EXPORT_SYMBOL(blk_mq_tag_to_rq);

511
struct blk_mq_timeout_data {
512 513
	unsigned long next;
	unsigned int next_set;
514 515
};

516
void blk_mq_rq_timed_out(struct request *req, bool reserved)
517
{
518 519
	struct blk_mq_ops *ops = req->q->mq_ops;
	enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
520 521 522 523 524 525 526 527 528 529

	/*
	 * We know that complete is set at this point. If STARTED isn't set
	 * anymore, then the request isn't active and the "timeout" should
	 * just be ignored. This can happen due to the bitflag ordering.
	 * Timeout first checks if STARTED is set, and if it is, assumes
	 * the request is active. But if we race with completion, then
	 * we both flags will get cleared. So check here again, and ignore
	 * a timeout event with a request that isn't active.
	 */
530 531
	if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
		return;
532

533
	if (ops->timeout)
534
		ret = ops->timeout(req, reserved);
535 536 537 538 539 540 541 542 543 544 545 546 547 548 549

	switch (ret) {
	case BLK_EH_HANDLED:
		__blk_mq_complete_request(req);
		break;
	case BLK_EH_RESET_TIMER:
		blk_add_timer(req);
		blk_clear_rq_complete(req);
		break;
	case BLK_EH_NOT_HANDLED:
		break;
	default:
		printk(KERN_ERR "block: bad eh return: %d\n", ret);
		break;
	}
550
}
551 552 553 554 555
		
static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
		struct request *rq, void *priv, bool reserved)
{
	struct blk_mq_timeout_data *data = priv;
556

557 558
	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
		return;
559

560 561
	if (time_after_eq(jiffies, rq->deadline)) {
		if (!blk_mark_rq_complete(rq))
562
			blk_mq_rq_timed_out(rq, reserved);
563 564 565 566
	} else if (!data->next_set || time_after(data->next, rq->deadline)) {
		data->next = rq->deadline;
		data->next_set = 1;
	}
567 568
}

569
static void blk_mq_rq_timer(unsigned long priv)
570
{
571 572 573 574 575
	struct request_queue *q = (struct request_queue *)priv;
	struct blk_mq_timeout_data data = {
		.next		= 0,
		.next_set	= 0,
	};
576
	struct blk_mq_hw_ctx *hctx;
577
	int i;
578

579 580 581 582 583 584 585 586
	queue_for_each_hw_ctx(q, hctx, i) {
		/*
		 * If not software queues are currently mapped to this
		 * hardware queue, there's nothing to check
		 */
		if (!hctx->nr_ctx || !hctx->tags)
			continue;

587
		blk_mq_tag_busy_iter(hctx, blk_mq_check_expired, &data);
588
	}
589

590 591 592
	if (data.next_set) {
		data.next = blk_rq_timeout(round_jiffies_up(data.next));
		mod_timer(&q->timeout, data.next);
593 594 595 596
	} else {
		queue_for_each_hw_ctx(q, hctx, i)
			blk_mq_tag_idle(hctx);
	}
597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637
}

/*
 * Reverse check our software queue for entries that we could potentially
 * merge with. Currently includes a hand-wavy stop count of 8, to not spend
 * too much time checking for merges.
 */
static bool blk_mq_attempt_merge(struct request_queue *q,
				 struct blk_mq_ctx *ctx, struct bio *bio)
{
	struct request *rq;
	int checked = 8;

	list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
		int el_ret;

		if (!checked--)
			break;

		if (!blk_rq_merge_ok(rq, bio))
			continue;

		el_ret = blk_try_merge(rq, bio);
		if (el_ret == ELEVATOR_BACK_MERGE) {
			if (bio_attempt_back_merge(q, rq, bio)) {
				ctx->rq_merged++;
				return true;
			}
			break;
		} else if (el_ret == ELEVATOR_FRONT_MERGE) {
			if (bio_attempt_front_merge(q, rq, bio)) {
				ctx->rq_merged++;
				return true;
			}
			break;
		}
	}

	return false;
}

638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671
/*
 * Process software queues that have been marked busy, splicing them
 * to the for-dispatch
 */
static void flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
{
	struct blk_mq_ctx *ctx;
	int i;

	for (i = 0; i < hctx->ctx_map.map_size; i++) {
		struct blk_align_bitmap *bm = &hctx->ctx_map.map[i];
		unsigned int off, bit;

		if (!bm->word)
			continue;

		bit = 0;
		off = i * hctx->ctx_map.bits_per_word;
		do {
			bit = find_next_bit(&bm->word, bm->depth, bit);
			if (bit >= bm->depth)
				break;

			ctx = hctx->ctxs[bit + off];
			clear_bit(bit, &bm->word);
			spin_lock(&ctx->lock);
			list_splice_tail_init(&ctx->rq_list, list);
			spin_unlock(&ctx->lock);

			bit++;
		} while (1);
	}
}

672 673 674 675 676 677 678 679 680 681 682
/*
 * Run this hardware queue, pulling any software queues mapped to it in.
 * Note that this function currently has various problems around ordering
 * of IO. In particular, we'd like FIFO behaviour on handling existing
 * items on the hctx->dispatch list. Ignore that for now.
 */
static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	struct request_queue *q = hctx->queue;
	struct request *rq;
	LIST_HEAD(rq_list);
683 684
	LIST_HEAD(driver_list);
	struct list_head *dptr;
685
	int queued;
686

687
	WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask));
688

689
	if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
690 691 692 693 694 695 696
		return;

	hctx->run++;

	/*
	 * Touch any software queue that has pending entries.
	 */
697
	flush_busy_ctxs(hctx, &rq_list);
698 699 700 701 702 703 704 705 706 707 708 709

	/*
	 * If we have previous entries on our dispatch list, grab them
	 * and stuff them at the front for more fair dispatch.
	 */
	if (!list_empty_careful(&hctx->dispatch)) {
		spin_lock(&hctx->lock);
		if (!list_empty(&hctx->dispatch))
			list_splice_init(&hctx->dispatch, &rq_list);
		spin_unlock(&hctx->lock);
	}

710 711 712 713 714 715
	/*
	 * Start off with dptr being NULL, so we start the first request
	 * immediately, even if we have more pending.
	 */
	dptr = NULL;

716 717 718
	/*
	 * Now process all the entries, sending them to the driver.
	 */
719
	queued = 0;
720
	while (!list_empty(&rq_list)) {
721
		struct blk_mq_queue_data bd;
722 723 724 725 726
		int ret;

		rq = list_first_entry(&rq_list, struct request, queuelist);
		list_del_init(&rq->queuelist);

727 728 729 730 731
		bd.rq = rq;
		bd.list = dptr;
		bd.last = list_empty(&rq_list);

		ret = q->mq_ops->queue_rq(hctx, &bd);
732 733 734 735 736 737
		switch (ret) {
		case BLK_MQ_RQ_QUEUE_OK:
			queued++;
			continue;
		case BLK_MQ_RQ_QUEUE_BUSY:
			list_add(&rq->queuelist, &rq_list);
738
			__blk_mq_requeue_request(rq);
739 740 741 742
			break;
		default:
			pr_err("blk-mq: bad return on queue: %d\n", ret);
		case BLK_MQ_RQ_QUEUE_ERROR:
743
			rq->errors = -EIO;
744
			blk_mq_end_request(rq, rq->errors);
745 746 747 748 749
			break;
		}

		if (ret == BLK_MQ_RQ_QUEUE_BUSY)
			break;
750 751 752 753 754 755 756

		/*
		 * We've done the first request. If we have more than 1
		 * left in the list, set dptr to defer issue.
		 */
		if (!dptr && rq_list.next != rq_list.prev)
			dptr = &driver_list;
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774
	}

	if (!queued)
		hctx->dispatched[0]++;
	else if (queued < (1 << (BLK_MQ_MAX_DISPATCH_ORDER - 1)))
		hctx->dispatched[ilog2(queued) + 1]++;

	/*
	 * Any items that need requeuing? Stuff them into hctx->dispatch,
	 * that is where we will continue on next queue run.
	 */
	if (!list_empty(&rq_list)) {
		spin_lock(&hctx->lock);
		list_splice(&rq_list, &hctx->dispatch);
		spin_unlock(&hctx->lock);
	}
}

775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798
/*
 * It'd be great if the workqueue API had a way to pass
 * in a mask and had some smarts for more clever placement.
 * For now we just round-robin here, switching for every
 * BLK_MQ_CPU_WORK_BATCH queued items.
 */
static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
{
	int cpu = hctx->next_cpu;

	if (--hctx->next_cpu_batch <= 0) {
		int next_cpu;

		next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
		if (next_cpu >= nr_cpu_ids)
			next_cpu = cpumask_first(hctx->cpumask);

		hctx->next_cpu = next_cpu;
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}

	return cpu;
}

799 800
void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
801
	if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
802 803
		return;

804
	if (!async) {
805 806
		int cpu = get_cpu();
		if (cpumask_test_cpu(cpu, hctx->cpumask)) {
807
			__blk_mq_run_hw_queue(hctx);
808
			put_cpu();
809 810 811
			return;
		}

812
		put_cpu();
813 814 815
	}

	if (hctx->queue->nr_hw_queues == 1)
816
		kblockd_schedule_delayed_work(&hctx->run_work, 0);
817 818 819
	else {
		unsigned int cpu;

820
		cpu = blk_mq_hctx_next_cpu(hctx);
821
		kblockd_schedule_delayed_work_on(cpu, &hctx->run_work, 0);
822
	}
823 824 825 826 827 828 829 830 831 832
}

void blk_mq_run_queues(struct request_queue *q, bool async)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if ((!blk_mq_hctx_has_pending(hctx) &&
		    list_empty_careful(&hctx->dispatch)) ||
833
		    test_bit(BLK_MQ_S_STOPPED, &hctx->state))
834 835 836 837 838 839 840 841 842
			continue;

		blk_mq_run_hw_queue(hctx, async);
	}
}
EXPORT_SYMBOL(blk_mq_run_queues);

void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
{
843 844
	cancel_delayed_work(&hctx->run_work);
	cancel_delayed_work(&hctx->delay_work);
845 846 847 848
	set_bit(BLK_MQ_S_STOPPED, &hctx->state);
}
EXPORT_SYMBOL(blk_mq_stop_hw_queue);

849 850 851 852 853 854 855 856 857 858
void blk_mq_stop_hw_queues(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_stop_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_stop_hw_queues);

859 860 861
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
862

863
	blk_mq_run_hw_queue(hctx, false);
864 865 866
}
EXPORT_SYMBOL(blk_mq_start_hw_queue);

867 868 869 870 871 872 873 874 875 876 877
void blk_mq_start_hw_queues(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_start_hw_queues);


878
void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
879 880 881 882 883 884 885 886 887
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if (!test_bit(BLK_MQ_S_STOPPED, &hctx->state))
			continue;

		clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
888
		blk_mq_run_hw_queue(hctx, async);
889 890 891 892
	}
}
EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);

893
static void blk_mq_run_work_fn(struct work_struct *work)
894 895 896
{
	struct blk_mq_hw_ctx *hctx;

897
	hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
898

899 900 901
	__blk_mq_run_hw_queue(hctx);
}

902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920
static void blk_mq_delay_work_fn(struct work_struct *work)
{
	struct blk_mq_hw_ctx *hctx;

	hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);

	if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
		__blk_mq_run_hw_queue(hctx);
}

void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
{
	unsigned long tmo = msecs_to_jiffies(msecs);

	if (hctx->queue->nr_hw_queues == 1)
		kblockd_schedule_delayed_work(&hctx->delay_work, tmo);
	else {
		unsigned int cpu;

921
		cpu = blk_mq_hctx_next_cpu(hctx);
922 923 924 925 926
		kblockd_schedule_delayed_work_on(cpu, &hctx->delay_work, tmo);
	}
}
EXPORT_SYMBOL(blk_mq_delay_queue);

927
static void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx,
928
				    struct request *rq, bool at_head)
929 930 931
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;

932 933
	trace_block_rq_insert(hctx->queue, rq);

934 935 936 937
	if (at_head)
		list_add(&rq->queuelist, &ctx->rq_list);
	else
		list_add_tail(&rq->queuelist, &ctx->rq_list);
938

939 940 941
	blk_mq_hctx_mark_pending(hctx, ctx);
}

942 943
void blk_mq_insert_request(struct request *rq, bool at_head, bool run_queue,
		bool async)
944
{
945
	struct request_queue *q = rq->q;
946
	struct blk_mq_hw_ctx *hctx;
947 948 949 950 951
	struct blk_mq_ctx *ctx = rq->mq_ctx, *current_ctx;

	current_ctx = blk_mq_get_ctx(q);
	if (!cpu_online(ctx->cpu))
		rq->mq_ctx = ctx = current_ctx;
952 953 954

	hctx = q->mq_ops->map_queue(q, ctx->cpu);

955 956 957
	spin_lock(&ctx->lock);
	__blk_mq_insert_request(hctx, rq, at_head);
	spin_unlock(&ctx->lock);
958 959 960

	if (run_queue)
		blk_mq_run_hw_queue(hctx, async);
961 962

	blk_mq_put_ctx(current_ctx);
963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993
}

static void blk_mq_insert_requests(struct request_queue *q,
				     struct blk_mq_ctx *ctx,
				     struct list_head *list,
				     int depth,
				     bool from_schedule)

{
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *current_ctx;

	trace_block_unplug(q, depth, !from_schedule);

	current_ctx = blk_mq_get_ctx(q);

	if (!cpu_online(ctx->cpu))
		ctx = current_ctx;
	hctx = q->mq_ops->map_queue(q, ctx->cpu);

	/*
	 * preemption doesn't flush plug list, so it's possible ctx->cpu is
	 * offline now
	 */
	spin_lock(&ctx->lock);
	while (!list_empty(list)) {
		struct request *rq;

		rq = list_first_entry(list, struct request, queuelist);
		list_del_init(&rq->queuelist);
		rq->mq_ctx = ctx;
994
		__blk_mq_insert_request(hctx, rq, false);
995 996 997 998
	}
	spin_unlock(&ctx->lock);

	blk_mq_run_hw_queue(hctx, from_schedule);
999
	blk_mq_put_ctx(current_ctx);
1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061
}

static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
{
	struct request *rqa = container_of(a, struct request, queuelist);
	struct request *rqb = container_of(b, struct request, queuelist);

	return !(rqa->mq_ctx < rqb->mq_ctx ||
		 (rqa->mq_ctx == rqb->mq_ctx &&
		  blk_rq_pos(rqa) < blk_rq_pos(rqb)));
}

void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
{
	struct blk_mq_ctx *this_ctx;
	struct request_queue *this_q;
	struct request *rq;
	LIST_HEAD(list);
	LIST_HEAD(ctx_list);
	unsigned int depth;

	list_splice_init(&plug->mq_list, &list);

	list_sort(NULL, &list, plug_ctx_cmp);

	this_q = NULL;
	this_ctx = NULL;
	depth = 0;

	while (!list_empty(&list)) {
		rq = list_entry_rq(list.next);
		list_del_init(&rq->queuelist);
		BUG_ON(!rq->q);
		if (rq->mq_ctx != this_ctx) {
			if (this_ctx) {
				blk_mq_insert_requests(this_q, this_ctx,
							&ctx_list, depth,
							from_schedule);
			}

			this_ctx = rq->mq_ctx;
			this_q = rq->q;
			depth = 0;
		}

		depth++;
		list_add_tail(&rq->queuelist, &ctx_list);
	}

	/*
	 * If 'this_ctx' is set, we know we have entries to complete
	 * on 'ctx_list'. Do those.
	 */
	if (this_ctx) {
		blk_mq_insert_requests(this_q, this_ctx, &ctx_list, depth,
				       from_schedule);
	}
}

static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
{
	init_request_from_bio(rq, bio);
1062

1063
	if (blk_do_io_stat(rq))
1064
		blk_account_io_start(rq, 1);
1065 1066
}

1067 1068 1069 1070 1071 1072
static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx)
{
	return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
		!blk_queue_nomerges(hctx->queue);
}

1073 1074 1075
static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
					 struct blk_mq_ctx *ctx,
					 struct request *rq, struct bio *bio)
1076
{
1077
	if (!hctx_allow_merges(hctx)) {
1078 1079 1080 1081 1082 1083 1084
		blk_mq_bio_to_request(rq, bio);
		spin_lock(&ctx->lock);
insert_rq:
		__blk_mq_insert_request(hctx, rq, false);
		spin_unlock(&ctx->lock);
		return false;
	} else {
1085 1086
		struct request_queue *q = hctx->queue;

1087 1088 1089 1090 1091
		spin_lock(&ctx->lock);
		if (!blk_mq_attempt_merge(q, ctx, bio)) {
			blk_mq_bio_to_request(rq, bio);
			goto insert_rq;
		}
1092

1093 1094 1095
		spin_unlock(&ctx->lock);
		__blk_mq_free_request(hctx, ctx, rq);
		return true;
1096
	}
1097
}
1098

1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111
struct blk_map_ctx {
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
};

static struct request *blk_mq_map_request(struct request_queue *q,
					  struct bio *bio,
					  struct blk_map_ctx *data)
{
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
	struct request *rq;
	int rw = bio_data_dir(bio);
1112
	struct blk_mq_alloc_data alloc_data;
1113

1114
	if (unlikely(blk_mq_queue_enter(q))) {
1115
		bio_endio(bio, -EIO);
1116
		return NULL;
1117 1118 1119 1120 1121
	}

	ctx = blk_mq_get_ctx(q);
	hctx = q->mq_ops->map_queue(q, ctx->cpu);

1122
	if (rw_is_sync(bio->bi_rw))
S
Shaohua Li 已提交
1123
		rw |= REQ_SYNC;
1124

1125
	trace_block_getrq(q, bio, rw);
1126 1127 1128
	blk_mq_set_alloc_data(&alloc_data, q, GFP_ATOMIC, false, ctx,
			hctx);
	rq = __blk_mq_alloc_request(&alloc_data, rw);
1129
	if (unlikely(!rq)) {
1130
		__blk_mq_run_hw_queue(hctx);
1131 1132
		blk_mq_put_ctx(ctx);
		trace_block_sleeprq(q, bio, rw);
1133 1134

		ctx = blk_mq_get_ctx(q);
1135
		hctx = q->mq_ops->map_queue(q, ctx->cpu);
1136 1137 1138 1139 1140
		blk_mq_set_alloc_data(&alloc_data, q,
				__GFP_WAIT|GFP_ATOMIC, false, ctx, hctx);
		rq = __blk_mq_alloc_request(&alloc_data, rw);
		ctx = alloc_data.ctx;
		hctx = alloc_data.hctx;
1141 1142 1143
	}

	hctx->queued++;
1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177
	data->hctx = hctx;
	data->ctx = ctx;
	return rq;
}

/*
 * Multiple hardware queue variant. This will not use per-process plugs,
 * but will attempt to bypass the hctx queueing if we can go straight to
 * hardware for SYNC IO.
 */
static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
{
	const int is_sync = rw_is_sync(bio->bi_rw);
	const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
	struct blk_map_ctx data;
	struct request *rq;

	blk_queue_bounce(q, &bio);

	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
		bio_endio(bio, -EIO);
		return;
	}

	rq = blk_mq_map_request(q, bio, &data);
	if (unlikely(!rq))
		return;

	if (unlikely(is_flush_fua)) {
		blk_mq_bio_to_request(rq, bio);
		blk_insert_flush(rq);
		goto run_queue;
	}

1178 1179 1180 1181 1182 1183
	/*
	 * If the driver supports defer issued based on 'last', then
	 * queue it up like normal since we can potentially save some
	 * CPU this way.
	 */
	if (is_sync && !(data.hctx->flags & BLK_MQ_F_DEFER_ISSUE)) {
1184 1185 1186 1187 1188
		struct blk_mq_queue_data bd = {
			.rq = rq,
			.list = NULL,
			.last = 1
		};
1189 1190 1191 1192 1193 1194 1195 1196 1197
		int ret;

		blk_mq_bio_to_request(rq, bio);

		/*
		 * For OK queue, we are done. For error, kill it. Any other
		 * error (busy), just add it to our list as we previously
		 * would have done
		 */
1198
		ret = q->mq_ops->queue_rq(data.hctx, &bd);
1199 1200 1201 1202 1203 1204 1205
		if (ret == BLK_MQ_RQ_QUEUE_OK)
			goto done;
		else {
			__blk_mq_requeue_request(rq);

			if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
				rq->errors = -EIO;
1206
				blk_mq_end_request(rq, rq->errors);
1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255
				goto done;
			}
		}
	}

	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
		/*
		 * For a SYNC request, send it to the hardware immediately. For
		 * an ASYNC request, just ensure that we run it later on. The
		 * latter allows for merging opportunities and more efficient
		 * dispatching.
		 */
run_queue:
		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
	}
done:
	blk_mq_put_ctx(data.ctx);
}

/*
 * Single hardware queue variant. This will attempt to use any per-process
 * plug for merging and IO deferral.
 */
static void blk_sq_make_request(struct request_queue *q, struct bio *bio)
{
	const int is_sync = rw_is_sync(bio->bi_rw);
	const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
	unsigned int use_plug, request_count = 0;
	struct blk_map_ctx data;
	struct request *rq;

	/*
	 * If we have multiple hardware queues, just go directly to
	 * one of those for sync IO.
	 */
	use_plug = !is_flush_fua && !is_sync;

	blk_queue_bounce(q, &bio);

	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
		bio_endio(bio, -EIO);
		return;
	}

	if (use_plug && !blk_queue_nomerges(q) &&
	    blk_attempt_plug_merge(q, bio, &request_count))
		return;

	rq = blk_mq_map_request(q, bio, &data);
1256 1257
	if (unlikely(!rq))
		return;
1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274

	if (unlikely(is_flush_fua)) {
		blk_mq_bio_to_request(rq, bio);
		blk_insert_flush(rq);
		goto run_queue;
	}

	/*
	 * A task plug currently exists. Since this is completely lockless,
	 * utilize that to temporarily store requests until the task is
	 * either done or scheduled away.
	 */
	if (use_plug) {
		struct blk_plug *plug = current->plug;

		if (plug) {
			blk_mq_bio_to_request(rq, bio);
S
Shaohua Li 已提交
1275
			if (list_empty(&plug->mq_list))
1276 1277 1278 1279 1280 1281
				trace_block_plug(q);
			else if (request_count >= BLK_MAX_REQUEST_COUNT) {
				blk_flush_plug_list(plug, false);
				trace_block_plug(q);
			}
			list_add_tail(&rq->queuelist, &plug->mq_list);
1282
			blk_mq_put_ctx(data.ctx);
1283 1284 1285 1286
			return;
		}
	}

1287 1288 1289 1290 1291 1292 1293 1294 1295
	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
		/*
		 * For a SYNC request, send it to the hardware immediately. For
		 * an ASYNC request, just ensure that we run it later on. The
		 * latter allows for merging opportunities and more efficient
		 * dispatching.
		 */
run_queue:
		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1296 1297
	}

1298
	blk_mq_put_ctx(data.ctx);
1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309
}

/*
 * Default mapping to a software queue, since we use one per CPU.
 */
struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, const int cpu)
{
	return q->queue_hw_ctx[q->mq_map[cpu]];
}
EXPORT_SYMBOL(blk_mq_map_queue);

1310 1311
static void blk_mq_free_rq_map(struct blk_mq_tag_set *set,
		struct blk_mq_tags *tags, unsigned int hctx_idx)
1312
{
1313
	struct page *page;
1314

1315
	if (tags->rqs && set->ops->exit_request) {
1316
		int i;
1317

1318 1319
		for (i = 0; i < tags->nr_tags; i++) {
			if (!tags->rqs[i])
1320
				continue;
1321 1322
			set->ops->exit_request(set->driver_data, tags->rqs[i],
						hctx_idx, i);
1323
			tags->rqs[i] = NULL;
1324
		}
1325 1326
	}

1327 1328
	while (!list_empty(&tags->page_list)) {
		page = list_first_entry(&tags->page_list, struct page, lru);
1329
		list_del_init(&page->lru);
1330 1331 1332
		__free_pages(page, page->private);
	}

1333
	kfree(tags->rqs);
1334

1335
	blk_mq_free_tags(tags);
1336 1337 1338 1339
}

static size_t order_to_size(unsigned int order)
{
1340
	return (size_t)PAGE_SIZE << order;
1341 1342
}

1343 1344
static struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
		unsigned int hctx_idx)
1345
{
1346
	struct blk_mq_tags *tags;
1347 1348 1349
	unsigned int i, j, entries_per_page, max_order = 4;
	size_t rq_size, left;

1350 1351 1352 1353
	tags = blk_mq_init_tags(set->queue_depth, set->reserved_tags,
				set->numa_node);
	if (!tags)
		return NULL;
1354

1355 1356
	INIT_LIST_HEAD(&tags->page_list);

1357 1358 1359
	tags->rqs = kzalloc_node(set->queue_depth * sizeof(struct request *),
				 GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY,
				 set->numa_node);
1360 1361 1362 1363
	if (!tags->rqs) {
		blk_mq_free_tags(tags);
		return NULL;
	}
1364 1365 1366 1367 1368

	/*
	 * rq_size is the size of the request plus driver payload, rounded
	 * to the cacheline size
	 */
1369
	rq_size = round_up(sizeof(struct request) + set->cmd_size,
1370
				cache_line_size());
1371
	left = rq_size * set->queue_depth;
1372

1373
	for (i = 0; i < set->queue_depth; ) {
1374 1375 1376 1377 1378 1379 1380 1381 1382
		int this_order = max_order;
		struct page *page;
		int to_do;
		void *p;

		while (left < order_to_size(this_order - 1) && this_order)
			this_order--;

		do {
1383 1384 1385
			page = alloc_pages_node(set->numa_node,
				GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY,
				this_order);
1386 1387 1388 1389 1390 1391 1392 1393 1394
			if (page)
				break;
			if (!this_order--)
				break;
			if (order_to_size(this_order) < rq_size)
				break;
		} while (1);

		if (!page)
1395
			goto fail;
1396 1397

		page->private = this_order;
1398
		list_add_tail(&page->lru, &tags->page_list);
1399 1400 1401

		p = page_address(page);
		entries_per_page = order_to_size(this_order) / rq_size;
1402
		to_do = min(entries_per_page, set->queue_depth - i);
1403 1404
		left -= to_do * rq_size;
		for (j = 0; j < to_do; j++) {
1405
			tags->rqs[i] = p;
1406 1407
			tags->rqs[i]->atomic_flags = 0;
			tags->rqs[i]->cmd_flags = 0;
1408 1409 1410
			if (set->ops->init_request) {
				if (set->ops->init_request(set->driver_data,
						tags->rqs[i], hctx_idx, i,
1411 1412
						set->numa_node)) {
					tags->rqs[i] = NULL;
1413
					goto fail;
1414
				}
1415 1416
			}

1417 1418 1419 1420 1421
			p += rq_size;
			i++;
		}
	}

1422
	return tags;
1423

1424 1425 1426
fail:
	blk_mq_free_rq_map(set, tags, hctx_idx);
	return NULL;
1427 1428
}

1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456
static void blk_mq_free_bitmap(struct blk_mq_ctxmap *bitmap)
{
	kfree(bitmap->map);
}

static int blk_mq_alloc_bitmap(struct blk_mq_ctxmap *bitmap, int node)
{
	unsigned int bpw = 8, total, num_maps, i;

	bitmap->bits_per_word = bpw;

	num_maps = ALIGN(nr_cpu_ids, bpw) / bpw;
	bitmap->map = kzalloc_node(num_maps * sizeof(struct blk_align_bitmap),
					GFP_KERNEL, node);
	if (!bitmap->map)
		return -ENOMEM;

	bitmap->map_size = num_maps;

	total = nr_cpu_ids;
	for (i = 0; i < num_maps; i++) {
		bitmap->map[i].depth = min(total, bitmap->bits_per_word);
		total -= bitmap->map[i].depth;
	}

	return 0;
}

1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527
static int blk_mq_hctx_cpu_offline(struct blk_mq_hw_ctx *hctx, int cpu)
{
	struct request_queue *q = hctx->queue;
	struct blk_mq_ctx *ctx;
	LIST_HEAD(tmp);

	/*
	 * Move ctx entries to new CPU, if this one is going away.
	 */
	ctx = __blk_mq_get_ctx(q, cpu);

	spin_lock(&ctx->lock);
	if (!list_empty(&ctx->rq_list)) {
		list_splice_init(&ctx->rq_list, &tmp);
		blk_mq_hctx_clear_pending(hctx, ctx);
	}
	spin_unlock(&ctx->lock);

	if (list_empty(&tmp))
		return NOTIFY_OK;

	ctx = blk_mq_get_ctx(q);
	spin_lock(&ctx->lock);

	while (!list_empty(&tmp)) {
		struct request *rq;

		rq = list_first_entry(&tmp, struct request, queuelist);
		rq->mq_ctx = ctx;
		list_move_tail(&rq->queuelist, &ctx->rq_list);
	}

	hctx = q->mq_ops->map_queue(q, ctx->cpu);
	blk_mq_hctx_mark_pending(hctx, ctx);

	spin_unlock(&ctx->lock);

	blk_mq_run_hw_queue(hctx, true);
	blk_mq_put_ctx(ctx);
	return NOTIFY_OK;
}

static int blk_mq_hctx_cpu_online(struct blk_mq_hw_ctx *hctx, int cpu)
{
	struct request_queue *q = hctx->queue;
	struct blk_mq_tag_set *set = q->tag_set;

	if (set->tags[hctx->queue_num])
		return NOTIFY_OK;

	set->tags[hctx->queue_num] = blk_mq_init_rq_map(set, hctx->queue_num);
	if (!set->tags[hctx->queue_num])
		return NOTIFY_STOP;

	hctx->tags = set->tags[hctx->queue_num];
	return NOTIFY_OK;
}

static int blk_mq_hctx_notify(void *data, unsigned long action,
			      unsigned int cpu)
{
	struct blk_mq_hw_ctx *hctx = data;

	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
		return blk_mq_hctx_cpu_offline(hctx, cpu);
	else if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN)
		return blk_mq_hctx_cpu_online(hctx, cpu);

	return NOTIFY_OK;
}

1528 1529 1530 1531
static void blk_mq_exit_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
1532 1533
	unsigned flush_start_tag = set->queue_depth;

1534 1535
	blk_mq_tag_idle(hctx);

1536 1537 1538 1539 1540
	if (set->ops->exit_request)
		set->ops->exit_request(set->driver_data,
				       hctx->fq->flush_rq, hctx_idx,
				       flush_start_tag + hctx_idx);

1541 1542 1543 1544
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);

	blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1545
	blk_free_flush_queue(hctx->fq);
1546 1547 1548 1549
	kfree(hctx->ctxs);
	blk_mq_free_bitmap(&hctx->ctx_map);
}

M
Ming Lei 已提交
1550 1551 1552 1553 1554 1555 1556 1557 1558
static void blk_mq_exit_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set, int nr_queue)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if (i == nr_queue)
			break;
1559
		blk_mq_exit_hctx(q, set, hctx, i);
M
Ming Lei 已提交
1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
	}
}

static void blk_mq_free_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		free_cpumask_var(hctx->cpumask);
1571
		kfree(hctx);
M
Ming Lei 已提交
1572 1573 1574
	}
}

1575 1576 1577
static int blk_mq_init_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
1578
{
1579
	int node;
1580
	unsigned flush_start_tag = set->queue_depth;
1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599

	node = hctx->numa_node;
	if (node == NUMA_NO_NODE)
		node = hctx->numa_node = set->numa_node;

	INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
	INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
	spin_lock_init(&hctx->lock);
	INIT_LIST_HEAD(&hctx->dispatch);
	hctx->queue = q;
	hctx->queue_num = hctx_idx;
	hctx->flags = set->flags;
	hctx->cmd_size = set->cmd_size;

	blk_mq_init_cpu_notifier(&hctx->cpu_notifier,
					blk_mq_hctx_notify, hctx);
	blk_mq_register_cpu_notifier(&hctx->cpu_notifier);

	hctx->tags = set->tags[hctx_idx];
1600 1601

	/*
1602 1603
	 * Allocate space for all possible cpus to avoid allocation at
	 * runtime
1604
	 */
1605 1606 1607 1608
	hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
					GFP_KERNEL, node);
	if (!hctx->ctxs)
		goto unregister_cpu_notifier;
1609

1610 1611
	if (blk_mq_alloc_bitmap(&hctx->ctx_map, node))
		goto free_ctxs;
1612

1613
	hctx->nr_ctx = 0;
1614

1615 1616 1617
	if (set->ops->init_hctx &&
	    set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
		goto free_bitmap;
1618

1619 1620 1621
	hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size);
	if (!hctx->fq)
		goto exit_hctx;
1622

1623 1624 1625 1626 1627
	if (set->ops->init_request &&
	    set->ops->init_request(set->driver_data,
				   hctx->fq->flush_rq, hctx_idx,
				   flush_start_tag + hctx_idx, node))
		goto free_fq;
1628

1629
	return 0;
1630

1631 1632 1633 1634 1635
 free_fq:
	kfree(hctx->fq);
 exit_hctx:
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);
1636 1637 1638 1639 1640 1641
 free_bitmap:
	blk_mq_free_bitmap(&hctx->ctx_map);
 free_ctxs:
	kfree(hctx->ctxs);
 unregister_cpu_notifier:
	blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1642

1643 1644
	return -1;
}
1645

1646 1647 1648 1649 1650
static int blk_mq_init_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;
1651

1652 1653 1654 1655 1656
	/*
	 * Initialize hardware queues
	 */
	queue_for_each_hw_ctx(q, hctx, i) {
		if (blk_mq_init_hctx(q, set, hctx, i))
1657 1658 1659 1660 1661 1662 1663 1664 1665
			break;
	}

	if (i == q->nr_hw_queues)
		return 0;

	/*
	 * Init failed
	 */
M
Ming Lei 已提交
1666
	blk_mq_exit_hw_queues(q, set, i);
1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689

	return 1;
}

static void blk_mq_init_cpu_queues(struct request_queue *q,
				   unsigned int nr_hw_queues)
{
	unsigned int i;

	for_each_possible_cpu(i) {
		struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
		struct blk_mq_hw_ctx *hctx;

		memset(__ctx, 0, sizeof(*__ctx));
		__ctx->cpu = i;
		spin_lock_init(&__ctx->lock);
		INIT_LIST_HEAD(&__ctx->rq_list);
		__ctx->queue = q;

		/* If the cpu isn't online, the cpu is mapped to first hctx */
		if (!cpu_online(i))
			continue;

1690 1691 1692 1693
		hctx = q->mq_ops->map_queue(q, i);
		cpumask_set_cpu(i, hctx->cpumask);
		hctx->nr_ctx++;

1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709
		/*
		 * Set local node, IFF we have more than one hw queue. If
		 * not, we remain on the home node of the device
		 */
		if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
			hctx->numa_node = cpu_to_node(i);
	}
}

static void blk_mq_map_swqueue(struct request_queue *q)
{
	unsigned int i;
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;

	queue_for_each_hw_ctx(q, hctx, i) {
1710
		cpumask_clear(hctx->cpumask);
1711 1712 1713 1714 1715 1716 1717 1718
		hctx->nr_ctx = 0;
	}

	/*
	 * Map software to hardware queues
	 */
	queue_for_each_ctx(q, ctx, i) {
		/* If the cpu isn't online, the cpu is mapped to first hctx */
1719 1720 1721
		if (!cpu_online(i))
			continue;

1722
		hctx = q->mq_ops->map_queue(q, i);
1723
		cpumask_set_cpu(i, hctx->cpumask);
1724 1725 1726
		ctx->index_hw = hctx->nr_ctx;
		hctx->ctxs[hctx->nr_ctx++] = ctx;
	}
1727 1728

	queue_for_each_hw_ctx(q, hctx, i) {
1729
		/*
1730 1731
		 * If no software queues are mapped to this hardware queue,
		 * disable it and free the request entries.
1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746
		 */
		if (!hctx->nr_ctx) {
			struct blk_mq_tag_set *set = q->tag_set;

			if (set->tags[i]) {
				blk_mq_free_rq_map(set, set->tags[i], i);
				set->tags[i] = NULL;
				hctx->tags = NULL;
			}
			continue;
		}

		/*
		 * Initialize batch roundrobin counts
		 */
1747 1748 1749
		hctx->next_cpu = cpumask_first(hctx->cpumask);
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}
1750 1751
}

1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797
static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set)
{
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	bool shared;
	int i;

	if (set->tag_list.next == set->tag_list.prev)
		shared = false;
	else
		shared = true;

	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_freeze_queue(q);

		queue_for_each_hw_ctx(q, hctx, i) {
			if (shared)
				hctx->flags |= BLK_MQ_F_TAG_SHARED;
			else
				hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
		}
		blk_mq_unfreeze_queue(q);
	}
}

static void blk_mq_del_queue_tag_set(struct request_queue *q)
{
	struct blk_mq_tag_set *set = q->tag_set;

	mutex_lock(&set->tag_list_lock);
	list_del_init(&q->tag_set_list);
	blk_mq_update_tag_set_depth(set);
	mutex_unlock(&set->tag_list_lock);
}

static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
				     struct request_queue *q)
{
	q->tag_set = set;

	mutex_lock(&set->tag_list_lock);
	list_add_tail(&q->tag_set_list, &set->tag_list);
	blk_mq_update_tag_set_depth(set);
	mutex_unlock(&set->tag_list_lock);
}

1798
struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
1799 1800
{
	struct blk_mq_hw_ctx **hctxs;
1801
	struct blk_mq_ctx __percpu *ctx;
1802
	struct request_queue *q;
1803
	unsigned int *map;
1804 1805 1806 1807 1808 1809
	int i;

	ctx = alloc_percpu(struct blk_mq_ctx);
	if (!ctx)
		return ERR_PTR(-ENOMEM);

1810 1811 1812 1813 1814 1815 1816 1817 1818 1819
	/*
	 * If a crashdump is active, then we are potentially in a very
	 * memory constrained environment. Limit us to 1 queue and
	 * 64 tags to prevent using too much memory.
	 */
	if (is_kdump_kernel()) {
		set->nr_hw_queues = 1;
		set->queue_depth = min(64U, set->queue_depth);
	}

1820 1821
	hctxs = kmalloc_node(set->nr_hw_queues * sizeof(*hctxs), GFP_KERNEL,
			set->numa_node);
1822 1823 1824 1825

	if (!hctxs)
		goto err_percpu;

1826 1827 1828 1829
	map = blk_mq_make_queue_map(set);
	if (!map)
		goto err_map;

1830
	for (i = 0; i < set->nr_hw_queues; i++) {
1831 1832
		int node = blk_mq_hw_queue_to_node(map, i);

1833 1834
		hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
					GFP_KERNEL, node);
1835 1836 1837
		if (!hctxs[i])
			goto err_hctxs;

1838 1839
		if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL,
						node))
1840 1841
			goto err_hctxs;

1842
		atomic_set(&hctxs[i]->nr_active, 0);
1843
		hctxs[i]->numa_node = node;
1844 1845 1846
		hctxs[i]->queue_num = i;
	}

1847
	q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
1848 1849 1850
	if (!q)
		goto err_hctxs;

1851 1852 1853 1854
	/*
	 * Init percpu_ref in atomic mode so that it's faster to shutdown.
	 * See blk_register_queue() for details.
	 */
1855
	if (percpu_ref_init(&q->mq_usage_counter, blk_mq_usage_counter_release,
1856
			    PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
1857 1858
		goto err_map;

1859 1860 1861 1862
	setup_timer(&q->timeout, blk_mq_rq_timer, (unsigned long) q);
	blk_queue_rq_timeout(q, 30000);

	q->nr_queues = nr_cpu_ids;
1863
	q->nr_hw_queues = set->nr_hw_queues;
1864
	q->mq_map = map;
1865 1866 1867 1868

	q->queue_ctx = ctx;
	q->queue_hw_ctx = hctxs;

1869
	q->mq_ops = set->ops;
1870
	q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
1871

1872 1873 1874
	if (!(set->flags & BLK_MQ_F_SG_MERGE))
		q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;

1875 1876
	q->sg_reserved_size = INT_MAX;

1877 1878 1879 1880
	INIT_WORK(&q->requeue_work, blk_mq_requeue_work);
	INIT_LIST_HEAD(&q->requeue_list);
	spin_lock_init(&q->requeue_lock);

1881 1882 1883 1884 1885
	if (q->nr_hw_queues > 1)
		blk_queue_make_request(q, blk_mq_make_request);
	else
		blk_queue_make_request(q, blk_sq_make_request);

1886 1887
	if (set->timeout)
		blk_queue_rq_timeout(q, set->timeout);
1888

1889 1890 1891 1892 1893
	/*
	 * Do this after blk_queue_make_request() overrides it...
	 */
	q->nr_requests = set->queue_depth;

1894 1895
	if (set->ops->complete)
		blk_queue_softirq_done(q, set->ops->complete);
1896

1897
	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
1898

1899
	if (blk_mq_init_hw_queues(q, set))
1900
		goto err_hw;
1901

1902 1903 1904 1905
	mutex_lock(&all_q_mutex);
	list_add_tail(&q->all_q_node, &all_q_list);
	mutex_unlock(&all_q_mutex);

1906 1907
	blk_mq_add_queue_tag_set(set, q);

1908 1909
	blk_mq_map_swqueue(q);

1910
	return q;
1911

1912 1913 1914
err_hw:
	blk_cleanup_queue(q);
err_hctxs:
1915
	kfree(map);
1916
	for (i = 0; i < set->nr_hw_queues; i++) {
1917 1918
		if (!hctxs[i])
			break;
1919
		free_cpumask_var(hctxs[i]->cpumask);
1920
		kfree(hctxs[i]);
1921
	}
1922
err_map:
1923 1924 1925 1926 1927 1928 1929 1930 1931
	kfree(hctxs);
err_percpu:
	free_percpu(ctx);
	return ERR_PTR(-ENOMEM);
}
EXPORT_SYMBOL(blk_mq_init_queue);

void blk_mq_free_queue(struct request_queue *q)
{
M
Ming Lei 已提交
1932
	struct blk_mq_tag_set	*set = q->tag_set;
1933

1934 1935
	blk_mq_del_queue_tag_set(q);

M
Ming Lei 已提交
1936 1937
	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
	blk_mq_free_hw_queues(q, set);
1938

1939
	percpu_ref_exit(&q->mq_usage_counter);
1940

1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954
	free_percpu(q->queue_ctx);
	kfree(q->queue_hw_ctx);
	kfree(q->mq_map);

	q->queue_ctx = NULL;
	q->queue_hw_ctx = NULL;
	q->mq_map = NULL;

	mutex_lock(&all_q_mutex);
	list_del_init(&q->all_q_node);
	mutex_unlock(&all_q_mutex);
}

/* Basically redo blk_mq_init_queue with queue frozen */
1955
static void blk_mq_queue_reinit(struct request_queue *q)
1956 1957 1958
{
	blk_mq_freeze_queue(q);

1959 1960
	blk_mq_sysfs_unregister(q);

1961 1962 1963 1964 1965 1966 1967 1968 1969 1970
	blk_mq_update_queue_map(q->mq_map, q->nr_hw_queues);

	/*
	 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
	 * we should change hctx numa_node according to new topology (this
	 * involves free and re-allocate memory, worthy doing?)
	 */

	blk_mq_map_swqueue(q);

1971 1972
	blk_mq_sysfs_register(q);

1973 1974 1975
	blk_mq_unfreeze_queue(q);
}

1976 1977
static int blk_mq_queue_reinit_notify(struct notifier_block *nb,
				      unsigned long action, void *hcpu)
1978 1979 1980 1981
{
	struct request_queue *q;

	/*
1982 1983 1984 1985
	 * Before new mappings are established, hotadded cpu might already
	 * start handling requests. This doesn't break anything as we map
	 * offline CPUs to first hardware queue. We will re-init the queue
	 * below to get optimal settings.
1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997
	 */
	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN &&
	    action != CPU_ONLINE && action != CPU_ONLINE_FROZEN)
		return NOTIFY_OK;

	mutex_lock(&all_q_mutex);
	list_for_each_entry(q, &all_q_list, all_q_node)
		blk_mq_queue_reinit(q);
	mutex_unlock(&all_q_mutex);
	return NOTIFY_OK;
}

1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051
static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	int i;

	for (i = 0; i < set->nr_hw_queues; i++) {
		set->tags[i] = blk_mq_init_rq_map(set, i);
		if (!set->tags[i])
			goto out_unwind;
	}

	return 0;

out_unwind:
	while (--i >= 0)
		blk_mq_free_rq_map(set, set->tags[i], i);

	return -ENOMEM;
}

/*
 * Allocate the request maps associated with this tag_set. Note that this
 * may reduce the depth asked for, if memory is tight. set->queue_depth
 * will be updated to reflect the allocated depth.
 */
static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	unsigned int depth;
	int err;

	depth = set->queue_depth;
	do {
		err = __blk_mq_alloc_rq_maps(set);
		if (!err)
			break;

		set->queue_depth >>= 1;
		if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
			err = -ENOMEM;
			break;
		}
	} while (set->queue_depth);

	if (!set->queue_depth || err) {
		pr_err("blk-mq: failed to allocate request map\n");
		return -ENOMEM;
	}

	if (depth != set->queue_depth)
		pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
						depth, set->queue_depth);

	return 0;
}

2052 2053 2054 2055 2056 2057
/*
 * Alloc a tag set to be associated with one or more request queues.
 * May fail with EINVAL for various error conditions. May adjust the
 * requested depth down, if if it too large. In that case, the set
 * value will be stored in set->queue_depth.
 */
2058 2059 2060 2061
int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
{
	if (!set->nr_hw_queues)
		return -EINVAL;
2062
	if (!set->queue_depth)
2063 2064 2065 2066
		return -EINVAL;
	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
		return -EINVAL;

2067
	if (!set->nr_hw_queues || !set->ops->queue_rq || !set->ops->map_queue)
2068 2069
		return -EINVAL;

2070 2071 2072 2073 2074
	if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
		pr_info("blk-mq: reduced tag depth to %u\n",
			BLK_MQ_MAX_DEPTH);
		set->queue_depth = BLK_MQ_MAX_DEPTH;
	}
2075

M
Ming Lei 已提交
2076 2077
	set->tags = kmalloc_node(set->nr_hw_queues *
				 sizeof(struct blk_mq_tags *),
2078 2079
				 GFP_KERNEL, set->numa_node);
	if (!set->tags)
2080
		return -ENOMEM;
2081

2082 2083
	if (blk_mq_alloc_rq_maps(set))
		goto enomem;
2084

2085 2086 2087
	mutex_init(&set->tag_list_lock);
	INIT_LIST_HEAD(&set->tag_list);

2088
	return 0;
2089
enomem:
2090 2091
	kfree(set->tags);
	set->tags = NULL;
2092 2093 2094 2095 2096 2097 2098 2099
	return -ENOMEM;
}
EXPORT_SYMBOL(blk_mq_alloc_tag_set);

void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
{
	int i;

2100 2101 2102 2103 2104
	for (i = 0; i < set->nr_hw_queues; i++) {
		if (set->tags[i])
			blk_mq_free_rq_map(set, set->tags[i], i);
	}

M
Ming Lei 已提交
2105
	kfree(set->tags);
2106
	set->tags = NULL;
2107 2108 2109
}
EXPORT_SYMBOL(blk_mq_free_tag_set);

2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
{
	struct blk_mq_tag_set *set = q->tag_set;
	struct blk_mq_hw_ctx *hctx;
	int i, ret;

	if (!set || nr > set->queue_depth)
		return -EINVAL;

	ret = 0;
	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_tag_update_depth(hctx->tags, nr);
		if (ret)
			break;
	}

	if (!ret)
		q->nr_requests = nr;

	return ret;
}

2132 2133 2134 2135 2136 2137 2138 2139 2140 2141
void blk_mq_disable_hotplug(void)
{
	mutex_lock(&all_q_mutex);
}

void blk_mq_enable_hotplug(void)
{
	mutex_unlock(&all_q_mutex);
}

2142 2143 2144 2145
static int __init blk_mq_init(void)
{
	blk_mq_cpu_init();

2146
	hotcpu_notifier(blk_mq_queue_reinit_notify, 0);
2147 2148 2149 2150

	return 0;
}
subsys_initcall(blk_mq_init);