blk-mq.c 52.4 KB
Newer Older
1 2 3 4 5 6
/*
 * Block multiqueue core code
 *
 * Copyright (C) 2013-2014 Jens Axboe
 * Copyright (C) 2013-2014 Christoph Hellwig
 */
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>
#include <linux/llist.h>
#include <linux/list_sort.h>
#include <linux/cpu.h>
#include <linux/cache.h>
#include <linux/sched/sysctl.h>
#include <linux/delay.h>
23
#include <linux/crash_dump.h>
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43

#include <trace/events/block.h>

#include <linux/blk-mq.h>
#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-tag.h"

static DEFINE_MUTEX(all_q_mutex);
static LIST_HEAD(all_q_list);

static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx);

/*
 * Check if any of the ctx's have pending work in this hardware queue
 */
static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
{
	unsigned int i;

44
	for (i = 0; i < hctx->ctx_map.size; i++)
45
		if (hctx->ctx_map.map[i].word)
46 47 48 49 50
			return true;

	return false;
}

51 52 53 54 55 56 57 58 59
static inline struct blk_align_bitmap *get_bm(struct blk_mq_hw_ctx *hctx,
					      struct blk_mq_ctx *ctx)
{
	return &hctx->ctx_map.map[ctx->index_hw / hctx->ctx_map.bits_per_word];
}

#define CTX_TO_BIT(hctx, ctx)	\
	((ctx)->index_hw & ((hctx)->ctx_map.bits_per_word - 1))

60 61 62 63 64 65
/*
 * Mark this ctx as having pending work in this hardware queue
 */
static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
				     struct blk_mq_ctx *ctx)
{
66 67 68 69 70 71 72 73 74 75 76 77
	struct blk_align_bitmap *bm = get_bm(hctx, ctx);

	if (!test_bit(CTX_TO_BIT(hctx, ctx), &bm->word))
		set_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
}

static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
				      struct blk_mq_ctx *ctx)
{
	struct blk_align_bitmap *bm = get_bm(hctx, ctx);

	clear_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
78 79
}

80
static int blk_mq_queue_enter(struct request_queue *q, gfp_t gfp)
81
{
82 83
	while (true) {
		int ret;
84

85 86
		if (percpu_ref_tryget_live(&q->mq_usage_counter))
			return 0;
87

88 89 90
		if (!(gfp & __GFP_WAIT))
			return -EBUSY;

91 92 93 94 95 96 97
		ret = wait_event_interruptible(q->mq_freeze_wq,
				!q->mq_freeze_depth || blk_queue_dying(q));
		if (blk_queue_dying(q))
			return -ENODEV;
		if (ret)
			return ret;
	}
98 99 100 101
}

static void blk_mq_queue_exit(struct request_queue *q)
{
102 103 104 105 106 107 108 109 110
	percpu_ref_put(&q->mq_usage_counter);
}

static void blk_mq_usage_counter_release(struct percpu_ref *ref)
{
	struct request_queue *q =
		container_of(ref, struct request_queue, mq_usage_counter);

	wake_up_all(&q->mq_freeze_wq);
111 112
}

113
void blk_mq_freeze_queue_start(struct request_queue *q)
114
{
115 116
	bool freeze;

117
	spin_lock_irq(q->queue_lock);
118
	freeze = !q->mq_freeze_depth++;
119 120
	spin_unlock_irq(q->queue_lock);

121
	if (freeze) {
122
		percpu_ref_kill(&q->mq_usage_counter);
123
		blk_mq_run_hw_queues(q, false);
124
	}
125
}
126
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_start);
127 128 129

static void blk_mq_freeze_queue_wait(struct request_queue *q)
{
130
	wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->mq_usage_counter));
131 132
}

133 134 135 136 137 138 139 140 141
/*
 * Guarantee no request is in use, so we can change any data structure of
 * the queue afterward.
 */
void blk_mq_freeze_queue(struct request_queue *q)
{
	blk_mq_freeze_queue_start(q);
	blk_mq_freeze_queue_wait(q);
}
142
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
143

144
void blk_mq_unfreeze_queue(struct request_queue *q)
145
{
146
	bool wake;
147 148

	spin_lock_irq(q->queue_lock);
149 150
	wake = !--q->mq_freeze_depth;
	WARN_ON_ONCE(q->mq_freeze_depth < 0);
151
	spin_unlock_irq(q->queue_lock);
152 153
	if (wake) {
		percpu_ref_reinit(&q->mq_usage_counter);
154
		wake_up_all(&q->mq_freeze_wq);
155
	}
156
}
157
EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
158

159 160 161 162 163 164 165 166
void blk_mq_wake_waiters(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i)
		if (blk_mq_hw_queue_mapped(hctx))
			blk_mq_tag_wakeup_all(hctx->tags, true);
167 168 169 170 171 172 173

	/*
	 * If we are called because the queue has now been marked as
	 * dying, we need to ensure that processes currently waiting on
	 * the queue are notified as well.
	 */
	wake_up_all(&q->mq_freeze_wq);
174 175
}

176 177 178 179 180 181
bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
{
	return blk_mq_has_free_tags(hctx->tags);
}
EXPORT_SYMBOL(blk_mq_can_queue);

182 183
static void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
			       struct request *rq, unsigned int rw_flags)
184
{
185 186 187
	if (blk_queue_io_stat(q))
		rw_flags |= REQ_IO_STAT;

188 189 190
	INIT_LIST_HEAD(&rq->queuelist);
	/* csd/requeue_work/fifo_time is initialized before use */
	rq->q = q;
191
	rq->mq_ctx = ctx;
192
	rq->cmd_flags |= rw_flags;
193 194 195 196 197 198
	/* do not touch atomic flags, it needs atomic ops against the timer */
	rq->cpu = -1;
	INIT_HLIST_NODE(&rq->hash);
	RB_CLEAR_NODE(&rq->rb_node);
	rq->rq_disk = NULL;
	rq->part = NULL;
199
	rq->start_time = jiffies;
200 201
#ifdef CONFIG_BLK_CGROUP
	rq->rl = NULL;
202
	set_start_time_ns(rq);
203 204 205 206 207 208 209 210 211 212
	rq->io_start_time_ns = 0;
#endif
	rq->nr_phys_segments = 0;
#if defined(CONFIG_BLK_DEV_INTEGRITY)
	rq->nr_integrity_segments = 0;
#endif
	rq->special = NULL;
	/* tag was already set */
	rq->errors = 0;

213 214
	rq->cmd = rq->__cmd;

215 216 217 218 219 220
	rq->extra_len = 0;
	rq->sense_len = 0;
	rq->resid_len = 0;
	rq->sense = NULL;

	INIT_LIST_HEAD(&rq->timeout_list);
221 222
	rq->timeout = 0;

223 224 225 226
	rq->end_io = NULL;
	rq->end_io_data = NULL;
	rq->next_rq = NULL;

227 228 229
	ctx->rq_dispatched[rw_is_sync(rw_flags)]++;
}

230
static struct request *
231
__blk_mq_alloc_request(struct blk_mq_alloc_data *data, int rw)
232 233 234 235
{
	struct request *rq;
	unsigned int tag;

236
	tag = blk_mq_get_tag(data);
237
	if (tag != BLK_MQ_TAG_FAIL) {
238
		rq = data->hctx->tags->rqs[tag];
239

240
		if (blk_mq_tag_busy(data->hctx)) {
241
			rq->cmd_flags = REQ_MQ_INFLIGHT;
242
			atomic_inc(&data->hctx->nr_active);
243 244 245
		}

		rq->tag = tag;
246
		blk_mq_rq_ctx_init(data->q, data->ctx, rq, rw);
247 248 249 250 251 252
		return rq;
	}

	return NULL;
}

253 254
struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp,
		bool reserved)
255
{
256 257
	struct blk_mq_ctx *ctx;
	struct blk_mq_hw_ctx *hctx;
258
	struct request *rq;
259
	struct blk_mq_alloc_data alloc_data;
260
	int ret;
261

262
	ret = blk_mq_queue_enter(q, gfp);
263 264
	if (ret)
		return ERR_PTR(ret);
265

266 267
	ctx = blk_mq_get_ctx(q);
	hctx = q->mq_ops->map_queue(q, ctx->cpu);
268 269
	blk_mq_set_alloc_data(&alloc_data, q, gfp & ~__GFP_WAIT,
			reserved, ctx, hctx);
270

271
	rq = __blk_mq_alloc_request(&alloc_data, rw);
272 273 274 275 276 277
	if (!rq && (gfp & __GFP_WAIT)) {
		__blk_mq_run_hw_queue(hctx);
		blk_mq_put_ctx(ctx);

		ctx = blk_mq_get_ctx(q);
		hctx = q->mq_ops->map_queue(q, ctx->cpu);
278 279 280 281
		blk_mq_set_alloc_data(&alloc_data, q, gfp, reserved, ctx,
				hctx);
		rq =  __blk_mq_alloc_request(&alloc_data, rw);
		ctx = alloc_data.ctx;
282 283
	}
	blk_mq_put_ctx(ctx);
K
Keith Busch 已提交
284 285
	if (!rq) {
		blk_mq_queue_exit(q);
286
		return ERR_PTR(-EWOULDBLOCK);
K
Keith Busch 已提交
287
	}
288 289
	return rq;
}
290
EXPORT_SYMBOL(blk_mq_alloc_request);
291 292 293 294 295 296 297

static void __blk_mq_free_request(struct blk_mq_hw_ctx *hctx,
				  struct blk_mq_ctx *ctx, struct request *rq)
{
	const int tag = rq->tag;
	struct request_queue *q = rq->q;

298 299
	if (rq->cmd_flags & REQ_MQ_INFLIGHT)
		atomic_dec(&hctx->nr_active);
300
	rq->cmd_flags = 0;
301

302
	clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
303
	blk_mq_put_tag(hctx, tag, &ctx->last_tag);
304 305 306
	blk_mq_queue_exit(q);
}

307
void blk_mq_free_hctx_request(struct blk_mq_hw_ctx *hctx, struct request *rq)
308 309 310 311 312
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;

	ctx->rq_completed[rq_is_sync(rq)]++;
	__blk_mq_free_request(hctx, ctx, rq);
313 314 315 316 317 318 319 320 321 322 323

}
EXPORT_SYMBOL_GPL(blk_mq_free_hctx_request);

void blk_mq_free_request(struct request *rq)
{
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q = rq->q;

	hctx = q->mq_ops->map_queue(q, rq->mq_ctx->cpu);
	blk_mq_free_hctx_request(hctx, rq);
324
}
J
Jens Axboe 已提交
325
EXPORT_SYMBOL_GPL(blk_mq_free_request);
326

327
inline void __blk_mq_end_request(struct request *rq, int error)
328
{
M
Ming Lei 已提交
329 330
	blk_account_io_done(rq);

C
Christoph Hellwig 已提交
331
	if (rq->end_io) {
332
		rq->end_io(rq, error);
C
Christoph Hellwig 已提交
333 334 335
	} else {
		if (unlikely(blk_bidi_rq(rq)))
			blk_mq_free_request(rq->next_rq);
336
		blk_mq_free_request(rq);
C
Christoph Hellwig 已提交
337
	}
338
}
339
EXPORT_SYMBOL(__blk_mq_end_request);
340

341
void blk_mq_end_request(struct request *rq, int error)
342 343 344
{
	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
		BUG();
345
	__blk_mq_end_request(rq, error);
346
}
347
EXPORT_SYMBOL(blk_mq_end_request);
348

349
static void __blk_mq_complete_request_remote(void *data)
350
{
351
	struct request *rq = data;
352

353
	rq->q->softirq_done_fn(rq);
354 355
}

356
static void blk_mq_ipi_complete_request(struct request *rq)
357 358
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;
C
Christoph Hellwig 已提交
359
	bool shared = false;
360 361
	int cpu;

C
Christoph Hellwig 已提交
362
	if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
363 364 365
		rq->q->softirq_done_fn(rq);
		return;
	}
366 367

	cpu = get_cpu();
C
Christoph Hellwig 已提交
368 369 370 371
	if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
		shared = cpus_share_cache(cpu, ctx->cpu);

	if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
372
		rq->csd.func = __blk_mq_complete_request_remote;
373 374
		rq->csd.info = rq;
		rq->csd.flags = 0;
375
		smp_call_function_single_async(ctx->cpu, &rq->csd);
376
	} else {
377
		rq->q->softirq_done_fn(rq);
378
	}
379 380
	put_cpu();
}
381

382 383 384 385 386
void __blk_mq_complete_request(struct request *rq)
{
	struct request_queue *q = rq->q;

	if (!q->softirq_done_fn)
387
		blk_mq_end_request(rq, rq->errors);
388 389 390 391
	else
		blk_mq_ipi_complete_request(rq);
}

392 393 394 395 396 397 398 399 400 401
/**
 * blk_mq_complete_request - end I/O on a request
 * @rq:		the request being processed
 *
 * Description:
 *	Ends all I/O on a request. It does not handle partial completions.
 *	The actual completion happens out-of-order, through a IPI handler.
 **/
void blk_mq_complete_request(struct request *rq)
{
402 403 404
	struct request_queue *q = rq->q;

	if (unlikely(blk_should_fake_timeout(q)))
405
		return;
406 407
	if (!blk_mark_rq_complete(rq))
		__blk_mq_complete_request(rq);
408 409
}
EXPORT_SYMBOL(blk_mq_complete_request);
410

411 412 413 414 415 416
int blk_mq_request_started(struct request *rq)
{
	return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
}
EXPORT_SYMBOL_GPL(blk_mq_request_started);

417
void blk_mq_start_request(struct request *rq)
418 419 420 421 422
{
	struct request_queue *q = rq->q;

	trace_block_rq_issue(q, rq);

C
Christoph Hellwig 已提交
423
	rq->resid_len = blk_rq_bytes(rq);
C
Christoph Hellwig 已提交
424 425
	if (unlikely(blk_bidi_rq(rq)))
		rq->next_rq->resid_len = blk_rq_bytes(rq->next_rq);
C
Christoph Hellwig 已提交
426

427
	blk_add_timer(rq);
428

429 430 431 432 433 434
	/*
	 * Ensure that ->deadline is visible before set the started
	 * flag and clear the completed flag.
	 */
	smp_mb__before_atomic();

435 436 437 438 439 440
	/*
	 * Mark us as started and clear complete. Complete might have been
	 * set if requeue raced with timeout, which then marked it as
	 * complete. So be sure to clear complete again when we start
	 * the request, otherwise we'll ignore the completion event.
	 */
441 442 443 444
	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
		set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
	if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
		clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
445 446 447 448 449 450 451 452 453

	if (q->dma_drain_size && blk_rq_bytes(rq)) {
		/*
		 * Make sure space for the drain appears.  We know we can do
		 * this because max_hw_segments has been adjusted to be one
		 * fewer than the device can handle.
		 */
		rq->nr_phys_segments++;
	}
454
}
455
EXPORT_SYMBOL(blk_mq_start_request);
456

457
static void __blk_mq_requeue_request(struct request *rq)
458 459 460 461
{
	struct request_queue *q = rq->q;

	trace_block_rq_requeue(q, rq);
462

463 464 465 466
	if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
		if (q->dma_drain_size && blk_rq_bytes(rq))
			rq->nr_phys_segments--;
	}
467 468
}

469 470 471 472 473
void blk_mq_requeue_request(struct request *rq)
{
	__blk_mq_requeue_request(rq);

	BUG_ON(blk_queued_rq(rq));
474
	blk_mq_add_to_requeue_list(rq, true);
475 476 477
}
EXPORT_SYMBOL(blk_mq_requeue_request);

478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
static void blk_mq_requeue_work(struct work_struct *work)
{
	struct request_queue *q =
		container_of(work, struct request_queue, requeue_work);
	LIST_HEAD(rq_list);
	struct request *rq, *next;
	unsigned long flags;

	spin_lock_irqsave(&q->requeue_lock, flags);
	list_splice_init(&q->requeue_list, &rq_list);
	spin_unlock_irqrestore(&q->requeue_lock, flags);

	list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
		if (!(rq->cmd_flags & REQ_SOFTBARRIER))
			continue;

		rq->cmd_flags &= ~REQ_SOFTBARRIER;
		list_del_init(&rq->queuelist);
		blk_mq_insert_request(rq, true, false, false);
	}

	while (!list_empty(&rq_list)) {
		rq = list_entry(rq_list.next, struct request, queuelist);
		list_del_init(&rq->queuelist);
		blk_mq_insert_request(rq, false, false, false);
	}

505 506 507 508 509
	/*
	 * Use the start variant of queue running here, so that running
	 * the requeue work will kick stopped queues.
	 */
	blk_mq_start_hw_queues(q);
510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533
}

void blk_mq_add_to_requeue_list(struct request *rq, bool at_head)
{
	struct request_queue *q = rq->q;
	unsigned long flags;

	/*
	 * We abuse this flag that is otherwise used by the I/O scheduler to
	 * request head insertation from the workqueue.
	 */
	BUG_ON(rq->cmd_flags & REQ_SOFTBARRIER);

	spin_lock_irqsave(&q->requeue_lock, flags);
	if (at_head) {
		rq->cmd_flags |= REQ_SOFTBARRIER;
		list_add(&rq->queuelist, &q->requeue_list);
	} else {
		list_add_tail(&rq->queuelist, &q->requeue_list);
	}
	spin_unlock_irqrestore(&q->requeue_lock, flags);
}
EXPORT_SYMBOL(blk_mq_add_to_requeue_list);

534 535 536 537 538 539
void blk_mq_cancel_requeue_work(struct request_queue *q)
{
	cancel_work_sync(&q->requeue_work);
}
EXPORT_SYMBOL_GPL(blk_mq_cancel_requeue_work);

540 541 542 543 544 545
void blk_mq_kick_requeue_list(struct request_queue *q)
{
	kblockd_schedule_work(&q->requeue_work);
}
EXPORT_SYMBOL(blk_mq_kick_requeue_list);

546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565
void blk_mq_abort_requeue_list(struct request_queue *q)
{
	unsigned long flags;
	LIST_HEAD(rq_list);

	spin_lock_irqsave(&q->requeue_lock, flags);
	list_splice_init(&q->requeue_list, &rq_list);
	spin_unlock_irqrestore(&q->requeue_lock, flags);

	while (!list_empty(&rq_list)) {
		struct request *rq;

		rq = list_first_entry(&rq_list, struct request, queuelist);
		list_del_init(&rq->queuelist);
		rq->errors = -EIO;
		blk_mq_end_request(rq, rq->errors);
	}
}
EXPORT_SYMBOL(blk_mq_abort_requeue_list);

566 567
static inline bool is_flush_request(struct request *rq,
		struct blk_flush_queue *fq, unsigned int tag)
568
{
569
	return ((rq->cmd_flags & REQ_FLUSH_SEQ) &&
570
			fq->flush_rq->tag == tag);
571 572 573 574 575
}

struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
{
	struct request *rq = tags->rqs[tag];
576 577
	/* mq_ctx of flush rq is always cloned from the corresponding req */
	struct blk_flush_queue *fq = blk_get_flush_queue(rq->q, rq->mq_ctx);
578

579
	if (!is_flush_request(rq, fq, tag))
580
		return rq;
581

582
	return fq->flush_rq;
583 584 585
}
EXPORT_SYMBOL(blk_mq_tag_to_rq);

586
struct blk_mq_timeout_data {
587 588
	unsigned long next;
	unsigned int next_set;
589 590
};

591
void blk_mq_rq_timed_out(struct request *req, bool reserved)
592
{
593 594
	struct blk_mq_ops *ops = req->q->mq_ops;
	enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
595 596 597 598 599 600 601 602 603 604

	/*
	 * We know that complete is set at this point. If STARTED isn't set
	 * anymore, then the request isn't active and the "timeout" should
	 * just be ignored. This can happen due to the bitflag ordering.
	 * Timeout first checks if STARTED is set, and if it is, assumes
	 * the request is active. But if we race with completion, then
	 * we both flags will get cleared. So check here again, and ignore
	 * a timeout event with a request that isn't active.
	 */
605 606
	if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
		return;
607

608
	if (ops->timeout)
609
		ret = ops->timeout(req, reserved);
610 611 612 613 614 615 616 617 618 619 620 621 622 623 624

	switch (ret) {
	case BLK_EH_HANDLED:
		__blk_mq_complete_request(req);
		break;
	case BLK_EH_RESET_TIMER:
		blk_add_timer(req);
		blk_clear_rq_complete(req);
		break;
	case BLK_EH_NOT_HANDLED:
		break;
	default:
		printk(KERN_ERR "block: bad eh return: %d\n", ret);
		break;
	}
625
}
626

627 628 629 630
static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
		struct request *rq, void *priv, bool reserved)
{
	struct blk_mq_timeout_data *data = priv;
631

632 633 634 635 636 637 638 639 640
	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
		/*
		 * If a request wasn't started before the queue was
		 * marked dying, kill it here or it'll go unnoticed.
		 */
		if (unlikely(blk_queue_dying(rq->q))) {
			rq->errors = -EIO;
			blk_mq_complete_request(rq);
		}
641
		return;
642
	}
643 644
	if (rq->cmd_flags & REQ_NO_TIMEOUT)
		return;
645

646 647
	if (time_after_eq(jiffies, rq->deadline)) {
		if (!blk_mark_rq_complete(rq))
648
			blk_mq_rq_timed_out(rq, reserved);
649 650 651 652
	} else if (!data->next_set || time_after(data->next, rq->deadline)) {
		data->next = rq->deadline;
		data->next_set = 1;
	}
653 654
}

655
static void blk_mq_rq_timer(unsigned long priv)
656
{
657 658 659 660 661
	struct request_queue *q = (struct request_queue *)priv;
	struct blk_mq_timeout_data data = {
		.next		= 0,
		.next_set	= 0,
	};
662
	struct blk_mq_hw_ctx *hctx;
663
	int i;
664

665 666 667 668 669
	queue_for_each_hw_ctx(q, hctx, i) {
		/*
		 * If not software queues are currently mapped to this
		 * hardware queue, there's nothing to check
		 */
670
		if (!blk_mq_hw_queue_mapped(hctx))
671 672
			continue;

673
		blk_mq_tag_busy_iter(hctx, blk_mq_check_expired, &data);
674
	}
675

676 677 678
	if (data.next_set) {
		data.next = blk_rq_timeout(round_jiffies_up(data.next));
		mod_timer(&q->timeout, data.next);
679
	} else {
680 681 682 683 684
		queue_for_each_hw_ctx(q, hctx, i) {
			/* the hctx may be unmapped, so check it here */
			if (blk_mq_hw_queue_mapped(hctx))
				blk_mq_tag_idle(hctx);
		}
685
	}
686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726
}

/*
 * Reverse check our software queue for entries that we could potentially
 * merge with. Currently includes a hand-wavy stop count of 8, to not spend
 * too much time checking for merges.
 */
static bool blk_mq_attempt_merge(struct request_queue *q,
				 struct blk_mq_ctx *ctx, struct bio *bio)
{
	struct request *rq;
	int checked = 8;

	list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
		int el_ret;

		if (!checked--)
			break;

		if (!blk_rq_merge_ok(rq, bio))
			continue;

		el_ret = blk_try_merge(rq, bio);
		if (el_ret == ELEVATOR_BACK_MERGE) {
			if (bio_attempt_back_merge(q, rq, bio)) {
				ctx->rq_merged++;
				return true;
			}
			break;
		} else if (el_ret == ELEVATOR_FRONT_MERGE) {
			if (bio_attempt_front_merge(q, rq, bio)) {
				ctx->rq_merged++;
				return true;
			}
			break;
		}
	}

	return false;
}

727 728 729 730 731 732 733 734 735
/*
 * Process software queues that have been marked busy, splicing them
 * to the for-dispatch
 */
static void flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
{
	struct blk_mq_ctx *ctx;
	int i;

736
	for (i = 0; i < hctx->ctx_map.size; i++) {
737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760
		struct blk_align_bitmap *bm = &hctx->ctx_map.map[i];
		unsigned int off, bit;

		if (!bm->word)
			continue;

		bit = 0;
		off = i * hctx->ctx_map.bits_per_word;
		do {
			bit = find_next_bit(&bm->word, bm->depth, bit);
			if (bit >= bm->depth)
				break;

			ctx = hctx->ctxs[bit + off];
			clear_bit(bit, &bm->word);
			spin_lock(&ctx->lock);
			list_splice_tail_init(&ctx->rq_list, list);
			spin_unlock(&ctx->lock);

			bit++;
		} while (1);
	}
}

761 762 763 764 765 766 767 768 769 770 771
/*
 * Run this hardware queue, pulling any software queues mapped to it in.
 * Note that this function currently has various problems around ordering
 * of IO. In particular, we'd like FIFO behaviour on handling existing
 * items on the hctx->dispatch list. Ignore that for now.
 */
static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	struct request_queue *q = hctx->queue;
	struct request *rq;
	LIST_HEAD(rq_list);
772 773
	LIST_HEAD(driver_list);
	struct list_head *dptr;
774
	int queued;
775

776
	WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask));
777

778
	if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
779 780 781 782 783 784 785
		return;

	hctx->run++;

	/*
	 * Touch any software queue that has pending entries.
	 */
786
	flush_busy_ctxs(hctx, &rq_list);
787 788 789 790 791 792 793 794 795 796 797 798

	/*
	 * If we have previous entries on our dispatch list, grab them
	 * and stuff them at the front for more fair dispatch.
	 */
	if (!list_empty_careful(&hctx->dispatch)) {
		spin_lock(&hctx->lock);
		if (!list_empty(&hctx->dispatch))
			list_splice_init(&hctx->dispatch, &rq_list);
		spin_unlock(&hctx->lock);
	}

799 800 801 802 803 804
	/*
	 * Start off with dptr being NULL, so we start the first request
	 * immediately, even if we have more pending.
	 */
	dptr = NULL;

805 806 807
	/*
	 * Now process all the entries, sending them to the driver.
	 */
808
	queued = 0;
809
	while (!list_empty(&rq_list)) {
810
		struct blk_mq_queue_data bd;
811 812 813 814 815
		int ret;

		rq = list_first_entry(&rq_list, struct request, queuelist);
		list_del_init(&rq->queuelist);

816 817 818 819 820
		bd.rq = rq;
		bd.list = dptr;
		bd.last = list_empty(&rq_list);

		ret = q->mq_ops->queue_rq(hctx, &bd);
821 822 823 824 825 826
		switch (ret) {
		case BLK_MQ_RQ_QUEUE_OK:
			queued++;
			continue;
		case BLK_MQ_RQ_QUEUE_BUSY:
			list_add(&rq->queuelist, &rq_list);
827
			__blk_mq_requeue_request(rq);
828 829 830 831
			break;
		default:
			pr_err("blk-mq: bad return on queue: %d\n", ret);
		case BLK_MQ_RQ_QUEUE_ERROR:
832
			rq->errors = -EIO;
833
			blk_mq_end_request(rq, rq->errors);
834 835 836 837 838
			break;
		}

		if (ret == BLK_MQ_RQ_QUEUE_BUSY)
			break;
839 840 841 842 843 844 845

		/*
		 * We've done the first request. If we have more than 1
		 * left in the list, set dptr to defer issue.
		 */
		if (!dptr && rq_list.next != rq_list.prev)
			dptr = &driver_list;
846 847 848 849 850 851 852 853 854 855 856 857 858 859 860
	}

	if (!queued)
		hctx->dispatched[0]++;
	else if (queued < (1 << (BLK_MQ_MAX_DISPATCH_ORDER - 1)))
		hctx->dispatched[ilog2(queued) + 1]++;

	/*
	 * Any items that need requeuing? Stuff them into hctx->dispatch,
	 * that is where we will continue on next queue run.
	 */
	if (!list_empty(&rq_list)) {
		spin_lock(&hctx->lock);
		list_splice(&rq_list, &hctx->dispatch);
		spin_unlock(&hctx->lock);
861 862 863 864 865 866 867 868 869 870
		/*
		 * the queue is expected stopped with BLK_MQ_RQ_QUEUE_BUSY, but
		 * it's possible the queue is stopped and restarted again
		 * before this. Queue restart will dispatch requests. And since
		 * requests in rq_list aren't added into hctx->dispatch yet,
		 * the requests in rq_list might get lost.
		 *
		 * blk_mq_run_hw_queue() already checks the STOPPED bit
		 **/
		blk_mq_run_hw_queue(hctx, true);
871 872 873
	}
}

874 875 876 877 878 879 880 881
/*
 * It'd be great if the workqueue API had a way to pass
 * in a mask and had some smarts for more clever placement.
 * For now we just round-robin here, switching for every
 * BLK_MQ_CPU_WORK_BATCH queued items.
 */
static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
{
882 883
	if (hctx->queue->nr_hw_queues == 1)
		return WORK_CPU_UNBOUND;
884 885

	if (--hctx->next_cpu_batch <= 0) {
886
		int cpu = hctx->next_cpu, next_cpu;
887 888 889 890 891 892 893

		next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
		if (next_cpu >= nr_cpu_ids)
			next_cpu = cpumask_first(hctx->cpumask);

		hctx->next_cpu = next_cpu;
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
894 895

		return cpu;
896 897
	}

898
	return hctx->next_cpu;
899 900
}

901 902
void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
903 904
	if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state) ||
	    !blk_mq_hw_queue_mapped(hctx)))
905 906
		return;

907
	if (!async) {
908 909
		int cpu = get_cpu();
		if (cpumask_test_cpu(cpu, hctx->cpumask)) {
910
			__blk_mq_run_hw_queue(hctx);
911
			put_cpu();
912 913
			return;
		}
914

915
		put_cpu();
916
	}
917

918 919
	kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
			&hctx->run_work, 0);
920 921
}

922
void blk_mq_run_hw_queues(struct request_queue *q, bool async)
923 924 925 926 927 928 929
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if ((!blk_mq_hctx_has_pending(hctx) &&
		    list_empty_careful(&hctx->dispatch)) ||
930
		    test_bit(BLK_MQ_S_STOPPED, &hctx->state))
931 932
			continue;

933
		blk_mq_run_hw_queue(hctx, async);
934 935
	}
}
936
EXPORT_SYMBOL(blk_mq_run_hw_queues);
937 938 939

void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
{
940 941
	cancel_delayed_work(&hctx->run_work);
	cancel_delayed_work(&hctx->delay_work);
942 943 944 945
	set_bit(BLK_MQ_S_STOPPED, &hctx->state);
}
EXPORT_SYMBOL(blk_mq_stop_hw_queue);

946 947 948 949 950 951 952 953 954 955
void blk_mq_stop_hw_queues(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_stop_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_stop_hw_queues);

956 957 958
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
959

960
	blk_mq_run_hw_queue(hctx, false);
961 962 963
}
EXPORT_SYMBOL(blk_mq_start_hw_queue);

964 965 966 967 968 969 970 971 972 973
void blk_mq_start_hw_queues(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_start_hw_queues);

974
void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
975 976 977 978 979 980 981 982 983
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if (!test_bit(BLK_MQ_S_STOPPED, &hctx->state))
			continue;

		clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
984
		blk_mq_run_hw_queue(hctx, async);
985 986 987 988
	}
}
EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);

989
static void blk_mq_run_work_fn(struct work_struct *work)
990 991 992
{
	struct blk_mq_hw_ctx *hctx;

993
	hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
994

995 996 997
	__blk_mq_run_hw_queue(hctx);
}

998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
static void blk_mq_delay_work_fn(struct work_struct *work)
{
	struct blk_mq_hw_ctx *hctx;

	hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);

	if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
		__blk_mq_run_hw_queue(hctx);
}

void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
{
1010 1011
	if (unlikely(!blk_mq_hw_queue_mapped(hctx)))
		return;
1012

1013 1014
	kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
			&hctx->delay_work, msecs_to_jiffies(msecs));
1015 1016 1017
}
EXPORT_SYMBOL(blk_mq_delay_queue);

1018
static void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx,
1019
				    struct request *rq, bool at_head)
1020 1021 1022
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;

1023 1024
	trace_block_rq_insert(hctx->queue, rq);

1025 1026 1027 1028
	if (at_head)
		list_add(&rq->queuelist, &ctx->rq_list);
	else
		list_add_tail(&rq->queuelist, &ctx->rq_list);
1029

1030 1031 1032
	blk_mq_hctx_mark_pending(hctx, ctx);
}

1033 1034
void blk_mq_insert_request(struct request *rq, bool at_head, bool run_queue,
		bool async)
1035
{
1036
	struct request_queue *q = rq->q;
1037
	struct blk_mq_hw_ctx *hctx;
1038 1039 1040 1041 1042
	struct blk_mq_ctx *ctx = rq->mq_ctx, *current_ctx;

	current_ctx = blk_mq_get_ctx(q);
	if (!cpu_online(ctx->cpu))
		rq->mq_ctx = ctx = current_ctx;
1043 1044 1045

	hctx = q->mq_ops->map_queue(q, ctx->cpu);

1046 1047 1048
	spin_lock(&ctx->lock);
	__blk_mq_insert_request(hctx, rq, at_head);
	spin_unlock(&ctx->lock);
1049 1050 1051

	if (run_queue)
		blk_mq_run_hw_queue(hctx, async);
1052 1053

	blk_mq_put_ctx(current_ctx);
1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
}

static void blk_mq_insert_requests(struct request_queue *q,
				     struct blk_mq_ctx *ctx,
				     struct list_head *list,
				     int depth,
				     bool from_schedule)

{
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *current_ctx;

	trace_block_unplug(q, depth, !from_schedule);

	current_ctx = blk_mq_get_ctx(q);

	if (!cpu_online(ctx->cpu))
		ctx = current_ctx;
	hctx = q->mq_ops->map_queue(q, ctx->cpu);

	/*
	 * preemption doesn't flush plug list, so it's possible ctx->cpu is
	 * offline now
	 */
	spin_lock(&ctx->lock);
	while (!list_empty(list)) {
		struct request *rq;

		rq = list_first_entry(list, struct request, queuelist);
		list_del_init(&rq->queuelist);
		rq->mq_ctx = ctx;
1085
		__blk_mq_insert_request(hctx, rq, false);
1086 1087 1088 1089
	}
	spin_unlock(&ctx->lock);

	blk_mq_run_hw_queue(hctx, from_schedule);
1090
	blk_mq_put_ctx(current_ctx);
1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152
}

static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
{
	struct request *rqa = container_of(a, struct request, queuelist);
	struct request *rqb = container_of(b, struct request, queuelist);

	return !(rqa->mq_ctx < rqb->mq_ctx ||
		 (rqa->mq_ctx == rqb->mq_ctx &&
		  blk_rq_pos(rqa) < blk_rq_pos(rqb)));
}

void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
{
	struct blk_mq_ctx *this_ctx;
	struct request_queue *this_q;
	struct request *rq;
	LIST_HEAD(list);
	LIST_HEAD(ctx_list);
	unsigned int depth;

	list_splice_init(&plug->mq_list, &list);

	list_sort(NULL, &list, plug_ctx_cmp);

	this_q = NULL;
	this_ctx = NULL;
	depth = 0;

	while (!list_empty(&list)) {
		rq = list_entry_rq(list.next);
		list_del_init(&rq->queuelist);
		BUG_ON(!rq->q);
		if (rq->mq_ctx != this_ctx) {
			if (this_ctx) {
				blk_mq_insert_requests(this_q, this_ctx,
							&ctx_list, depth,
							from_schedule);
			}

			this_ctx = rq->mq_ctx;
			this_q = rq->q;
			depth = 0;
		}

		depth++;
		list_add_tail(&rq->queuelist, &ctx_list);
	}

	/*
	 * If 'this_ctx' is set, we know we have entries to complete
	 * on 'ctx_list'. Do those.
	 */
	if (this_ctx) {
		blk_mq_insert_requests(this_q, this_ctx, &ctx_list, depth,
				       from_schedule);
	}
}

static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
{
	init_request_from_bio(rq, bio);
1153

1154
	if (blk_do_io_stat(rq))
1155
		blk_account_io_start(rq, 1);
1156 1157
}

1158 1159 1160 1161 1162 1163
static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx)
{
	return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
		!blk_queue_nomerges(hctx->queue);
}

1164 1165 1166
static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
					 struct blk_mq_ctx *ctx,
					 struct request *rq, struct bio *bio)
1167
{
1168
	if (!hctx_allow_merges(hctx)) {
1169 1170 1171 1172 1173 1174 1175
		blk_mq_bio_to_request(rq, bio);
		spin_lock(&ctx->lock);
insert_rq:
		__blk_mq_insert_request(hctx, rq, false);
		spin_unlock(&ctx->lock);
		return false;
	} else {
1176 1177
		struct request_queue *q = hctx->queue;

1178 1179 1180 1181 1182
		spin_lock(&ctx->lock);
		if (!blk_mq_attempt_merge(q, ctx, bio)) {
			blk_mq_bio_to_request(rq, bio);
			goto insert_rq;
		}
1183

1184 1185 1186
		spin_unlock(&ctx->lock);
		__blk_mq_free_request(hctx, ctx, rq);
		return true;
1187
	}
1188
}
1189

1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202
struct blk_map_ctx {
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
};

static struct request *blk_mq_map_request(struct request_queue *q,
					  struct bio *bio,
					  struct blk_map_ctx *data)
{
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
	struct request *rq;
	int rw = bio_data_dir(bio);
1203
	struct blk_mq_alloc_data alloc_data;
1204

1205
	if (unlikely(blk_mq_queue_enter(q, GFP_KERNEL))) {
1206
		bio_endio(bio, -EIO);
1207
		return NULL;
1208 1209 1210 1211 1212
	}

	ctx = blk_mq_get_ctx(q);
	hctx = q->mq_ops->map_queue(q, ctx->cpu);

1213
	if (rw_is_sync(bio->bi_rw))
S
Shaohua Li 已提交
1214
		rw |= REQ_SYNC;
1215

1216
	trace_block_getrq(q, bio, rw);
1217 1218 1219
	blk_mq_set_alloc_data(&alloc_data, q, GFP_ATOMIC, false, ctx,
			hctx);
	rq = __blk_mq_alloc_request(&alloc_data, rw);
1220
	if (unlikely(!rq)) {
1221
		__blk_mq_run_hw_queue(hctx);
1222 1223
		blk_mq_put_ctx(ctx);
		trace_block_sleeprq(q, bio, rw);
1224 1225

		ctx = blk_mq_get_ctx(q);
1226
		hctx = q->mq_ops->map_queue(q, ctx->cpu);
1227 1228 1229 1230 1231
		blk_mq_set_alloc_data(&alloc_data, q,
				__GFP_WAIT|GFP_ATOMIC, false, ctx, hctx);
		rq = __blk_mq_alloc_request(&alloc_data, rw);
		ctx = alloc_data.ctx;
		hctx = alloc_data.hctx;
1232 1233 1234
	}

	hctx->queued++;
1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268
	data->hctx = hctx;
	data->ctx = ctx;
	return rq;
}

/*
 * Multiple hardware queue variant. This will not use per-process plugs,
 * but will attempt to bypass the hctx queueing if we can go straight to
 * hardware for SYNC IO.
 */
static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
{
	const int is_sync = rw_is_sync(bio->bi_rw);
	const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
	struct blk_map_ctx data;
	struct request *rq;

	blk_queue_bounce(q, &bio);

	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
		bio_endio(bio, -EIO);
		return;
	}

	rq = blk_mq_map_request(q, bio, &data);
	if (unlikely(!rq))
		return;

	if (unlikely(is_flush_fua)) {
		blk_mq_bio_to_request(rq, bio);
		blk_insert_flush(rq);
		goto run_queue;
	}

1269 1270 1271 1272 1273 1274
	/*
	 * If the driver supports defer issued based on 'last', then
	 * queue it up like normal since we can potentially save some
	 * CPU this way.
	 */
	if (is_sync && !(data.hctx->flags & BLK_MQ_F_DEFER_ISSUE)) {
1275 1276 1277 1278 1279
		struct blk_mq_queue_data bd = {
			.rq = rq,
			.list = NULL,
			.last = 1
		};
1280 1281 1282 1283 1284 1285 1286 1287 1288
		int ret;

		blk_mq_bio_to_request(rq, bio);

		/*
		 * For OK queue, we are done. For error, kill it. Any other
		 * error (busy), just add it to our list as we previously
		 * would have done
		 */
1289
		ret = q->mq_ops->queue_rq(data.hctx, &bd);
1290 1291 1292 1293 1294 1295 1296
		if (ret == BLK_MQ_RQ_QUEUE_OK)
			goto done;
		else {
			__blk_mq_requeue_request(rq);

			if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
				rq->errors = -EIO;
1297
				blk_mq_end_request(rq, rq->errors);
1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346
				goto done;
			}
		}
	}

	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
		/*
		 * For a SYNC request, send it to the hardware immediately. For
		 * an ASYNC request, just ensure that we run it later on. The
		 * latter allows for merging opportunities and more efficient
		 * dispatching.
		 */
run_queue:
		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
	}
done:
	blk_mq_put_ctx(data.ctx);
}

/*
 * Single hardware queue variant. This will attempt to use any per-process
 * plug for merging and IO deferral.
 */
static void blk_sq_make_request(struct request_queue *q, struct bio *bio)
{
	const int is_sync = rw_is_sync(bio->bi_rw);
	const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
	unsigned int use_plug, request_count = 0;
	struct blk_map_ctx data;
	struct request *rq;

	/*
	 * If we have multiple hardware queues, just go directly to
	 * one of those for sync IO.
	 */
	use_plug = !is_flush_fua && !is_sync;

	blk_queue_bounce(q, &bio);

	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
		bio_endio(bio, -EIO);
		return;
	}

	if (use_plug && !blk_queue_nomerges(q) &&
	    blk_attempt_plug_merge(q, bio, &request_count))
		return;

	rq = blk_mq_map_request(q, bio, &data);
1347 1348
	if (unlikely(!rq))
		return;
1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365

	if (unlikely(is_flush_fua)) {
		blk_mq_bio_to_request(rq, bio);
		blk_insert_flush(rq);
		goto run_queue;
	}

	/*
	 * A task plug currently exists. Since this is completely lockless,
	 * utilize that to temporarily store requests until the task is
	 * either done or scheduled away.
	 */
	if (use_plug) {
		struct blk_plug *plug = current->plug;

		if (plug) {
			blk_mq_bio_to_request(rq, bio);
S
Shaohua Li 已提交
1366
			if (list_empty(&plug->mq_list))
1367 1368 1369 1370 1371 1372
				trace_block_plug(q);
			else if (request_count >= BLK_MAX_REQUEST_COUNT) {
				blk_flush_plug_list(plug, false);
				trace_block_plug(q);
			}
			list_add_tail(&rq->queuelist, &plug->mq_list);
1373
			blk_mq_put_ctx(data.ctx);
1374 1375 1376 1377
			return;
		}
	}

1378 1379 1380 1381 1382 1383 1384 1385 1386
	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
		/*
		 * For a SYNC request, send it to the hardware immediately. For
		 * an ASYNC request, just ensure that we run it later on. The
		 * latter allows for merging opportunities and more efficient
		 * dispatching.
		 */
run_queue:
		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1387 1388
	}

1389
	blk_mq_put_ctx(data.ctx);
1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400
}

/*
 * Default mapping to a software queue, since we use one per CPU.
 */
struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *q, const int cpu)
{
	return q->queue_hw_ctx[q->mq_map[cpu]];
}
EXPORT_SYMBOL(blk_mq_map_queue);

1401 1402
static void blk_mq_free_rq_map(struct blk_mq_tag_set *set,
		struct blk_mq_tags *tags, unsigned int hctx_idx)
1403
{
1404
	struct page *page;
1405

1406
	if (tags->rqs && set->ops->exit_request) {
1407
		int i;
1408

1409 1410
		for (i = 0; i < tags->nr_tags; i++) {
			if (!tags->rqs[i])
1411
				continue;
1412 1413
			set->ops->exit_request(set->driver_data, tags->rqs[i],
						hctx_idx, i);
1414
			tags->rqs[i] = NULL;
1415
		}
1416 1417
	}

1418 1419
	while (!list_empty(&tags->page_list)) {
		page = list_first_entry(&tags->page_list, struct page, lru);
1420
		list_del_init(&page->lru);
1421 1422 1423
		__free_pages(page, page->private);
	}

1424
	kfree(tags->rqs);
1425

1426
	blk_mq_free_tags(tags);
1427 1428 1429 1430
}

static size_t order_to_size(unsigned int order)
{
1431
	return (size_t)PAGE_SIZE << order;
1432 1433
}

1434 1435
static struct blk_mq_tags *blk_mq_init_rq_map(struct blk_mq_tag_set *set,
		unsigned int hctx_idx)
1436
{
1437
	struct blk_mq_tags *tags;
1438 1439 1440
	unsigned int i, j, entries_per_page, max_order = 4;
	size_t rq_size, left;

1441
	tags = blk_mq_init_tags(set->queue_depth, set->reserved_tags,
S
Shaohua Li 已提交
1442 1443
				set->numa_node,
				BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
1444 1445
	if (!tags)
		return NULL;
1446

1447 1448
	INIT_LIST_HEAD(&tags->page_list);

1449 1450 1451
	tags->rqs = kzalloc_node(set->queue_depth * sizeof(struct request *),
				 GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY,
				 set->numa_node);
1452 1453 1454 1455
	if (!tags->rqs) {
		blk_mq_free_tags(tags);
		return NULL;
	}
1456 1457 1458 1459 1460

	/*
	 * rq_size is the size of the request plus driver payload, rounded
	 * to the cacheline size
	 */
1461
	rq_size = round_up(sizeof(struct request) + set->cmd_size,
1462
				cache_line_size());
1463
	left = rq_size * set->queue_depth;
1464

1465
	for (i = 0; i < set->queue_depth; ) {
1466 1467 1468 1469 1470 1471 1472 1473 1474
		int this_order = max_order;
		struct page *page;
		int to_do;
		void *p;

		while (left < order_to_size(this_order - 1) && this_order)
			this_order--;

		do {
1475
			page = alloc_pages_node(set->numa_node,
1476
				GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
1477
				this_order);
1478 1479 1480 1481 1482 1483 1484 1485 1486
			if (page)
				break;
			if (!this_order--)
				break;
			if (order_to_size(this_order) < rq_size)
				break;
		} while (1);

		if (!page)
1487
			goto fail;
1488 1489

		page->private = this_order;
1490
		list_add_tail(&page->lru, &tags->page_list);
1491 1492 1493

		p = page_address(page);
		entries_per_page = order_to_size(this_order) / rq_size;
1494
		to_do = min(entries_per_page, set->queue_depth - i);
1495 1496
		left -= to_do * rq_size;
		for (j = 0; j < to_do; j++) {
1497 1498 1499 1500
			tags->rqs[i] = p;
			if (set->ops->init_request) {
				if (set->ops->init_request(set->driver_data,
						tags->rqs[i], hctx_idx, i,
1501 1502
						set->numa_node)) {
					tags->rqs[i] = NULL;
1503
					goto fail;
1504
				}
1505 1506
			}

1507 1508 1509 1510 1511
			p += rq_size;
			i++;
		}
	}

1512
	return tags;
1513

1514 1515 1516
fail:
	blk_mq_free_rq_map(set, tags, hctx_idx);
	return NULL;
1517 1518
}

1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544
static void blk_mq_free_bitmap(struct blk_mq_ctxmap *bitmap)
{
	kfree(bitmap->map);
}

static int blk_mq_alloc_bitmap(struct blk_mq_ctxmap *bitmap, int node)
{
	unsigned int bpw = 8, total, num_maps, i;

	bitmap->bits_per_word = bpw;

	num_maps = ALIGN(nr_cpu_ids, bpw) / bpw;
	bitmap->map = kzalloc_node(num_maps * sizeof(struct blk_align_bitmap),
					GFP_KERNEL, node);
	if (!bitmap->map)
		return -ENOMEM;

	total = nr_cpu_ids;
	for (i = 0; i < num_maps; i++) {
		bitmap->map[i].depth = min(total, bitmap->bits_per_word);
		total -= bitmap->map[i].depth;
	}

	return 0;
}

1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593
static int blk_mq_hctx_cpu_offline(struct blk_mq_hw_ctx *hctx, int cpu)
{
	struct request_queue *q = hctx->queue;
	struct blk_mq_ctx *ctx;
	LIST_HEAD(tmp);

	/*
	 * Move ctx entries to new CPU, if this one is going away.
	 */
	ctx = __blk_mq_get_ctx(q, cpu);

	spin_lock(&ctx->lock);
	if (!list_empty(&ctx->rq_list)) {
		list_splice_init(&ctx->rq_list, &tmp);
		blk_mq_hctx_clear_pending(hctx, ctx);
	}
	spin_unlock(&ctx->lock);

	if (list_empty(&tmp))
		return NOTIFY_OK;

	ctx = blk_mq_get_ctx(q);
	spin_lock(&ctx->lock);

	while (!list_empty(&tmp)) {
		struct request *rq;

		rq = list_first_entry(&tmp, struct request, queuelist);
		rq->mq_ctx = ctx;
		list_move_tail(&rq->queuelist, &ctx->rq_list);
	}

	hctx = q->mq_ops->map_queue(q, ctx->cpu);
	blk_mq_hctx_mark_pending(hctx, ctx);

	spin_unlock(&ctx->lock);

	blk_mq_run_hw_queue(hctx, true);
	blk_mq_put_ctx(ctx);
	return NOTIFY_OK;
}

static int blk_mq_hctx_notify(void *data, unsigned long action,
			      unsigned int cpu)
{
	struct blk_mq_hw_ctx *hctx = data;

	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
		return blk_mq_hctx_cpu_offline(hctx, cpu);
M
Ming Lei 已提交
1594 1595 1596 1597 1598

	/*
	 * In case of CPU online, tags may be reallocated
	 * in blk_mq_map_swqueue() after mapping is updated.
	 */
1599 1600 1601 1602

	return NOTIFY_OK;
}

1603 1604 1605 1606
static void blk_mq_exit_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
1607 1608
	unsigned flush_start_tag = set->queue_depth;

1609 1610
	blk_mq_tag_idle(hctx);

1611 1612 1613 1614 1615
	if (set->ops->exit_request)
		set->ops->exit_request(set->driver_data,
				       hctx->fq->flush_rq, hctx_idx,
				       flush_start_tag + hctx_idx);

1616 1617 1618 1619
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);

	blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1620
	blk_free_flush_queue(hctx->fq);
1621 1622 1623 1624
	kfree(hctx->ctxs);
	blk_mq_free_bitmap(&hctx->ctx_map);
}

M
Ming Lei 已提交
1625 1626 1627 1628 1629 1630 1631 1632 1633
static void blk_mq_exit_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set, int nr_queue)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if (i == nr_queue)
			break;
1634
		blk_mq_exit_hctx(q, set, hctx, i);
M
Ming Lei 已提交
1635 1636 1637 1638 1639 1640 1641 1642 1643
	}
}

static void blk_mq_free_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

1644
	queue_for_each_hw_ctx(q, hctx, i)
M
Ming Lei 已提交
1645 1646 1647
		free_cpumask_var(hctx->cpumask);
}

1648 1649 1650
static int blk_mq_init_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
1651
{
1652
	int node;
1653
	unsigned flush_start_tag = set->queue_depth;
1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671

	node = hctx->numa_node;
	if (node == NUMA_NO_NODE)
		node = hctx->numa_node = set->numa_node;

	INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
	INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
	spin_lock_init(&hctx->lock);
	INIT_LIST_HEAD(&hctx->dispatch);
	hctx->queue = q;
	hctx->queue_num = hctx_idx;
	hctx->flags = set->flags;

	blk_mq_init_cpu_notifier(&hctx->cpu_notifier,
					blk_mq_hctx_notify, hctx);
	blk_mq_register_cpu_notifier(&hctx->cpu_notifier);

	hctx->tags = set->tags[hctx_idx];
1672 1673

	/*
1674 1675
	 * Allocate space for all possible cpus to avoid allocation at
	 * runtime
1676
	 */
1677 1678 1679 1680
	hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
					GFP_KERNEL, node);
	if (!hctx->ctxs)
		goto unregister_cpu_notifier;
1681

1682 1683
	if (blk_mq_alloc_bitmap(&hctx->ctx_map, node))
		goto free_ctxs;
1684

1685
	hctx->nr_ctx = 0;
1686

1687 1688 1689
	if (set->ops->init_hctx &&
	    set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
		goto free_bitmap;
1690

1691 1692 1693
	hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size);
	if (!hctx->fq)
		goto exit_hctx;
1694

1695 1696 1697 1698 1699
	if (set->ops->init_request &&
	    set->ops->init_request(set->driver_data,
				   hctx->fq->flush_rq, hctx_idx,
				   flush_start_tag + hctx_idx, node))
		goto free_fq;
1700

1701
	return 0;
1702

1703 1704 1705 1706 1707
 free_fq:
	kfree(hctx->fq);
 exit_hctx:
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);
1708 1709 1710 1711 1712 1713
 free_bitmap:
	blk_mq_free_bitmap(&hctx->ctx_map);
 free_ctxs:
	kfree(hctx->ctxs);
 unregister_cpu_notifier:
	blk_mq_unregister_cpu_notifier(&hctx->cpu_notifier);
1714

1715 1716
	return -1;
}
1717

1718 1719 1720 1721 1722
static int blk_mq_init_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;
1723

1724 1725 1726 1727 1728
	/*
	 * Initialize hardware queues
	 */
	queue_for_each_hw_ctx(q, hctx, i) {
		if (blk_mq_init_hctx(q, set, hctx, i))
1729 1730 1731 1732 1733 1734 1735 1736 1737
			break;
	}

	if (i == q->nr_hw_queues)
		return 0;

	/*
	 * Init failed
	 */
M
Ming Lei 已提交
1738
	blk_mq_exit_hw_queues(q, set, i);
1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761

	return 1;
}

static void blk_mq_init_cpu_queues(struct request_queue *q,
				   unsigned int nr_hw_queues)
{
	unsigned int i;

	for_each_possible_cpu(i) {
		struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
		struct blk_mq_hw_ctx *hctx;

		memset(__ctx, 0, sizeof(*__ctx));
		__ctx->cpu = i;
		spin_lock_init(&__ctx->lock);
		INIT_LIST_HEAD(&__ctx->rq_list);
		__ctx->queue = q;

		/* If the cpu isn't online, the cpu is mapped to first hctx */
		if (!cpu_online(i))
			continue;

1762 1763
		hctx = q->mq_ops->map_queue(q, i);

1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777
		/*
		 * Set local node, IFF we have more than one hw queue. If
		 * not, we remain on the home node of the device
		 */
		if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
			hctx->numa_node = cpu_to_node(i);
	}
}

static void blk_mq_map_swqueue(struct request_queue *q)
{
	unsigned int i;
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
M
Ming Lei 已提交
1778
	struct blk_mq_tag_set *set = q->tag_set;
1779 1780

	queue_for_each_hw_ctx(q, hctx, i) {
1781
		cpumask_clear(hctx->cpumask);
1782 1783 1784 1785 1786 1787 1788 1789
		hctx->nr_ctx = 0;
	}

	/*
	 * Map software to hardware queues
	 */
	queue_for_each_ctx(q, ctx, i) {
		/* If the cpu isn't online, the cpu is mapped to first hctx */
1790 1791 1792
		if (!cpu_online(i))
			continue;

1793
		hctx = q->mq_ops->map_queue(q, i);
1794
		cpumask_set_cpu(i, hctx->cpumask);
1795 1796 1797
		ctx->index_hw = hctx->nr_ctx;
		hctx->ctxs[hctx->nr_ctx++] = ctx;
	}
1798 1799

	queue_for_each_hw_ctx(q, hctx, i) {
1800 1801
		struct blk_mq_ctxmap *map = &hctx->ctx_map;

1802
		/*
1803 1804
		 * If no software queues are mapped to this hardware queue,
		 * disable it and free the request entries.
1805 1806 1807 1808 1809 1810
		 */
		if (!hctx->nr_ctx) {
			if (set->tags[i]) {
				blk_mq_free_rq_map(set, set->tags[i], i);
				set->tags[i] = NULL;
			}
M
Ming Lei 已提交
1811
			hctx->tags = NULL;
1812 1813 1814
			continue;
		}

M
Ming Lei 已提交
1815 1816 1817 1818 1819 1820
		/* unmapped hw queue can be remapped after CPU topo changed */
		if (!set->tags[i])
			set->tags[i] = blk_mq_init_rq_map(set, i);
		hctx->tags = set->tags[i];
		WARN_ON(!hctx->tags);

1821 1822 1823 1824 1825
		/*
		 * Set the map size to the number of mapped software queues.
		 * This is more accurate and more efficient than looping
		 * over all possibly mapped software queues.
		 */
1826
		map->size = DIV_ROUND_UP(hctx->nr_ctx, map->bits_per_word);
1827

1828 1829 1830
		/*
		 * Initialize batch roundrobin counts
		 */
1831 1832 1833
		hctx->next_cpu = cpumask_first(hctx->cpumask);
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}
1834 1835
}

1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881
static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set)
{
	struct blk_mq_hw_ctx *hctx;
	struct request_queue *q;
	bool shared;
	int i;

	if (set->tag_list.next == set->tag_list.prev)
		shared = false;
	else
		shared = true;

	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_freeze_queue(q);

		queue_for_each_hw_ctx(q, hctx, i) {
			if (shared)
				hctx->flags |= BLK_MQ_F_TAG_SHARED;
			else
				hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
		}
		blk_mq_unfreeze_queue(q);
	}
}

static void blk_mq_del_queue_tag_set(struct request_queue *q)
{
	struct blk_mq_tag_set *set = q->tag_set;

	mutex_lock(&set->tag_list_lock);
	list_del_init(&q->tag_set_list);
	blk_mq_update_tag_set_depth(set);
	mutex_unlock(&set->tag_list_lock);
}

static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
				     struct request_queue *q)
{
	q->tag_set = set;

	mutex_lock(&set->tag_list_lock);
	list_add_tail(&q->tag_set_list, &set->tag_list);
	blk_mq_update_tag_set_depth(set);
	mutex_unlock(&set->tag_list_lock);
}

1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902
/*
 * It is the actual release handler for mq, but we do it from
 * request queue's release handler for avoiding use-after-free
 * and headache because q->mq_kobj shouldn't have been introduced,
 * but we can't group ctx/kctx kobj without it.
 */
void blk_mq_release(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	/* hctx kobj stays in hctx */
	queue_for_each_hw_ctx(q, hctx, i)
		kfree(hctx);

	kfree(q->queue_hw_ctx);

	/* ctx kobj stays in queue_ctx */
	free_percpu(q->queue_ctx);
}

1903
struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920
{
	struct request_queue *uninit_q, *q;

	uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
	if (!uninit_q)
		return ERR_PTR(-ENOMEM);

	q = blk_mq_init_allocated_queue(set, uninit_q);
	if (IS_ERR(q))
		blk_cleanup_queue(uninit_q);

	return q;
}
EXPORT_SYMBOL(blk_mq_init_queue);

struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
						  struct request_queue *q)
1921 1922
{
	struct blk_mq_hw_ctx **hctxs;
1923
	struct blk_mq_ctx __percpu *ctx;
1924
	unsigned int *map;
1925 1926 1927 1928 1929 1930
	int i;

	ctx = alloc_percpu(struct blk_mq_ctx);
	if (!ctx)
		return ERR_PTR(-ENOMEM);

1931 1932
	hctxs = kmalloc_node(set->nr_hw_queues * sizeof(*hctxs), GFP_KERNEL,
			set->numa_node);
1933 1934 1935 1936

	if (!hctxs)
		goto err_percpu;

1937 1938 1939 1940
	map = blk_mq_make_queue_map(set);
	if (!map)
		goto err_map;

1941
	for (i = 0; i < set->nr_hw_queues; i++) {
1942 1943
		int node = blk_mq_hw_queue_to_node(map, i);

1944 1945
		hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
					GFP_KERNEL, node);
1946 1947 1948
		if (!hctxs[i])
			goto err_hctxs;

1949 1950
		if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL,
						node))
1951 1952
			goto err_hctxs;

1953
		atomic_set(&hctxs[i]->nr_active, 0);
1954
		hctxs[i]->numa_node = node;
1955 1956 1957
		hctxs[i]->queue_num = i;
	}

1958 1959 1960 1961
	/*
	 * Init percpu_ref in atomic mode so that it's faster to shutdown.
	 * See blk_register_queue() for details.
	 */
1962
	if (percpu_ref_init(&q->mq_usage_counter, blk_mq_usage_counter_release,
1963
			    PERCPU_REF_INIT_ATOMIC, GFP_KERNEL))
1964
		goto err_hctxs;
1965

1966
	setup_timer(&q->timeout, blk_mq_rq_timer, (unsigned long) q);
1967
	blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30000);
1968 1969

	q->nr_queues = nr_cpu_ids;
1970
	q->nr_hw_queues = set->nr_hw_queues;
1971
	q->mq_map = map;
1972 1973 1974 1975

	q->queue_ctx = ctx;
	q->queue_hw_ctx = hctxs;

1976
	q->mq_ops = set->ops;
1977
	q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
1978

1979 1980 1981
	if (!(set->flags & BLK_MQ_F_SG_MERGE))
		q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;

1982 1983
	q->sg_reserved_size = INT_MAX;

1984 1985 1986 1987
	INIT_WORK(&q->requeue_work, blk_mq_requeue_work);
	INIT_LIST_HEAD(&q->requeue_list);
	spin_lock_init(&q->requeue_lock);

1988 1989 1990 1991 1992
	if (q->nr_hw_queues > 1)
		blk_queue_make_request(q, blk_mq_make_request);
	else
		blk_queue_make_request(q, blk_sq_make_request);

1993 1994 1995 1996 1997
	/*
	 * Do this after blk_queue_make_request() overrides it...
	 */
	q->nr_requests = set->queue_depth;

1998 1999
	if (set->ops->complete)
		blk_queue_softirq_done(q, set->ops->complete);
2000

2001
	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
2002

2003
	if (blk_mq_init_hw_queues(q, set))
2004
		goto err_hctxs;
2005

2006 2007 2008 2009
	mutex_lock(&all_q_mutex);
	list_add_tail(&q->all_q_node, &all_q_list);
	mutex_unlock(&all_q_mutex);

2010 2011
	blk_mq_add_queue_tag_set(set, q);

2012 2013
	blk_mq_map_swqueue(q);

2014
	return q;
2015

2016
err_hctxs:
2017
	kfree(map);
2018
	for (i = 0; i < set->nr_hw_queues; i++) {
2019 2020
		if (!hctxs[i])
			break;
2021
		free_cpumask_var(hctxs[i]->cpumask);
2022
		kfree(hctxs[i]);
2023
	}
2024
err_map:
2025 2026 2027 2028 2029
	kfree(hctxs);
err_percpu:
	free_percpu(ctx);
	return ERR_PTR(-ENOMEM);
}
2030
EXPORT_SYMBOL(blk_mq_init_allocated_queue);
2031 2032 2033

void blk_mq_free_queue(struct request_queue *q)
{
M
Ming Lei 已提交
2034
	struct blk_mq_tag_set	*set = q->tag_set;
2035

2036 2037
	blk_mq_del_queue_tag_set(q);

M
Ming Lei 已提交
2038 2039
	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
	blk_mq_free_hw_queues(q, set);
2040

2041
	percpu_ref_exit(&q->mq_usage_counter);
2042

2043 2044 2045 2046 2047 2048 2049 2050 2051 2052
	kfree(q->mq_map);

	q->mq_map = NULL;

	mutex_lock(&all_q_mutex);
	list_del_init(&q->all_q_node);
	mutex_unlock(&all_q_mutex);
}

/* Basically redo blk_mq_init_queue with queue frozen */
2053
static void blk_mq_queue_reinit(struct request_queue *q)
2054
{
2055
	WARN_ON_ONCE(!q->mq_freeze_depth);
2056

2057 2058
	blk_mq_sysfs_unregister(q);

2059 2060 2061 2062 2063 2064 2065 2066 2067 2068
	blk_mq_update_queue_map(q->mq_map, q->nr_hw_queues);

	/*
	 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
	 * we should change hctx numa_node according to new topology (this
	 * involves free and re-allocate memory, worthy doing?)
	 */

	blk_mq_map_swqueue(q);

2069
	blk_mq_sysfs_register(q);
2070 2071
}

2072 2073
static int blk_mq_queue_reinit_notify(struct notifier_block *nb,
				      unsigned long action, void *hcpu)
2074 2075 2076 2077
{
	struct request_queue *q;

	/*
2078 2079 2080 2081
	 * Before new mappings are established, hotadded cpu might already
	 * start handling requests. This doesn't break anything as we map
	 * offline CPUs to first hardware queue. We will re-init the queue
	 * below to get optimal settings.
2082 2083 2084 2085 2086 2087
	 */
	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN &&
	    action != CPU_ONLINE && action != CPU_ONLINE_FROZEN)
		return NOTIFY_OK;

	mutex_lock(&all_q_mutex);
2088 2089 2090 2091 2092 2093 2094 2095 2096 2097

	/*
	 * We need to freeze and reinit all existing queues.  Freezing
	 * involves synchronous wait for an RCU grace period and doing it
	 * one by one may take a long time.  Start freezing all queues in
	 * one swoop and then wait for the completions so that freezing can
	 * take place in parallel.
	 */
	list_for_each_entry(q, &all_q_list, all_q_node)
		blk_mq_freeze_queue_start(q);
2098
	list_for_each_entry(q, &all_q_list, all_q_node) {
2099 2100
		blk_mq_freeze_queue_wait(q);

2101 2102 2103 2104 2105 2106 2107
		/*
		 * timeout handler can't touch hw queue during the
		 * reinitialization
		 */
		del_timer_sync(&q->timeout);
	}

2108 2109
	list_for_each_entry(q, &all_q_list, all_q_node)
		blk_mq_queue_reinit(q);
2110 2111 2112 2113

	list_for_each_entry(q, &all_q_list, all_q_node)
		blk_mq_unfreeze_queue(q);

2114 2115 2116 2117
	mutex_unlock(&all_q_mutex);
	return NOTIFY_OK;
}

2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171
static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	int i;

	for (i = 0; i < set->nr_hw_queues; i++) {
		set->tags[i] = blk_mq_init_rq_map(set, i);
		if (!set->tags[i])
			goto out_unwind;
	}

	return 0;

out_unwind:
	while (--i >= 0)
		blk_mq_free_rq_map(set, set->tags[i], i);

	return -ENOMEM;
}

/*
 * Allocate the request maps associated with this tag_set. Note that this
 * may reduce the depth asked for, if memory is tight. set->queue_depth
 * will be updated to reflect the allocated depth.
 */
static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	unsigned int depth;
	int err;

	depth = set->queue_depth;
	do {
		err = __blk_mq_alloc_rq_maps(set);
		if (!err)
			break;

		set->queue_depth >>= 1;
		if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
			err = -ENOMEM;
			break;
		}
	} while (set->queue_depth);

	if (!set->queue_depth || err) {
		pr_err("blk-mq: failed to allocate request map\n");
		return -ENOMEM;
	}

	if (depth != set->queue_depth)
		pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
						depth, set->queue_depth);

	return 0;
}

2172 2173 2174 2175 2176 2177
/*
 * Alloc a tag set to be associated with one or more request queues.
 * May fail with EINVAL for various error conditions. May adjust the
 * requested depth down, if if it too large. In that case, the set
 * value will be stored in set->queue_depth.
 */
2178 2179
int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
{
B
Bart Van Assche 已提交
2180 2181
	BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);

2182 2183
	if (!set->nr_hw_queues)
		return -EINVAL;
2184
	if (!set->queue_depth)
2185 2186 2187 2188
		return -EINVAL;
	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
		return -EINVAL;

2189
	if (!set->ops->queue_rq || !set->ops->map_queue)
2190 2191
		return -EINVAL;

2192 2193 2194 2195 2196
	if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
		pr_info("blk-mq: reduced tag depth to %u\n",
			BLK_MQ_MAX_DEPTH);
		set->queue_depth = BLK_MQ_MAX_DEPTH;
	}
2197

2198 2199 2200 2201 2202 2203 2204 2205 2206 2207
	/*
	 * If a crashdump is active, then we are potentially in a very
	 * memory constrained environment. Limit us to 1 queue and
	 * 64 tags to prevent using too much memory.
	 */
	if (is_kdump_kernel()) {
		set->nr_hw_queues = 1;
		set->queue_depth = min(64U, set->queue_depth);
	}

M
Ming Lei 已提交
2208 2209
	set->tags = kmalloc_node(set->nr_hw_queues *
				 sizeof(struct blk_mq_tags *),
2210 2211
				 GFP_KERNEL, set->numa_node);
	if (!set->tags)
2212
		return -ENOMEM;
2213

2214 2215
	if (blk_mq_alloc_rq_maps(set))
		goto enomem;
2216

2217 2218 2219
	mutex_init(&set->tag_list_lock);
	INIT_LIST_HEAD(&set->tag_list);

2220
	return 0;
2221
enomem:
2222 2223
	kfree(set->tags);
	set->tags = NULL;
2224 2225 2226 2227 2228 2229 2230 2231
	return -ENOMEM;
}
EXPORT_SYMBOL(blk_mq_alloc_tag_set);

void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
{
	int i;

2232 2233 2234 2235 2236
	for (i = 0; i < set->nr_hw_queues; i++) {
		if (set->tags[i])
			blk_mq_free_rq_map(set, set->tags[i], i);
	}

M
Ming Lei 已提交
2237
	kfree(set->tags);
2238
	set->tags = NULL;
2239 2240 2241
}
EXPORT_SYMBOL(blk_mq_free_tag_set);

2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
{
	struct blk_mq_tag_set *set = q->tag_set;
	struct blk_mq_hw_ctx *hctx;
	int i, ret;

	if (!set || nr > set->queue_depth)
		return -EINVAL;

	ret = 0;
	queue_for_each_hw_ctx(q, hctx, i) {
		ret = blk_mq_tag_update_depth(hctx->tags, nr);
		if (ret)
			break;
	}

	if (!ret)
		q->nr_requests = nr;

	return ret;
}

2264 2265 2266 2267 2268 2269 2270 2271 2272 2273
void blk_mq_disable_hotplug(void)
{
	mutex_lock(&all_q_mutex);
}

void blk_mq_enable_hotplug(void)
{
	mutex_unlock(&all_q_mutex);
}

2274 2275 2276 2277
static int __init blk_mq_init(void)
{
	blk_mq_cpu_init();

2278
	hotcpu_notifier(blk_mq_queue_reinit_notify, 0);
2279 2280 2281 2282

	return 0;
}
subsys_initcall(blk_mq_init);