blk-mq.c 69.9 KB
Newer Older
1 2 3 4 5 6
/*
 * Block multiqueue core code
 *
 * Copyright (C) 2013-2014 Jens Axboe
 * Copyright (C) 2013-2014 Christoph Hellwig
 */
7 8 9 10 11
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
12
#include <linux/kmemleak.h>
13 14 15 16 17 18 19 20 21 22
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>
#include <linux/llist.h>
#include <linux/list_sort.h>
#include <linux/cpu.h>
#include <linux/cache.h>
#include <linux/sched/sysctl.h>
23
#include <linux/sched/topology.h>
24
#include <linux/sched/signal.h>
25
#include <linux/delay.h>
26
#include <linux/crash_dump.h>
27
#include <linux/prefetch.h>
28 29 30 31 32 33 34

#include <trace/events/block.h>

#include <linux/blk-mq.h>
#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-tag.h"
35
#include "blk-stat.h"
J
Jens Axboe 已提交
36
#include "blk-wbt.h"
37
#include "blk-mq-sched.h"
38 39 40 41 42 43 44

static DEFINE_MUTEX(all_q_mutex);
static LIST_HEAD(all_q_list);

/*
 * Check if any of the ctx's have pending work in this hardware queue
 */
45
bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
46
{
47 48 49
	return sbitmap_any_bit_set(&hctx->ctx_map) ||
			!list_empty_careful(&hctx->dispatch) ||
			blk_mq_sched_has_work(hctx);
50 51
}

52 53 54 55 56 57
/*
 * Mark this ctx as having pending work in this hardware queue
 */
static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
				     struct blk_mq_ctx *ctx)
{
58 59
	if (!sbitmap_test_bit(&hctx->ctx_map, ctx->index_hw))
		sbitmap_set_bit(&hctx->ctx_map, ctx->index_hw);
60 61 62 63 64
}

static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
				      struct blk_mq_ctx *ctx)
{
65
	sbitmap_clear_bit(&hctx->ctx_map, ctx->index_hw);
66 67
}

68
void blk_mq_freeze_queue_start(struct request_queue *q)
69
{
70
	int freeze_depth;
71

72 73
	freeze_depth = atomic_inc_return(&q->mq_freeze_depth);
	if (freeze_depth == 1) {
74
		percpu_ref_kill(&q->q_usage_counter);
75
		blk_mq_run_hw_queues(q, false);
76
	}
77
}
78
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_start);
79

80
void blk_mq_freeze_queue_wait(struct request_queue *q)
81
{
82
	wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
83
}
84
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
85

86 87 88 89 90 91 92 93
int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
				     unsigned long timeout)
{
	return wait_event_timeout(q->mq_freeze_wq,
					percpu_ref_is_zero(&q->q_usage_counter),
					timeout);
}
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
94

95 96 97 98
/*
 * Guarantee no request is in use, so we can change any data structure of
 * the queue afterward.
 */
99
void blk_freeze_queue(struct request_queue *q)
100
{
101 102 103 104 105 106 107
	/*
	 * In the !blk_mq case we are only calling this to kill the
	 * q_usage_counter, otherwise this increases the freeze depth
	 * and waits for it to return to zero.  For this reason there is
	 * no blk_unfreeze_queue(), and blk_freeze_queue() is not
	 * exported to drivers as the only user for unfreeze is blk_mq.
	 */
108 109 110
	blk_mq_freeze_queue_start(q);
	blk_mq_freeze_queue_wait(q);
}
111 112 113 114 115 116 117 118 119

void blk_mq_freeze_queue(struct request_queue *q)
{
	/*
	 * ...just an alias to keep freeze and unfreeze actions balanced
	 * in the blk_mq_* namespace
	 */
	blk_freeze_queue(q);
}
120
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
121

122
void blk_mq_unfreeze_queue(struct request_queue *q)
123
{
124
	int freeze_depth;
125

126 127 128
	freeze_depth = atomic_dec_return(&q->mq_freeze_depth);
	WARN_ON_ONCE(freeze_depth < 0);
	if (!freeze_depth) {
129
		percpu_ref_reinit(&q->q_usage_counter);
130
		wake_up_all(&q->mq_freeze_wq);
131
	}
132
}
133
EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
134

135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
/**
 * blk_mq_quiesce_queue() - wait until all ongoing queue_rq calls have finished
 * @q: request queue.
 *
 * Note: this function does not prevent that the struct request end_io()
 * callback function is invoked. Additionally, it is not prevented that
 * new queue_rq() calls occur unless the queue has been stopped first.
 */
void blk_mq_quiesce_queue(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;
	bool rcu = false;

	blk_mq_stop_hw_queues(q);

	queue_for_each_hw_ctx(q, hctx, i) {
		if (hctx->flags & BLK_MQ_F_BLOCKING)
			synchronize_srcu(&hctx->queue_rq_srcu);
		else
			rcu = true;
	}
	if (rcu)
		synchronize_rcu();
}
EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);

162 163 164 165 166 167 168 169
void blk_mq_wake_waiters(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i)
		if (blk_mq_hw_queue_mapped(hctx))
			blk_mq_tag_wakeup_all(hctx->tags, true);
170 171 172 173 174 175 176

	/*
	 * If we are called because the queue has now been marked as
	 * dying, we need to ensure that processes currently waiting on
	 * the queue are notified as well.
	 */
	wake_up_all(&q->mq_freeze_wq);
177 178
}

179 180 181 182 183 184
bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
{
	return blk_mq_has_free_tags(hctx->tags);
}
EXPORT_SYMBOL(blk_mq_can_queue);

185 186
void blk_mq_rq_ctx_init(struct request_queue *q, struct blk_mq_ctx *ctx,
			struct request *rq, unsigned int op)
187
{
188 189 190
	INIT_LIST_HEAD(&rq->queuelist);
	/* csd/requeue_work/fifo_time is initialized before use */
	rq->q = q;
191
	rq->mq_ctx = ctx;
192
	rq->cmd_flags = op;
193 194
	if (blk_queue_io_stat(q))
		rq->rq_flags |= RQF_IO_STAT;
195 196 197 198 199 200
	/* do not touch atomic flags, it needs atomic ops against the timer */
	rq->cpu = -1;
	INIT_HLIST_NODE(&rq->hash);
	RB_CLEAR_NODE(&rq->rb_node);
	rq->rq_disk = NULL;
	rq->part = NULL;
201
	rq->start_time = jiffies;
202 203
#ifdef CONFIG_BLK_CGROUP
	rq->rl = NULL;
204
	set_start_time_ns(rq);
205 206 207 208 209 210 211 212 213 214 215 216
	rq->io_start_time_ns = 0;
#endif
	rq->nr_phys_segments = 0;
#if defined(CONFIG_BLK_DEV_INTEGRITY)
	rq->nr_integrity_segments = 0;
#endif
	rq->special = NULL;
	/* tag was already set */
	rq->errors = 0;
	rq->extra_len = 0;

	INIT_LIST_HEAD(&rq->timeout_list);
217 218
	rq->timeout = 0;

219 220 221 222
	rq->end_io = NULL;
	rq->end_io_data = NULL;
	rq->next_rq = NULL;

223
	ctx->rq_dispatched[op_is_sync(op)]++;
224
}
225
EXPORT_SYMBOL_GPL(blk_mq_rq_ctx_init);
226

227 228
struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data,
				       unsigned int op)
229 230 231 232
{
	struct request *rq;
	unsigned int tag;

233
	tag = blk_mq_get_tag(data);
234
	if (tag != BLK_MQ_TAG_FAIL) {
235 236 237
		struct blk_mq_tags *tags = blk_mq_tags_from_data(data);

		rq = tags->static_rqs[tag];
238

239 240 241 242
		if (data->flags & BLK_MQ_REQ_INTERNAL) {
			rq->tag = -1;
			rq->internal_tag = tag;
		} else {
243 244 245 246
			if (blk_mq_tag_busy(data->hctx)) {
				rq->rq_flags = RQF_MQ_INFLIGHT;
				atomic_inc(&data->hctx->nr_active);
			}
247 248
			rq->tag = tag;
			rq->internal_tag = -1;
249
			data->hctx->tags->rqs[rq->tag] = rq;
250 251
		}

252
		blk_mq_rq_ctx_init(data->q, data->ctx, rq, op);
253 254 255 256 257
		return rq;
	}

	return NULL;
}
258
EXPORT_SYMBOL_GPL(__blk_mq_alloc_request);
259

260 261
struct request *blk_mq_alloc_request(struct request_queue *q, int rw,
		unsigned int flags)
262
{
263
	struct blk_mq_alloc_data alloc_data = { .flags = flags };
264
	struct request *rq;
265
	int ret;
266

267
	ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT);
268 269
	if (ret)
		return ERR_PTR(ret);
270

271
	rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data);
272

273 274 275 276
	blk_mq_put_ctx(alloc_data.ctx);
	blk_queue_exit(q);

	if (!rq)
277
		return ERR_PTR(-EWOULDBLOCK);
278 279 280 281

	rq->__data_len = 0;
	rq->__sector = (sector_t) -1;
	rq->bio = rq->biotail = NULL;
282 283
	return rq;
}
284
EXPORT_SYMBOL(blk_mq_alloc_request);
285

M
Ming Lin 已提交
286 287 288
struct request *blk_mq_alloc_request_hctx(struct request_queue *q, int rw,
		unsigned int flags, unsigned int hctx_idx)
{
289
	struct blk_mq_alloc_data alloc_data = { .flags = flags };
M
Ming Lin 已提交
290
	struct request *rq;
291
	unsigned int cpu;
M
Ming Lin 已提交
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309
	int ret;

	/*
	 * If the tag allocator sleeps we could get an allocation for a
	 * different hardware context.  No need to complicate the low level
	 * allocator for this for the rare use case of a command tied to
	 * a specific queue.
	 */
	if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)))
		return ERR_PTR(-EINVAL);

	if (hctx_idx >= q->nr_hw_queues)
		return ERR_PTR(-EIO);

	ret = blk_queue_enter(q, true);
	if (ret)
		return ERR_PTR(ret);

310 311 312 313
	/*
	 * Check if the hardware context is actually mapped to anything.
	 * If not tell the caller that it should skip this queue.
	 */
314 315 316 317
	alloc_data.hctx = q->queue_hw_ctx[hctx_idx];
	if (!blk_mq_hw_queue_mapped(alloc_data.hctx)) {
		blk_queue_exit(q);
		return ERR_PTR(-EXDEV);
318
	}
319 320
	cpu = cpumask_first(alloc_data.hctx->cpumask);
	alloc_data.ctx = __blk_mq_get_ctx(q, cpu);
M
Ming Lin 已提交
321

322
	rq = blk_mq_sched_get_request(q, NULL, rw, &alloc_data);
323

324
	blk_mq_put_ctx(alloc_data.ctx);
325
	blk_queue_exit(q);
326 327 328 329 330

	if (!rq)
		return ERR_PTR(-EWOULDBLOCK);

	return rq;
M
Ming Lin 已提交
331 332 333
}
EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);

334 335
void __blk_mq_finish_request(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
			     struct request *rq)
336
{
337
	const int sched_tag = rq->internal_tag;
338 339
	struct request_queue *q = rq->q;

340
	if (rq->rq_flags & RQF_MQ_INFLIGHT)
341
		atomic_dec(&hctx->nr_active);
J
Jens Axboe 已提交
342 343

	wbt_done(q->rq_wb, &rq->issue_stat);
344
	rq->rq_flags = 0;
345

346
	clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
347
	clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
348 349 350 351
	if (rq->tag != -1)
		blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
	if (sched_tag != -1)
		blk_mq_sched_completed_request(hctx, rq);
352
	blk_mq_sched_restart_queues(hctx);
353
	blk_queue_exit(q);
354 355
}

356
static void blk_mq_finish_hctx_request(struct blk_mq_hw_ctx *hctx,
357
				     struct request *rq)
358 359 360 361
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;

	ctx->rq_completed[rq_is_sync(rq)]++;
362 363 364 365 366 367
	__blk_mq_finish_request(hctx, ctx, rq);
}

void blk_mq_finish_request(struct request *rq)
{
	blk_mq_finish_hctx_request(blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), rq);
368 369 370 371
}

void blk_mq_free_request(struct request *rq)
{
372
	blk_mq_sched_put_request(rq);
373
}
J
Jens Axboe 已提交
374
EXPORT_SYMBOL_GPL(blk_mq_free_request);
375

376
inline void __blk_mq_end_request(struct request *rq, int error)
377
{
M
Ming Lei 已提交
378 379
	blk_account_io_done(rq);

C
Christoph Hellwig 已提交
380
	if (rq->end_io) {
J
Jens Axboe 已提交
381
		wbt_done(rq->q->rq_wb, &rq->issue_stat);
382
		rq->end_io(rq, error);
C
Christoph Hellwig 已提交
383 384 385
	} else {
		if (unlikely(blk_bidi_rq(rq)))
			blk_mq_free_request(rq->next_rq);
386
		blk_mq_free_request(rq);
C
Christoph Hellwig 已提交
387
	}
388
}
389
EXPORT_SYMBOL(__blk_mq_end_request);
390

391
void blk_mq_end_request(struct request *rq, int error)
392 393 394
{
	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
		BUG();
395
	__blk_mq_end_request(rq, error);
396
}
397
EXPORT_SYMBOL(blk_mq_end_request);
398

399
static void __blk_mq_complete_request_remote(void *data)
400
{
401
	struct request *rq = data;
402

403
	rq->q->softirq_done_fn(rq);
404 405
}

406
static void blk_mq_ipi_complete_request(struct request *rq)
407 408
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;
C
Christoph Hellwig 已提交
409
	bool shared = false;
410 411
	int cpu;

C
Christoph Hellwig 已提交
412
	if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
413 414 415
		rq->q->softirq_done_fn(rq);
		return;
	}
416 417

	cpu = get_cpu();
C
Christoph Hellwig 已提交
418 419 420 421
	if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
		shared = cpus_share_cache(cpu, ctx->cpu);

	if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
422
		rq->csd.func = __blk_mq_complete_request_remote;
423 424
		rq->csd.info = rq;
		rq->csd.flags = 0;
425
		smp_call_function_single_async(ctx->cpu, &rq->csd);
426
	} else {
427
		rq->q->softirq_done_fn(rq);
428
	}
429 430
	put_cpu();
}
431

432 433 434 435 436 437 438 439 440 441 442 443 444 445 446
static void blk_mq_stat_add(struct request *rq)
{
	if (rq->rq_flags & RQF_STATS) {
		/*
		 * We could rq->mq_ctx here, but there's less of a risk
		 * of races if we have the completion event add the stats
		 * to the local software queue.
		 */
		struct blk_mq_ctx *ctx;

		ctx = __blk_mq_get_ctx(rq->q, raw_smp_processor_id());
		blk_stat_add(&ctx->stat[rq_data_dir(rq)], rq);
	}
}

447
static void __blk_mq_complete_request(struct request *rq)
448 449 450
{
	struct request_queue *q = rq->q;

451 452
	blk_mq_stat_add(rq);

453
	if (!q->softirq_done_fn)
454
		blk_mq_end_request(rq, rq->errors);
455 456 457 458
	else
		blk_mq_ipi_complete_request(rq);
}

459 460 461 462 463 464 465 466
/**
 * blk_mq_complete_request - end I/O on a request
 * @rq:		the request being processed
 *
 * Description:
 *	Ends all I/O on a request. It does not handle partial completions.
 *	The actual completion happens out-of-order, through a IPI handler.
 **/
467
void blk_mq_complete_request(struct request *rq, int error)
468
{
469 470 471
	struct request_queue *q = rq->q;

	if (unlikely(blk_should_fake_timeout(q)))
472
		return;
473 474
	if (!blk_mark_rq_complete(rq)) {
		rq->errors = error;
475
		__blk_mq_complete_request(rq);
476
	}
477 478
}
EXPORT_SYMBOL(blk_mq_complete_request);
479

480 481 482 483 484 485
int blk_mq_request_started(struct request *rq)
{
	return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
}
EXPORT_SYMBOL_GPL(blk_mq_request_started);

486
void blk_mq_start_request(struct request *rq)
487 488 489
{
	struct request_queue *q = rq->q;

490 491
	blk_mq_sched_started_request(rq);

492 493
	trace_block_rq_issue(q, rq);

494 495 496
	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
		blk_stat_set_issue_time(&rq->issue_stat);
		rq->rq_flags |= RQF_STATS;
J
Jens Axboe 已提交
497
		wbt_issue(q->rq_wb, &rq->issue_stat);
498 499
	}

500
	blk_add_timer(rq);
501

502 503 504 505 506 507
	/*
	 * Ensure that ->deadline is visible before set the started
	 * flag and clear the completed flag.
	 */
	smp_mb__before_atomic();

508 509 510 511 512 513
	/*
	 * Mark us as started and clear complete. Complete might have been
	 * set if requeue raced with timeout, which then marked it as
	 * complete. So be sure to clear complete again when we start
	 * the request, otherwise we'll ignore the completion event.
	 */
514 515 516 517
	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
		set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
	if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
		clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
518 519 520 521 522 523 524 525 526

	if (q->dma_drain_size && blk_rq_bytes(rq)) {
		/*
		 * Make sure space for the drain appears.  We know we can do
		 * this because max_hw_segments has been adjusted to be one
		 * fewer than the device can handle.
		 */
		rq->nr_phys_segments++;
	}
527
}
528
EXPORT_SYMBOL(blk_mq_start_request);
529

530
static void __blk_mq_requeue_request(struct request *rq)
531 532 533 534
{
	struct request_queue *q = rq->q;

	trace_block_rq_requeue(q, rq);
J
Jens Axboe 已提交
535
	wbt_requeue(q->rq_wb, &rq->issue_stat);
536
	blk_mq_sched_requeue_request(rq);
537

538 539 540 541
	if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
		if (q->dma_drain_size && blk_rq_bytes(rq))
			rq->nr_phys_segments--;
	}
542 543
}

544
void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
545 546 547 548
{
	__blk_mq_requeue_request(rq);

	BUG_ON(blk_queued_rq(rq));
549
	blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
550 551 552
}
EXPORT_SYMBOL(blk_mq_requeue_request);

553 554 555
static void blk_mq_requeue_work(struct work_struct *work)
{
	struct request_queue *q =
556
		container_of(work, struct request_queue, requeue_work.work);
557 558 559 560 561 562 563 564 565
	LIST_HEAD(rq_list);
	struct request *rq, *next;
	unsigned long flags;

	spin_lock_irqsave(&q->requeue_lock, flags);
	list_splice_init(&q->requeue_list, &rq_list);
	spin_unlock_irqrestore(&q->requeue_lock, flags);

	list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
566
		if (!(rq->rq_flags & RQF_SOFTBARRIER))
567 568
			continue;

569
		rq->rq_flags &= ~RQF_SOFTBARRIER;
570
		list_del_init(&rq->queuelist);
571
		blk_mq_sched_insert_request(rq, true, false, false, true);
572 573 574 575 576
	}

	while (!list_empty(&rq_list)) {
		rq = list_entry(rq_list.next, struct request, queuelist);
		list_del_init(&rq->queuelist);
577
		blk_mq_sched_insert_request(rq, false, false, false, true);
578 579
	}

580
	blk_mq_run_hw_queues(q, false);
581 582
}

583 584
void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
				bool kick_requeue_list)
585 586 587 588 589 590 591 592
{
	struct request_queue *q = rq->q;
	unsigned long flags;

	/*
	 * We abuse this flag that is otherwise used by the I/O scheduler to
	 * request head insertation from the workqueue.
	 */
593
	BUG_ON(rq->rq_flags & RQF_SOFTBARRIER);
594 595 596

	spin_lock_irqsave(&q->requeue_lock, flags);
	if (at_head) {
597
		rq->rq_flags |= RQF_SOFTBARRIER;
598 599 600 601 602
		list_add(&rq->queuelist, &q->requeue_list);
	} else {
		list_add_tail(&rq->queuelist, &q->requeue_list);
	}
	spin_unlock_irqrestore(&q->requeue_lock, flags);
603 604 605

	if (kick_requeue_list)
		blk_mq_kick_requeue_list(q);
606 607 608 609 610
}
EXPORT_SYMBOL(blk_mq_add_to_requeue_list);

void blk_mq_kick_requeue_list(struct request_queue *q)
{
611
	kblockd_schedule_delayed_work(&q->requeue_work, 0);
612 613 614
}
EXPORT_SYMBOL(blk_mq_kick_requeue_list);

615 616 617 618 619 620 621 622
void blk_mq_delay_kick_requeue_list(struct request_queue *q,
				    unsigned long msecs)
{
	kblockd_schedule_delayed_work(&q->requeue_work,
				      msecs_to_jiffies(msecs));
}
EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);

623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642
void blk_mq_abort_requeue_list(struct request_queue *q)
{
	unsigned long flags;
	LIST_HEAD(rq_list);

	spin_lock_irqsave(&q->requeue_lock, flags);
	list_splice_init(&q->requeue_list, &rq_list);
	spin_unlock_irqrestore(&q->requeue_lock, flags);

	while (!list_empty(&rq_list)) {
		struct request *rq;

		rq = list_first_entry(&rq_list, struct request, queuelist);
		list_del_init(&rq->queuelist);
		rq->errors = -EIO;
		blk_mq_end_request(rq, rq->errors);
	}
}
EXPORT_SYMBOL(blk_mq_abort_requeue_list);

643 644
struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
{
645 646
	if (tag < tags->nr_tags) {
		prefetch(tags->rqs[tag]);
647
		return tags->rqs[tag];
648
	}
649 650

	return NULL;
651 652 653
}
EXPORT_SYMBOL(blk_mq_tag_to_rq);

654
struct blk_mq_timeout_data {
655 656
	unsigned long next;
	unsigned int next_set;
657 658
};

659
void blk_mq_rq_timed_out(struct request *req, bool reserved)
660
{
J
Jens Axboe 已提交
661
	const struct blk_mq_ops *ops = req->q->mq_ops;
662
	enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
663 664 665 666 667 668 669 670 671 672

	/*
	 * We know that complete is set at this point. If STARTED isn't set
	 * anymore, then the request isn't active and the "timeout" should
	 * just be ignored. This can happen due to the bitflag ordering.
	 * Timeout first checks if STARTED is set, and if it is, assumes
	 * the request is active. But if we race with completion, then
	 * we both flags will get cleared. So check here again, and ignore
	 * a timeout event with a request that isn't active.
	 */
673 674
	if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
		return;
675

676
	if (ops->timeout)
677
		ret = ops->timeout(req, reserved);
678 679 680 681 682 683 684 685 686 687 688 689 690 691 692

	switch (ret) {
	case BLK_EH_HANDLED:
		__blk_mq_complete_request(req);
		break;
	case BLK_EH_RESET_TIMER:
		blk_add_timer(req);
		blk_clear_rq_complete(req);
		break;
	case BLK_EH_NOT_HANDLED:
		break;
	default:
		printk(KERN_ERR "block: bad eh return: %d\n", ret);
		break;
	}
693
}
694

695 696 697 698
static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
		struct request *rq, void *priv, bool reserved)
{
	struct blk_mq_timeout_data *data = priv;
699

700 701 702 703 704
	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
		/*
		 * If a request wasn't started before the queue was
		 * marked dying, kill it here or it'll go unnoticed.
		 */
705 706 707 708
		if (unlikely(blk_queue_dying(rq->q))) {
			rq->errors = -EIO;
			blk_mq_end_request(rq, rq->errors);
		}
709
		return;
710
	}
711

712 713
	if (time_after_eq(jiffies, rq->deadline)) {
		if (!blk_mark_rq_complete(rq))
714
			blk_mq_rq_timed_out(rq, reserved);
715 716 717 718
	} else if (!data->next_set || time_after(data->next, rq->deadline)) {
		data->next = rq->deadline;
		data->next_set = 1;
	}
719 720
}

721
static void blk_mq_timeout_work(struct work_struct *work)
722
{
723 724
	struct request_queue *q =
		container_of(work, struct request_queue, timeout_work);
725 726 727 728 729
	struct blk_mq_timeout_data data = {
		.next		= 0,
		.next_set	= 0,
	};
	int i;
730

731 732 733 734 735 736 737 738 739 740 741 742 743 744
	/* A deadlock might occur if a request is stuck requiring a
	 * timeout at the same time a queue freeze is waiting
	 * completion, since the timeout code would not be able to
	 * acquire the queue reference here.
	 *
	 * That's why we don't use blk_queue_enter here; instead, we use
	 * percpu_ref_tryget directly, because we need to be able to
	 * obtain a reference even in the short window between the queue
	 * starting to freeze, by dropping the first reference in
	 * blk_mq_freeze_queue_start, and the moment the last request is
	 * consumed, marked by the instant q_usage_counter reaches
	 * zero.
	 */
	if (!percpu_ref_tryget(&q->q_usage_counter))
745 746
		return;

747
	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data);
748

749 750 751
	if (data.next_set) {
		data.next = blk_rq_timeout(round_jiffies_up(data.next));
		mod_timer(&q->timeout, data.next);
752
	} else {
753 754
		struct blk_mq_hw_ctx *hctx;

755 756 757 758 759
		queue_for_each_hw_ctx(q, hctx, i) {
			/* the hctx may be unmapped, so check it here */
			if (blk_mq_hw_queue_mapped(hctx))
				blk_mq_tag_idle(hctx);
		}
760
	}
761
	blk_queue_exit(q);
762 763 764 765 766 767 768 769 770 771 772 773 774 775
}

/*
 * Reverse check our software queue for entries that we could potentially
 * merge with. Currently includes a hand-wavy stop count of 8, to not spend
 * too much time checking for merges.
 */
static bool blk_mq_attempt_merge(struct request_queue *q,
				 struct blk_mq_ctx *ctx, struct bio *bio)
{
	struct request *rq;
	int checked = 8;

	list_for_each_entry_reverse(rq, &ctx->rq_list, queuelist) {
776
		bool merged = false;
777 778 779 780 781 782 783

		if (!checked--)
			break;

		if (!blk_rq_merge_ok(rq, bio))
			continue;

784 785 786 787
		switch (blk_try_merge(rq, bio)) {
		case ELEVATOR_BACK_MERGE:
			if (blk_mq_sched_allow_merge(q, rq, bio))
				merged = bio_attempt_back_merge(q, rq, bio);
788
			break;
789 790 791
		case ELEVATOR_FRONT_MERGE:
			if (blk_mq_sched_allow_merge(q, rq, bio))
				merged = bio_attempt_front_merge(q, rq, bio);
792
			break;
793 794
		case ELEVATOR_DISCARD_MERGE:
			merged = bio_attempt_discard_merge(q, rq, bio);
795
			break;
796 797
		default:
			continue;
798
		}
799 800 801 802

		if (merged)
			ctx->rq_merged++;
		return merged;
803 804 805 806 807
	}

	return false;
}

808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825
struct flush_busy_ctx_data {
	struct blk_mq_hw_ctx *hctx;
	struct list_head *list;
};

static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
{
	struct flush_busy_ctx_data *flush_data = data;
	struct blk_mq_hw_ctx *hctx = flush_data->hctx;
	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];

	sbitmap_clear_bit(sb, bitnr);
	spin_lock(&ctx->lock);
	list_splice_tail_init(&ctx->rq_list, flush_data->list);
	spin_unlock(&ctx->lock);
	return true;
}

826 827 828 829
/*
 * Process software queues that have been marked busy, splicing them
 * to the for-dispatch
 */
830
void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
831
{
832 833 834 835
	struct flush_busy_ctx_data data = {
		.hctx = hctx,
		.list = list,
	};
836

837
	sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
838
}
839
EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
840

841 842 843 844
static inline unsigned int queued_to_index(unsigned int queued)
{
	if (!queued)
		return 0;
845

846
	return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
847 848
}

849 850
bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx,
			   bool wait)
851 852 853 854 855 856 857 858 859 860 861 862 863 864
{
	struct blk_mq_alloc_data data = {
		.q = rq->q,
		.hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu),
		.flags = wait ? 0 : BLK_MQ_REQ_NOWAIT,
	};

	if (rq->tag != -1) {
done:
		if (hctx)
			*hctx = data.hctx;
		return true;
	}

865 866 867
	if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag))
		data.flags |= BLK_MQ_REQ_RESERVED;

868 869
	rq->tag = blk_mq_get_tag(&data);
	if (rq->tag >= 0) {
870 871 872 873
		if (blk_mq_tag_busy(data.hctx)) {
			rq->rq_flags |= RQF_MQ_INFLIGHT;
			atomic_inc(&data.hctx->nr_active);
		}
874 875 876 877 878 879 880
		data.hctx->tags->rqs[rq->tag] = rq;
		goto done;
	}

	return false;
}

881 882
static void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx,
				    struct request *rq)
883 884 885 886 887 888 889 890 891 892
{
	blk_mq_put_tag(hctx, hctx->tags, rq->mq_ctx, rq->tag);
	rq->tag = -1;

	if (rq->rq_flags & RQF_MQ_INFLIGHT) {
		rq->rq_flags &= ~RQF_MQ_INFLIGHT;
		atomic_dec(&hctx->nr_active);
	}
}

893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912
static void blk_mq_put_driver_tag_hctx(struct blk_mq_hw_ctx *hctx,
				       struct request *rq)
{
	if (rq->tag == -1 || rq->internal_tag == -1)
		return;

	__blk_mq_put_driver_tag(hctx, rq);
}

static void blk_mq_put_driver_tag(struct request *rq)
{
	struct blk_mq_hw_ctx *hctx;

	if (rq->tag == -1 || rq->internal_tag == -1)
		return;

	hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu);
	__blk_mq_put_driver_tag(hctx, rq);
}

913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936
/*
 * If we fail getting a driver tag because all the driver tags are already
 * assigned and on the dispatch list, BUT the first entry does not have a
 * tag, then we could deadlock. For that case, move entries with assigned
 * driver tags to the front, leaving the set of tagged requests in the
 * same order, and the untagged set in the same order.
 */
static bool reorder_tags_to_front(struct list_head *list)
{
	struct request *rq, *tmp, *first = NULL;

	list_for_each_entry_safe_reverse(rq, tmp, list, queuelist) {
		if (rq == first)
			break;
		if (rq->tag != -1) {
			list_move(&rq->queuelist, list);
			if (!first)
				first = rq;
		}
	}

	return first != NULL;
}

937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974
static int blk_mq_dispatch_wake(wait_queue_t *wait, unsigned mode, int flags,
				void *key)
{
	struct blk_mq_hw_ctx *hctx;

	hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);

	list_del(&wait->task_list);
	clear_bit_unlock(BLK_MQ_S_TAG_WAITING, &hctx->state);
	blk_mq_run_hw_queue(hctx, true);
	return 1;
}

static bool blk_mq_dispatch_wait_add(struct blk_mq_hw_ctx *hctx)
{
	struct sbq_wait_state *ws;

	/*
	 * The TAG_WAITING bit serves as a lock protecting hctx->dispatch_wait.
	 * The thread which wins the race to grab this bit adds the hardware
	 * queue to the wait queue.
	 */
	if (test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state) ||
	    test_and_set_bit_lock(BLK_MQ_S_TAG_WAITING, &hctx->state))
		return false;

	init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
	ws = bt_wait_ptr(&hctx->tags->bitmap_tags, hctx);

	/*
	 * As soon as this returns, it's no longer safe to fiddle with
	 * hctx->dispatch_wait, since a completion can wake up the wait queue
	 * and unlock the bit.
	 */
	add_wait_queue(&ws->wait, &hctx->dispatch_wait);
	return true;
}

975
bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list)
976 977 978
{
	struct request_queue *q = hctx->queue;
	struct request *rq;
979 980
	LIST_HEAD(driver_list);
	struct list_head *dptr;
981
	int queued, ret = BLK_MQ_RQ_QUEUE_OK;
982

983 984 985 986 987 988
	/*
	 * Start off with dptr being NULL, so we start the first request
	 * immediately, even if we have more pending.
	 */
	dptr = NULL;

989 990 991
	/*
	 * Now process all the entries, sending them to the driver.
	 */
992
	queued = 0;
993
	while (!list_empty(list)) {
994
		struct blk_mq_queue_data bd;
995

996
		rq = list_first_entry(list, struct request, queuelist);
997 998 999
		if (!blk_mq_get_driver_tag(rq, &hctx, false)) {
			if (!queued && reorder_tags_to_front(list))
				continue;
1000 1001

			/*
1002 1003
			 * The initial allocation attempt failed, so we need to
			 * rerun the hardware queue when a tag is freed.
1004
			 */
1005 1006 1007 1008 1009 1010 1011 1012 1013
			if (blk_mq_dispatch_wait_add(hctx)) {
				/*
				 * It's possible that a tag was freed in the
				 * window between the allocation failure and
				 * adding the hardware queue to the wait queue.
				 */
				if (!blk_mq_get_driver_tag(rq, &hctx, false))
					break;
			} else {
1014
				break;
1015
			}
1016
		}
1017

1018 1019
		list_del_init(&rq->queuelist);

1020 1021
		bd.rq = rq;
		bd.list = dptr;
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034

		/*
		 * Flag last if we have no more requests, or if we have more
		 * but can't assign a driver tag to it.
		 */
		if (list_empty(list))
			bd.last = true;
		else {
			struct request *nxt;

			nxt = list_first_entry(list, struct request, queuelist);
			bd.last = !blk_mq_get_driver_tag(nxt, NULL, false);
		}
1035 1036

		ret = q->mq_ops->queue_rq(hctx, &bd);
1037 1038 1039
		switch (ret) {
		case BLK_MQ_RQ_QUEUE_OK:
			queued++;
1040
			break;
1041
		case BLK_MQ_RQ_QUEUE_BUSY:
1042
			blk_mq_put_driver_tag_hctx(hctx, rq);
1043
			list_add(&rq->queuelist, list);
1044
			__blk_mq_requeue_request(rq);
1045 1046 1047 1048
			break;
		default:
			pr_err("blk-mq: bad return on queue: %d\n", ret);
		case BLK_MQ_RQ_QUEUE_ERROR:
1049
			rq->errors = -EIO;
1050
			blk_mq_end_request(rq, rq->errors);
1051 1052 1053 1054 1055
			break;
		}

		if (ret == BLK_MQ_RQ_QUEUE_BUSY)
			break;
1056 1057 1058 1059 1060

		/*
		 * We've done the first request. If we have more than 1
		 * left in the list, set dptr to defer issue.
		 */
1061
		if (!dptr && list->next != list->prev)
1062
			dptr = &driver_list;
1063 1064
	}

1065
	hctx->dispatched[queued_to_index(queued)]++;
1066 1067 1068 1069 1070

	/*
	 * Any items that need requeuing? Stuff them into hctx->dispatch,
	 * that is where we will continue on next queue run.
	 */
1071
	if (!list_empty(list)) {
1072 1073 1074 1075 1076 1077 1078
		/*
		 * If we got a driver tag for the next request already,
		 * free it again.
		 */
		rq = list_first_entry(list, struct request, queuelist);
		blk_mq_put_driver_tag(rq);

1079
		spin_lock(&hctx->lock);
1080
		list_splice_init(list, &hctx->dispatch);
1081
		spin_unlock(&hctx->lock);
1082

1083 1084 1085 1086 1087 1088 1089 1090
		/*
		 * the queue is expected stopped with BLK_MQ_RQ_QUEUE_BUSY, but
		 * it's possible the queue is stopped and restarted again
		 * before this. Queue restart will dispatch requests. And since
		 * requests in rq_list aren't added into hctx->dispatch yet,
		 * the requests in rq_list might get lost.
		 *
		 * blk_mq_run_hw_queue() already checks the STOPPED bit
1091
		 *
1092 1093
		 * If RESTART or TAG_WAITING is set, then let completion restart
		 * the queue instead of potentially looping here.
1094
		 */
1095 1096
		if (!blk_mq_sched_needs_restart(hctx) &&
		    !test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state))
1097
			blk_mq_run_hw_queue(hctx, true);
1098
	}
1099

1100
	return queued != 0;
1101 1102
}

1103 1104 1105 1106 1107 1108 1109 1110 1111
static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	int srcu_idx;

	WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) &&
		cpu_online(hctx->next_cpu));

	if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
		rcu_read_lock();
1112
		blk_mq_sched_dispatch_requests(hctx);
1113 1114 1115
		rcu_read_unlock();
	} else {
		srcu_idx = srcu_read_lock(&hctx->queue_rq_srcu);
1116
		blk_mq_sched_dispatch_requests(hctx);
1117 1118 1119 1120
		srcu_read_unlock(&hctx->queue_rq_srcu, srcu_idx);
	}
}

1121 1122 1123 1124 1125 1126 1127 1128
/*
 * It'd be great if the workqueue API had a way to pass
 * in a mask and had some smarts for more clever placement.
 * For now we just round-robin here, switching for every
 * BLK_MQ_CPU_WORK_BATCH queued items.
 */
static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
{
1129 1130
	if (hctx->queue->nr_hw_queues == 1)
		return WORK_CPU_UNBOUND;
1131 1132

	if (--hctx->next_cpu_batch <= 0) {
1133
		int next_cpu;
1134 1135 1136 1137 1138 1139 1140 1141 1142

		next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
		if (next_cpu >= nr_cpu_ids)
			next_cpu = cpumask_first(hctx->cpumask);

		hctx->next_cpu = next_cpu;
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}

1143
	return hctx->next_cpu;
1144 1145
}

1146 1147
void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
1148 1149
	if (unlikely(blk_mq_hctx_stopped(hctx) ||
		     !blk_mq_hw_queue_mapped(hctx)))
1150 1151
		return;

1152
	if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
1153 1154
		int cpu = get_cpu();
		if (cpumask_test_cpu(cpu, hctx->cpumask)) {
1155
			__blk_mq_run_hw_queue(hctx);
1156
			put_cpu();
1157 1158
			return;
		}
1159

1160
		put_cpu();
1161
	}
1162

1163
	kblockd_schedule_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work);
1164 1165
}

1166
void blk_mq_run_hw_queues(struct request_queue *q, bool async)
1167 1168 1169 1170 1171
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i) {
1172
		if (!blk_mq_hctx_has_pending(hctx) ||
1173
		    blk_mq_hctx_stopped(hctx))
1174 1175
			continue;

1176
		blk_mq_run_hw_queue(hctx, async);
1177 1178
	}
}
1179
EXPORT_SYMBOL(blk_mq_run_hw_queues);
1180

1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200
/**
 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped
 * @q: request queue.
 *
 * The caller is responsible for serializing this function against
 * blk_mq_{start,stop}_hw_queue().
 */
bool blk_mq_queue_stopped(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		if (blk_mq_hctx_stopped(hctx))
			return true;

	return false;
}
EXPORT_SYMBOL(blk_mq_queue_stopped);

1201 1202
void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
{
1203
	cancel_work(&hctx->run_work);
1204
	cancel_delayed_work(&hctx->delay_work);
1205 1206 1207 1208
	set_bit(BLK_MQ_S_STOPPED, &hctx->state);
}
EXPORT_SYMBOL(blk_mq_stop_hw_queue);

1209 1210 1211 1212 1213 1214 1215 1216 1217 1218
void blk_mq_stop_hw_queues(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_stop_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_stop_hw_queues);

1219 1220 1221
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1222

1223
	blk_mq_run_hw_queue(hctx, false);
1224 1225 1226
}
EXPORT_SYMBOL(blk_mq_start_hw_queue);

1227 1228 1229 1230 1231 1232 1233 1234 1235 1236
void blk_mq_start_hw_queues(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_start_hw_queues);

1237 1238 1239 1240 1241 1242 1243 1244 1245 1246
void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
	if (!blk_mq_hctx_stopped(hctx))
		return;

	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
	blk_mq_run_hw_queue(hctx, async);
}
EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);

1247
void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
1248 1249 1250 1251
{
	struct blk_mq_hw_ctx *hctx;
	int i;

1252 1253
	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_stopped_hw_queue(hctx, async);
1254 1255 1256
}
EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);

1257
static void blk_mq_run_work_fn(struct work_struct *work)
1258 1259 1260
{
	struct blk_mq_hw_ctx *hctx;

1261
	hctx = container_of(work, struct blk_mq_hw_ctx, run_work);
1262

1263 1264 1265
	__blk_mq_run_hw_queue(hctx);
}

1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277
static void blk_mq_delay_work_fn(struct work_struct *work)
{
	struct blk_mq_hw_ctx *hctx;

	hctx = container_of(work, struct blk_mq_hw_ctx, delay_work.work);

	if (test_and_clear_bit(BLK_MQ_S_STOPPED, &hctx->state))
		__blk_mq_run_hw_queue(hctx);
}

void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
{
1278 1279
	if (unlikely(!blk_mq_hw_queue_mapped(hctx)))
		return;
1280

1281
	blk_mq_stop_hw_queue(hctx);
1282 1283
	kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
			&hctx->delay_work, msecs_to_jiffies(msecs));
1284 1285 1286
}
EXPORT_SYMBOL(blk_mq_delay_queue);

1287 1288 1289
static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,
					    struct request *rq,
					    bool at_head)
1290
{
J
Jens Axboe 已提交
1291 1292
	struct blk_mq_ctx *ctx = rq->mq_ctx;

1293 1294
	trace_block_rq_insert(hctx->queue, rq);

1295 1296 1297 1298
	if (at_head)
		list_add(&rq->queuelist, &ctx->rq_list);
	else
		list_add_tail(&rq->queuelist, &ctx->rq_list);
1299
}
1300

1301 1302
void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
			     bool at_head)
1303 1304 1305
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;

J
Jens Axboe 已提交
1306
	__blk_mq_insert_req_list(hctx, rq, at_head);
1307 1308 1309
	blk_mq_hctx_mark_pending(hctx, ctx);
}

1310 1311
void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
			    struct list_head *list)
1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322

{
	/*
	 * preemption doesn't flush plug list, so it's possible ctx->cpu is
	 * offline now
	 */
	spin_lock(&ctx->lock);
	while (!list_empty(list)) {
		struct request *rq;

		rq = list_first_entry(list, struct request, queuelist);
J
Jens Axboe 已提交
1323
		BUG_ON(rq->mq_ctx != ctx);
1324
		list_del_init(&rq->queuelist);
J
Jens Axboe 已提交
1325
		__blk_mq_insert_req_list(hctx, rq, false);
1326
	}
1327
	blk_mq_hctx_mark_pending(hctx, ctx);
1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363
	spin_unlock(&ctx->lock);
}

static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
{
	struct request *rqa = container_of(a, struct request, queuelist);
	struct request *rqb = container_of(b, struct request, queuelist);

	return !(rqa->mq_ctx < rqb->mq_ctx ||
		 (rqa->mq_ctx == rqb->mq_ctx &&
		  blk_rq_pos(rqa) < blk_rq_pos(rqb)));
}

void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
{
	struct blk_mq_ctx *this_ctx;
	struct request_queue *this_q;
	struct request *rq;
	LIST_HEAD(list);
	LIST_HEAD(ctx_list);
	unsigned int depth;

	list_splice_init(&plug->mq_list, &list);

	list_sort(NULL, &list, plug_ctx_cmp);

	this_q = NULL;
	this_ctx = NULL;
	depth = 0;

	while (!list_empty(&list)) {
		rq = list_entry_rq(list.next);
		list_del_init(&rq->queuelist);
		BUG_ON(!rq->q);
		if (rq->mq_ctx != this_ctx) {
			if (this_ctx) {
1364 1365 1366 1367
				trace_block_unplug(this_q, depth, from_schedule);
				blk_mq_sched_insert_requests(this_q, this_ctx,
								&ctx_list,
								from_schedule);
1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383
			}

			this_ctx = rq->mq_ctx;
			this_q = rq->q;
			depth = 0;
		}

		depth++;
		list_add_tail(&rq->queuelist, &ctx_list);
	}

	/*
	 * If 'this_ctx' is set, we know we have entries to complete
	 * on 'ctx_list'. Do those.
	 */
	if (this_ctx) {
1384 1385 1386
		trace_block_unplug(this_q, depth, from_schedule);
		blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list,
						from_schedule);
1387 1388 1389 1390 1391 1392
	}
}

static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
{
	init_request_from_bio(rq, bio);
1393

1394
	blk_account_io_start(rq, true);
1395 1396
}

1397 1398 1399 1400 1401 1402
static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx)
{
	return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
		!blk_queue_nomerges(hctx->queue);
}

1403 1404 1405
static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
					 struct blk_mq_ctx *ctx,
					 struct request *rq, struct bio *bio)
1406
{
1407
	if (!hctx_allow_merges(hctx) || !bio_mergeable(bio)) {
1408 1409 1410 1411 1412 1413 1414
		blk_mq_bio_to_request(rq, bio);
		spin_lock(&ctx->lock);
insert_rq:
		__blk_mq_insert_request(hctx, rq, false);
		spin_unlock(&ctx->lock);
		return false;
	} else {
1415 1416
		struct request_queue *q = hctx->queue;

1417 1418 1419 1420 1421
		spin_lock(&ctx->lock);
		if (!blk_mq_attempt_merge(q, ctx, bio)) {
			blk_mq_bio_to_request(rq, bio);
			goto insert_rq;
		}
1422

1423
		spin_unlock(&ctx->lock);
1424
		__blk_mq_finish_request(hctx, ctx, rq);
1425
		return true;
1426
	}
1427
}
1428

1429 1430
static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq)
{
1431 1432 1433 1434
	if (rq->tag != -1)
		return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false);

	return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true);
1435 1436
}

1437 1438
static void blk_mq_try_issue_directly(struct request *rq, blk_qc_t *cookie,
				      bool may_sleep)
1439 1440 1441 1442 1443 1444 1445
{
	struct request_queue *q = rq->q;
	struct blk_mq_queue_data bd = {
		.rq = rq,
		.list = NULL,
		.last = 1
	};
1446 1447 1448
	struct blk_mq_hw_ctx *hctx;
	blk_qc_t new_cookie;
	int ret;
1449

1450
	if (q->elevator)
1451 1452
		goto insert;

1453 1454 1455 1456 1457
	if (!blk_mq_get_driver_tag(rq, &hctx, false))
		goto insert;

	new_cookie = request_to_qc_t(hctx, rq);

1458 1459 1460 1461 1462 1463
	/*
	 * For OK queue, we are done. For error, kill it. Any other
	 * error (busy), just add it to our list as we previously
	 * would have done
	 */
	ret = q->mq_ops->queue_rq(hctx, &bd);
1464 1465
	if (ret == BLK_MQ_RQ_QUEUE_OK) {
		*cookie = new_cookie;
1466
		return;
1467
	}
1468

1469 1470 1471 1472 1473 1474
	__blk_mq_requeue_request(rq);

	if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
		*cookie = BLK_QC_T_NONE;
		rq->errors = -EIO;
		blk_mq_end_request(rq, rq->errors);
1475
		return;
1476
	}
1477

1478
insert:
1479
	blk_mq_sched_insert_request(rq, false, true, false, may_sleep);
1480 1481
}

1482 1483 1484 1485 1486
/*
 * Multiple hardware queue variant. This will not use per-process plugs,
 * but will attempt to bypass the hctx queueing if we can go straight to
 * hardware for SYNC IO.
 */
1487
static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
1488
{
1489
	const int is_sync = op_is_sync(bio->bi_opf);
1490
	const int is_flush_fua = op_is_flush(bio->bi_opf);
1491
	struct blk_mq_alloc_data data = { .flags = 0 };
1492
	struct request *rq;
1493
	unsigned int request_count = 0, srcu_idx;
1494
	struct blk_plug *plug;
1495
	struct request *same_queue_rq = NULL;
1496
	blk_qc_t cookie;
J
Jens Axboe 已提交
1497
	unsigned int wb_acct;
1498 1499 1500 1501

	blk_queue_bounce(q, &bio);

	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1502
		bio_io_error(bio);
1503
		return BLK_QC_T_NONE;
1504 1505
	}

1506 1507
	blk_queue_split(q, &bio, q->bio_split);

1508 1509 1510
	if (!is_flush_fua && !blk_queue_nomerges(q) &&
	    blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq))
		return BLK_QC_T_NONE;
1511

1512 1513 1514
	if (blk_mq_sched_bio_merge(q, bio))
		return BLK_QC_T_NONE;

J
Jens Axboe 已提交
1515 1516
	wb_acct = wbt_wait(q->rq_wb, bio, NULL);

1517 1518 1519
	trace_block_getrq(q, bio, bio->bi_opf);

	rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
J
Jens Axboe 已提交
1520 1521
	if (unlikely(!rq)) {
		__wbt_done(q->rq_wb, wb_acct);
1522
		return BLK_QC_T_NONE;
J
Jens Axboe 已提交
1523 1524 1525
	}

	wbt_track(&rq->issue_stat, wb_acct);
1526

1527
	cookie = request_to_qc_t(data.hctx, rq);
1528 1529

	if (unlikely(is_flush_fua)) {
1530 1531
		if (q->elevator)
			goto elv_insert;
1532 1533
		blk_mq_bio_to_request(rq, bio);
		blk_insert_flush(rq);
1534
		goto run_queue;
1535 1536
	}

1537
	plug = current->plug;
1538 1539 1540 1541 1542
	/*
	 * If the driver supports defer issued based on 'last', then
	 * queue it up like normal since we can potentially save some
	 * CPU this way.
	 */
1543 1544 1545
	if (((plug && !blk_queue_nomerges(q)) || is_sync) &&
	    !(data.hctx->flags & BLK_MQ_F_DEFER_ISSUE)) {
		struct request *old_rq = NULL;
1546 1547 1548 1549

		blk_mq_bio_to_request(rq, bio);

		/*
1550
		 * We do limited plugging. If the bio can be merged, do that.
1551 1552
		 * Otherwise the existing request in the plug list will be
		 * issued. So the plug list will have one request at most
1553
		 */
1554
		if (plug) {
1555 1556
			/*
			 * The plug list might get flushed before this. If that
1557 1558 1559
			 * happens, same_queue_rq is invalid and plug list is
			 * empty
			 */
1560 1561
			if (same_queue_rq && !list_empty(&plug->mq_list)) {
				old_rq = same_queue_rq;
1562
				list_del_init(&old_rq->queuelist);
1563
			}
1564 1565 1566 1567 1568
			list_add_tail(&rq->queuelist, &plug->mq_list);
		} else /* is_sync */
			old_rq = rq;
		blk_mq_put_ctx(data.ctx);
		if (!old_rq)
1569
			goto done;
1570 1571 1572

		if (!(data.hctx->flags & BLK_MQ_F_BLOCKING)) {
			rcu_read_lock();
1573
			blk_mq_try_issue_directly(old_rq, &cookie, false);
1574 1575 1576
			rcu_read_unlock();
		} else {
			srcu_idx = srcu_read_lock(&data.hctx->queue_rq_srcu);
1577
			blk_mq_try_issue_directly(old_rq, &cookie, true);
1578 1579
			srcu_read_unlock(&data.hctx->queue_rq_srcu, srcu_idx);
		}
1580
		goto done;
1581 1582
	}

1583
	if (q->elevator) {
1584
elv_insert:
1585 1586
		blk_mq_put_ctx(data.ctx);
		blk_mq_bio_to_request(rq, bio);
1587
		blk_mq_sched_insert_request(rq, false, true,
1588
						!is_sync || is_flush_fua, true);
1589 1590
		goto done;
	}
1591 1592 1593 1594 1595 1596 1597
	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
		/*
		 * For a SYNC request, send it to the hardware immediately. For
		 * an ASYNC request, just ensure that we run it later on. The
		 * latter allows for merging opportunities and more efficient
		 * dispatching.
		 */
1598
run_queue:
1599 1600 1601
		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
	}
	blk_mq_put_ctx(data.ctx);
1602 1603
done:
	return cookie;
1604 1605 1606 1607 1608 1609
}

/*
 * Single hardware queue variant. This will attempt to use any per-process
 * plug for merging and IO deferral.
 */
1610
static blk_qc_t blk_sq_make_request(struct request_queue *q, struct bio *bio)
1611
{
1612
	const int is_sync = op_is_sync(bio->bi_opf);
1613
	const int is_flush_fua = op_is_flush(bio->bi_opf);
1614 1615
	struct blk_plug *plug;
	unsigned int request_count = 0;
1616
	struct blk_mq_alloc_data data = { .flags = 0 };
1617
	struct request *rq;
1618
	blk_qc_t cookie;
J
Jens Axboe 已提交
1619
	unsigned int wb_acct;
1620 1621 1622 1623

	blk_queue_bounce(q, &bio);

	if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1624
		bio_io_error(bio);
1625
		return BLK_QC_T_NONE;
1626 1627
	}

1628 1629
	blk_queue_split(q, &bio, q->bio_split);

1630 1631 1632 1633 1634
	if (!is_flush_fua && !blk_queue_nomerges(q)) {
		if (blk_attempt_plug_merge(q, bio, &request_count, NULL))
			return BLK_QC_T_NONE;
	} else
		request_count = blk_plug_queued_count(q);
1635

1636 1637 1638
	if (blk_mq_sched_bio_merge(q, bio))
		return BLK_QC_T_NONE;

J
Jens Axboe 已提交
1639 1640
	wb_acct = wbt_wait(q->rq_wb, bio, NULL);

1641 1642 1643
	trace_block_getrq(q, bio, bio->bi_opf);

	rq = blk_mq_sched_get_request(q, bio, bio->bi_opf, &data);
J
Jens Axboe 已提交
1644 1645
	if (unlikely(!rq)) {
		__wbt_done(q->rq_wb, wb_acct);
1646
		return BLK_QC_T_NONE;
J
Jens Axboe 已提交
1647 1648 1649
	}

	wbt_track(&rq->issue_stat, wb_acct);
1650

1651
	cookie = request_to_qc_t(data.hctx, rq);
1652 1653

	if (unlikely(is_flush_fua)) {
1654 1655
		if (q->elevator)
			goto elv_insert;
1656 1657
		blk_mq_bio_to_request(rq, bio);
		blk_insert_flush(rq);
1658
		goto run_queue;
1659 1660 1661 1662 1663 1664 1665
	}

	/*
	 * A task plug currently exists. Since this is completely lockless,
	 * utilize that to temporarily store requests until the task is
	 * either done or scheduled away.
	 */
1666 1667
	plug = current->plug;
	if (plug) {
1668 1669
		struct request *last = NULL;

1670
		blk_mq_bio_to_request(rq, bio);
1671 1672 1673 1674 1675 1676 1677

		/*
		 * @request_count may become stale because of schedule
		 * out, so check the list again.
		 */
		if (list_empty(&plug->mq_list))
			request_count = 0;
M
Ming Lei 已提交
1678
		if (!request_count)
1679
			trace_block_plug(q);
1680 1681
		else
			last = list_entry_rq(plug->mq_list.prev);
1682 1683 1684

		blk_mq_put_ctx(data.ctx);

1685 1686
		if (request_count >= BLK_MAX_REQUEST_COUNT || (last &&
		    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
1687 1688
			blk_flush_plug_list(plug, false);
			trace_block_plug(q);
1689
		}
1690

1691
		list_add_tail(&rq->queuelist, &plug->mq_list);
1692
		return cookie;
1693 1694
	}

1695
	if (q->elevator) {
1696
elv_insert:
1697 1698
		blk_mq_put_ctx(data.ctx);
		blk_mq_bio_to_request(rq, bio);
1699
		blk_mq_sched_insert_request(rq, false, true,
1700
						!is_sync || is_flush_fua, true);
1701 1702
		goto done;
	}
1703 1704 1705 1706 1707 1708 1709
	if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
		/*
		 * For a SYNC request, send it to the hardware immediately. For
		 * an ASYNC request, just ensure that we run it later on. The
		 * latter allows for merging opportunities and more efficient
		 * dispatching.
		 */
1710
run_queue:
1711
		blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
1712 1713
	}

1714
	blk_mq_put_ctx(data.ctx);
1715
done:
1716
	return cookie;
1717 1718
}

1719 1720
void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx)
1721
{
1722
	struct page *page;
1723

1724
	if (tags->rqs && set->ops->exit_request) {
1725
		int i;
1726

1727
		for (i = 0; i < tags->nr_tags; i++) {
J
Jens Axboe 已提交
1728 1729 1730
			struct request *rq = tags->static_rqs[i];

			if (!rq)
1731
				continue;
J
Jens Axboe 已提交
1732
			set->ops->exit_request(set->driver_data, rq,
1733
						hctx_idx, i);
J
Jens Axboe 已提交
1734
			tags->static_rqs[i] = NULL;
1735
		}
1736 1737
	}

1738 1739
	while (!list_empty(&tags->page_list)) {
		page = list_first_entry(&tags->page_list, struct page, lru);
1740
		list_del_init(&page->lru);
1741 1742 1743 1744 1745
		/*
		 * Remove kmemleak object previously allocated in
		 * blk_mq_init_rq_map().
		 */
		kmemleak_free(page_address(page));
1746 1747
		__free_pages(page, page->private);
	}
1748
}
1749

1750 1751
void blk_mq_free_rq_map(struct blk_mq_tags *tags)
{
1752
	kfree(tags->rqs);
1753
	tags->rqs = NULL;
J
Jens Axboe 已提交
1754 1755
	kfree(tags->static_rqs);
	tags->static_rqs = NULL;
1756

1757
	blk_mq_free_tags(tags);
1758 1759
}

1760 1761 1762 1763
struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
					unsigned int hctx_idx,
					unsigned int nr_tags,
					unsigned int reserved_tags)
1764
{
1765
	struct blk_mq_tags *tags;
1766
	int node;
1767

1768 1769 1770 1771 1772
	node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx);
	if (node == NUMA_NO_NODE)
		node = set->numa_node;

	tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
S
Shaohua Li 已提交
1773
				BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
1774 1775
	if (!tags)
		return NULL;
1776

1777
	tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *),
1778
				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
1779
				 node);
1780 1781 1782 1783
	if (!tags->rqs) {
		blk_mq_free_tags(tags);
		return NULL;
	}
1784

J
Jens Axboe 已提交
1785 1786
	tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *),
				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
1787
				 node);
J
Jens Axboe 已提交
1788 1789 1790 1791 1792 1793
	if (!tags->static_rqs) {
		kfree(tags->rqs);
		blk_mq_free_tags(tags);
		return NULL;
	}

1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806
	return tags;
}

static size_t order_to_size(unsigned int order)
{
	return (size_t)PAGE_SIZE << order;
}

int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx, unsigned int depth)
{
	unsigned int i, j, entries_per_page, max_order = 4;
	size_t rq_size, left;
1807 1808 1809 1810 1811
	int node;

	node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx);
	if (node == NUMA_NO_NODE)
		node = set->numa_node;
1812 1813 1814

	INIT_LIST_HEAD(&tags->page_list);

1815 1816 1817 1818
	/*
	 * rq_size is the size of the request plus driver payload, rounded
	 * to the cacheline size
	 */
1819
	rq_size = round_up(sizeof(struct request) + set->cmd_size,
1820
				cache_line_size());
1821
	left = rq_size * depth;
1822

1823
	for (i = 0; i < depth; ) {
1824 1825 1826 1827 1828
		int this_order = max_order;
		struct page *page;
		int to_do;
		void *p;

1829
		while (this_order && left < order_to_size(this_order - 1))
1830 1831 1832
			this_order--;

		do {
1833
			page = alloc_pages_node(node,
1834
				GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
1835
				this_order);
1836 1837 1838 1839 1840 1841 1842 1843 1844
			if (page)
				break;
			if (!this_order--)
				break;
			if (order_to_size(this_order) < rq_size)
				break;
		} while (1);

		if (!page)
1845
			goto fail;
1846 1847

		page->private = this_order;
1848
		list_add_tail(&page->lru, &tags->page_list);
1849 1850

		p = page_address(page);
1851 1852 1853 1854
		/*
		 * Allow kmemleak to scan these pages as they contain pointers
		 * to additional allocations like via ops->init_request().
		 */
1855
		kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
1856
		entries_per_page = order_to_size(this_order) / rq_size;
1857
		to_do = min(entries_per_page, depth - i);
1858 1859
		left -= to_do * rq_size;
		for (j = 0; j < to_do; j++) {
J
Jens Axboe 已提交
1860 1861 1862
			struct request *rq = p;

			tags->static_rqs[i] = rq;
1863 1864
			if (set->ops->init_request) {
				if (set->ops->init_request(set->driver_data,
J
Jens Axboe 已提交
1865
						rq, hctx_idx, i,
1866
						node)) {
J
Jens Axboe 已提交
1867
					tags->static_rqs[i] = NULL;
1868
					goto fail;
1869
				}
1870 1871
			}

1872 1873 1874 1875
			p += rq_size;
			i++;
		}
	}
1876
	return 0;
1877

1878
fail:
1879 1880
	blk_mq_free_rqs(set, tags, hctx_idx);
	return -ENOMEM;
1881 1882
}

J
Jens Axboe 已提交
1883 1884 1885 1886 1887
/*
 * 'cpu' is going away. splice any existing rq_list entries from this
 * software queue to the hw queue dispatch list, and ensure that it
 * gets run.
 */
1888
static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
1889
{
1890
	struct blk_mq_hw_ctx *hctx;
1891 1892 1893
	struct blk_mq_ctx *ctx;
	LIST_HEAD(tmp);

1894
	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
J
Jens Axboe 已提交
1895
	ctx = __blk_mq_get_ctx(hctx->queue, cpu);
1896 1897 1898 1899 1900 1901 1902 1903 1904

	spin_lock(&ctx->lock);
	if (!list_empty(&ctx->rq_list)) {
		list_splice_init(&ctx->rq_list, &tmp);
		blk_mq_hctx_clear_pending(hctx, ctx);
	}
	spin_unlock(&ctx->lock);

	if (list_empty(&tmp))
1905
		return 0;
1906

J
Jens Axboe 已提交
1907 1908 1909
	spin_lock(&hctx->lock);
	list_splice_tail_init(&tmp, &hctx->dispatch);
	spin_unlock(&hctx->lock);
1910 1911

	blk_mq_run_hw_queue(hctx, true);
1912
	return 0;
1913 1914
}

1915
static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
1916
{
1917 1918
	cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
					    &hctx->cpuhp_dead);
1919 1920
}

1921
/* hctx->ctxs will be freed in queue's release handler */
1922 1923 1924 1925
static void blk_mq_exit_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
1926 1927
	unsigned flush_start_tag = set->queue_depth;

1928 1929
	blk_mq_tag_idle(hctx);

1930 1931 1932 1933 1934
	if (set->ops->exit_request)
		set->ops->exit_request(set->driver_data,
				       hctx->fq->flush_rq, hctx_idx,
				       flush_start_tag + hctx_idx);

1935 1936 1937
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);

1938 1939 1940
	if (hctx->flags & BLK_MQ_F_BLOCKING)
		cleanup_srcu_struct(&hctx->queue_rq_srcu);

1941
	blk_mq_remove_cpuhp(hctx);
1942
	blk_free_flush_queue(hctx->fq);
1943
	sbitmap_free(&hctx->ctx_map);
1944 1945
}

M
Ming Lei 已提交
1946 1947 1948 1949 1950 1951 1952 1953 1954
static void blk_mq_exit_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set, int nr_queue)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if (i == nr_queue)
			break;
1955
		blk_mq_exit_hctx(q, set, hctx, i);
M
Ming Lei 已提交
1956 1957 1958
	}
}

1959 1960 1961
static int blk_mq_init_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
1962
{
1963
	int node;
1964
	unsigned flush_start_tag = set->queue_depth;
1965 1966 1967 1968 1969

	node = hctx->numa_node;
	if (node == NUMA_NO_NODE)
		node = hctx->numa_node = set->numa_node;

1970
	INIT_WORK(&hctx->run_work, blk_mq_run_work_fn);
1971 1972 1973 1974 1975
	INIT_DELAYED_WORK(&hctx->delay_work, blk_mq_delay_work_fn);
	spin_lock_init(&hctx->lock);
	INIT_LIST_HEAD(&hctx->dispatch);
	hctx->queue = q;
	hctx->queue_num = hctx_idx;
1976
	hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED;
1977

1978
	cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
1979 1980

	hctx->tags = set->tags[hctx_idx];
1981 1982

	/*
1983 1984
	 * Allocate space for all possible cpus to avoid allocation at
	 * runtime
1985
	 */
1986 1987 1988 1989
	hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
					GFP_KERNEL, node);
	if (!hctx->ctxs)
		goto unregister_cpu_notifier;
1990

1991 1992
	if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL,
			      node))
1993
		goto free_ctxs;
1994

1995
	hctx->nr_ctx = 0;
1996

1997 1998 1999
	if (set->ops->init_hctx &&
	    set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
		goto free_bitmap;
2000

2001 2002 2003
	hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size);
	if (!hctx->fq)
		goto exit_hctx;
2004

2005 2006 2007 2008 2009
	if (set->ops->init_request &&
	    set->ops->init_request(set->driver_data,
				   hctx->fq->flush_rq, hctx_idx,
				   flush_start_tag + hctx_idx, node))
		goto free_fq;
2010

2011 2012 2013
	if (hctx->flags & BLK_MQ_F_BLOCKING)
		init_srcu_struct(&hctx->queue_rq_srcu);

2014
	return 0;
2015

2016 2017 2018 2019 2020
 free_fq:
	kfree(hctx->fq);
 exit_hctx:
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);
2021
 free_bitmap:
2022
	sbitmap_free(&hctx->ctx_map);
2023 2024 2025
 free_ctxs:
	kfree(hctx->ctxs);
 unregister_cpu_notifier:
2026
	blk_mq_remove_cpuhp(hctx);
2027 2028
	return -1;
}
2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042

static void blk_mq_init_cpu_queues(struct request_queue *q,
				   unsigned int nr_hw_queues)
{
	unsigned int i;

	for_each_possible_cpu(i) {
		struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
		struct blk_mq_hw_ctx *hctx;

		__ctx->cpu = i;
		spin_lock_init(&__ctx->lock);
		INIT_LIST_HEAD(&__ctx->rq_list);
		__ctx->queue = q;
2043 2044
		blk_stat_init(&__ctx->stat[BLK_STAT_READ]);
		blk_stat_init(&__ctx->stat[BLK_STAT_WRITE]);
2045 2046 2047 2048 2049

		/* If the cpu isn't online, the cpu is mapped to first hctx */
		if (!cpu_online(i))
			continue;

C
Christoph Hellwig 已提交
2050
		hctx = blk_mq_map_queue(q, i);
2051

2052 2053 2054 2055 2056
		/*
		 * Set local node, IFF we have more than one hw queue. If
		 * not, we remain on the home node of the device
		 */
		if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
2057
			hctx->numa_node = local_memory_node(cpu_to_node(i));
2058 2059 2060
	}
}

2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082
static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx)
{
	int ret = 0;

	set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
					set->queue_depth, set->reserved_tags);
	if (!set->tags[hctx_idx])
		return false;

	ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
				set->queue_depth);
	if (!ret)
		return true;

	blk_mq_free_rq_map(set->tags[hctx_idx]);
	set->tags[hctx_idx] = NULL;
	return false;
}

static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
					 unsigned int hctx_idx)
{
2083 2084 2085 2086 2087
	if (set->tags[hctx_idx]) {
		blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
		blk_mq_free_rq_map(set->tags[hctx_idx]);
		set->tags[hctx_idx] = NULL;
	}
2088 2089
}

2090 2091
static void blk_mq_map_swqueue(struct request_queue *q,
			       const struct cpumask *online_mask)
2092
{
2093
	unsigned int i, hctx_idx;
2094 2095
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
M
Ming Lei 已提交
2096
	struct blk_mq_tag_set *set = q->tag_set;
2097

2098 2099 2100 2101 2102
	/*
	 * Avoid others reading imcomplete hctx->cpumask through sysfs
	 */
	mutex_lock(&q->sysfs_lock);

2103
	queue_for_each_hw_ctx(q, hctx, i) {
2104
		cpumask_clear(hctx->cpumask);
2105 2106 2107 2108 2109 2110
		hctx->nr_ctx = 0;
	}

	/*
	 * Map software to hardware queues
	 */
2111
	for_each_possible_cpu(i) {
2112
		/* If the cpu isn't online, the cpu is mapped to first hctx */
2113
		if (!cpumask_test_cpu(i, online_mask))
2114 2115
			continue;

2116 2117
		hctx_idx = q->mq_map[i];
		/* unmapped hw queue can be remapped after CPU topo changed */
2118 2119
		if (!set->tags[hctx_idx] &&
		    !__blk_mq_alloc_rq_map(set, hctx_idx)) {
2120 2121 2122 2123 2124 2125
			/*
			 * If tags initialization fail for some hctx,
			 * that hctx won't be brought online.  In this
			 * case, remap the current ctx to hctx[0] which
			 * is guaranteed to always have tags allocated
			 */
2126
			q->mq_map[i] = 0;
2127 2128
		}

2129
		ctx = per_cpu_ptr(q->queue_ctx, i);
C
Christoph Hellwig 已提交
2130
		hctx = blk_mq_map_queue(q, i);
K
Keith Busch 已提交
2131

2132
		cpumask_set_cpu(i, hctx->cpumask);
2133 2134 2135
		ctx->index_hw = hctx->nr_ctx;
		hctx->ctxs[hctx->nr_ctx++] = ctx;
	}
2136

2137 2138
	mutex_unlock(&q->sysfs_lock);

2139
	queue_for_each_hw_ctx(q, hctx, i) {
2140
		/*
2141 2142
		 * If no software queues are mapped to this hardware queue,
		 * disable it and free the request entries.
2143 2144
		 */
		if (!hctx->nr_ctx) {
2145 2146 2147 2148
			/* Never unmap queue 0.  We need it as a
			 * fallback in case of a new remap fails
			 * allocation
			 */
2149 2150 2151
			if (i && set->tags[i])
				blk_mq_free_map_and_requests(set, i);

M
Ming Lei 已提交
2152
			hctx->tags = NULL;
2153 2154 2155
			continue;
		}

M
Ming Lei 已提交
2156 2157 2158
		hctx->tags = set->tags[i];
		WARN_ON(!hctx->tags);

2159 2160 2161 2162 2163
		/*
		 * Set the map size to the number of mapped software queues.
		 * This is more accurate and more efficient than looping
		 * over all possibly mapped software queues.
		 */
2164
		sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
2165

2166 2167 2168
		/*
		 * Initialize batch roundrobin counts
		 */
2169 2170 2171
		hctx->next_cpu = cpumask_first(hctx->cpumask);
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}
2172 2173
}

2174
static void queue_set_hctx_shared(struct request_queue *q, bool shared)
2175 2176 2177 2178
{
	struct blk_mq_hw_ctx *hctx;
	int i;

2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189
	queue_for_each_hw_ctx(q, hctx, i) {
		if (shared)
			hctx->flags |= BLK_MQ_F_TAG_SHARED;
		else
			hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
	}
}

static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, bool shared)
{
	struct request_queue *q;
2190 2191 2192

	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_freeze_queue(q);
2193
		queue_set_hctx_shared(q, shared);
2194 2195 2196 2197 2198 2199 2200 2201 2202 2203
		blk_mq_unfreeze_queue(q);
	}
}

static void blk_mq_del_queue_tag_set(struct request_queue *q)
{
	struct blk_mq_tag_set *set = q->tag_set;

	mutex_lock(&set->tag_list_lock);
	list_del_init(&q->tag_set_list);
2204 2205 2206 2207 2208 2209
	if (list_is_singular(&set->tag_list)) {
		/* just transitioned to unshared */
		set->flags &= ~BLK_MQ_F_TAG_SHARED;
		/* update existing queue */
		blk_mq_update_tag_set_depth(set, false);
	}
2210 2211 2212 2213 2214 2215 2216 2217 2218
	mutex_unlock(&set->tag_list_lock);
}

static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
				     struct request_queue *q)
{
	q->tag_set = set;

	mutex_lock(&set->tag_list_lock);
2219 2220 2221 2222 2223 2224 2225 2226 2227

	/* Check to see if we're transitioning to shared (from 1 to 2 queues). */
	if (!list_empty(&set->tag_list) && !(set->flags & BLK_MQ_F_TAG_SHARED)) {
		set->flags |= BLK_MQ_F_TAG_SHARED;
		/* update existing queue */
		blk_mq_update_tag_set_depth(set, true);
	}
	if (set->flags & BLK_MQ_F_TAG_SHARED)
		queue_set_hctx_shared(q, true);
2228
	list_add_tail(&q->tag_set_list, &set->tag_list);
2229

2230 2231 2232
	mutex_unlock(&set->tag_list_lock);
}

2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243
/*
 * It is the actual release handler for mq, but we do it from
 * request queue's release handler for avoiding use-after-free
 * and headache because q->mq_kobj shouldn't have been introduced,
 * but we can't group ctx/kctx kobj without it.
 */
void blk_mq_release(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

2244 2245
	blk_mq_sched_teardown(q);

2246
	/* hctx kobj stays in hctx */
2247 2248 2249
	queue_for_each_hw_ctx(q, hctx, i) {
		if (!hctx)
			continue;
2250
		kobject_put(&hctx->kobj);
2251
	}
2252

2253 2254
	q->mq_map = NULL;

2255 2256
	kfree(q->queue_hw_ctx);

2257 2258 2259 2260 2261 2262
	/*
	 * release .mq_kobj and sw queue's kobject now because
	 * both share lifetime with request queue.
	 */
	blk_mq_sysfs_deinit(q);

2263 2264 2265
	free_percpu(q->queue_ctx);
}

2266
struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281
{
	struct request_queue *uninit_q, *q;

	uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
	if (!uninit_q)
		return ERR_PTR(-ENOMEM);

	q = blk_mq_init_allocated_queue(set, uninit_q);
	if (IS_ERR(q))
		blk_cleanup_queue(uninit_q);

	return q;
}
EXPORT_SYMBOL(blk_mq_init_queue);

K
Keith Busch 已提交
2282 2283
static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
						struct request_queue *q)
2284
{
K
Keith Busch 已提交
2285 2286
	int i, j;
	struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
2287

K
Keith Busch 已提交
2288
	blk_mq_sysfs_unregister(q);
2289
	for (i = 0; i < set->nr_hw_queues; i++) {
K
Keith Busch 已提交
2290
		int node;
2291

K
Keith Busch 已提交
2292 2293 2294 2295
		if (hctxs[i])
			continue;

		node = blk_mq_hw_queue_to_node(q->mq_map, i);
2296 2297
		hctxs[i] = kzalloc_node(sizeof(struct blk_mq_hw_ctx),
					GFP_KERNEL, node);
2298
		if (!hctxs[i])
K
Keith Busch 已提交
2299
			break;
2300

2301
		if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL,
K
Keith Busch 已提交
2302 2303 2304 2305 2306
						node)) {
			kfree(hctxs[i]);
			hctxs[i] = NULL;
			break;
		}
2307

2308
		atomic_set(&hctxs[i]->nr_active, 0);
2309
		hctxs[i]->numa_node = node;
2310
		hctxs[i]->queue_num = i;
K
Keith Busch 已提交
2311 2312 2313 2314 2315 2316 2317 2318

		if (blk_mq_init_hctx(q, set, hctxs[i], i)) {
			free_cpumask_var(hctxs[i]->cpumask);
			kfree(hctxs[i]);
			hctxs[i] = NULL;
			break;
		}
		blk_mq_hctx_kobj_init(hctxs[i]);
2319
	}
K
Keith Busch 已提交
2320 2321 2322 2323
	for (j = i; j < q->nr_hw_queues; j++) {
		struct blk_mq_hw_ctx *hctx = hctxs[j];

		if (hctx) {
2324 2325
			if (hctx->tags)
				blk_mq_free_map_and_requests(set, j);
K
Keith Busch 已提交
2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338
			blk_mq_exit_hctx(q, set, hctx, j);
			kobject_put(&hctx->kobj);
			hctxs[j] = NULL;

		}
	}
	q->nr_hw_queues = i;
	blk_mq_sysfs_register(q);
}

struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
						  struct request_queue *q)
{
M
Ming Lei 已提交
2339 2340 2341
	/* mark the queue as mq asap */
	q->mq_ops = set->ops;

K
Keith Busch 已提交
2342 2343
	q->queue_ctx = alloc_percpu(struct blk_mq_ctx);
	if (!q->queue_ctx)
M
Ming Lin 已提交
2344
		goto err_exit;
K
Keith Busch 已提交
2345

2346 2347 2348
	/* init q->mq_kobj and sw queues' kobjects */
	blk_mq_sysfs_init(q);

K
Keith Busch 已提交
2349 2350 2351 2352 2353
	q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)),
						GFP_KERNEL, set->numa_node);
	if (!q->queue_hw_ctx)
		goto err_percpu;

2354
	q->mq_map = set->mq_map;
K
Keith Busch 已提交
2355 2356 2357 2358

	blk_mq_realloc_hw_ctxs(set, q);
	if (!q->nr_hw_queues)
		goto err_hctxs;
2359

2360
	INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
2361
	blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
2362 2363 2364

	q->nr_queues = nr_cpu_ids;

2365
	q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
2366

2367 2368 2369
	if (!(set->flags & BLK_MQ_F_SG_MERGE))
		q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;

2370 2371
	q->sg_reserved_size = INT_MAX;

2372
	INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
2373 2374 2375
	INIT_LIST_HEAD(&q->requeue_list);
	spin_lock_init(&q->requeue_lock);

2376 2377 2378 2379 2380
	if (q->nr_hw_queues > 1)
		blk_queue_make_request(q, blk_mq_make_request);
	else
		blk_queue_make_request(q, blk_sq_make_request);

2381 2382 2383 2384 2385
	/*
	 * Do this after blk_queue_make_request() overrides it...
	 */
	q->nr_requests = set->queue_depth;

2386 2387 2388 2389 2390
	/*
	 * Default to classic polling
	 */
	q->poll_nsec = -1;

2391 2392
	if (set->ops->complete)
		blk_queue_softirq_done(q, set->ops->complete);
2393

2394
	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
2395

2396
	get_online_cpus();
2397 2398
	mutex_lock(&all_q_mutex);

2399
	list_add_tail(&q->all_q_node, &all_q_list);
2400
	blk_mq_add_queue_tag_set(set, q);
2401
	blk_mq_map_swqueue(q, cpu_online_mask);
2402

2403
	mutex_unlock(&all_q_mutex);
2404
	put_online_cpus();
2405

2406 2407 2408 2409 2410 2411 2412 2413
	if (!(set->flags & BLK_MQ_F_NO_SCHED)) {
		int ret;

		ret = blk_mq_sched_init(q);
		if (ret)
			return ERR_PTR(ret);
	}

2414
	return q;
2415

2416
err_hctxs:
K
Keith Busch 已提交
2417
	kfree(q->queue_hw_ctx);
2418
err_percpu:
K
Keith Busch 已提交
2419
	free_percpu(q->queue_ctx);
M
Ming Lin 已提交
2420 2421
err_exit:
	q->mq_ops = NULL;
2422 2423
	return ERR_PTR(-ENOMEM);
}
2424
EXPORT_SYMBOL(blk_mq_init_allocated_queue);
2425 2426 2427

void blk_mq_free_queue(struct request_queue *q)
{
M
Ming Lei 已提交
2428
	struct blk_mq_tag_set	*set = q->tag_set;
2429

2430 2431 2432 2433
	mutex_lock(&all_q_mutex);
	list_del_init(&q->all_q_node);
	mutex_unlock(&all_q_mutex);

J
Jens Axboe 已提交
2434 2435
	wbt_exit(q);

2436 2437
	blk_mq_del_queue_tag_set(q);

M
Ming Lei 已提交
2438
	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
2439 2440 2441
}

/* Basically redo blk_mq_init_queue with queue frozen */
2442 2443
static void blk_mq_queue_reinit(struct request_queue *q,
				const struct cpumask *online_mask)
2444
{
2445
	WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth));
2446

2447 2448
	blk_mq_sysfs_unregister(q);

2449 2450 2451 2452 2453 2454
	/*
	 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
	 * we should change hctx numa_node according to new topology (this
	 * involves free and re-allocate memory, worthy doing?)
	 */

2455
	blk_mq_map_swqueue(q, online_mask);
2456

2457
	blk_mq_sysfs_register(q);
2458 2459
}

2460 2461 2462 2463 2464 2465 2466 2467
/*
 * New online cpumask which is going to be set in this hotplug event.
 * Declare this cpumasks as global as cpu-hotplug operation is invoked
 * one-by-one and dynamically allocating this could result in a failure.
 */
static struct cpumask cpuhp_online_new;

static void blk_mq_queue_reinit_work(void)
2468 2469 2470 2471
{
	struct request_queue *q;

	mutex_lock(&all_q_mutex);
2472 2473 2474 2475 2476 2477 2478 2479 2480
	/*
	 * We need to freeze and reinit all existing queues.  Freezing
	 * involves synchronous wait for an RCU grace period and doing it
	 * one by one may take a long time.  Start freezing all queues in
	 * one swoop and then wait for the completions so that freezing can
	 * take place in parallel.
	 */
	list_for_each_entry(q, &all_q_list, all_q_node)
		blk_mq_freeze_queue_start(q);
2481
	list_for_each_entry(q, &all_q_list, all_q_node)
2482 2483
		blk_mq_freeze_queue_wait(q);

2484
	list_for_each_entry(q, &all_q_list, all_q_node)
2485
		blk_mq_queue_reinit(q, &cpuhp_online_new);
2486 2487 2488 2489

	list_for_each_entry(q, &all_q_list, all_q_node)
		blk_mq_unfreeze_queue(q);

2490
	mutex_unlock(&all_q_mutex);
2491 2492 2493 2494
}

static int blk_mq_queue_reinit_dead(unsigned int cpu)
{
2495
	cpumask_copy(&cpuhp_online_new, cpu_online_mask);
2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510
	blk_mq_queue_reinit_work();
	return 0;
}

/*
 * Before hotadded cpu starts handling requests, new mappings must be
 * established.  Otherwise, these requests in hw queue might never be
 * dispatched.
 *
 * For example, there is a single hw queue (hctx) and two CPU queues (ctx0
 * for CPU0, and ctx1 for CPU1).
 *
 * Now CPU1 is just onlined and a request is inserted into ctx1->rq_list
 * and set bit0 in pending bitmap as ctx1->index_hw is still zero.
 *
2511 2512 2513 2514
 * And then while running hw queue, blk_mq_flush_busy_ctxs() finds bit0 is set
 * in pending bitmap and tries to retrieve requests in hctx->ctxs[0]->rq_list.
 * But htx->ctxs[0] is a pointer to ctx0, so the request in ctx1->rq_list is
 * ignored.
2515 2516 2517 2518 2519 2520 2521
 */
static int blk_mq_queue_reinit_prepare(unsigned int cpu)
{
	cpumask_copy(&cpuhp_online_new, cpu_online_mask);
	cpumask_set_cpu(cpu, &cpuhp_online_new);
	blk_mq_queue_reinit_work();
	return 0;
2522 2523
}

2524 2525 2526 2527
static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	int i;

2528 2529
	for (i = 0; i < set->nr_hw_queues; i++)
		if (!__blk_mq_alloc_rq_map(set, i))
2530 2531 2532 2533 2534 2535
			goto out_unwind;

	return 0;

out_unwind:
	while (--i >= 0)
2536
		blk_mq_free_rq_map(set->tags[i]);
2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575

	return -ENOMEM;
}

/*
 * Allocate the request maps associated with this tag_set. Note that this
 * may reduce the depth asked for, if memory is tight. set->queue_depth
 * will be updated to reflect the allocated depth.
 */
static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	unsigned int depth;
	int err;

	depth = set->queue_depth;
	do {
		err = __blk_mq_alloc_rq_maps(set);
		if (!err)
			break;

		set->queue_depth >>= 1;
		if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
			err = -ENOMEM;
			break;
		}
	} while (set->queue_depth);

	if (!set->queue_depth || err) {
		pr_err("blk-mq: failed to allocate request map\n");
		return -ENOMEM;
	}

	if (depth != set->queue_depth)
		pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
						depth, set->queue_depth);

	return 0;
}

2576 2577 2578 2579 2580 2581
/*
 * Alloc a tag set to be associated with one or more request queues.
 * May fail with EINVAL for various error conditions. May adjust the
 * requested depth down, if if it too large. In that case, the set
 * value will be stored in set->queue_depth.
 */
2582 2583
int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
{
2584 2585
	int ret;

B
Bart Van Assche 已提交
2586 2587
	BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);

2588 2589
	if (!set->nr_hw_queues)
		return -EINVAL;
2590
	if (!set->queue_depth)
2591 2592 2593 2594
		return -EINVAL;
	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
		return -EINVAL;

C
Christoph Hellwig 已提交
2595
	if (!set->ops->queue_rq)
2596 2597
		return -EINVAL;

2598 2599 2600 2601 2602
	if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
		pr_info("blk-mq: reduced tag depth to %u\n",
			BLK_MQ_MAX_DEPTH);
		set->queue_depth = BLK_MQ_MAX_DEPTH;
	}
2603

2604 2605 2606 2607 2608 2609 2610 2611 2612
	/*
	 * If a crashdump is active, then we are potentially in a very
	 * memory constrained environment. Limit us to 1 queue and
	 * 64 tags to prevent using too much memory.
	 */
	if (is_kdump_kernel()) {
		set->nr_hw_queues = 1;
		set->queue_depth = min(64U, set->queue_depth);
	}
K
Keith Busch 已提交
2613 2614 2615 2616 2617
	/*
	 * There is no use for more h/w queues than cpus.
	 */
	if (set->nr_hw_queues > nr_cpu_ids)
		set->nr_hw_queues = nr_cpu_ids;
2618

K
Keith Busch 已提交
2619
	set->tags = kzalloc_node(nr_cpu_ids * sizeof(struct blk_mq_tags *),
2620 2621
				 GFP_KERNEL, set->numa_node);
	if (!set->tags)
2622
		return -ENOMEM;
2623

2624 2625 2626
	ret = -ENOMEM;
	set->mq_map = kzalloc_node(sizeof(*set->mq_map) * nr_cpu_ids,
			GFP_KERNEL, set->numa_node);
2627 2628 2629
	if (!set->mq_map)
		goto out_free_tags;

2630 2631 2632 2633 2634 2635 2636 2637 2638
	if (set->ops->map_queues)
		ret = set->ops->map_queues(set);
	else
		ret = blk_mq_map_queues(set);
	if (ret)
		goto out_free_mq_map;

	ret = blk_mq_alloc_rq_maps(set);
	if (ret)
2639
		goto out_free_mq_map;
2640

2641 2642 2643
	mutex_init(&set->tag_list_lock);
	INIT_LIST_HEAD(&set->tag_list);

2644
	return 0;
2645 2646 2647 2648 2649

out_free_mq_map:
	kfree(set->mq_map);
	set->mq_map = NULL;
out_free_tags:
2650 2651
	kfree(set->tags);
	set->tags = NULL;
2652
	return ret;
2653 2654 2655 2656 2657 2658 2659
}
EXPORT_SYMBOL(blk_mq_alloc_tag_set);

void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
{
	int i;

2660 2661
	for (i = 0; i < nr_cpu_ids; i++)
		blk_mq_free_map_and_requests(set, i);
2662

2663 2664 2665
	kfree(set->mq_map);
	set->mq_map = NULL;

M
Ming Lei 已提交
2666
	kfree(set->tags);
2667
	set->tags = NULL;
2668 2669 2670
}
EXPORT_SYMBOL(blk_mq_free_tag_set);

2671 2672 2673 2674 2675 2676
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
{
	struct blk_mq_tag_set *set = q->tag_set;
	struct blk_mq_hw_ctx *hctx;
	int i, ret;

2677
	if (!set)
2678 2679
		return -EINVAL;

2680 2681 2682
	blk_mq_freeze_queue(q);
	blk_mq_quiesce_queue(q);

2683 2684
	ret = 0;
	queue_for_each_hw_ctx(q, hctx, i) {
2685 2686
		if (!hctx->tags)
			continue;
2687 2688 2689 2690
		/*
		 * If we're using an MQ scheduler, just update the scheduler
		 * queue depth. This is similar to what the old code would do.
		 */
2691 2692 2693 2694 2695 2696 2697 2698
		if (!hctx->sched_tags) {
			ret = blk_mq_tag_update_depth(hctx, &hctx->tags,
							min(nr, set->queue_depth),
							false);
		} else {
			ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
							nr, true);
		}
2699 2700 2701 2702 2703 2704 2705
		if (ret)
			break;
	}

	if (!ret)
		q->nr_requests = nr;

2706 2707 2708
	blk_mq_unfreeze_queue(q);
	blk_mq_start_stopped_hw_queues(q, true);

2709 2710 2711
	return ret;
}

K
Keith Busch 已提交
2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727
void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
{
	struct request_queue *q;

	if (nr_hw_queues > nr_cpu_ids)
		nr_hw_queues = nr_cpu_ids;
	if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues)
		return;

	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_freeze_queue(q);

	set->nr_hw_queues = nr_hw_queues;
	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_realloc_hw_ctxs(set, q);

2728 2729 2730 2731
		/*
		 * Manually set the make_request_fn as blk_queue_make_request
		 * resets a lot of the queue settings.
		 */
K
Keith Busch 已提交
2732
		if (q->nr_hw_queues > 1)
2733
			q->make_request_fn = blk_mq_make_request;
K
Keith Busch 已提交
2734
		else
2735
			q->make_request_fn = blk_sq_make_request;
K
Keith Busch 已提交
2736 2737 2738 2739 2740 2741 2742 2743 2744

		blk_mq_queue_reinit(q, cpu_online_mask);
	}

	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_unfreeze_queue(q);
}
EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);

2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781
static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
				       struct blk_mq_hw_ctx *hctx,
				       struct request *rq)
{
	struct blk_rq_stat stat[2];
	unsigned long ret = 0;

	/*
	 * If stats collection isn't on, don't sleep but turn it on for
	 * future users
	 */
	if (!blk_stat_enable(q))
		return 0;

	/*
	 * We don't have to do this once per IO, should optimize this
	 * to just use the current window of stats until it changes
	 */
	memset(&stat, 0, sizeof(stat));
	blk_hctx_stat_get(hctx, stat);

	/*
	 * As an optimistic guess, use half of the mean service time
	 * for this type of request. We can (and should) make this smarter.
	 * For instance, if the completion latencies are tight, we can
	 * get closer than just half the mean. This is especially
	 * important on devices where the completion latencies are longer
	 * than ~10 usec.
	 */
	if (req_op(rq) == REQ_OP_READ && stat[BLK_STAT_READ].nr_samples)
		ret = (stat[BLK_STAT_READ].mean + 1) / 2;
	else if (req_op(rq) == REQ_OP_WRITE && stat[BLK_STAT_WRITE].nr_samples)
		ret = (stat[BLK_STAT_WRITE].mean + 1) / 2;

	return ret;
}

2782
static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
2783
				     struct blk_mq_hw_ctx *hctx,
2784 2785 2786 2787
				     struct request *rq)
{
	struct hrtimer_sleeper hs;
	enum hrtimer_mode mode;
2788
	unsigned int nsecs;
2789 2790
	ktime_t kt;

2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808
	if (test_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags))
		return false;

	/*
	 * poll_nsec can be:
	 *
	 * -1:	don't ever hybrid sleep
	 *  0:	use half of prev avg
	 * >0:	use this specific value
	 */
	if (q->poll_nsec == -1)
		return false;
	else if (q->poll_nsec > 0)
		nsecs = q->poll_nsec;
	else
		nsecs = blk_mq_poll_nsecs(q, hctx, rq);

	if (!nsecs)
2809 2810 2811 2812 2813 2814 2815 2816
		return false;

	set_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);

	/*
	 * This will be replaced with the stats tracking code, using
	 * 'avg_completion_time / 2' as the pre-sleep target.
	 */
T
Thomas Gleixner 已提交
2817
	kt = nsecs;
2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839

	mode = HRTIMER_MODE_REL;
	hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode);
	hrtimer_set_expires(&hs.timer, kt);

	hrtimer_init_sleeper(&hs, current);
	do {
		if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
			break;
		set_current_state(TASK_UNINTERRUPTIBLE);
		hrtimer_start_expires(&hs.timer, mode);
		if (hs.task)
			io_schedule();
		hrtimer_cancel(&hs.timer);
		mode = HRTIMER_MODE_ABS;
	} while (hs.task && !signal_pending(current));

	__set_current_state(TASK_RUNNING);
	destroy_hrtimer_on_stack(&hs.timer);
	return true;
}

J
Jens Axboe 已提交
2840 2841 2842 2843 2844
static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq)
{
	struct request_queue *q = hctx->queue;
	long state;

2845 2846 2847 2848 2849 2850 2851
	/*
	 * If we sleep, have the caller restart the poll loop to reset
	 * the state. Like for the other success return cases, the
	 * caller is responsible for checking if the IO completed. If
	 * the IO isn't complete, we'll get called again and will go
	 * straight to the busy poll loop.
	 */
2852
	if (blk_mq_poll_hybrid_sleep(q, hctx, rq))
2853 2854
		return true;

J
Jens Axboe 已提交
2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897
	hctx->poll_considered++;

	state = current->state;
	while (!need_resched()) {
		int ret;

		hctx->poll_invoked++;

		ret = q->mq_ops->poll(hctx, rq->tag);
		if (ret > 0) {
			hctx->poll_success++;
			set_current_state(TASK_RUNNING);
			return true;
		}

		if (signal_pending_state(state, current))
			set_current_state(TASK_RUNNING);

		if (current->state == TASK_RUNNING)
			return true;
		if (ret < 0)
			break;
		cpu_relax();
	}

	return false;
}

bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie)
{
	struct blk_mq_hw_ctx *hctx;
	struct blk_plug *plug;
	struct request *rq;

	if (!q->mq_ops || !q->mq_ops->poll || !blk_qc_t_valid(cookie) ||
	    !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
		return false;

	plug = current->plug;
	if (plug)
		blk_flush_plug_list(plug, false);

	hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
2898 2899 2900 2901
	if (!blk_qc_t_is_internal(cookie))
		rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
	else
		rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
J
Jens Axboe 已提交
2902 2903 2904 2905 2906

	return __blk_mq_poll(hctx, rq);
}
EXPORT_SYMBOL_GPL(blk_mq_poll);

2907 2908 2909 2910 2911 2912 2913 2914 2915 2916
void blk_mq_disable_hotplug(void)
{
	mutex_lock(&all_q_mutex);
}

void blk_mq_enable_hotplug(void)
{
	mutex_unlock(&all_q_mutex);
}

2917 2918
static int __init blk_mq_init(void)
{
2919 2920
	cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
				blk_mq_hctx_notify_dead);
2921

2922 2923 2924
	cpuhp_setup_state_nocalls(CPUHP_BLK_MQ_PREPARE, "block/mq:prepare",
				  blk_mq_queue_reinit_prepare,
				  blk_mq_queue_reinit_dead);
2925 2926 2927
	return 0;
}
subsys_initcall(blk_mq_init);