blk-mq.c 100.2 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2 3 4 5 6 7
/*
 * Block multiqueue core code
 *
 * Copyright (C) 2013-2014 Jens Axboe
 * Copyright (C) 2013-2014 Christoph Hellwig
 */
8 9 10 11 12
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
13
#include <linux/kmemleak.h>
14 15 16 17 18 19 20 21 22 23
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>
#include <linux/llist.h>
#include <linux/list_sort.h>
#include <linux/cpu.h>
#include <linux/cache.h>
#include <linux/sched/sysctl.h>
24
#include <linux/sched/topology.h>
25
#include <linux/sched/signal.h>
26
#include <linux/delay.h>
27
#include <linux/crash_dump.h>
28
#include <linux/prefetch.h>
29
#include <linux/blk-crypto.h>
30 31 32 33

#include <trace/events/block.h>

#include <linux/blk-mq.h>
34
#include <linux/t10-pi.h>
35 36
#include "blk.h"
#include "blk-mq.h"
37
#include "blk-mq-debugfs.h"
38
#include "blk-mq-tag.h"
39
#include "blk-pm.h"
40
#include "blk-stat.h"
41
#include "blk-mq-sched.h"
42
#include "blk-rq-qos.h"
43

44
static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);
45

46 47 48
static void blk_mq_poll_stats_start(struct request_queue *q);
static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb);

49 50
static int blk_mq_poll_stats_bkt(const struct request *rq)
{
51
	int ddir, sectors, bucket;
52

J
Jens Axboe 已提交
53
	ddir = rq_data_dir(rq);
54
	sectors = blk_rq_stats_sectors(rq);
55

56
	bucket = ddir + 2 * ilog2(sectors);
57 58 59 60 61 62 63 64 65

	if (bucket < 0)
		return -1;
	else if (bucket >= BLK_MQ_POLL_STATS_BKTS)
		return ddir + BLK_MQ_POLL_STATS_BKTS - 2;

	return bucket;
}

66
/*
67 68
 * Check if any of the ctx, dispatch list or elevator
 * have pending work in this hardware queue.
69
 */
70
static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
71
{
72 73
	return !list_empty_careful(&hctx->dispatch) ||
		sbitmap_any_bit_set(&hctx->ctx_map) ||
74
			blk_mq_sched_has_work(hctx);
75 76
}

77 78 79 80 81 82
/*
 * Mark this ctx as having pending work in this hardware queue
 */
static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
				     struct blk_mq_ctx *ctx)
{
83 84 85 86
	const int bit = ctx->index_hw[hctx->type];

	if (!sbitmap_test_bit(&hctx->ctx_map, bit))
		sbitmap_set_bit(&hctx->ctx_map, bit);
87 88 89 90 91
}

static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
				      struct blk_mq_ctx *ctx)
{
92 93 94
	const int bit = ctx->index_hw[hctx->type];

	sbitmap_clear_bit(&hctx->ctx_map, bit);
95 96
}

97
struct mq_inflight {
98
	struct block_device *part;
99
	unsigned int inflight[2];
100 101
};

102
static bool blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx,
103 104 105 106 107
				  struct request *rq, void *priv,
				  bool reserved)
{
	struct mq_inflight *mi = priv;

108 109
	if ((!mi->part->bd_partno || rq->part == mi->part) &&
	    blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
110
		mi->inflight[rq_data_dir(rq)]++;
111 112

	return true;
113 114
}

115 116
unsigned int blk_mq_in_flight(struct request_queue *q,
		struct block_device *part)
117
{
118
	struct mq_inflight mi = { .part = part };
119 120

	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
121

122
	return mi.inflight[0] + mi.inflight[1];
123 124
}

125 126
void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part,
		unsigned int inflight[2])
127
{
128
	struct mq_inflight mi = { .part = part };
129

130
	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
131 132
	inflight[0] = mi.inflight[0];
	inflight[1] = mi.inflight[1];
133 134
}

135
void blk_freeze_queue_start(struct request_queue *q)
136
{
137 138
	mutex_lock(&q->mq_freeze_lock);
	if (++q->mq_freeze_depth == 1) {
139
		percpu_ref_kill(&q->q_usage_counter);
140
		mutex_unlock(&q->mq_freeze_lock);
J
Jens Axboe 已提交
141
		if (queue_is_mq(q))
142
			blk_mq_run_hw_queues(q, false);
143 144
	} else {
		mutex_unlock(&q->mq_freeze_lock);
145
	}
146
}
147
EXPORT_SYMBOL_GPL(blk_freeze_queue_start);
148

149
void blk_mq_freeze_queue_wait(struct request_queue *q)
150
{
151
	wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
152
}
153
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
154

155 156 157 158 159 160 161 162
int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
				     unsigned long timeout)
{
	return wait_event_timeout(q->mq_freeze_wq,
					percpu_ref_is_zero(&q->q_usage_counter),
					timeout);
}
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
163

164 165 166 167
/*
 * Guarantee no request is in use, so we can change any data structure of
 * the queue afterward.
 */
168
void blk_freeze_queue(struct request_queue *q)
169
{
170 171 172 173 174 175 176
	/*
	 * In the !blk_mq case we are only calling this to kill the
	 * q_usage_counter, otherwise this increases the freeze depth
	 * and waits for it to return to zero.  For this reason there is
	 * no blk_unfreeze_queue(), and blk_freeze_queue() is not
	 * exported to drivers as the only user for unfreeze is blk_mq.
	 */
177
	blk_freeze_queue_start(q);
178 179
	blk_mq_freeze_queue_wait(q);
}
180 181 182 183 184 185 186 187 188

void blk_mq_freeze_queue(struct request_queue *q)
{
	/*
	 * ...just an alias to keep freeze and unfreeze actions balanced
	 * in the blk_mq_* namespace
	 */
	blk_freeze_queue(q);
}
189
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
190

191
void __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic)
192
{
193
	mutex_lock(&q->mq_freeze_lock);
194 195
	if (force_atomic)
		q->q_usage_counter.data->force_atomic = true;
196 197 198
	q->mq_freeze_depth--;
	WARN_ON_ONCE(q->mq_freeze_depth < 0);
	if (!q->mq_freeze_depth) {
199
		percpu_ref_resurrect(&q->q_usage_counter);
200
		wake_up_all(&q->mq_freeze_wq);
201
	}
202
	mutex_unlock(&q->mq_freeze_lock);
203
}
204 205 206 207 208

void blk_mq_unfreeze_queue(struct request_queue *q)
{
	__blk_mq_unfreeze_queue(q, false);
}
209
EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
210

211 212 213 214 215 216
/*
 * FIXME: replace the scsi_internal_device_*block_nowait() calls in the
 * mpt3sas driver such that this function can be removed.
 */
void blk_mq_quiesce_queue_nowait(struct request_queue *q)
{
217
	blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q);
218 219 220
}
EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait);

221
/**
222
 * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
223 224 225
 * @q: request queue.
 *
 * Note: this function does not prevent that the struct request end_io()
226 227 228
 * callback function is invoked. Once this function is returned, we make
 * sure no dispatch can happen until the queue is unquiesced via
 * blk_mq_unquiesce_queue().
229 230 231 232 233 234 235
 */
void blk_mq_quiesce_queue(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;
	bool rcu = false;

236
	blk_mq_quiesce_queue_nowait(q);
237

238 239
	queue_for_each_hw_ctx(q, hctx, i) {
		if (hctx->flags & BLK_MQ_F_BLOCKING)
240
			synchronize_srcu(hctx->srcu);
241 242 243 244 245 246 247 248
		else
			rcu = true;
	}
	if (rcu)
		synchronize_rcu();
}
EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);

249 250 251 252 253 254 255 256 257
/*
 * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue()
 * @q: request queue.
 *
 * This function recovers queue into the state before quiescing
 * which is done by blk_mq_quiesce_queue.
 */
void blk_mq_unquiesce_queue(struct request_queue *q)
{
258
	blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q);
259

260 261
	/* dispatch requests which are inserted during quiescing */
	blk_mq_run_hw_queues(q, true);
262 263 264
}
EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue);

265 266 267 268 269 270 271 272 273 274
void blk_mq_wake_waiters(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i)
		if (blk_mq_hw_queue_mapped(hctx))
			blk_mq_tag_wakeup_all(hctx->tags, true);
}

275
/*
276 277
 * Only need start/end time stamping if we have iostat or
 * blk stats enabled, or using an IO scheduler.
278 279 280
 */
static inline bool blk_mq_need_time_stamp(struct request *rq)
{
281
	return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS)) || rq->q->elevator;
282 283
}

284
static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
285
		unsigned int tag, u64 alloc_time_ns)
286
{
287 288
	struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
	struct request *rq = tags->static_rqs[tag];
289

290
	if (data->q->elevator) {
291
		rq->tag = BLK_MQ_NO_TAG;
292 293 294
		rq->internal_tag = tag;
	} else {
		rq->tag = tag;
295
		rq->internal_tag = BLK_MQ_NO_TAG;
296 297
	}

298
	/* csd/requeue_work/fifo_time is initialized before use */
299 300
	rq->q = data->q;
	rq->mq_ctx = data->ctx;
301
	rq->mq_hctx = data->hctx;
302
	rq->rq_flags = 0;
303
	rq->cmd_flags = data->cmd_flags;
304 305
	if (data->flags & BLK_MQ_REQ_PM)
		rq->rq_flags |= RQF_PM;
306
	if (blk_queue_io_stat(data->q))
307
		rq->rq_flags |= RQF_IO_STAT;
308
	INIT_LIST_HEAD(&rq->queuelist);
309 310 311 312
	INIT_HLIST_NODE(&rq->hash);
	RB_CLEAR_NODE(&rq->rb_node);
	rq->rq_disk = NULL;
	rq->part = NULL;
313 314 315
#ifdef CONFIG_BLK_RQ_ALLOC_TIME
	rq->alloc_time_ns = alloc_time_ns;
#endif
316 317 318 319
	if (blk_mq_need_time_stamp(rq))
		rq->start_time_ns = ktime_get_ns();
	else
		rq->start_time_ns = 0;
320
	rq->io_start_time_ns = 0;
321
	rq->stats_sectors = 0;
322 323 324 325
	rq->nr_phys_segments = 0;
#if defined(CONFIG_BLK_DEV_INTEGRITY)
	rq->nr_integrity_segments = 0;
#endif
326
	blk_crypto_rq_set_defaults(rq);
327
	/* tag was already set */
328
	WRITE_ONCE(rq->deadline, 0);
329

330 331
	rq->timeout = 0;

332 333 334
	rq->end_io = NULL;
	rq->end_io_data = NULL;

335
	data->ctx->rq_dispatched[op_is_sync(data->cmd_flags)]++;
K
Keith Busch 已提交
336
	refcount_set(&rq->ref, 1);
337 338 339 340 341 342 343 344 345 346 347 348 349 350 351

	if (!op_is_flush(data->cmd_flags)) {
		struct elevator_queue *e = data->q->elevator;

		rq->elv.icq = NULL;
		if (e && e->type->ops.prepare_request) {
			if (e->type->icq_cache)
				blk_mq_sched_assign_ioc(rq);

			e->type->ops.prepare_request(rq);
			rq->rq_flags |= RQF_ELVPRIV;
		}
	}

	data->hctx->queued++;
352
	return rq;
353 354
}

355
static struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data)
356
{
357
	struct request_queue *q = data->q;
358
	struct elevator_queue *e = q->elevator;
359
	u64 alloc_time_ns = 0;
360
	unsigned int tag;
361

362 363 364 365
	/* alloc_time includes depth and tag waits */
	if (blk_queue_rq_alloc_time(q))
		alloc_time_ns = ktime_get_ns();

366
	if (data->cmd_flags & REQ_NOWAIT)
367
		data->flags |= BLK_MQ_REQ_NOWAIT;
368 369 370

	if (e) {
		/*
371
		 * Flush/passthrough requests are special and go directly to the
372 373
		 * dispatch list. Don't include reserved tags in the
		 * limiting, as it isn't useful.
374
		 */
375
		if (!op_is_flush(data->cmd_flags) &&
376
		    !blk_op_is_passthrough(data->cmd_flags) &&
377
		    e->type->ops.limit_depth &&
378
		    !(data->flags & BLK_MQ_REQ_RESERVED))
379
			e->type->ops.limit_depth(data->cmd_flags, data);
380 381
	}

382
retry:
383 384
	data->ctx = blk_mq_get_ctx(q);
	data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx);
385
	if (!e)
386 387
		blk_mq_tag_busy(data->hctx);

388 389 390 391 392
	/*
	 * Waiting allocations only fail because of an inactive hctx.  In that
	 * case just retry the hctx assignment and tag allocation as CPU hotplug
	 * should have migrated us to an online CPU by now.
	 */
393
	tag = blk_mq_get_tag(data);
394 395 396 397 398 399 400
	if (tag == BLK_MQ_NO_TAG) {
		if (data->flags & BLK_MQ_REQ_NOWAIT)
			return NULL;

		/*
		 * Give up the CPU and sleep for a random short time to ensure
		 * that thread using a realtime scheduling class are migrated
401
		 * off the CPU, and thus off the hctx that is going away.
402 403 404 405
		 */
		msleep(3);
		goto retry;
	}
406
	return blk_mq_rq_ctx_init(data, tag, alloc_time_ns);
407 408
}

409
struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op,
410
		blk_mq_req_flags_t flags)
411
{
412 413 414 415 416
	struct blk_mq_alloc_data data = {
		.q		= q,
		.flags		= flags,
		.cmd_flags	= op,
	};
417
	struct request *rq;
418
	int ret;
419

420
	ret = blk_queue_enter(q, flags);
421 422
	if (ret)
		return ERR_PTR(ret);
423

424
	rq = __blk_mq_alloc_request(&data);
425
	if (!rq)
426
		goto out_queue_exit;
427 428 429
	rq->__data_len = 0;
	rq->__sector = (sector_t) -1;
	rq->bio = rq->biotail = NULL;
430
	return rq;
431 432 433
out_queue_exit:
	blk_queue_exit(q);
	return ERR_PTR(-EWOULDBLOCK);
434
}
435
EXPORT_SYMBOL(blk_mq_alloc_request);
436

437
struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
438
	unsigned int op, blk_mq_req_flags_t flags, unsigned int hctx_idx)
M
Ming Lin 已提交
439
{
440 441 442 443 444
	struct blk_mq_alloc_data data = {
		.q		= q,
		.flags		= flags,
		.cmd_flags	= op,
	};
445
	u64 alloc_time_ns = 0;
446
	unsigned int cpu;
447
	unsigned int tag;
M
Ming Lin 已提交
448 449
	int ret;

450 451 452 453
	/* alloc_time includes depth and tag waits */
	if (blk_queue_rq_alloc_time(q))
		alloc_time_ns = ktime_get_ns();

M
Ming Lin 已提交
454 455 456 457 458 459
	/*
	 * If the tag allocator sleeps we could get an allocation for a
	 * different hardware context.  No need to complicate the low level
	 * allocator for this for the rare use case of a command tied to
	 * a specific queue.
	 */
460
	if (WARN_ON_ONCE(!(flags & (BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED))))
M
Ming Lin 已提交
461 462 463 464 465
		return ERR_PTR(-EINVAL);

	if (hctx_idx >= q->nr_hw_queues)
		return ERR_PTR(-EIO);

466
	ret = blk_queue_enter(q, flags);
M
Ming Lin 已提交
467 468 469
	if (ret)
		return ERR_PTR(ret);

470 471 472 473
	/*
	 * Check if the hardware context is actually mapped to anything.
	 * If not tell the caller that it should skip this queue.
	 */
474
	ret = -EXDEV;
475 476
	data.hctx = q->queue_hw_ctx[hctx_idx];
	if (!blk_mq_hw_queue_mapped(data.hctx))
477
		goto out_queue_exit;
478 479
	cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask);
	data.ctx = __blk_mq_get_ctx(q, cpu);
M
Ming Lin 已提交
480

481
	if (!q->elevator)
482 483
		blk_mq_tag_busy(data.hctx);

484
	ret = -EWOULDBLOCK;
485 486
	tag = blk_mq_get_tag(&data);
	if (tag == BLK_MQ_NO_TAG)
487
		goto out_queue_exit;
488 489
	return blk_mq_rq_ctx_init(&data, tag, alloc_time_ns);

490 491 492
out_queue_exit:
	blk_queue_exit(q);
	return ERR_PTR(ret);
M
Ming Lin 已提交
493 494 495
}
EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);

K
Keith Busch 已提交
496 497 498 499
static void __blk_mq_free_request(struct request *rq)
{
	struct request_queue *q = rq->q;
	struct blk_mq_ctx *ctx = rq->mq_ctx;
500
	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
K
Keith Busch 已提交
501 502
	const int sched_tag = rq->internal_tag;

503
	blk_crypto_free_request(rq);
504
	blk_pm_mark_last_busy(rq);
505
	rq->mq_hctx = NULL;
506
	if (rq->tag != BLK_MQ_NO_TAG)
507
		blk_mq_put_tag(hctx->tags, ctx, rq->tag);
508
	if (sched_tag != BLK_MQ_NO_TAG)
509
		blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag);
K
Keith Busch 已提交
510 511 512 513
	blk_mq_sched_restart(hctx);
	blk_queue_exit(q);
}

514
void blk_mq_free_request(struct request *rq)
515 516
{
	struct request_queue *q = rq->q;
517 518
	struct elevator_queue *e = q->elevator;
	struct blk_mq_ctx *ctx = rq->mq_ctx;
519
	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
520

521
	if (rq->rq_flags & RQF_ELVPRIV) {
522 523
		if (e && e->type->ops.finish_request)
			e->type->ops.finish_request(rq);
524 525 526 527 528
		if (rq->elv.icq) {
			put_io_context(rq->elv.icq->ioc);
			rq->elv.icq = NULL;
		}
	}
529

530
	ctx->rq_completed[rq_is_sync(rq)]++;
531
	if (rq->rq_flags & RQF_MQ_INFLIGHT)
532
		__blk_mq_dec_active_requests(hctx);
J
Jens Axboe 已提交
533

534
	if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
535
		laptop_io_completion(q->disk->bdi);
536

537
	rq_qos_done(q, rq);
538

K
Keith Busch 已提交
539 540 541
	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
	if (refcount_dec_and_test(&rq->ref))
		__blk_mq_free_request(rq);
542
}
J
Jens Axboe 已提交
543
EXPORT_SYMBOL_GPL(blk_mq_free_request);
544

545
inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
546
{
547 548 549 550
	u64 now = 0;

	if (blk_mq_need_time_stamp(rq))
		now = ktime_get_ns();
551

552 553
	if (rq->rq_flags & RQF_STATS) {
		blk_mq_poll_stats_start(rq->q);
554
		blk_stat_add(rq, now);
555 556
	}

557
	blk_mq_sched_completed_request(rq, now);
558

559
	blk_account_io_done(rq, now);
M
Ming Lei 已提交
560

C
Christoph Hellwig 已提交
561
	if (rq->end_io) {
562
		rq_qos_done(rq->q, rq);
563
		rq->end_io(rq, error);
C
Christoph Hellwig 已提交
564
	} else {
565
		blk_mq_free_request(rq);
C
Christoph Hellwig 已提交
566
	}
567
}
568
EXPORT_SYMBOL(__blk_mq_end_request);
569

570
void blk_mq_end_request(struct request *rq, blk_status_t error)
571 572 573
{
	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
		BUG();
574
	__blk_mq_end_request(rq, error);
575
}
576
EXPORT_SYMBOL(blk_mq_end_request);
577

578
static void blk_complete_reqs(struct llist_head *list)
579
{
580 581
	struct llist_node *entry = llist_reverse_order(llist_del_all(list));
	struct request *rq, *next;
582

583
	llist_for_each_entry_safe(rq, next, entry, ipi_list)
584
		rq->q->mq_ops->complete(rq);
585 586
}

587
static __latent_entropy void blk_done_softirq(struct softirq_action *h)
588
{
589
	blk_complete_reqs(this_cpu_ptr(&blk_cpu_done));
590 591
}

592 593
static int blk_softirq_cpu_dead(unsigned int cpu)
{
594
	blk_complete_reqs(&per_cpu(blk_cpu_done, cpu));
595 596 597
	return 0;
}

598
static void __blk_mq_complete_request_remote(void *data)
599
{
600
	__raise_softirq_irqoff(BLOCK_SOFTIRQ);
601 602
}

603 604 605 606 607 608 609
static inline bool blk_mq_complete_need_ipi(struct request *rq)
{
	int cpu = raw_smp_processor_id();

	if (!IS_ENABLED(CONFIG_SMP) ||
	    !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags))
		return false;
610 611 612 613 614 615
	/*
	 * With force threaded interrupts enabled, raising softirq from an SMP
	 * function call will always result in waking the ksoftirqd thread.
	 * This is probably worse than completing the request on a different
	 * cache domain.
	 */
616
	if (force_irqthreads())
617
		return false;
618 619 620 621 622 623 624 625 626 627 628

	/* same CPU or cache domain?  Complete locally */
	if (cpu == rq->mq_ctx->cpu ||
	    (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) &&
	     cpus_share_cache(cpu, rq->mq_ctx->cpu)))
		return false;

	/* don't try to IPI to an offline CPU */
	return cpu_online(rq->mq_ctx->cpu);
}

629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652
static void blk_mq_complete_send_ipi(struct request *rq)
{
	struct llist_head *list;
	unsigned int cpu;

	cpu = rq->mq_ctx->cpu;
	list = &per_cpu(blk_cpu_done, cpu);
	if (llist_add(&rq->ipi_list, list)) {
		INIT_CSD(&rq->csd, __blk_mq_complete_request_remote, rq);
		smp_call_function_single_async(cpu, &rq->csd);
	}
}

static void blk_mq_raise_softirq(struct request *rq)
{
	struct llist_head *list;

	preempt_disable();
	list = this_cpu_ptr(&blk_cpu_done);
	if (llist_add(&rq->ipi_list, list))
		raise_softirq(BLOCK_SOFTIRQ);
	preempt_enable();
}

653
bool blk_mq_complete_request_remote(struct request *rq)
654
{
655
	WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
656

657 658 659 660
	/*
	 * For a polled request, always complete locallly, it's pointless
	 * to redirect the completion.
	 */
661 662
	if (rq->cmd_flags & REQ_HIPRI)
		return false;
C
Christoph Hellwig 已提交
663

664
	if (blk_mq_complete_need_ipi(rq)) {
665 666
		blk_mq_complete_send_ipi(rq);
		return true;
667
	}
668

669 670 671 672 673
	if (rq->q->nr_hw_queues == 1) {
		blk_mq_raise_softirq(rq);
		return true;
	}
	return false;
674 675 676 677 678 679 680 681 682 683 684 685 686 687
}
EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote);

/**
 * blk_mq_complete_request - end I/O on a request
 * @rq:		the request being processed
 *
 * Description:
 *	Complete a request by scheduling the ->complete_rq operation.
 **/
void blk_mq_complete_request(struct request *rq)
{
	if (!blk_mq_complete_request_remote(rq))
		rq->q->mq_ops->complete(rq);
688
}
689
EXPORT_SYMBOL(blk_mq_complete_request);
690

691
static void hctx_unlock(struct blk_mq_hw_ctx *hctx, int srcu_idx)
692
	__releases(hctx->srcu)
693 694 695 696
{
	if (!(hctx->flags & BLK_MQ_F_BLOCKING))
		rcu_read_unlock();
	else
697
		srcu_read_unlock(hctx->srcu, srcu_idx);
698 699 700
}

static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx)
701
	__acquires(hctx->srcu)
702
{
703 704 705
	if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
		/* shut up gcc false positive */
		*srcu_idx = 0;
706
		rcu_read_lock();
707
	} else
708
		*srcu_idx = srcu_read_lock(hctx->srcu);
709 710
}

711 712 713 714 715 716 717 718
/**
 * blk_mq_start_request - Start processing a request
 * @rq: Pointer to request to be started
 *
 * Function used by device drivers to notify the block layer that a request
 * is going to be processed now, so blk layer can do proper initializations
 * such as starting the timeout timer.
 */
719
void blk_mq_start_request(struct request *rq)
720 721 722
{
	struct request_queue *q = rq->q;

723
	trace_block_rq_issue(rq);
724

725
	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
726
		rq->io_start_time_ns = ktime_get_ns();
727
		rq->stats_sectors = blk_rq_sectors(rq);
728
		rq->rq_flags |= RQF_STATS;
729
		rq_qos_issue(q, rq);
730 731
	}

732
	WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
733

734
	blk_add_timer(rq);
K
Keith Busch 已提交
735
	WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT);
736

737 738 739 740
#ifdef CONFIG_BLK_DEV_INTEGRITY
	if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE)
		q->integrity.profile->prepare_fn(rq);
#endif
741
}
742
EXPORT_SYMBOL(blk_mq_start_request);
743

744
static void __blk_mq_requeue_request(struct request *rq)
745 746 747
{
	struct request_queue *q = rq->q;

748 749
	blk_mq_put_driver_tag(rq);

750
	trace_block_rq_requeue(rq);
751
	rq_qos_requeue(q, rq);
752

K
Keith Busch 已提交
753 754
	if (blk_mq_request_started(rq)) {
		WRITE_ONCE(rq->state, MQ_RQ_IDLE);
755
		rq->rq_flags &= ~RQF_TIMED_OUT;
756
	}
757 758
}

759
void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
760 761 762
{
	__blk_mq_requeue_request(rq);

763 764 765
	/* this request will be re-inserted to io scheduler queue */
	blk_mq_sched_requeue_request(rq);

J
Jens Axboe 已提交
766
	BUG_ON(!list_empty(&rq->queuelist));
767
	blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
768 769 770
}
EXPORT_SYMBOL(blk_mq_requeue_request);

771 772 773
static void blk_mq_requeue_work(struct work_struct *work)
{
	struct request_queue *q =
774
		container_of(work, struct request_queue, requeue_work.work);
775 776 777
	LIST_HEAD(rq_list);
	struct request *rq, *next;

778
	spin_lock_irq(&q->requeue_lock);
779
	list_splice_init(&q->requeue_list, &rq_list);
780
	spin_unlock_irq(&q->requeue_lock);
781 782

	list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
783
		if (!(rq->rq_flags & (RQF_SOFTBARRIER | RQF_DONTPREP)))
784 785
			continue;

786
		rq->rq_flags &= ~RQF_SOFTBARRIER;
787
		list_del_init(&rq->queuelist);
788 789 790 791 792 793
		/*
		 * If RQF_DONTPREP, rq has contained some driver specific
		 * data, so insert it to hctx dispatch list to avoid any
		 * merge.
		 */
		if (rq->rq_flags & RQF_DONTPREP)
794
			blk_mq_request_bypass_insert(rq, false, false);
795 796
		else
			blk_mq_sched_insert_request(rq, true, false, false);
797 798 799 800 801
	}

	while (!list_empty(&rq_list)) {
		rq = list_entry(rq_list.next, struct request, queuelist);
		list_del_init(&rq->queuelist);
802
		blk_mq_sched_insert_request(rq, false, false, false);
803 804
	}

805
	blk_mq_run_hw_queues(q, false);
806 807
}

808 809
void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
				bool kick_requeue_list)
810 811 812 813 814 815
{
	struct request_queue *q = rq->q;
	unsigned long flags;

	/*
	 * We abuse this flag that is otherwise used by the I/O scheduler to
816
	 * request head insertion from the workqueue.
817
	 */
818
	BUG_ON(rq->rq_flags & RQF_SOFTBARRIER);
819 820 821

	spin_lock_irqsave(&q->requeue_lock, flags);
	if (at_head) {
822
		rq->rq_flags |= RQF_SOFTBARRIER;
823 824 825 826 827
		list_add(&rq->queuelist, &q->requeue_list);
	} else {
		list_add_tail(&rq->queuelist, &q->requeue_list);
	}
	spin_unlock_irqrestore(&q->requeue_lock, flags);
828 829 830

	if (kick_requeue_list)
		blk_mq_kick_requeue_list(q);
831 832 833 834
}

void blk_mq_kick_requeue_list(struct request_queue *q)
{
835
	kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0);
836 837 838
}
EXPORT_SYMBOL(blk_mq_kick_requeue_list);

839 840 841
void blk_mq_delay_kick_requeue_list(struct request_queue *q,
				    unsigned long msecs)
{
842 843
	kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work,
				    msecs_to_jiffies(msecs));
844 845 846
}
EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);

847 848
struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
{
849 850
	if (tag < tags->nr_tags) {
		prefetch(tags->rqs[tag]);
851
		return tags->rqs[tag];
852
	}
853 854

	return NULL;
855 856 857
}
EXPORT_SYMBOL(blk_mq_tag_to_rq);

858 859
static bool blk_mq_rq_inflight(struct blk_mq_hw_ctx *hctx, struct request *rq,
			       void *priv, bool reserved)
860 861
{
	/*
862
	 * If we find a request that isn't idle and the queue matches,
863
	 * we know the queue is busy. Return false to stop the iteration.
864
	 */
865
	if (blk_mq_request_started(rq) && rq->q == hctx->queue) {
866 867 868 869 870 871 872 873 874
		bool *busy = priv;

		*busy = true;
		return false;
	}

	return true;
}

875
bool blk_mq_queue_inflight(struct request_queue *q)
876 877 878
{
	bool busy = false;

879
	blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy);
880 881
	return busy;
}
882
EXPORT_SYMBOL_GPL(blk_mq_queue_inflight);
883

884
static void blk_mq_rq_timed_out(struct request *req, bool reserved)
885
{
886
	req->rq_flags |= RQF_TIMED_OUT;
887 888 889 890 891 892 893
	if (req->q->mq_ops->timeout) {
		enum blk_eh_timer_return ret;

		ret = req->q->mq_ops->timeout(req, reserved);
		if (ret == BLK_EH_DONE)
			return;
		WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER);
894
	}
895 896

	blk_add_timer(req);
897
}
898

K
Keith Busch 已提交
899
static bool blk_mq_req_expired(struct request *rq, unsigned long *next)
900
{
K
Keith Busch 已提交
901
	unsigned long deadline;
902

K
Keith Busch 已提交
903 904
	if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT)
		return false;
905 906
	if (rq->rq_flags & RQF_TIMED_OUT)
		return false;
907

908
	deadline = READ_ONCE(rq->deadline);
K
Keith Busch 已提交
909 910
	if (time_after_eq(jiffies, deadline))
		return true;
911

K
Keith Busch 已提交
912 913 914 915 916
	if (*next == 0)
		*next = deadline;
	else if (time_after(*next, deadline))
		*next = deadline;
	return false;
917 918
}

919 920
void blk_mq_put_rq_ref(struct request *rq)
{
M
Ming Lei 已提交
921
	if (is_flush_rq(rq))
922 923 924 925 926
		rq->end_io(rq, 0);
	else if (refcount_dec_and_test(&rq->ref))
		__blk_mq_free_request(rq);
}

927
static bool blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
928 929
		struct request *rq, void *priv, bool reserved)
{
K
Keith Busch 已提交
930 931 932
	unsigned long *next = priv;

	/*
933 934 935 936 937
	 * blk_mq_queue_tag_busy_iter() has locked the request, so it cannot
	 * be reallocated underneath the timeout handler's processing, then
	 * the expire check is reliable. If the request is not expired, then
	 * it was completed and reallocated as a new request after returning
	 * from blk_mq_check_expired().
938
	 */
K
Keith Busch 已提交
939
	if (blk_mq_req_expired(rq, next))
940
		blk_mq_rq_timed_out(rq, reserved);
941
	return true;
942 943
}

944
static void blk_mq_timeout_work(struct work_struct *work)
945
{
946 947
	struct request_queue *q =
		container_of(work, struct request_queue, timeout_work);
K
Keith Busch 已提交
948
	unsigned long next = 0;
949
	struct blk_mq_hw_ctx *hctx;
950
	int i;
951

952 953 954 955 956 957 958 959 960
	/* A deadlock might occur if a request is stuck requiring a
	 * timeout at the same time a queue freeze is waiting
	 * completion, since the timeout code would not be able to
	 * acquire the queue reference here.
	 *
	 * That's why we don't use blk_queue_enter here; instead, we use
	 * percpu_ref_tryget directly, because we need to be able to
	 * obtain a reference even in the short window between the queue
	 * starting to freeze, by dropping the first reference in
961
	 * blk_freeze_queue_start, and the moment the last request is
962 963 964 965
	 * consumed, marked by the instant q_usage_counter reaches
	 * zero.
	 */
	if (!percpu_ref_tryget(&q->q_usage_counter))
966 967
		return;

K
Keith Busch 已提交
968
	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next);
969

K
Keith Busch 已提交
970 971
	if (next != 0) {
		mod_timer(&q->timeout, next);
972
	} else {
973 974 975 976 977 978
		/*
		 * Request timeouts are handled as a forward rolling timer. If
		 * we end up here it means that no requests are pending and
		 * also that no request has been pending for a while. Mark
		 * each hctx as idle.
		 */
979 980 981 982 983
		queue_for_each_hw_ctx(q, hctx, i) {
			/* the hctx may be unmapped, so check it here */
			if (blk_mq_hw_queue_mapped(hctx))
				blk_mq_tag_idle(hctx);
		}
984
	}
985
	blk_queue_exit(q);
986 987
}

988 989 990 991 992 993 994 995 996 997
struct flush_busy_ctx_data {
	struct blk_mq_hw_ctx *hctx;
	struct list_head *list;
};

static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
{
	struct flush_busy_ctx_data *flush_data = data;
	struct blk_mq_hw_ctx *hctx = flush_data->hctx;
	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
M
Ming Lei 已提交
998
	enum hctx_type type = hctx->type;
999 1000

	spin_lock(&ctx->lock);
M
Ming Lei 已提交
1001
	list_splice_tail_init(&ctx->rq_lists[type], flush_data->list);
1002
	sbitmap_clear_bit(sb, bitnr);
1003 1004 1005 1006
	spin_unlock(&ctx->lock);
	return true;
}

1007 1008 1009 1010
/*
 * Process software queues that have been marked busy, splicing them
 * to the for-dispatch
 */
1011
void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
1012
{
1013 1014 1015 1016
	struct flush_busy_ctx_data data = {
		.hctx = hctx,
		.list = list,
	};
1017

1018
	sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
1019
}
1020
EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
1021

1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032
struct dispatch_rq_data {
	struct blk_mq_hw_ctx *hctx;
	struct request *rq;
};

static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr,
		void *data)
{
	struct dispatch_rq_data *dispatch_data = data;
	struct blk_mq_hw_ctx *hctx = dispatch_data->hctx;
	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
M
Ming Lei 已提交
1033
	enum hctx_type type = hctx->type;
1034 1035

	spin_lock(&ctx->lock);
M
Ming Lei 已提交
1036 1037
	if (!list_empty(&ctx->rq_lists[type])) {
		dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next);
1038
		list_del_init(&dispatch_data->rq->queuelist);
M
Ming Lei 已提交
1039
		if (list_empty(&ctx->rq_lists[type]))
1040 1041 1042 1043 1044 1045 1046 1047 1048 1049
			sbitmap_clear_bit(sb, bitnr);
	}
	spin_unlock(&ctx->lock);

	return !dispatch_data->rq;
}

struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
					struct blk_mq_ctx *start)
{
1050
	unsigned off = start ? start->index_hw[hctx->type] : 0;
1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061
	struct dispatch_rq_data data = {
		.hctx = hctx,
		.rq   = NULL,
	};

	__sbitmap_for_each_set(&hctx->ctx_map, off,
			       dispatch_rq_from_ctx, &data);

	return data.rq;
}

1062 1063 1064 1065
static inline unsigned int queued_to_index(unsigned int queued)
{
	if (!queued)
		return 0;
1066

1067
	return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
1068 1069
}

1070 1071
static bool __blk_mq_get_driver_tag(struct request *rq)
{
1072
	struct sbitmap_queue *bt = rq->mq_hctx->tags->bitmap_tags;
1073 1074 1075
	unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
	int tag;

1076 1077
	blk_mq_tag_busy(rq->mq_hctx);

1078
	if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
1079
		bt = rq->mq_hctx->tags->breserved_tags;
1080
		tag_offset = 0;
1081 1082 1083
	} else {
		if (!hctx_may_queue(rq->mq_hctx, bt))
			return false;
1084 1085 1086 1087 1088 1089 1090 1091 1092 1093
	}

	tag = __sbitmap_queue_get(bt);
	if (tag == BLK_MQ_NO_TAG)
		return false;

	rq->tag = tag + tag_offset;
	return true;
}

1094
bool blk_mq_get_driver_tag(struct request *rq)
1095
{
1096 1097 1098 1099 1100
	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;

	if (rq->tag == BLK_MQ_NO_TAG && !__blk_mq_get_driver_tag(rq))
		return false;

1101
	if ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
1102 1103
			!(rq->rq_flags & RQF_MQ_INFLIGHT)) {
		rq->rq_flags |= RQF_MQ_INFLIGHT;
1104
		__blk_mq_inc_active_requests(hctx);
1105 1106 1107
	}
	hctx->tags->rqs[rq->tag] = rq;
	return true;
1108 1109
}

1110 1111
static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
				int flags, void *key)
1112 1113 1114 1115 1116
{
	struct blk_mq_hw_ctx *hctx;

	hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);

1117
	spin_lock(&hctx->dispatch_wait_lock);
1118 1119 1120 1121
	if (!list_empty(&wait->entry)) {
		struct sbitmap_queue *sbq;

		list_del_init(&wait->entry);
1122
		sbq = hctx->tags->bitmap_tags;
1123 1124
		atomic_dec(&sbq->ws_active);
	}
1125 1126
	spin_unlock(&hctx->dispatch_wait_lock);

1127 1128 1129 1130
	blk_mq_run_hw_queue(hctx, true);
	return 1;
}

1131 1132
/*
 * Mark us waiting for a tag. For shared tags, this involves hooking us into
1133 1134
 * the tag wakeups. For non-shared tags, we can simply mark us needing a
 * restart. For both cases, take care to check the condition again after
1135 1136
 * marking us as waiting.
 */
1137
static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
1138
				 struct request *rq)
1139
{
1140
	struct sbitmap_queue *sbq = hctx->tags->bitmap_tags;
1141
	struct wait_queue_head *wq;
1142 1143
	wait_queue_entry_t *wait;
	bool ret;
1144

1145
	if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
1146
		blk_mq_sched_mark_restart_hctx(hctx);
1147

1148 1149 1150 1151 1152 1153 1154 1155
		/*
		 * It's possible that a tag was freed in the window between the
		 * allocation failure and adding the hardware queue to the wait
		 * queue.
		 *
		 * Don't clear RESTART here, someone else could have set it.
		 * At most this will cost an extra queue run.
		 */
1156
		return blk_mq_get_driver_tag(rq);
1157 1158
	}

1159
	wait = &hctx->dispatch_wait;
1160 1161 1162
	if (!list_empty_careful(&wait->entry))
		return false;

1163
	wq = &bt_wait_ptr(sbq, hctx)->wait;
1164 1165 1166

	spin_lock_irq(&wq->lock);
	spin_lock(&hctx->dispatch_wait_lock);
1167
	if (!list_empty(&wait->entry)) {
1168 1169
		spin_unlock(&hctx->dispatch_wait_lock);
		spin_unlock_irq(&wq->lock);
1170
		return false;
1171 1172
	}

1173
	atomic_inc(&sbq->ws_active);
1174 1175
	wait->flags &= ~WQ_FLAG_EXCLUSIVE;
	__add_wait_queue(wq, wait);
1176

1177
	/*
1178 1179 1180
	 * It's possible that a tag was freed in the window between the
	 * allocation failure and adding the hardware queue to the wait
	 * queue.
1181
	 */
1182
	ret = blk_mq_get_driver_tag(rq);
1183
	if (!ret) {
1184 1185
		spin_unlock(&hctx->dispatch_wait_lock);
		spin_unlock_irq(&wq->lock);
1186
		return false;
1187
	}
1188 1189 1190 1191 1192 1193

	/*
	 * We got a tag, remove ourselves from the wait queue to ensure
	 * someone else gets the wakeup.
	 */
	list_del_init(&wait->entry);
1194
	atomic_dec(&sbq->ws_active);
1195 1196
	spin_unlock(&hctx->dispatch_wait_lock);
	spin_unlock_irq(&wq->lock);
1197 1198

	return true;
1199 1200
}

1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226
#define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT  8
#define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR  4
/*
 * Update dispatch busy with the Exponential Weighted Moving Average(EWMA):
 * - EWMA is one simple way to compute running average value
 * - weight(7/8 and 1/8) is applied so that it can decrease exponentially
 * - take 4 as factor for avoiding to get too small(0) result, and this
 *   factor doesn't matter because EWMA decreases exponentially
 */
static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy)
{
	unsigned int ewma;

	ewma = hctx->dispatch_busy;

	if (!ewma && !busy)
		return;

	ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1;
	if (busy)
		ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR;
	ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT;

	hctx->dispatch_busy = ewma;
}

1227 1228
#define BLK_MQ_RESOURCE_DELAY	3		/* ms units */

1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245
static void blk_mq_handle_dev_resource(struct request *rq,
				       struct list_head *list)
{
	struct request *next =
		list_first_entry_or_null(list, struct request, queuelist);

	/*
	 * If an I/O scheduler has been configured and we got a driver tag for
	 * the next request already, free it.
	 */
	if (next)
		blk_mq_put_driver_tag(next);

	list_add(&rq->queuelist, list);
	__blk_mq_requeue_request(rq);
}

1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258
static void blk_mq_handle_zone_resource(struct request *rq,
					struct list_head *zone_list)
{
	/*
	 * If we end up here it is because we cannot dispatch a request to a
	 * specific zone due to LLD level zone-write locking or other zone
	 * related resource not being available. In this case, set the request
	 * aside in zone_list for retrying it later.
	 */
	list_add(&rq->queuelist, zone_list);
	__blk_mq_requeue_request(rq);
}

1259 1260 1261 1262 1263 1264 1265 1266 1267 1268
enum prep_dispatch {
	PREP_DISPATCH_OK,
	PREP_DISPATCH_NO_TAG,
	PREP_DISPATCH_NO_BUDGET,
};

static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
						  bool need_budget)
{
	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1269
	int budget_token = -1;
1270

1271 1272 1273 1274 1275 1276 1277
	if (need_budget) {
		budget_token = blk_mq_get_dispatch_budget(rq->q);
		if (budget_token < 0) {
			blk_mq_put_driver_tag(rq);
			return PREP_DISPATCH_NO_BUDGET;
		}
		blk_mq_set_rq_budget_token(rq, budget_token);
1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288
	}

	if (!blk_mq_get_driver_tag(rq)) {
		/*
		 * The initial allocation attempt failed, so we need to
		 * rerun the hardware queue when a tag is freed. The
		 * waitqueue takes care of that. If the queue is run
		 * before we add this entry back on the dispatch list,
		 * we'll re-run it below.
		 */
		if (!blk_mq_mark_tag_wait(hctx, rq)) {
1289 1290 1291 1292 1293
			/*
			 * All budgets not got from this function will be put
			 * together during handling partial dispatch
			 */
			if (need_budget)
1294
				blk_mq_put_dispatch_budget(rq->q, budget_token);
1295 1296 1297 1298 1299 1300 1301
			return PREP_DISPATCH_NO_TAG;
		}
	}

	return PREP_DISPATCH_OK;
}

1302 1303
/* release all allocated budgets before calling to blk_mq_dispatch_rq_list */
static void blk_mq_release_budgets(struct request_queue *q,
1304
		struct list_head *list)
1305
{
1306
	struct request *rq;
1307

1308 1309
	list_for_each_entry(rq, list, queuelist) {
		int budget_token = blk_mq_get_rq_budget_token(rq);
1310

1311 1312 1313
		if (budget_token >= 0)
			blk_mq_put_dispatch_budget(q, budget_token);
	}
1314 1315
}

1316 1317 1318
/*
 * Returns true if we did some work AND can potentially do more.
 */
1319
bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
1320
			     unsigned int nr_budgets)
1321
{
1322
	enum prep_dispatch prep;
1323
	struct request_queue *q = hctx->queue;
1324
	struct request *rq, *nxt;
1325
	int errors, queued;
1326
	blk_status_t ret = BLK_STS_OK;
1327
	LIST_HEAD(zone_list);
1328

1329 1330 1331
	if (list_empty(list))
		return false;

1332 1333 1334
	/*
	 * Now process all the entries, sending them to the driver.
	 */
1335
	errors = queued = 0;
1336
	do {
1337
		struct blk_mq_queue_data bd;
1338

1339
		rq = list_first_entry(list, struct request, queuelist);
1340

1341
		WARN_ON_ONCE(hctx != rq->mq_hctx);
1342
		prep = blk_mq_prep_dispatch_rq(rq, !nr_budgets);
1343
		if (prep != PREP_DISPATCH_OK)
1344
			break;
1345

1346 1347
		list_del_init(&rq->queuelist);

1348
		bd.rq = rq;
1349 1350 1351 1352 1353 1354 1355 1356 1357

		/*
		 * Flag last if we have no more requests, or if we have more
		 * but can't assign a driver tag to it.
		 */
		if (list_empty(list))
			bd.last = true;
		else {
			nxt = list_first_entry(list, struct request, queuelist);
1358
			bd.last = !blk_mq_get_driver_tag(nxt);
1359
		}
1360

1361 1362 1363 1364 1365 1366
		/*
		 * once the request is queued to lld, no need to cover the
		 * budget any more
		 */
		if (nr_budgets)
			nr_budgets--;
1367
		ret = q->mq_ops->queue_rq(hctx, &bd);
1368 1369 1370
		switch (ret) {
		case BLK_STS_OK:
			queued++;
1371
			break;
1372 1373 1374 1375 1376
		case BLK_STS_RESOURCE:
		case BLK_STS_DEV_RESOURCE:
			blk_mq_handle_dev_resource(rq, list);
			goto out;
		case BLK_STS_ZONE_RESOURCE:
1377 1378 1379 1380 1381 1382
			/*
			 * Move the request to zone_list and keep going through
			 * the dispatch list to find more requests the drive can
			 * accept.
			 */
			blk_mq_handle_zone_resource(rq, &zone_list);
1383 1384
			break;
		default:
1385
			errors++;
1386
			blk_mq_end_request(rq, ret);
1387
		}
1388
	} while (!list_empty(list));
1389
out:
1390 1391 1392
	if (!list_empty(&zone_list))
		list_splice_tail_init(&zone_list, list);

1393
	hctx->dispatched[queued_to_index(queued)]++;
1394

1395 1396 1397 1398 1399
	/* If we didn't flush the entire list, we could have told the driver
	 * there was more coming, but that turned out to be a lie.
	 */
	if ((!list_empty(list) || errors) && q->mq_ops->commit_rqs && queued)
		q->mq_ops->commit_rqs(hctx);
1400 1401 1402 1403
	/*
	 * Any items that need requeuing? Stuff them into hctx->dispatch,
	 * that is where we will continue on next queue run.
	 */
1404
	if (!list_empty(list)) {
1405
		bool needs_restart;
1406 1407
		/* For non-shared tags, the RESTART check will suffice */
		bool no_tag = prep == PREP_DISPATCH_NO_TAG &&
1408
			(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED);
1409
		bool no_budget_avail = prep == PREP_DISPATCH_NO_BUDGET;
1410

1411 1412
		if (nr_budgets)
			blk_mq_release_budgets(q, list);
1413

1414
		spin_lock(&hctx->lock);
1415
		list_splice_tail_init(list, &hctx->dispatch);
1416
		spin_unlock(&hctx->lock);
1417

1418 1419 1420 1421 1422 1423 1424 1425 1426
		/*
		 * Order adding requests to hctx->dispatch and checking
		 * SCHED_RESTART flag. The pair of this smp_mb() is the one
		 * in blk_mq_sched_restart(). Avoid restart code path to
		 * miss the new added requests to hctx->dispatch, meantime
		 * SCHED_RESTART is observed here.
		 */
		smp_mb();

1427
		/*
1428 1429 1430
		 * If SCHED_RESTART was set by the caller of this function and
		 * it is no longer set that means that it was cleared by another
		 * thread and hence that a queue rerun is needed.
1431
		 *
1432 1433 1434 1435
		 * If 'no_tag' is set, that means that we failed getting
		 * a driver tag with an I/O scheduler attached. If our dispatch
		 * waitqueue is no longer active, ensure that we run the queue
		 * AFTER adding our entries back to the list.
1436
		 *
1437 1438 1439 1440 1441 1442 1443
		 * If no I/O scheduler has been configured it is possible that
		 * the hardware queue got stopped and restarted before requests
		 * were pushed back onto the dispatch list. Rerun the queue to
		 * avoid starvation. Notes:
		 * - blk_mq_run_hw_queue() checks whether or not a queue has
		 *   been stopped before rerunning a queue.
		 * - Some but not all block drivers stop a queue before
1444
		 *   returning BLK_STS_RESOURCE. Two exceptions are scsi-mq
1445
		 *   and dm-rq.
1446 1447 1448
		 *
		 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART
		 * bit is set, run queue after a delay to avoid IO stalls
1449 1450
		 * that could otherwise occur if the queue is idle.  We'll do
		 * similar if we couldn't get budget and SCHED_RESTART is set.
1451
		 */
1452 1453
		needs_restart = blk_mq_sched_needs_restart(hctx);
		if (!needs_restart ||
1454
		    (no_tag && list_empty_careful(&hctx->dispatch_wait.entry)))
1455
			blk_mq_run_hw_queue(hctx, true);
1456 1457
		else if (needs_restart && (ret == BLK_STS_RESOURCE ||
					   no_budget_avail))
1458
			blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY);
1459

1460
		blk_mq_update_dispatch_busy(hctx, true);
1461
		return false;
1462 1463
	} else
		blk_mq_update_dispatch_busy(hctx, false);
1464

1465
	return (queued + errors) != 0;
1466 1467
}

1468 1469 1470 1471 1472 1473
/**
 * __blk_mq_run_hw_queue - Run a hardware queue.
 * @hctx: Pointer to the hardware queue to run.
 *
 * Send pending requests to the hardware.
 */
1474 1475 1476 1477
static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	int srcu_idx;

1478 1479 1480 1481 1482 1483
	/*
	 * We can't run the queue inline with ints disabled. Ensure that
	 * we catch bad users of this early.
	 */
	WARN_ON_ONCE(in_interrupt());

1484
	might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
1485

1486 1487 1488
	hctx_lock(hctx, &srcu_idx);
	blk_mq_sched_dispatch_requests(hctx);
	hctx_unlock(hctx, srcu_idx);
1489 1490
}

1491 1492 1493 1494 1495 1496 1497 1498 1499
static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)
{
	int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask);

	if (cpu >= nr_cpu_ids)
		cpu = cpumask_first(hctx->cpumask);
	return cpu;
}

1500 1501 1502 1503 1504 1505 1506 1507
/*
 * It'd be great if the workqueue API had a way to pass
 * in a mask and had some smarts for more clever placement.
 * For now we just round-robin here, switching for every
 * BLK_MQ_CPU_WORK_BATCH queued items.
 */
static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
{
1508
	bool tried = false;
1509
	int next_cpu = hctx->next_cpu;
1510

1511 1512
	if (hctx->queue->nr_hw_queues == 1)
		return WORK_CPU_UNBOUND;
1513 1514

	if (--hctx->next_cpu_batch <= 0) {
1515
select_cpu:
1516
		next_cpu = cpumask_next_and(next_cpu, hctx->cpumask,
1517
				cpu_online_mask);
1518
		if (next_cpu >= nr_cpu_ids)
1519
			next_cpu = blk_mq_first_mapped_cpu(hctx);
1520 1521 1522
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}

1523 1524 1525 1526
	/*
	 * Do unbound schedule if we can't find a online CPU for this hctx,
	 * and it should only happen in the path of handling CPU DEAD.
	 */
1527
	if (!cpu_online(next_cpu)) {
1528 1529 1530 1531 1532 1533 1534 1535 1536
		if (!tried) {
			tried = true;
			goto select_cpu;
		}

		/*
		 * Make sure to re-select CPU next time once after CPUs
		 * in hctx->cpumask become online again.
		 */
1537
		hctx->next_cpu = next_cpu;
1538 1539 1540
		hctx->next_cpu_batch = 1;
		return WORK_CPU_UNBOUND;
	}
1541 1542 1543

	hctx->next_cpu = next_cpu;
	return next_cpu;
1544 1545
}

1546 1547 1548 1549
/**
 * __blk_mq_delay_run_hw_queue - Run (or schedule to run) a hardware queue.
 * @hctx: Pointer to the hardware queue to run.
 * @async: If we want to run the queue asynchronously.
1550
 * @msecs: Milliseconds of delay to wait before running the queue.
1551 1552 1553 1554
 *
 * If !@async, try to run the queue now. Else, run the queue asynchronously and
 * with a delay of @msecs.
 */
1555 1556
static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async,
					unsigned long msecs)
1557
{
1558
	if (unlikely(blk_mq_hctx_stopped(hctx)))
1559 1560
		return;

1561
	if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
1562 1563
		int cpu = get_cpu();
		if (cpumask_test_cpu(cpu, hctx->cpumask)) {
1564
			__blk_mq_run_hw_queue(hctx);
1565
			put_cpu();
1566 1567
			return;
		}
1568

1569
		put_cpu();
1570
	}
1571

1572 1573
	kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work,
				    msecs_to_jiffies(msecs));
1574 1575
}

1576 1577 1578
/**
 * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously.
 * @hctx: Pointer to the hardware queue to run.
1579
 * @msecs: Milliseconds of delay to wait before running the queue.
1580 1581 1582
 *
 * Run a hardware queue asynchronously with a delay of @msecs.
 */
1583 1584 1585 1586 1587 1588
void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
{
	__blk_mq_delay_run_hw_queue(hctx, true, msecs);
}
EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);

1589 1590 1591 1592 1593 1594 1595 1596 1597
/**
 * blk_mq_run_hw_queue - Start to run a hardware queue.
 * @hctx: Pointer to the hardware queue to run.
 * @async: If we want to run the queue asynchronously.
 *
 * Check if the request queue is not in a quiesced state and if there are
 * pending requests to be sent. If this is true, run the queue to send requests
 * to hardware.
 */
1598
void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
1599
{
1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610
	int srcu_idx;
	bool need_run;

	/*
	 * When queue is quiesced, we may be switching io scheduler, or
	 * updating nr_hw_queues, or other things, and we can't run queue
	 * any more, even __blk_mq_hctx_has_pending() can't be called safely.
	 *
	 * And queue will be rerun in blk_mq_unquiesce_queue() if it is
	 * quiesced.
	 */
1611 1612 1613 1614
	hctx_lock(hctx, &srcu_idx);
	need_run = !blk_queue_quiesced(hctx->queue) &&
		blk_mq_hctx_has_pending(hctx);
	hctx_unlock(hctx, srcu_idx);
1615

1616
	if (need_run)
1617
		__blk_mq_delay_run_hw_queue(hctx, async, 0);
1618
}
O
Omar Sandoval 已提交
1619
EXPORT_SYMBOL(blk_mq_run_hw_queue);
1620

1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656
/*
 * Is the request queue handled by an IO scheduler that does not respect
 * hardware queues when dispatching?
 */
static bool blk_mq_has_sqsched(struct request_queue *q)
{
	struct elevator_queue *e = q->elevator;

	if (e && e->type->ops.dispatch_request &&
	    !(e->type->elevator_features & ELEVATOR_F_MQ_AWARE))
		return true;
	return false;
}

/*
 * Return prefered queue to dispatch from (if any) for non-mq aware IO
 * scheduler.
 */
static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;

	/*
	 * If the IO scheduler does not respect hardware queues when
	 * dispatching, we just don't bother with multiple HW queues and
	 * dispatch from hctx for the current CPU since running multiple queues
	 * just causes lock contention inside the scheduler and pointless cache
	 * bouncing.
	 */
	hctx = blk_mq_map_queue_type(q, HCTX_TYPE_DEFAULT,
				     raw_smp_processor_id());
	if (!blk_mq_hctx_stopped(hctx))
		return hctx;
	return NULL;
}

1657
/**
1658
 * blk_mq_run_hw_queues - Run all hardware queues in a request queue.
1659 1660 1661
 * @q: Pointer to the request queue to run.
 * @async: If we want to run the queue asynchronously.
 */
1662
void blk_mq_run_hw_queues(struct request_queue *q, bool async)
1663
{
1664
	struct blk_mq_hw_ctx *hctx, *sq_hctx;
1665 1666
	int i;

1667 1668 1669
	sq_hctx = NULL;
	if (blk_mq_has_sqsched(q))
		sq_hctx = blk_mq_get_sq_hctx(q);
1670
	queue_for_each_hw_ctx(q, hctx, i) {
1671
		if (blk_mq_hctx_stopped(hctx))
1672
			continue;
1673 1674 1675 1676 1677 1678 1679 1680
		/*
		 * Dispatch from this hctx either if there's no hctx preferred
		 * by IO scheduler or if it has requests that bypass the
		 * scheduler.
		 */
		if (!sq_hctx || sq_hctx == hctx ||
		    !list_empty_careful(&hctx->dispatch))
			blk_mq_run_hw_queue(hctx, async);
1681 1682
	}
}
1683
EXPORT_SYMBOL(blk_mq_run_hw_queues);
1684

1685 1686 1687
/**
 * blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously.
 * @q: Pointer to the request queue to run.
1688
 * @msecs: Milliseconds of delay to wait before running the queues.
1689 1690 1691
 */
void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
{
1692
	struct blk_mq_hw_ctx *hctx, *sq_hctx;
1693 1694
	int i;

1695 1696 1697
	sq_hctx = NULL;
	if (blk_mq_has_sqsched(q))
		sq_hctx = blk_mq_get_sq_hctx(q);
1698 1699 1700
	queue_for_each_hw_ctx(q, hctx, i) {
		if (blk_mq_hctx_stopped(hctx))
			continue;
1701 1702 1703 1704 1705 1706 1707 1708
		/*
		 * Dispatch from this hctx either if there's no hctx preferred
		 * by IO scheduler or if it has requests that bypass the
		 * scheduler.
		 */
		if (!sq_hctx || sq_hctx == hctx ||
		    !list_empty_careful(&hctx->dispatch))
			blk_mq_delay_run_hw_queue(hctx, msecs);
1709 1710 1711 1712
	}
}
EXPORT_SYMBOL(blk_mq_delay_run_hw_queues);

1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732
/**
 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped
 * @q: request queue.
 *
 * The caller is responsible for serializing this function against
 * blk_mq_{start,stop}_hw_queue().
 */
bool blk_mq_queue_stopped(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		if (blk_mq_hctx_stopped(hctx))
			return true;

	return false;
}
EXPORT_SYMBOL(blk_mq_queue_stopped);

1733 1734 1735
/*
 * This function is often used for pausing .queue_rq() by driver when
 * there isn't enough resource or some conditions aren't satisfied, and
1736
 * BLK_STS_RESOURCE is usually returned.
1737 1738 1739 1740 1741
 *
 * We do not guarantee that dispatch can be drained or blocked
 * after blk_mq_stop_hw_queue() returns. Please use
 * blk_mq_quiesce_queue() for that requirement.
 */
1742 1743
void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
{
1744
	cancel_delayed_work(&hctx->run_work);
1745

1746
	set_bit(BLK_MQ_S_STOPPED, &hctx->state);
1747
}
1748
EXPORT_SYMBOL(blk_mq_stop_hw_queue);
1749

1750 1751 1752
/*
 * This function is often used for pausing .queue_rq() by driver when
 * there isn't enough resource or some conditions aren't satisfied, and
1753
 * BLK_STS_RESOURCE is usually returned.
1754 1755 1756 1757 1758
 *
 * We do not guarantee that dispatch can be drained or blocked
 * after blk_mq_stop_hw_queues() returns. Please use
 * blk_mq_quiesce_queue() for that requirement.
 */
1759 1760
void blk_mq_stop_hw_queues(struct request_queue *q)
{
1761 1762 1763 1764 1765
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_stop_hw_queue(hctx);
1766 1767 1768
}
EXPORT_SYMBOL(blk_mq_stop_hw_queues);

1769 1770 1771
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1772

1773
	blk_mq_run_hw_queue(hctx, false);
1774 1775 1776
}
EXPORT_SYMBOL(blk_mq_start_hw_queue);

1777 1778 1779 1780 1781 1782 1783 1784 1785 1786
void blk_mq_start_hw_queues(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_start_hw_queues);

1787 1788 1789 1790 1791 1792 1793 1794 1795 1796
void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
	if (!blk_mq_hctx_stopped(hctx))
		return;

	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
	blk_mq_run_hw_queue(hctx, async);
}
EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);

1797
void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
1798 1799 1800 1801
{
	struct blk_mq_hw_ctx *hctx;
	int i;

1802 1803
	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_stopped_hw_queue(hctx, async);
1804 1805 1806
}
EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);

1807
static void blk_mq_run_work_fn(struct work_struct *work)
1808 1809 1810
{
	struct blk_mq_hw_ctx *hctx;

1811
	hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
1812

1813
	/*
M
Ming Lei 已提交
1814
	 * If we are stopped, don't run the queue.
1815
	 */
1816
	if (blk_mq_hctx_stopped(hctx))
1817
		return;
1818 1819 1820 1821

	__blk_mq_run_hw_queue(hctx);
}

1822 1823 1824
static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,
					    struct request *rq,
					    bool at_head)
1825
{
J
Jens Axboe 已提交
1826
	struct blk_mq_ctx *ctx = rq->mq_ctx;
M
Ming Lei 已提交
1827
	enum hctx_type type = hctx->type;
J
Jens Axboe 已提交
1828

1829 1830
	lockdep_assert_held(&ctx->lock);

1831
	trace_block_rq_insert(rq);
1832

1833
	if (at_head)
M
Ming Lei 已提交
1834
		list_add(&rq->queuelist, &ctx->rq_lists[type]);
1835
	else
M
Ming Lei 已提交
1836
		list_add_tail(&rq->queuelist, &ctx->rq_lists[type]);
1837
}
1838

1839 1840
void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
			     bool at_head)
1841 1842 1843
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;

1844 1845
	lockdep_assert_held(&ctx->lock);

J
Jens Axboe 已提交
1846
	__blk_mq_insert_req_list(hctx, rq, at_head);
1847 1848 1849
	blk_mq_hctx_mark_pending(hctx, ctx);
}

1850 1851 1852
/**
 * blk_mq_request_bypass_insert - Insert a request at dispatch list.
 * @rq: Pointer to request to be inserted.
1853
 * @at_head: true if the request should be inserted at the head of the list.
1854 1855
 * @run_queue: If we should run the hardware queue after inserting the request.
 *
1856 1857 1858
 * Should only be used carefully, when the caller knows we want to
 * bypass a potential IO scheduler on the target device.
 */
1859 1860
void blk_mq_request_bypass_insert(struct request *rq, bool at_head,
				  bool run_queue)
1861
{
1862
	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1863 1864

	spin_lock(&hctx->lock);
1865 1866 1867 1868
	if (at_head)
		list_add(&rq->queuelist, &hctx->dispatch);
	else
		list_add_tail(&rq->queuelist, &hctx->dispatch);
1869 1870
	spin_unlock(&hctx->lock);

1871 1872
	if (run_queue)
		blk_mq_run_hw_queue(hctx, false);
1873 1874
}

1875 1876
void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
			    struct list_head *list)
1877 1878

{
1879
	struct request *rq;
M
Ming Lei 已提交
1880
	enum hctx_type type = hctx->type;
1881

1882 1883 1884 1885
	/*
	 * preemption doesn't flush plug list, so it's possible ctx->cpu is
	 * offline now
	 */
1886
	list_for_each_entry(rq, list, queuelist) {
J
Jens Axboe 已提交
1887
		BUG_ON(rq->mq_ctx != ctx);
1888
		trace_block_rq_insert(rq);
1889
	}
1890 1891

	spin_lock(&ctx->lock);
M
Ming Lei 已提交
1892
	list_splice_tail_init(list, &ctx->rq_lists[type]);
1893
	blk_mq_hctx_mark_pending(hctx, ctx);
1894 1895 1896
	spin_unlock(&ctx->lock);
}

1897 1898
static int plug_rq_cmp(void *priv, const struct list_head *a,
		       const struct list_head *b)
1899 1900 1901 1902
{
	struct request *rqa = container_of(a, struct request, queuelist);
	struct request *rqb = container_of(b, struct request, queuelist);

P
Pavel Begunkov 已提交
1903 1904 1905 1906
	if (rqa->mq_ctx != rqb->mq_ctx)
		return rqa->mq_ctx > rqb->mq_ctx;
	if (rqa->mq_hctx != rqb->mq_hctx)
		return rqa->mq_hctx > rqb->mq_hctx;
J
Jens Axboe 已提交
1907 1908

	return blk_rq_pos(rqa) > blk_rq_pos(rqb);
1909 1910 1911 1912 1913 1914
}

void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
{
	LIST_HEAD(list);

1915 1916
	if (list_empty(&plug->mq_list))
		return;
1917 1918
	list_splice_init(&plug->mq_list, &list);

1919 1920
	if (plug->rq_count > 2 && plug->multiple_queues)
		list_sort(NULL, &list, plug_rq_cmp);
1921

1922 1923
	plug->rq_count = 0;

1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937
	do {
		struct list_head rq_list;
		struct request *rq, *head_rq = list_entry_rq(list.next);
		struct list_head *pos = &head_rq->queuelist; /* skip first */
		struct blk_mq_hw_ctx *this_hctx = head_rq->mq_hctx;
		struct blk_mq_ctx *this_ctx = head_rq->mq_ctx;
		unsigned int depth = 1;

		list_for_each_continue(pos, &list) {
			rq = list_entry_rq(pos);
			BUG_ON(!rq->q);
			if (rq->mq_hctx != this_hctx || rq->mq_ctx != this_ctx)
				break;
			depth++;
1938 1939
		}

1940 1941
		list_cut_before(&rq_list, &list, pos);
		trace_block_unplug(head_rq->q, depth, !from_schedule);
1942
		blk_mq_sched_insert_requests(this_hctx, this_ctx, &rq_list,
1943
						from_schedule);
1944
	} while(!list_empty(&list));
1945 1946
}

1947 1948
static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
		unsigned int nr_segs)
1949
{
1950 1951
	int err;

1952 1953 1954 1955 1956
	if (bio->bi_opf & REQ_RAHEAD)
		rq->cmd_flags |= REQ_FAILFAST_MASK;

	rq->__sector = bio->bi_iter.bi_sector;
	rq->write_hint = bio->bi_write_hint;
1957
	blk_rq_bio_prep(rq, bio, nr_segs);
1958 1959 1960 1961

	/* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */
	err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
	WARN_ON_ONCE(err);
1962

1963
	blk_account_io_start(rq);
1964 1965
}

1966 1967
static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
					    struct request *rq,
1968
					    blk_qc_t *cookie, bool last)
1969 1970 1971 1972
{
	struct request_queue *q = rq->q;
	struct blk_mq_queue_data bd = {
		.rq = rq,
1973
		.last = last,
1974
	};
1975
	blk_qc_t new_cookie;
1976
	blk_status_t ret;
1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987

	new_cookie = request_to_qc_t(hctx, rq);

	/*
	 * For OK queue, we are done. For error, caller may kill it.
	 * Any other error (busy), just add it to our list as we
	 * previously would have done.
	 */
	ret = q->mq_ops->queue_rq(hctx, &bd);
	switch (ret) {
	case BLK_STS_OK:
1988
		blk_mq_update_dispatch_busy(hctx, false);
1989 1990 1991
		*cookie = new_cookie;
		break;
	case BLK_STS_RESOURCE:
1992
	case BLK_STS_DEV_RESOURCE:
1993
		blk_mq_update_dispatch_busy(hctx, true);
1994 1995 1996
		__blk_mq_requeue_request(rq);
		break;
	default:
1997
		blk_mq_update_dispatch_busy(hctx, false);
1998 1999 2000 2001 2002 2003 2004
		*cookie = BLK_QC_T_NONE;
		break;
	}

	return ret;
}

2005
static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
2006
						struct request *rq,
2007
						blk_qc_t *cookie,
2008
						bool bypass_insert, bool last)
2009 2010
{
	struct request_queue *q = rq->q;
M
Ming Lei 已提交
2011
	bool run_queue = true;
2012
	int budget_token;
M
Ming Lei 已提交
2013

2014
	/*
2015
	 * RCU or SRCU read lock is needed before checking quiesced flag.
2016
	 *
2017 2018 2019
	 * When queue is stopped or quiesced, ignore 'bypass_insert' from
	 * blk_mq_request_issue_directly(), and return BLK_STS_OK to caller,
	 * and avoid driver to try to dispatch again.
2020
	 */
2021
	if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q)) {
M
Ming Lei 已提交
2022
		run_queue = false;
2023 2024
		bypass_insert = false;
		goto insert;
M
Ming Lei 已提交
2025
	}
2026

2027 2028
	if (q->elevator && !bypass_insert)
		goto insert;
2029

2030 2031
	budget_token = blk_mq_get_dispatch_budget(q);
	if (budget_token < 0)
2032
		goto insert;
2033

2034 2035
	blk_mq_set_rq_budget_token(rq, budget_token);

2036
	if (!blk_mq_get_driver_tag(rq)) {
2037
		blk_mq_put_dispatch_budget(q, budget_token);
2038
		goto insert;
2039
	}
2040

2041 2042 2043 2044 2045
	return __blk_mq_issue_directly(hctx, rq, cookie, last);
insert:
	if (bypass_insert)
		return BLK_STS_RESOURCE;

2046 2047
	blk_mq_sched_insert_request(rq, false, run_queue, false);

2048 2049 2050
	return BLK_STS_OK;
}

2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061
/**
 * blk_mq_try_issue_directly - Try to send a request directly to device driver.
 * @hctx: Pointer of the associated hardware queue.
 * @rq: Pointer to request to be sent.
 * @cookie: Request queue cookie.
 *
 * If the device has enough resources to accept a new request now, send the
 * request directly to device driver. Else, insert at hctx->dispatch queue, so
 * we can try send it another time in the future. Requests inserted at this
 * queue have higher priority.
 */
2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073
static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
		struct request *rq, blk_qc_t *cookie)
{
	blk_status_t ret;
	int srcu_idx;

	might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);

	hctx_lock(hctx, &srcu_idx);

	ret = __blk_mq_try_issue_directly(hctx, rq, cookie, false, true);
	if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE)
2074
		blk_mq_request_bypass_insert(rq, false, true);
2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089
	else if (ret != BLK_STS_OK)
		blk_mq_end_request(rq, ret);

	hctx_unlock(hctx, srcu_idx);
}

blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)
{
	blk_status_t ret;
	int srcu_idx;
	blk_qc_t unused_cookie;
	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;

	hctx_lock(hctx, &srcu_idx);
	ret = __blk_mq_try_issue_directly(hctx, rq, &unused_cookie, true, last);
2090
	hctx_unlock(hctx, srcu_idx);
2091 2092

	return ret;
2093 2094
}

2095 2096 2097
void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
		struct list_head *list)
{
2098
	int queued = 0;
2099
	int errors = 0;
2100

2101
	while (!list_empty(list)) {
2102
		blk_status_t ret;
2103 2104 2105 2106
		struct request *rq = list_first_entry(list, struct request,
				queuelist);

		list_del_init(&rq->queuelist);
2107 2108 2109 2110
		ret = blk_mq_request_issue_directly(rq, list_empty(list));
		if (ret != BLK_STS_OK) {
			if (ret == BLK_STS_RESOURCE ||
					ret == BLK_STS_DEV_RESOURCE) {
2111
				blk_mq_request_bypass_insert(rq, false,
2112
							list_empty(list));
2113 2114 2115
				break;
			}
			blk_mq_end_request(rq, ret);
2116
			errors++;
2117 2118
		} else
			queued++;
2119
	}
J
Jens Axboe 已提交
2120 2121 2122 2123 2124 2125

	/*
	 * If we didn't flush the entire list, we could have told
	 * the driver there was more coming, but that turned out to
	 * be a lie.
	 */
2126 2127
	if ((!list_empty(list) || errors) &&
	     hctx->queue->mq_ops->commit_rqs && queued)
J
Jens Axboe 已提交
2128
		hctx->queue->mq_ops->commit_rqs(hctx);
2129 2130
}

2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144
static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
{
	list_add_tail(&rq->queuelist, &plug->mq_list);
	plug->rq_count++;
	if (!plug->multiple_queues && !list_is_singular(&plug->mq_list)) {
		struct request *tmp;

		tmp = list_first_entry(&plug->mq_list, struct request,
						queuelist);
		if (tmp->q != rq->q)
			plug->multiple_queues = true;
	}
}

2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156
/*
 * Allow 4x BLK_MAX_REQUEST_COUNT requests on plug queue for multiple
 * queues. This is important for md arrays to benefit from merging
 * requests.
 */
static inline unsigned short blk_plug_max_rq_count(struct blk_plug *plug)
{
	if (plug->multiple_queues)
		return BLK_MAX_REQUEST_COUNT * 4;
	return BLK_MAX_REQUEST_COUNT;
}

2157
/**
2158
 * blk_mq_submit_bio - Create and send a request to block device.
2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171
 * @bio: Bio pointer.
 *
 * Builds up a request structure from @q and @bio and send to the device. The
 * request may not be queued directly to hardware if:
 * * This request can be merged with another one
 * * We want to place request at plug queue for possible future merging
 * * There is an IO scheduler active at this queue
 *
 * It will not queue the request if there is an error with the bio, or at the
 * request creation.
 *
 * Returns: Request queue cookie.
 */
2172
blk_qc_t blk_mq_submit_bio(struct bio *bio)
2173
{
2174
	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
2175
	const int is_sync = op_is_sync(bio->bi_opf);
2176
	const int is_flush_fua = op_is_flush(bio->bi_opf);
2177 2178 2179
	struct blk_mq_alloc_data data = {
		.q		= q,
	};
2180
	struct request *rq;
2181
	struct blk_plug *plug;
2182
	struct request *same_queue_rq = NULL;
2183
	unsigned int nr_segs;
2184
	blk_qc_t cookie;
2185
	blk_status_t ret;
J
Jeffle Xu 已提交
2186
	bool hipri;
2187 2188

	blk_queue_bounce(q, &bio);
2189
	__blk_queue_split(&bio, &nr_segs);
2190

2191
	if (!bio_integrity_prep(bio))
2192
		goto queue_exit;
2193

2194
	if (!is_flush_fua && !blk_queue_nomerges(q) &&
2195
	    blk_attempt_plug_merge(q, bio, nr_segs, &same_queue_rq))
2196
		goto queue_exit;
2197

2198
	if (blk_mq_sched_bio_merge(q, bio, nr_segs))
2199
		goto queue_exit;
2200

2201
	rq_qos_throttle(q, bio);
J
Jens Axboe 已提交
2202

J
Jeffle Xu 已提交
2203 2204
	hipri = bio->bi_opf & REQ_HIPRI;

2205
	data.cmd_flags = bio->bi_opf;
2206
	rq = __blk_mq_alloc_request(&data);
J
Jens Axboe 已提交
2207
	if (unlikely(!rq)) {
2208
		rq_qos_cleanup(q, bio);
J
Jens Axboe 已提交
2209
		if (bio->bi_opf & REQ_NOWAIT)
2210
			bio_wouldblock_error(bio);
2211
		goto queue_exit;
J
Jens Axboe 已提交
2212 2213
	}

2214
	trace_block_getrq(bio);
2215

2216
	rq_qos_track(q, rq, bio);
2217

2218
	cookie = request_to_qc_t(data.hctx, rq);
2219

2220 2221
	blk_mq_bio_to_request(rq, bio, nr_segs);

2222 2223 2224 2225 2226 2227 2228 2229
	ret = blk_crypto_init_request(rq);
	if (ret != BLK_STS_OK) {
		bio->bi_status = ret;
		bio_endio(bio);
		blk_mq_free_request(rq);
		return BLK_QC_T_NONE;
	}

2230
	plug = blk_mq_plug(q, bio);
2231
	if (unlikely(is_flush_fua)) {
2232
		/* Bypass scheduler for flush requests */
2233 2234
		blk_insert_flush(rq);
		blk_mq_run_hw_queue(data.hctx, true);
2235 2236 2237
	} else if (plug && (q->nr_hw_queues == 1 ||
		   blk_mq_is_sbitmap_shared(rq->mq_hctx->flags) ||
		   q->mq_ops->commit_rqs || !blk_queue_nonrot(q))) {
2238 2239 2240
		/*
		 * Use plugging if we have a ->commit_rqs() hook as well, as
		 * we know the driver uses bd->last in a smart fashion.
M
Ming Lei 已提交
2241 2242 2243
		 *
		 * Use normal plugging if this disk is slow HDD, as sequential
		 * IO may benefit a lot from plug merging.
2244
		 */
2245
		unsigned int request_count = plug->rq_count;
2246 2247
		struct request *last = NULL;

M
Ming Lei 已提交
2248
		if (!request_count)
2249
			trace_block_plug(q);
2250 2251
		else
			last = list_entry_rq(plug->mq_list.prev);
2252

2253
		if (request_count >= blk_plug_max_rq_count(plug) || (last &&
2254
		    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
2255 2256
			blk_flush_plug_list(plug, false);
			trace_block_plug(q);
2257
		}
2258

2259
		blk_add_rq_to_plug(plug, rq);
2260
	} else if (q->elevator) {
2261
		/* Insert the request at the IO scheduler queue */
2262
		blk_mq_sched_insert_request(rq, false, true, true);
2263
	} else if (plug && !blk_queue_nomerges(q)) {
2264
		/*
2265
		 * We do limited plugging. If the bio can be merged, do that.
2266 2267
		 * Otherwise the existing request in the plug list will be
		 * issued. So the plug list will have one request at most
2268 2269
		 * The plug list might get flushed before this. If that happens,
		 * the plug list is empty, and same_queue_rq is invalid.
2270
		 */
2271 2272
		if (list_empty(&plug->mq_list))
			same_queue_rq = NULL;
2273
		if (same_queue_rq) {
2274
			list_del_init(&same_queue_rq->queuelist);
2275 2276
			plug->rq_count--;
		}
2277
		blk_add_rq_to_plug(plug, rq);
2278
		trace_block_plug(q);
2279

2280
		if (same_queue_rq) {
2281
			data.hctx = same_queue_rq->mq_hctx;
2282
			trace_block_unplug(q, 1, true);
2283
			blk_mq_try_issue_directly(data.hctx, same_queue_rq,
2284
					&cookie);
2285
		}
2286 2287
	} else if ((q->nr_hw_queues > 1 && is_sync) ||
			!data.hctx->dispatch_busy) {
2288 2289 2290 2291
		/*
		 * There is no scheduler and we can try to send directly
		 * to the hardware.
		 */
2292
		blk_mq_try_issue_directly(data.hctx, rq, &cookie);
2293
	} else {
2294
		/* Default case. */
2295
		blk_mq_sched_insert_request(rq, false, true, true);
2296
	}
2297

J
Jeffle Xu 已提交
2298 2299
	if (!hipri)
		return BLK_QC_T_NONE;
2300
	return cookie;
2301 2302 2303
queue_exit:
	blk_queue_exit(q);
	return BLK_QC_T_NONE;
2304 2305
}

2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344
static size_t order_to_size(unsigned int order)
{
	return (size_t)PAGE_SIZE << order;
}

/* called before freeing request pool in @tags */
static void blk_mq_clear_rq_mapping(struct blk_mq_tag_set *set,
		struct blk_mq_tags *tags, unsigned int hctx_idx)
{
	struct blk_mq_tags *drv_tags = set->tags[hctx_idx];
	struct page *page;
	unsigned long flags;

	list_for_each_entry(page, &tags->page_list, lru) {
		unsigned long start = (unsigned long)page_address(page);
		unsigned long end = start + order_to_size(page->private);
		int i;

		for (i = 0; i < set->queue_depth; i++) {
			struct request *rq = drv_tags->rqs[i];
			unsigned long rq_addr = (unsigned long)rq;

			if (rq_addr >= start && rq_addr < end) {
				WARN_ON_ONCE(refcount_read(&rq->ref) != 0);
				cmpxchg(&drv_tags->rqs[i], rq, NULL);
			}
		}
	}

	/*
	 * Wait until all pending iteration is done.
	 *
	 * Request reference is cleared and it is guaranteed to be observed
	 * after the ->lock is released.
	 */
	spin_lock_irqsave(&drv_tags->lock, flags);
	spin_unlock_irqrestore(&drv_tags->lock, flags);
}

2345 2346
void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx)
2347
{
2348
	struct page *page;
2349

2350
	if (tags->rqs && set->ops->exit_request) {
2351
		int i;
2352

2353
		for (i = 0; i < tags->nr_tags; i++) {
J
Jens Axboe 已提交
2354 2355 2356
			struct request *rq = tags->static_rqs[i];

			if (!rq)
2357
				continue;
2358
			set->ops->exit_request(set, rq, hctx_idx);
J
Jens Axboe 已提交
2359
			tags->static_rqs[i] = NULL;
2360
		}
2361 2362
	}

2363 2364
	blk_mq_clear_rq_mapping(set, tags, hctx_idx);

2365 2366
	while (!list_empty(&tags->page_list)) {
		page = list_first_entry(&tags->page_list, struct page, lru);
2367
		list_del_init(&page->lru);
2368 2369
		/*
		 * Remove kmemleak object previously allocated in
2370
		 * blk_mq_alloc_rqs().
2371 2372
		 */
		kmemleak_free(page_address(page));
2373 2374
		__free_pages(page, page->private);
	}
2375
}
2376

2377
void blk_mq_free_rq_map(struct blk_mq_tags *tags, unsigned int flags)
2378
{
2379
	kfree(tags->rqs);
2380
	tags->rqs = NULL;
J
Jens Axboe 已提交
2381 2382
	kfree(tags->static_rqs);
	tags->static_rqs = NULL;
2383

2384
	blk_mq_free_tags(tags, flags);
2385 2386
}

2387 2388 2389
struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
					unsigned int hctx_idx,
					unsigned int nr_tags,
2390 2391
					unsigned int reserved_tags,
					unsigned int flags)
2392
{
2393
	struct blk_mq_tags *tags;
2394
	int node;
2395

2396
	node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx);
2397 2398 2399
	if (node == NUMA_NO_NODE)
		node = set->numa_node;

2400
	tags = blk_mq_init_tags(nr_tags, reserved_tags, node, flags);
2401 2402
	if (!tags)
		return NULL;
2403

2404
	tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *),
2405
				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
2406
				 node);
2407
	if (!tags->rqs) {
2408
		blk_mq_free_tags(tags, flags);
2409 2410
		return NULL;
	}
2411

2412 2413 2414
	tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *),
					GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
					node);
J
Jens Axboe 已提交
2415 2416
	if (!tags->static_rqs) {
		kfree(tags->rqs);
2417
		blk_mq_free_tags(tags, flags);
J
Jens Axboe 已提交
2418 2419 2420
		return NULL;
	}

2421 2422 2423
	return tags;
}

2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434
static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
			       unsigned int hctx_idx, int node)
{
	int ret;

	if (set->ops->init_request) {
		ret = set->ops->init_request(set, rq, hctx_idx, node);
		if (ret)
			return ret;
	}

K
Keith Busch 已提交
2435
	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
2436 2437 2438
	return 0;
}

2439 2440 2441 2442 2443
int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx, unsigned int depth)
{
	unsigned int i, j, entries_per_page, max_order = 4;
	size_t rq_size, left;
2444 2445
	int node;

2446
	node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx);
2447 2448
	if (node == NUMA_NO_NODE)
		node = set->numa_node;
2449 2450 2451

	INIT_LIST_HEAD(&tags->page_list);

2452 2453 2454 2455
	/*
	 * rq_size is the size of the request plus driver payload, rounded
	 * to the cacheline size
	 */
2456
	rq_size = round_up(sizeof(struct request) + set->cmd_size,
2457
				cache_line_size());
2458
	left = rq_size * depth;
2459

2460
	for (i = 0; i < depth; ) {
2461 2462 2463 2464 2465
		int this_order = max_order;
		struct page *page;
		int to_do;
		void *p;

2466
		while (this_order && left < order_to_size(this_order - 1))
2467 2468 2469
			this_order--;

		do {
2470
			page = alloc_pages_node(node,
2471
				GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
2472
				this_order);
2473 2474 2475 2476 2477 2478 2479 2480 2481
			if (page)
				break;
			if (!this_order--)
				break;
			if (order_to_size(this_order) < rq_size)
				break;
		} while (1);

		if (!page)
2482
			goto fail;
2483 2484

		page->private = this_order;
2485
		list_add_tail(&page->lru, &tags->page_list);
2486 2487

		p = page_address(page);
2488 2489 2490 2491
		/*
		 * Allow kmemleak to scan these pages as they contain pointers
		 * to additional allocations like via ops->init_request().
		 */
2492
		kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
2493
		entries_per_page = order_to_size(this_order) / rq_size;
2494
		to_do = min(entries_per_page, depth - i);
2495 2496
		left -= to_do * rq_size;
		for (j = 0; j < to_do; j++) {
J
Jens Axboe 已提交
2497 2498 2499
			struct request *rq = p;

			tags->static_rqs[i] = rq;
2500 2501 2502
			if (blk_mq_init_request(set, rq, hctx_idx, node)) {
				tags->static_rqs[i] = NULL;
				goto fail;
2503 2504
			}

2505 2506 2507 2508
			p += rq_size;
			i++;
		}
	}
2509
	return 0;
2510

2511
fail:
2512 2513
	blk_mq_free_rqs(set, tags, hctx_idx);
	return -ENOMEM;
2514 2515
}

2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595
struct rq_iter_data {
	struct blk_mq_hw_ctx *hctx;
	bool has_rq;
};

static bool blk_mq_has_request(struct request *rq, void *data, bool reserved)
{
	struct rq_iter_data *iter_data = data;

	if (rq->mq_hctx != iter_data->hctx)
		return true;
	iter_data->has_rq = true;
	return false;
}

static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)
{
	struct blk_mq_tags *tags = hctx->sched_tags ?
			hctx->sched_tags : hctx->tags;
	struct rq_iter_data data = {
		.hctx	= hctx,
	};

	blk_mq_all_tag_iter(tags, blk_mq_has_request, &data);
	return data.has_rq;
}

static inline bool blk_mq_last_cpu_in_hctx(unsigned int cpu,
		struct blk_mq_hw_ctx *hctx)
{
	if (cpumask_next_and(-1, hctx->cpumask, cpu_online_mask) != cpu)
		return false;
	if (cpumask_next_and(cpu, hctx->cpumask, cpu_online_mask) < nr_cpu_ids)
		return false;
	return true;
}

static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node)
{
	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
			struct blk_mq_hw_ctx, cpuhp_online);

	if (!cpumask_test_cpu(cpu, hctx->cpumask) ||
	    !blk_mq_last_cpu_in_hctx(cpu, hctx))
		return 0;

	/*
	 * Prevent new request from being allocated on the current hctx.
	 *
	 * The smp_mb__after_atomic() Pairs with the implied barrier in
	 * test_and_set_bit_lock in sbitmap_get().  Ensures the inactive flag is
	 * seen once we return from the tag allocator.
	 */
	set_bit(BLK_MQ_S_INACTIVE, &hctx->state);
	smp_mb__after_atomic();

	/*
	 * Try to grab a reference to the queue and wait for any outstanding
	 * requests.  If we could not grab a reference the queue has been
	 * frozen and there are no requests.
	 */
	if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) {
		while (blk_mq_hctx_has_requests(hctx))
			msleep(5);
		percpu_ref_put(&hctx->queue->q_usage_counter);
	}

	return 0;
}

static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node)
{
	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
			struct blk_mq_hw_ctx, cpuhp_online);

	if (cpumask_test_cpu(cpu, hctx->cpumask))
		clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
	return 0;
}

J
Jens Axboe 已提交
2596 2597 2598 2599 2600
/*
 * 'cpu' is going away. splice any existing rq_list entries from this
 * software queue to the hw queue dispatch list, and ensure that it
 * gets run.
 */
2601
static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
2602
{
2603
	struct blk_mq_hw_ctx *hctx;
2604 2605
	struct blk_mq_ctx *ctx;
	LIST_HEAD(tmp);
M
Ming Lei 已提交
2606
	enum hctx_type type;
2607

2608
	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
2609 2610 2611
	if (!cpumask_test_cpu(cpu, hctx->cpumask))
		return 0;

J
Jens Axboe 已提交
2612
	ctx = __blk_mq_get_ctx(hctx->queue, cpu);
M
Ming Lei 已提交
2613
	type = hctx->type;
2614 2615

	spin_lock(&ctx->lock);
M
Ming Lei 已提交
2616 2617
	if (!list_empty(&ctx->rq_lists[type])) {
		list_splice_init(&ctx->rq_lists[type], &tmp);
2618 2619 2620 2621 2622
		blk_mq_hctx_clear_pending(hctx, ctx);
	}
	spin_unlock(&ctx->lock);

	if (list_empty(&tmp))
2623
		return 0;
2624

J
Jens Axboe 已提交
2625 2626 2627
	spin_lock(&hctx->lock);
	list_splice_tail_init(&tmp, &hctx->dispatch);
	spin_unlock(&hctx->lock);
2628 2629

	blk_mq_run_hw_queue(hctx, true);
2630
	return 0;
2631 2632
}

2633
static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
2634
{
2635 2636 2637
	if (!(hctx->flags & BLK_MQ_F_STACKING))
		cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
						    &hctx->cpuhp_online);
2638 2639
	cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
					    &hctx->cpuhp_dead);
2640 2641
}

2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670
/*
 * Before freeing hw queue, clearing the flush request reference in
 * tags->rqs[] for avoiding potential UAF.
 */
static void blk_mq_clear_flush_rq_mapping(struct blk_mq_tags *tags,
		unsigned int queue_depth, struct request *flush_rq)
{
	int i;
	unsigned long flags;

	/* The hw queue may not be mapped yet */
	if (!tags)
		return;

	WARN_ON_ONCE(refcount_read(&flush_rq->ref) != 0);

	for (i = 0; i < queue_depth; i++)
		cmpxchg(&tags->rqs[i], flush_rq, NULL);

	/*
	 * Wait until all pending iteration is done.
	 *
	 * Request reference is cleared and it is guaranteed to be observed
	 * after the ->lock is released.
	 */
	spin_lock_irqsave(&tags->lock, flags);
	spin_unlock_irqrestore(&tags->lock, flags);
}

2671
/* hctx->ctxs will be freed in queue's release handler */
2672 2673 2674 2675
static void blk_mq_exit_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
2676 2677
	struct request *flush_rq = hctx->fq->flush_rq;

2678 2679
	if (blk_mq_hw_queue_mapped(hctx))
		blk_mq_tag_idle(hctx);
2680

2681 2682
	blk_mq_clear_flush_rq_mapping(set->tags[hctx_idx],
			set->queue_depth, flush_rq);
2683
	if (set->ops->exit_request)
2684
		set->ops->exit_request(set, flush_rq, hctx_idx);
2685

2686 2687 2688
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);

2689
	blk_mq_remove_cpuhp(hctx);
2690 2691 2692 2693

	spin_lock(&q->unused_hctx_lock);
	list_add(&hctx->hctx_list, &q->unused_hctx_list);
	spin_unlock(&q->unused_hctx_lock);
2694 2695
}

M
Ming Lei 已提交
2696 2697 2698 2699 2700 2701 2702 2703 2704
static void blk_mq_exit_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set, int nr_queue)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if (i == nr_queue)
			break;
2705
		blk_mq_debugfs_unregister_hctx(hctx);
2706
		blk_mq_exit_hctx(q, set, hctx, i);
M
Ming Lei 已提交
2707 2708 2709
	}
}

2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723
static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set)
{
	int hw_ctx_size = sizeof(struct blk_mq_hw_ctx);

	BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, srcu),
			   __alignof__(struct blk_mq_hw_ctx)) !=
		     sizeof(struct blk_mq_hw_ctx));

	if (tag_set->flags & BLK_MQ_F_BLOCKING)
		hw_ctx_size += sizeof(struct srcu_struct);

	return hw_ctx_size;
}

2724 2725 2726
static int blk_mq_init_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
2727
{
2728 2729
	hctx->queue_num = hctx_idx;

2730 2731 2732
	if (!(hctx->flags & BLK_MQ_F_STACKING))
		cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
				&hctx->cpuhp_online);
2733 2734 2735 2736 2737 2738 2739
	cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);

	hctx->tags = set->tags[hctx_idx];

	if (set->ops->init_hctx &&
	    set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
		goto unregister_cpu_notifier;
2740

2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768
	if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx,
				hctx->numa_node))
		goto exit_hctx;
	return 0;

 exit_hctx:
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);
 unregister_cpu_notifier:
	blk_mq_remove_cpuhp(hctx);
	return -1;
}

static struct blk_mq_hw_ctx *
blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
		int node)
{
	struct blk_mq_hw_ctx *hctx;
	gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY;

	hctx = kzalloc_node(blk_mq_hw_ctx_size(set), gfp, node);
	if (!hctx)
		goto fail_alloc_hctx;

	if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node))
		goto free_hctx;

	atomic_set(&hctx->nr_active, 0);
2769
	if (node == NUMA_NO_NODE)
2770 2771
		node = set->numa_node;
	hctx->numa_node = node;
2772

2773
	INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
2774 2775 2776
	spin_lock_init(&hctx->lock);
	INIT_LIST_HEAD(&hctx->dispatch);
	hctx->queue = q;
2777
	hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED;
2778

2779 2780
	INIT_LIST_HEAD(&hctx->hctx_list);

2781
	/*
2782 2783
	 * Allocate space for all possible cpus to avoid allocation at
	 * runtime
2784
	 */
2785
	hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *),
2786
			gfp, node);
2787
	if (!hctx->ctxs)
2788
		goto free_cpumask;
2789

2790
	if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8),
2791
				gfp, node, false, false))
2792 2793
		goto free_ctxs;
	hctx->nr_ctx = 0;
2794

2795
	spin_lock_init(&hctx->dispatch_wait_lock);
2796 2797 2798
	init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
	INIT_LIST_HEAD(&hctx->dispatch_wait.entry);

2799
	hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp);
2800
	if (!hctx->fq)
2801
		goto free_bitmap;
2802

2803
	if (hctx->flags & BLK_MQ_F_BLOCKING)
2804
		init_srcu_struct(hctx->srcu);
2805
	blk_mq_hctx_kobj_init(hctx);
2806

2807
	return hctx;
2808

2809
 free_bitmap:
2810
	sbitmap_free(&hctx->ctx_map);
2811 2812
 free_ctxs:
	kfree(hctx->ctxs);
2813 2814 2815 2816 2817 2818
 free_cpumask:
	free_cpumask_var(hctx->cpumask);
 free_hctx:
	kfree(hctx);
 fail_alloc_hctx:
	return NULL;
2819
}
2820 2821 2822 2823

static void blk_mq_init_cpu_queues(struct request_queue *q,
				   unsigned int nr_hw_queues)
{
J
Jens Axboe 已提交
2824 2825
	struct blk_mq_tag_set *set = q->tag_set;
	unsigned int i, j;
2826 2827 2828 2829

	for_each_possible_cpu(i) {
		struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
		struct blk_mq_hw_ctx *hctx;
M
Ming Lei 已提交
2830
		int k;
2831 2832 2833

		__ctx->cpu = i;
		spin_lock_init(&__ctx->lock);
M
Ming Lei 已提交
2834 2835 2836
		for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++)
			INIT_LIST_HEAD(&__ctx->rq_lists[k]);

2837 2838 2839 2840 2841 2842
		__ctx->queue = q;

		/*
		 * Set local node, IFF we have more than one hw queue. If
		 * not, we remain on the home node of the device
		 */
J
Jens Axboe 已提交
2843 2844 2845
		for (j = 0; j < set->nr_maps; j++) {
			hctx = blk_mq_map_queue_type(q, j, i);
			if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
2846
				hctx->numa_node = cpu_to_node(i);
J
Jens Axboe 已提交
2847
		}
2848 2849 2850
	}
}

2851 2852
static bool __blk_mq_alloc_map_and_request(struct blk_mq_tag_set *set,
					int hctx_idx)
2853
{
2854
	unsigned int flags = set->flags;
2855 2856 2857
	int ret = 0;

	set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
2858
					set->queue_depth, set->reserved_tags, flags);
2859 2860 2861 2862 2863 2864 2865 2866
	if (!set->tags[hctx_idx])
		return false;

	ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
				set->queue_depth);
	if (!ret)
		return true;

2867
	blk_mq_free_rq_map(set->tags[hctx_idx], flags);
2868 2869 2870 2871 2872 2873 2874
	set->tags[hctx_idx] = NULL;
	return false;
}

static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
					 unsigned int hctx_idx)
{
2875 2876
	unsigned int flags = set->flags;

2877
	if (set->tags && set->tags[hctx_idx]) {
2878
		blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
2879
		blk_mq_free_rq_map(set->tags[hctx_idx], flags);
2880 2881
		set->tags[hctx_idx] = NULL;
	}
2882 2883
}

2884
static void blk_mq_map_swqueue(struct request_queue *q)
2885
{
J
Jens Axboe 已提交
2886
	unsigned int i, j, hctx_idx;
2887 2888
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
M
Ming Lei 已提交
2889
	struct blk_mq_tag_set *set = q->tag_set;
2890 2891

	queue_for_each_hw_ctx(q, hctx, i) {
2892
		cpumask_clear(hctx->cpumask);
2893
		hctx->nr_ctx = 0;
2894
		hctx->dispatch_from = NULL;
2895 2896 2897
	}

	/*
2898
	 * Map software to hardware queues.
2899 2900
	 *
	 * If the cpu isn't present, the cpu is mapped to first hctx.
2901
	 */
2902
	for_each_possible_cpu(i) {
2903

2904
		ctx = per_cpu_ptr(q->queue_ctx, i);
J
Jens Axboe 已提交
2905
		for (j = 0; j < set->nr_maps; j++) {
2906 2907 2908
			if (!set->map[j].nr_queues) {
				ctx->hctxs[j] = blk_mq_map_queue_type(q,
						HCTX_TYPE_DEFAULT, i);
2909
				continue;
2910
			}
2911 2912 2913
			hctx_idx = set->map[j].mq_map[i];
			/* unmapped hw queue can be remapped after CPU topo changed */
			if (!set->tags[hctx_idx] &&
2914
			    !__blk_mq_alloc_map_and_request(set, hctx_idx)) {
2915 2916 2917 2918 2919 2920 2921 2922
				/*
				 * If tags initialization fail for some hctx,
				 * that hctx won't be brought online.  In this
				 * case, remap the current ctx to hctx[0] which
				 * is guaranteed to always have tags allocated
				 */
				set->map[j].mq_map[i] = 0;
			}
2923

J
Jens Axboe 已提交
2924
			hctx = blk_mq_map_queue_type(q, j, i);
2925
			ctx->hctxs[j] = hctx;
J
Jens Axboe 已提交
2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944
			/*
			 * If the CPU is already set in the mask, then we've
			 * mapped this one already. This can happen if
			 * devices share queues across queue maps.
			 */
			if (cpumask_test_cpu(i, hctx->cpumask))
				continue;

			cpumask_set_cpu(i, hctx->cpumask);
			hctx->type = j;
			ctx->index_hw[hctx->type] = hctx->nr_ctx;
			hctx->ctxs[hctx->nr_ctx++] = ctx;

			/*
			 * If the nr_ctx type overflows, we have exceeded the
			 * amount of sw queues we can support.
			 */
			BUG_ON(!hctx->nr_ctx);
		}
2945 2946 2947 2948

		for (; j < HCTX_MAX_TYPES; j++)
			ctx->hctxs[j] = blk_mq_map_queue_type(q,
					HCTX_TYPE_DEFAULT, i);
2949
	}
2950 2951

	queue_for_each_hw_ctx(q, hctx, i) {
2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966
		/*
		 * If no software queues are mapped to this hardware queue,
		 * disable it and free the request entries.
		 */
		if (!hctx->nr_ctx) {
			/* Never unmap queue 0.  We need it as a
			 * fallback in case of a new remap fails
			 * allocation
			 */
			if (i && set->tags[i])
				blk_mq_free_map_and_requests(set, i);

			hctx->tags = NULL;
			continue;
		}
2967

M
Ming Lei 已提交
2968 2969 2970
		hctx->tags = set->tags[i];
		WARN_ON(!hctx->tags);

2971 2972 2973 2974 2975
		/*
		 * Set the map size to the number of mapped software queues.
		 * This is more accurate and more efficient than looping
		 * over all possibly mapped software queues.
		 */
2976
		sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
2977

2978 2979 2980
		/*
		 * Initialize batch roundrobin counts
		 */
2981
		hctx->next_cpu = blk_mq_first_mapped_cpu(hctx);
2982 2983
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}
2984 2985
}

2986 2987 2988 2989
/*
 * Caller needs to ensure that we're either frozen/quiesced, or that
 * the queue isn't live yet.
 */
2990
static void queue_set_hctx_shared(struct request_queue *q, bool shared)
2991 2992 2993 2994
{
	struct blk_mq_hw_ctx *hctx;
	int i;

2995
	queue_for_each_hw_ctx(q, hctx, i) {
2996
		if (shared) {
2997
			hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
2998 2999
		} else {
			blk_mq_tag_idle(hctx);
3000
			hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
3001
		}
3002 3003 3004
	}
}

3005 3006
static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set,
					 bool shared)
3007 3008
{
	struct request_queue *q;
3009

3010 3011
	lockdep_assert_held(&set->tag_list_lock);

3012 3013
	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_freeze_queue(q);
3014
		queue_set_hctx_shared(q, shared);
3015 3016 3017 3018 3019 3020 3021 3022 3023
		blk_mq_unfreeze_queue(q);
	}
}

static void blk_mq_del_queue_tag_set(struct request_queue *q)
{
	struct blk_mq_tag_set *set = q->tag_set;

	mutex_lock(&set->tag_list_lock);
3024
	list_del(&q->tag_set_list);
3025 3026
	if (list_is_singular(&set->tag_list)) {
		/* just transitioned to unshared */
3027
		set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
3028
		/* update existing queue */
3029
		blk_mq_update_tag_set_shared(set, false);
3030
	}
3031
	mutex_unlock(&set->tag_list_lock);
3032
	INIT_LIST_HEAD(&q->tag_set_list);
3033 3034 3035 3036 3037 3038
}

static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
				     struct request_queue *q)
{
	mutex_lock(&set->tag_list_lock);
3039

3040 3041 3042 3043
	/*
	 * Check to see if we're transitioning to shared (from 1 to 2 queues).
	 */
	if (!list_empty(&set->tag_list) &&
3044 3045
	    !(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
		set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
3046
		/* update existing queue */
3047
		blk_mq_update_tag_set_shared(set, true);
3048
	}
3049
	if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
3050
		queue_set_hctx_shared(q, true);
3051
	list_add_tail(&q->tag_set_list, &set->tag_list);
3052

3053 3054 3055
	mutex_unlock(&set->tag_list_lock);
}

3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083
/* All allocations will be freed in release handler of q->mq_kobj */
static int blk_mq_alloc_ctxs(struct request_queue *q)
{
	struct blk_mq_ctxs *ctxs;
	int cpu;

	ctxs = kzalloc(sizeof(*ctxs), GFP_KERNEL);
	if (!ctxs)
		return -ENOMEM;

	ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx);
	if (!ctxs->queue_ctx)
		goto fail;

	for_each_possible_cpu(cpu) {
		struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu);
		ctx->ctxs = ctxs;
	}

	q->mq_kobj = &ctxs->kobj;
	q->queue_ctx = ctxs->queue_ctx;

	return 0;
 fail:
	kfree(ctxs);
	return -ENOMEM;
}

3084 3085 3086 3087 3088 3089 3090 3091
/*
 * It is the actual release handler for mq, but we do it from
 * request queue's release handler for avoiding use-after-free
 * and headache because q->mq_kobj shouldn't have been introduced,
 * but we can't group ctx/kctx kobj without it.
 */
void blk_mq_release(struct request_queue *q)
{
3092 3093
	struct blk_mq_hw_ctx *hctx, *next;
	int i;
3094

3095 3096 3097 3098 3099 3100
	queue_for_each_hw_ctx(q, hctx, i)
		WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list));

	/* all hctx are in .unused_hctx_list now */
	list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) {
		list_del_init(&hctx->hctx_list);
3101
		kobject_put(&hctx->kobj);
3102
	}
3103 3104 3105

	kfree(q->queue_hw_ctx);

3106 3107 3108 3109 3110
	/*
	 * release .mq_kobj and sw queue's kobject now because
	 * both share lifetime with request queue.
	 */
	blk_mq_sysfs_deinit(q);
3111 3112
}

3113
static struct request_queue *blk_mq_init_queue_data(struct blk_mq_tag_set *set,
3114
		void *queuedata)
3115
{
3116 3117
	struct request_queue *q;
	int ret;
3118

3119 3120
	q = blk_alloc_queue(set->numa_node);
	if (!q)
3121
		return ERR_PTR(-ENOMEM);
3122 3123 3124 3125 3126 3127
	q->queuedata = queuedata;
	ret = blk_mq_init_allocated_queue(set, q);
	if (ret) {
		blk_cleanup_queue(q);
		return ERR_PTR(ret);
	}
3128 3129
	return q;
}
3130 3131 3132 3133 3134

struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
{
	return blk_mq_init_queue_data(set, NULL);
}
3135 3136
EXPORT_SYMBOL(blk_mq_init_queue);

3137 3138
struct gendisk *__blk_mq_alloc_disk(struct blk_mq_tag_set *set, void *queuedata,
		struct lock_class_key *lkclass)
3139 3140
{
	struct request_queue *q;
3141
	struct gendisk *disk;
3142

3143 3144 3145
	q = blk_mq_init_queue_data(set, queuedata);
	if (IS_ERR(q))
		return ERR_CAST(q);
3146

3147
	disk = __alloc_disk_node(q, set->numa_node, lkclass);
3148 3149 3150
	if (!disk) {
		blk_cleanup_queue(q);
		return ERR_PTR(-ENOMEM);
3151
	}
3152
	return disk;
3153
}
3154
EXPORT_SYMBOL(__blk_mq_alloc_disk);
3155

3156 3157 3158 3159
static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(
		struct blk_mq_tag_set *set, struct request_queue *q,
		int hctx_idx, int node)
{
3160
	struct blk_mq_hw_ctx *hctx = NULL, *tmp;
3161

3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175
	/* reuse dead hctx first */
	spin_lock(&q->unused_hctx_lock);
	list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) {
		if (tmp->numa_node == node) {
			hctx = tmp;
			break;
		}
	}
	if (hctx)
		list_del_init(&hctx->hctx_list);
	spin_unlock(&q->unused_hctx_lock);

	if (!hctx)
		hctx = blk_mq_alloc_hctx(q, set, node);
3176
	if (!hctx)
3177
		goto fail;
3178

3179 3180
	if (blk_mq_init_hctx(q, set, hctx, hctx_idx))
		goto free_hctx;
3181 3182

	return hctx;
3183 3184 3185 3186 3187

 free_hctx:
	kobject_put(&hctx->kobj);
 fail:
	return NULL;
3188 3189
}

K
Keith Busch 已提交
3190 3191
static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
						struct request_queue *q)
3192
{
3193
	int i, j, end;
K
Keith Busch 已提交
3194
	struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
3195

3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211
	if (q->nr_hw_queues < set->nr_hw_queues) {
		struct blk_mq_hw_ctx **new_hctxs;

		new_hctxs = kcalloc_node(set->nr_hw_queues,
				       sizeof(*new_hctxs), GFP_KERNEL,
				       set->numa_node);
		if (!new_hctxs)
			return;
		if (hctxs)
			memcpy(new_hctxs, hctxs, q->nr_hw_queues *
			       sizeof(*hctxs));
		q->queue_hw_ctx = new_hctxs;
		kfree(hctxs);
		hctxs = new_hctxs;
	}

3212 3213
	/* protect against switching io scheduler  */
	mutex_lock(&q->sysfs_lock);
3214
	for (i = 0; i < set->nr_hw_queues; i++) {
K
Keith Busch 已提交
3215
		int node;
3216
		struct blk_mq_hw_ctx *hctx;
K
Keith Busch 已提交
3217

3218
		node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], i);
3219 3220 3221 3222 3223 3224 3225
		/*
		 * If the hw queue has been mapped to another numa node,
		 * we need to realloc the hctx. If allocation fails, fallback
		 * to use the previous one.
		 */
		if (hctxs[i] && (hctxs[i]->numa_node == node))
			continue;
K
Keith Busch 已提交
3226

3227 3228
		hctx = blk_mq_alloc_and_init_hctx(set, q, i, node);
		if (hctx) {
3229
			if (hctxs[i])
3230 3231 3232 3233 3234 3235 3236 3237 3238
				blk_mq_exit_hctx(q, set, hctxs[i], i);
			hctxs[i] = hctx;
		} else {
			if (hctxs[i])
				pr_warn("Allocate new hctx on node %d fails,\
						fallback to previous one on node %d\n",
						node, hctxs[i]->numa_node);
			else
				break;
K
Keith Busch 已提交
3239
		}
3240
	}
3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252
	/*
	 * Increasing nr_hw_queues fails. Free the newly allocated
	 * hctxs and keep the previous q->nr_hw_queues.
	 */
	if (i != set->nr_hw_queues) {
		j = q->nr_hw_queues;
		end = i;
	} else {
		j = i;
		end = q->nr_hw_queues;
		q->nr_hw_queues = set->nr_hw_queues;
	}
3253

3254
	for (; j < end; j++) {
K
Keith Busch 已提交
3255 3256 3257
		struct blk_mq_hw_ctx *hctx = hctxs[j];

		if (hctx) {
3258 3259
			if (hctx->tags)
				blk_mq_free_map_and_requests(set, j);
K
Keith Busch 已提交
3260 3261 3262 3263
			blk_mq_exit_hctx(q, set, hctx, j);
			hctxs[j] = NULL;
		}
	}
3264
	mutex_unlock(&q->sysfs_lock);
K
Keith Busch 已提交
3265 3266
}

3267 3268
int blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
		struct request_queue *q)
K
Keith Busch 已提交
3269
{
M
Ming Lei 已提交
3270 3271 3272
	/* mark the queue as mq asap */
	q->mq_ops = set->ops;

3273
	q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn,
3274 3275
					     blk_mq_poll_stats_bkt,
					     BLK_MQ_POLL_STATS_BKTS, q);
3276 3277 3278
	if (!q->poll_cb)
		goto err_exit;

3279
	if (blk_mq_alloc_ctxs(q))
3280
		goto err_poll;
K
Keith Busch 已提交
3281

3282 3283 3284
	/* init q->mq_kobj and sw queues' kobjects */
	blk_mq_sysfs_init(q);

3285 3286 3287
	INIT_LIST_HEAD(&q->unused_hctx_list);
	spin_lock_init(&q->unused_hctx_lock);

K
Keith Busch 已提交
3288 3289 3290
	blk_mq_realloc_hw_ctxs(set, q);
	if (!q->nr_hw_queues)
		goto err_hctxs;
3291

3292
	INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
3293
	blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
3294

J
Jens Axboe 已提交
3295
	q->tag_set = set;
3296

3297
	q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
3298 3299
	if (set->nr_maps > HCTX_TYPE_POLL &&
	    set->map[HCTX_TYPE_POLL].nr_queues)
3300
		blk_queue_flag_set(QUEUE_FLAG_POLL, q);
3301

3302
	INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
3303 3304 3305
	INIT_LIST_HEAD(&q->requeue_list);
	spin_lock_init(&q->requeue_lock);

3306 3307
	q->nr_requests = set->queue_depth;

3308 3309 3310
	/*
	 * Default to classic polling
	 */
3311
	q->poll_nsec = BLK_MQ_POLL_CLASSIC;
3312

3313
	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
3314
	blk_mq_add_queue_tag_set(set, q);
3315
	blk_mq_map_swqueue(q);
3316
	return 0;
3317

3318
err_hctxs:
K
Keith Busch 已提交
3319
	kfree(q->queue_hw_ctx);
3320
	q->nr_hw_queues = 0;
3321
	blk_mq_sysfs_deinit(q);
3322 3323 3324
err_poll:
	blk_stat_free_callback(q->poll_cb);
	q->poll_cb = NULL;
M
Ming Lin 已提交
3325 3326
err_exit:
	q->mq_ops = NULL;
3327
	return -ENOMEM;
3328
}
3329
EXPORT_SYMBOL(blk_mq_init_allocated_queue);
3330

3331 3332
/* tags can _not_ be used after returning from blk_mq_exit_queue */
void blk_mq_exit_queue(struct request_queue *q)
3333
{
3334
	struct blk_mq_tag_set *set = q->tag_set;
3335

3336
	/* Checks hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED. */
M
Ming Lei 已提交
3337
	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
3338 3339
	/* May clear BLK_MQ_F_TAG_QUEUE_SHARED in hctx->flags. */
	blk_mq_del_queue_tag_set(q);
3340 3341
}

3342 3343 3344 3345
static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	int i;

3346
	for (i = 0; i < set->nr_hw_queues; i++) {
3347
		if (!__blk_mq_alloc_map_and_request(set, i))
3348
			goto out_unwind;
3349 3350
		cond_resched();
	}
3351 3352 3353 3354 3355

	return 0;

out_unwind:
	while (--i >= 0)
3356
		blk_mq_free_map_and_requests(set, i);
3357 3358 3359 3360 3361 3362 3363 3364 3365

	return -ENOMEM;
}

/*
 * Allocate the request maps associated with this tag_set. Note that this
 * may reduce the depth asked for, if memory is tight. set->queue_depth
 * will be updated to reflect the allocated depth.
 */
3366
static int blk_mq_alloc_map_and_requests(struct blk_mq_tag_set *set)
3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395
{
	unsigned int depth;
	int err;

	depth = set->queue_depth;
	do {
		err = __blk_mq_alloc_rq_maps(set);
		if (!err)
			break;

		set->queue_depth >>= 1;
		if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
			err = -ENOMEM;
			break;
		}
	} while (set->queue_depth);

	if (!set->queue_depth || err) {
		pr_err("blk-mq: failed to allocate request map\n");
		return -ENOMEM;
	}

	if (depth != set->queue_depth)
		pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
						depth, set->queue_depth);

	return 0;
}

3396 3397
static int blk_mq_update_queue_map(struct blk_mq_tag_set *set)
{
3398 3399 3400 3401 3402 3403 3404 3405
	/*
	 * blk_mq_map_queues() and multiple .map_queues() implementations
	 * expect that set->map[HCTX_TYPE_DEFAULT].nr_queues is set to the
	 * number of hardware queues.
	 */
	if (set->nr_maps == 1)
		set->map[HCTX_TYPE_DEFAULT].nr_queues = set->nr_hw_queues;

3406
	if (set->ops->map_queues && !is_kdump_kernel()) {
J
Jens Axboe 已提交
3407 3408
		int i;

3409 3410 3411 3412 3413 3414 3415
		/*
		 * transport .map_queues is usually done in the following
		 * way:
		 *
		 * for (queue = 0; queue < set->nr_hw_queues; queue++) {
		 * 	mask = get_cpu_mask(queue)
		 * 	for_each_cpu(cpu, mask)
J
Jens Axboe 已提交
3416
		 * 		set->map[x].mq_map[cpu] = queue;
3417 3418 3419 3420 3421 3422
		 * }
		 *
		 * When we need to remap, the table has to be cleared for
		 * killing stale mapping since one CPU may not be mapped
		 * to any hw queue.
		 */
J
Jens Axboe 已提交
3423 3424
		for (i = 0; i < set->nr_maps; i++)
			blk_mq_clear_mq_map(&set->map[i]);
3425

3426
		return set->ops->map_queues(set);
J
Jens Axboe 已提交
3427 3428
	} else {
		BUG_ON(set->nr_maps > 1);
3429
		return blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
J
Jens Axboe 已提交
3430
	}
3431 3432
}

3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455
static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set,
				  int cur_nr_hw_queues, int new_nr_hw_queues)
{
	struct blk_mq_tags **new_tags;

	if (cur_nr_hw_queues >= new_nr_hw_queues)
		return 0;

	new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *),
				GFP_KERNEL, set->numa_node);
	if (!new_tags)
		return -ENOMEM;

	if (set->tags)
		memcpy(new_tags, set->tags, cur_nr_hw_queues *
		       sizeof(*set->tags));
	kfree(set->tags);
	set->tags = new_tags;
	set->nr_hw_queues = new_nr_hw_queues;

	return 0;
}

3456 3457 3458 3459 3460 3461
static int blk_mq_alloc_tag_set_tags(struct blk_mq_tag_set *set,
				int new_nr_hw_queues)
{
	return blk_mq_realloc_tag_set_tags(set, 0, new_nr_hw_queues);
}

3462 3463 3464
/*
 * Alloc a tag set to be associated with one or more request queues.
 * May fail with EINVAL for various error conditions. May adjust the
3465
 * requested depth down, if it's too large. In that case, the set
3466 3467
 * value will be stored in set->queue_depth.
 */
3468 3469
int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
{
J
Jens Axboe 已提交
3470
	int i, ret;
3471

B
Bart Van Assche 已提交
3472 3473
	BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);

3474 3475
	if (!set->nr_hw_queues)
		return -EINVAL;
3476
	if (!set->queue_depth)
3477 3478 3479 3480
		return -EINVAL;
	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
		return -EINVAL;

C
Christoph Hellwig 已提交
3481
	if (!set->ops->queue_rq)
3482 3483
		return -EINVAL;

3484 3485 3486
	if (!set->ops->get_budget ^ !set->ops->put_budget)
		return -EINVAL;

3487 3488 3489 3490 3491
	if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
		pr_info("blk-mq: reduced tag depth to %u\n",
			BLK_MQ_MAX_DEPTH);
		set->queue_depth = BLK_MQ_MAX_DEPTH;
	}
3492

J
Jens Axboe 已提交
3493 3494 3495 3496 3497
	if (!set->nr_maps)
		set->nr_maps = 1;
	else if (set->nr_maps > HCTX_MAX_TYPES)
		return -EINVAL;

3498 3499 3500 3501 3502 3503 3504
	/*
	 * If a crashdump is active, then we are potentially in a very
	 * memory constrained environment. Limit us to 1 queue and
	 * 64 tags to prevent using too much memory.
	 */
	if (is_kdump_kernel()) {
		set->nr_hw_queues = 1;
3505
		set->nr_maps = 1;
3506 3507
		set->queue_depth = min(64U, set->queue_depth);
	}
K
Keith Busch 已提交
3508
	/*
3509 3510
	 * There is no use for more h/w queues than cpus if we just have
	 * a single map
K
Keith Busch 已提交
3511
	 */
3512
	if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids)
K
Keith Busch 已提交
3513
		set->nr_hw_queues = nr_cpu_ids;
3514

3515
	if (blk_mq_alloc_tag_set_tags(set, set->nr_hw_queues) < 0)
3516
		return -ENOMEM;
3517

3518
	ret = -ENOMEM;
J
Jens Axboe 已提交
3519 3520
	for (i = 0; i < set->nr_maps; i++) {
		set->map[i].mq_map = kcalloc_node(nr_cpu_ids,
3521
						  sizeof(set->map[i].mq_map[0]),
J
Jens Axboe 已提交
3522 3523 3524
						  GFP_KERNEL, set->numa_node);
		if (!set->map[i].mq_map)
			goto out_free_mq_map;
3525
		set->map[i].nr_queues = is_kdump_kernel() ? 1 : set->nr_hw_queues;
J
Jens Axboe 已提交
3526
	}
3527

3528
	ret = blk_mq_update_queue_map(set);
3529 3530 3531
	if (ret)
		goto out_free_mq_map;

3532
	ret = blk_mq_alloc_map_and_requests(set);
3533
	if (ret)
3534
		goto out_free_mq_map;
3535

3536
	if (blk_mq_is_sbitmap_shared(set->flags)) {
3537 3538
		atomic_set(&set->active_queues_shared_sbitmap, 0);

3539
		if (blk_mq_init_shared_sbitmap(set)) {
3540 3541 3542 3543 3544
			ret = -ENOMEM;
			goto out_free_mq_rq_maps;
		}
	}

3545 3546 3547
	mutex_init(&set->tag_list_lock);
	INIT_LIST_HEAD(&set->tag_list);

3548
	return 0;
3549

3550 3551 3552
out_free_mq_rq_maps:
	for (i = 0; i < set->nr_hw_queues; i++)
		blk_mq_free_map_and_requests(set, i);
3553
out_free_mq_map:
J
Jens Axboe 已提交
3554 3555 3556 3557
	for (i = 0; i < set->nr_maps; i++) {
		kfree(set->map[i].mq_map);
		set->map[i].mq_map = NULL;
	}
3558 3559
	kfree(set->tags);
	set->tags = NULL;
3560
	return ret;
3561 3562 3563
}
EXPORT_SYMBOL(blk_mq_alloc_tag_set);

3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579
/* allocate and initialize a tagset for a simple single-queue device */
int blk_mq_alloc_sq_tag_set(struct blk_mq_tag_set *set,
		const struct blk_mq_ops *ops, unsigned int queue_depth,
		unsigned int set_flags)
{
	memset(set, 0, sizeof(*set));
	set->ops = ops;
	set->nr_hw_queues = 1;
	set->nr_maps = 1;
	set->queue_depth = queue_depth;
	set->numa_node = NUMA_NO_NODE;
	set->flags = set_flags;
	return blk_mq_alloc_tag_set(set);
}
EXPORT_SYMBOL_GPL(blk_mq_alloc_sq_tag_set);

3580 3581
void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
{
J
Jens Axboe 已提交
3582
	int i, j;
3583

3584
	for (i = 0; i < set->nr_hw_queues; i++)
3585
		blk_mq_free_map_and_requests(set, i);
3586

3587 3588 3589
	if (blk_mq_is_sbitmap_shared(set->flags))
		blk_mq_exit_shared_sbitmap(set);

J
Jens Axboe 已提交
3590 3591 3592 3593
	for (j = 0; j < set->nr_maps; j++) {
		kfree(set->map[j].mq_map);
		set->map[j].mq_map = NULL;
	}
3594

M
Ming Lei 已提交
3595
	kfree(set->tags);
3596
	set->tags = NULL;
3597 3598 3599
}
EXPORT_SYMBOL(blk_mq_free_tag_set);

3600 3601 3602 3603 3604 3605
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
{
	struct blk_mq_tag_set *set = q->tag_set;
	struct blk_mq_hw_ctx *hctx;
	int i, ret;

3606
	if (!set)
3607 3608
		return -EINVAL;

3609 3610 3611
	if (q->nr_requests == nr)
		return 0;

3612
	blk_mq_freeze_queue(q);
3613
	blk_mq_quiesce_queue(q);
3614

3615 3616
	ret = 0;
	queue_for_each_hw_ctx(q, hctx, i) {
3617 3618
		if (!hctx->tags)
			continue;
3619 3620 3621 3622
		/*
		 * If we're using an MQ scheduler, just update the scheduler
		 * queue depth. This is similar to what the old code would do.
		 */
3623
		if (!hctx->sched_tags) {
3624
			ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr,
3625
							false);
3626 3627
			if (!ret && blk_mq_is_sbitmap_shared(set->flags))
				blk_mq_tag_resize_shared_sbitmap(set, nr);
3628 3629 3630
		} else {
			ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
							nr, true);
3631 3632 3633 3634 3635 3636
			if (blk_mq_is_sbitmap_shared(set->flags)) {
				hctx->sched_tags->bitmap_tags =
					&q->sched_bitmap_tags;
				hctx->sched_tags->breserved_tags =
					&q->sched_breserved_tags;
			}
3637
		}
3638 3639
		if (ret)
			break;
3640 3641
		if (q->elevator && q->elevator->type->ops.depth_updated)
			q->elevator->type->ops.depth_updated(hctx);
3642
	}
3643
	if (!ret) {
3644
		q->nr_requests = nr;
3645 3646 3647 3648
		if (q->elevator && blk_mq_is_sbitmap_shared(set->flags))
			sbitmap_queue_resize(&q->sched_bitmap_tags,
					     nr - set->reserved_tags);
	}
3649

3650
	blk_mq_unquiesce_queue(q);
3651 3652
	blk_mq_unfreeze_queue(q);

3653 3654 3655
	return ret;
}

3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725
/*
 * request_queue and elevator_type pair.
 * It is just used by __blk_mq_update_nr_hw_queues to cache
 * the elevator_type associated with a request_queue.
 */
struct blk_mq_qe_pair {
	struct list_head node;
	struct request_queue *q;
	struct elevator_type *type;
};

/*
 * Cache the elevator_type in qe pair list and switch the
 * io scheduler to 'none'
 */
static bool blk_mq_elv_switch_none(struct list_head *head,
		struct request_queue *q)
{
	struct blk_mq_qe_pair *qe;

	if (!q->elevator)
		return true;

	qe = kmalloc(sizeof(*qe), GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY);
	if (!qe)
		return false;

	INIT_LIST_HEAD(&qe->node);
	qe->q = q;
	qe->type = q->elevator->type;
	list_add(&qe->node, head);

	mutex_lock(&q->sysfs_lock);
	/*
	 * After elevator_switch_mq, the previous elevator_queue will be
	 * released by elevator_release. The reference of the io scheduler
	 * module get by elevator_get will also be put. So we need to get
	 * a reference of the io scheduler module here to prevent it to be
	 * removed.
	 */
	__module_get(qe->type->elevator_owner);
	elevator_switch_mq(q, NULL);
	mutex_unlock(&q->sysfs_lock);

	return true;
}

static void blk_mq_elv_switch_back(struct list_head *head,
		struct request_queue *q)
{
	struct blk_mq_qe_pair *qe;
	struct elevator_type *t = NULL;

	list_for_each_entry(qe, head, node)
		if (qe->q == q) {
			t = qe->type;
			break;
		}

	if (!t)
		return;

	list_del(&qe->node);
	kfree(qe);

	mutex_lock(&q->sysfs_lock);
	elevator_switch_mq(q, t);
	mutex_unlock(&q->sysfs_lock);
}

3726 3727
static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
							int nr_hw_queues)
K
Keith Busch 已提交
3728 3729
{
	struct request_queue *q;
3730
	LIST_HEAD(head);
3731
	int prev_nr_hw_queues;
K
Keith Busch 已提交
3732

3733 3734
	lockdep_assert_held(&set->tag_list_lock);

3735
	if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids)
K
Keith Busch 已提交
3736
		nr_hw_queues = nr_cpu_ids;
3737 3738 3739
	if (nr_hw_queues < 1)
		return;
	if (set->nr_maps == 1 && nr_hw_queues == set->nr_hw_queues)
K
Keith Busch 已提交
3740 3741 3742 3743
		return;

	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_freeze_queue(q);
3744 3745 3746 3747 3748 3749 3750 3751
	/*
	 * Switch IO scheduler to 'none', cleaning up the data associated
	 * with the previous scheduler. We will switch back once we are done
	 * updating the new sw to hw queue mappings.
	 */
	list_for_each_entry(q, &set->tag_list, tag_set_list)
		if (!blk_mq_elv_switch_none(&head, q))
			goto switch_back;
K
Keith Busch 已提交
3752

3753 3754 3755 3756 3757
	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_debugfs_unregister_hctxs(q);
		blk_mq_sysfs_unregister(q);
	}

3758
	prev_nr_hw_queues = set->nr_hw_queues;
3759 3760 3761 3762
	if (blk_mq_realloc_tag_set_tags(set, set->nr_hw_queues, nr_hw_queues) <
	    0)
		goto reregister;

K
Keith Busch 已提交
3763
	set->nr_hw_queues = nr_hw_queues;
3764
fallback:
3765
	blk_mq_update_queue_map(set);
K
Keith Busch 已提交
3766 3767
	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_realloc_hw_ctxs(set, q);
3768 3769 3770 3771
		if (q->nr_hw_queues != set->nr_hw_queues) {
			pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n",
					nr_hw_queues, prev_nr_hw_queues);
			set->nr_hw_queues = prev_nr_hw_queues;
3772
			blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
3773 3774
			goto fallback;
		}
3775 3776 3777
		blk_mq_map_swqueue(q);
	}

3778
reregister:
3779 3780 3781
	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_sysfs_register(q);
		blk_mq_debugfs_register_hctxs(q);
K
Keith Busch 已提交
3782 3783
	}

3784 3785 3786 3787
switch_back:
	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_elv_switch_back(&head, q);

K
Keith Busch 已提交
3788 3789 3790
	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_unfreeze_queue(q);
}
3791 3792 3793 3794 3795 3796 3797

void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
{
	mutex_lock(&set->tag_list_lock);
	__blk_mq_update_nr_hw_queues(set, nr_hw_queues);
	mutex_unlock(&set->tag_list_lock);
}
K
Keith Busch 已提交
3798 3799
EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);

3800 3801 3802 3803
/* Enable polling stats and return whether they were already enabled. */
static bool blk_poll_stats_enable(struct request_queue *q)
{
	if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
3804
	    blk_queue_flag_test_and_set(QUEUE_FLAG_POLL_STATS, q))
3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825
		return true;
	blk_stat_add_callback(q, q->poll_cb);
	return false;
}

static void blk_mq_poll_stats_start(struct request_queue *q)
{
	/*
	 * We don't arm the callback if polling stats are not enabled or the
	 * callback is already active.
	 */
	if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
	    blk_stat_is_active(q->poll_cb))
		return;

	blk_stat_activate_msecs(q->poll_cb, 100);
}

static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb)
{
	struct request_queue *q = cb->data;
3826
	int bucket;
3827

3828 3829 3830 3831
	for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) {
		if (cb->stat[bucket].nr_samples)
			q->poll_stat[bucket] = cb->stat[bucket];
	}
3832 3833
}

3834 3835 3836 3837
static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
				       struct request *rq)
{
	unsigned long ret = 0;
3838
	int bucket;
3839 3840 3841 3842 3843

	/*
	 * If stats collection isn't on, don't sleep but turn it on for
	 * future users
	 */
3844
	if (!blk_poll_stats_enable(q))
3845 3846 3847 3848 3849 3850 3851 3852
		return 0;

	/*
	 * As an optimistic guess, use half of the mean service time
	 * for this type of request. We can (and should) make this smarter.
	 * For instance, if the completion latencies are tight, we can
	 * get closer than just half the mean. This is especially
	 * important on devices where the completion latencies are longer
3853 3854
	 * than ~10 usec. We do use the stats for the relevant IO size
	 * if available which does lead to better estimates.
3855
	 */
3856 3857 3858 3859 3860 3861
	bucket = blk_mq_poll_stats_bkt(rq);
	if (bucket < 0)
		return ret;

	if (q->poll_stat[bucket].nr_samples)
		ret = (q->poll_stat[bucket].mean + 1) / 2;
3862 3863 3864 3865

	return ret;
}

3866 3867 3868 3869 3870
static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
				     struct request *rq)
{
	struct hrtimer_sleeper hs;
	enum hrtimer_mode mode;
3871
	unsigned int nsecs;
3872 3873
	ktime_t kt;

J
Jens Axboe 已提交
3874
	if (rq->rq_flags & RQF_MQ_POLL_SLEPT)
3875 3876 3877
		return false;

	/*
3878
	 * If we get here, hybrid polling is enabled. Hence poll_nsec can be:
3879 3880 3881 3882
	 *
	 *  0:	use half of prev avg
	 * >0:	use this specific value
	 */
3883
	if (q->poll_nsec > 0)
3884 3885
		nsecs = q->poll_nsec;
	else
3886
		nsecs = blk_mq_poll_nsecs(q, rq);
3887 3888

	if (!nsecs)
3889 3890
		return false;

J
Jens Axboe 已提交
3891
	rq->rq_flags |= RQF_MQ_POLL_SLEPT;
3892 3893 3894 3895 3896

	/*
	 * This will be replaced with the stats tracking code, using
	 * 'avg_completion_time / 2' as the pre-sleep target.
	 */
T
Thomas Gleixner 已提交
3897
	kt = nsecs;
3898 3899

	mode = HRTIMER_MODE_REL;
3900
	hrtimer_init_sleeper_on_stack(&hs, CLOCK_MONOTONIC, mode);
3901 3902 3903
	hrtimer_set_expires(&hs.timer, kt);

	do {
T
Tejun Heo 已提交
3904
		if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE)
3905 3906
			break;
		set_current_state(TASK_UNINTERRUPTIBLE);
3907
		hrtimer_sleeper_start_expires(&hs, mode);
3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918
		if (hs.task)
			io_schedule();
		hrtimer_cancel(&hs.timer);
		mode = HRTIMER_MODE_ABS;
	} while (hs.task && !signal_pending(current));

	__set_current_state(TASK_RUNNING);
	destroy_hrtimer_on_stack(&hs.timer);
	return true;
}

3919 3920
static bool blk_mq_poll_hybrid(struct request_queue *q,
			       struct blk_mq_hw_ctx *hctx, blk_qc_t cookie)
J
Jens Axboe 已提交
3921
{
3922 3923
	struct request *rq;

3924
	if (q->poll_nsec == BLK_MQ_POLL_CLASSIC)
3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940
		return false;

	if (!blk_qc_t_is_internal(cookie))
		rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
	else {
		rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
		/*
		 * With scheduling, if the request has completed, we'll
		 * get a NULL return here, as we clear the sched tag when
		 * that happens. The request still remains valid, like always,
		 * so we should be safe with just the NULL check.
		 */
		if (!rq)
			return false;
	}

3941
	return blk_mq_poll_hybrid_sleep(q, rq);
3942 3943
}

C
Christoph Hellwig 已提交
3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956
/**
 * blk_poll - poll for IO completions
 * @q:  the queue
 * @cookie: cookie passed back at IO submission time
 * @spin: whether to spin for completions
 *
 * Description:
 *    Poll for completions on the passed in queue. Returns number of
 *    completed entries found. If @spin is true, then blk_poll will continue
 *    looping until at least one completion is found, unless the task is
 *    otherwise marked running (or we need to reschedule).
 */
int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin)
3957 3958
{
	struct blk_mq_hw_ctx *hctx;
3959
	unsigned int state;
J
Jens Axboe 已提交
3960

C
Christoph Hellwig 已提交
3961 3962
	if (!blk_qc_t_valid(cookie) ||
	    !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
3963 3964
		return 0;

C
Christoph Hellwig 已提交
3965 3966 3967
	if (current->plug)
		blk_flush_plug_list(current->plug, false);

3968 3969
	hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];

3970 3971 3972 3973 3974
	/*
	 * If we sleep, have the caller restart the poll loop to reset
	 * the state. Like for the other success return cases, the
	 * caller is responsible for checking if the IO completed. If
	 * the IO isn't complete, we'll get called again and will go
3975 3976
	 * straight to the busy poll loop. If specified not to spin,
	 * we also should not sleep.
3977
	 */
3978
	if (spin && blk_mq_poll_hybrid(q, hctx, cookie))
3979
		return 1;
3980

J
Jens Axboe 已提交
3981 3982
	hctx->poll_considered++;

P
Peter Zijlstra 已提交
3983
	state = get_current_state();
3984
	do {
J
Jens Axboe 已提交
3985 3986 3987 3988
		int ret;

		hctx->poll_invoked++;

3989
		ret = q->mq_ops->poll(hctx);
J
Jens Axboe 已提交
3990 3991
		if (ret > 0) {
			hctx->poll_success++;
3992
			__set_current_state(TASK_RUNNING);
3993
			return ret;
J
Jens Axboe 已提交
3994 3995 3996
		}

		if (signal_pending_state(state, current))
3997
			__set_current_state(TASK_RUNNING);
J
Jens Axboe 已提交
3998

3999
		if (task_is_running(current))
4000
			return 1;
4001
		if (ret < 0 || !spin)
J
Jens Axboe 已提交
4002 4003
			break;
		cpu_relax();
4004
	} while (!need_resched());
J
Jens Axboe 已提交
4005

4006
	__set_current_state(TASK_RUNNING);
4007
	return 0;
J
Jens Axboe 已提交
4008
}
C
Christoph Hellwig 已提交
4009
EXPORT_SYMBOL_GPL(blk_poll);
J
Jens Axboe 已提交
4010

J
Jens Axboe 已提交
4011 4012 4013 4014 4015 4016
unsigned int blk_mq_rq_cpu(struct request *rq)
{
	return rq->mq_ctx->cpu;
}
EXPORT_SYMBOL(blk_mq_rq_cpu);

4017 4018
static int __init blk_mq_init(void)
{
4019 4020 4021
	int i;

	for_each_possible_cpu(i)
4022
		init_llist_head(&per_cpu(blk_cpu_done, i));
4023 4024 4025 4026 4027
	open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);

	cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD,
				  "block/softirq:dead", NULL,
				  blk_softirq_cpu_dead);
4028 4029
	cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
				blk_mq_hctx_notify_dead);
4030 4031 4032
	cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online",
				blk_mq_hctx_notify_online,
				blk_mq_hctx_notify_offline);
4033 4034 4035
	return 0;
}
subsys_initcall(blk_mq_init);